b8d4d94b18
- Replace SetRanking/GetRanking/SetRankingPageHTML/GetRankingPageHTML blob methods with WriteRankingItem/ReadRankingItems/RankingFreshEnough per-item operations - Add 24h staleness gate in ScrapeRanking to skip re-scraping fresh data - Add GET /api/ranking endpoint returning []RankingItem sorted by rank - Remove RankingPageCacher interface and rankingCacheAdapter adapter - Update integration tests to use new per-item upsert semantics - Include e2e test suite (scraper/internal/e2e/)
817 lines
27 KiB
Go
817 lines
27 KiB
Go
//go:build integration
|
||
|
||
// End-to-end integration test for libnovel.
|
||
//
|
||
// Scenario (executed in order):
|
||
// 1. Health-check all Docker services (PocketBase, MinIO, Browserless, scraper).
|
||
// 2. Register a test user in the app_users PocketBase collection.
|
||
// 3. Scrape the popular-ranking page 1 and capture the first book.
|
||
// 4. Scrape full metadata for that book and persist it; verify in PocketBase.
|
||
// 5. Scrape chapters 1–3 and persist them; verify in MinIO + PocketBase.
|
||
// 6. Generate TTS audio for the first 100 chars of each chapter via the scraper
|
||
// HTTP API; verify MinIO object + PocketBase audio_cache entry.
|
||
// 7. Fetch presigned URLs for each chapter's markdown and audio; verify HTTP 200.
|
||
//
|
||
// Prerequisites (all must be running):
|
||
//
|
||
// docker-compose up -d minio pocketbase browserless scraper
|
||
//
|
||
// Run with:
|
||
//
|
||
// BROWSERLESS_URL=http://localhost:3030 \
|
||
// MINIO_ENDPOINT=localhost:9000 \
|
||
// POCKETBASE_URL=http://localhost:8090 \
|
||
// SCRAPER_URL=http://localhost:8080 \
|
||
// go test -v -tags integration -timeout 900s \
|
||
// github.com/libnovel/scraper/internal/e2e
|
||
package e2e
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"log/slog"
|
||
"net/http"
|
||
"os"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/libnovel/scraper/internal/browser"
|
||
"github.com/libnovel/scraper/internal/novelfire"
|
||
"github.com/libnovel/scraper/internal/scraper"
|
||
"github.com/libnovel/scraper/internal/storage"
|
||
)
|
||
|
||
// ─── env helpers ─────────────────────────────────────────────────────────────
|
||
|
||
func envOr(key, def string) string {
|
||
if v := os.Getenv(key); v != "" {
|
||
return v
|
||
}
|
||
return def
|
||
}
|
||
|
||
// ─── fixture ─────────────────────────────────────────────────────────────────
|
||
|
||
type e2eFixture struct {
|
||
sc *novelfire.Scraper
|
||
hs *storage.HybridStore
|
||
scraperURL string // base URL of the running scraper HTTP server
|
||
pbBaseURL string
|
||
pbEmail string
|
||
pbPassword string
|
||
}
|
||
|
||
func newE2EFixture(t *testing.T) *e2eFixture {
|
||
t.Helper()
|
||
|
||
browserlessURL := envOr("BROWSERLESS_URL", "")
|
||
if browserlessURL == "" {
|
||
t.Skip("BROWSERLESS_URL not set — skipping e2e test")
|
||
}
|
||
if os.Getenv("MINIO_ENDPOINT") == "" {
|
||
t.Skip("MINIO_ENDPOINT not set — skipping e2e test")
|
||
}
|
||
if os.Getenv("POCKETBASE_URL") == "" {
|
||
t.Skip("POCKETBASE_URL not set — skipping e2e test")
|
||
}
|
||
scraperURL := envOr("SCRAPER_URL", "http://localhost:8080")
|
||
|
||
pbBaseURL := envOr("POCKETBASE_URL", "http://localhost:8090")
|
||
pbEmail := envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local")
|
||
pbPassword := envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123")
|
||
|
||
pbCfg := storage.PocketBaseConfig{
|
||
BaseURL: pbBaseURL,
|
||
AdminEmail: pbEmail,
|
||
AdminPassword: pbPassword,
|
||
}
|
||
minioCfg := storage.MinioConfig{
|
||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||
UseSSL: envOr("MINIO_USE_SSL", "false") == "true",
|
||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||
}
|
||
|
||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||
defer cancel()
|
||
|
||
hs, err := storage.NewHybridStore(ctx, pbCfg, minioCfg)
|
||
if err != nil {
|
||
t.Fatalf("NewHybridStore: %v", err)
|
||
}
|
||
|
||
client := browser.NewContentClient(browser.Config{
|
||
BaseURL: browserlessURL,
|
||
Token: os.Getenv("BROWSERLESS_TOKEN"),
|
||
Timeout: 120 * time.Second,
|
||
MaxConcurrent: 2,
|
||
})
|
||
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn}))
|
||
sc := novelfire.New(client, log, client, nil)
|
||
|
||
return &e2eFixture{
|
||
sc: sc,
|
||
hs: hs,
|
||
scraperURL: scraperURL,
|
||
pbBaseURL: pbBaseURL,
|
||
pbEmail: pbEmail,
|
||
pbPassword: pbPassword,
|
||
}
|
||
}
|
||
|
||
// ─── The single end-to-end test ───────────────────────────────────────────────
|
||
|
||
// TestE2E_FullScenario executes the complete end-to-end scenario in order.
|
||
func TestE2E_FullScenario(t *testing.T) {
|
||
f := newE2EFixture(t)
|
||
|
||
// ── Step 1: Health-check services ────────────────────────────────────────
|
||
t.Run("step1_health_checks", func(t *testing.T) {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||
defer cancel()
|
||
|
||
// PocketBase health
|
||
pbHealth := f.pbBaseURL + "/api/health"
|
||
checkHTTP(t, ctx, pbHealth, "PocketBase")
|
||
|
||
// MinIO health — the MinIO console liveness endpoint
|
||
minioEndpoint := envOr("MINIO_ENDPOINT", "localhost:9000")
|
||
scheme := "http"
|
||
if envOr("MINIO_USE_SSL", "false") == "true" {
|
||
scheme = "https"
|
||
}
|
||
minioHealth := fmt.Sprintf("%s://%s/minio/health/live", scheme, minioEndpoint)
|
||
checkHTTP(t, ctx, minioHealth, "MinIO")
|
||
|
||
// Browserless health — /pressure is the liveness endpoint
|
||
browserlessURL := envOr("BROWSERLESS_URL", "http://localhost:3030")
|
||
blHealth := browserlessURL + "/pressure"
|
||
checkHTTP(t, ctx, blHealth, "Browserless")
|
||
|
||
// Scraper server health — wait up to 10 s for it to be ready
|
||
scraperHealth := f.scraperURL + "/health"
|
||
waitForHTTP(t, ctx, scraperHealth, "scraper server", 10*time.Second)
|
||
})
|
||
|
||
// ── Step 2: Register test user ────────────────────────────────────────────
|
||
var testUsername string
|
||
t.Run("step2_register_user", func(t *testing.T) {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||
defer cancel()
|
||
|
||
testUsername = fmt.Sprintf("e2euser-%d", time.Now().UnixMilli()%100000)
|
||
passwordHash := "pbkdf2:sha256:dummy-hash-for-test"
|
||
|
||
t.Cleanup(func() {
|
||
cleanCtx, cleanCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||
defer cleanCancel()
|
||
deleteAppUser(t, f, cleanCtx, testUsername)
|
||
})
|
||
|
||
if err := createAppUser(ctx, f, testUsername, passwordHash, "reader"); err != nil {
|
||
t.Fatalf("createAppUser: %v", err)
|
||
}
|
||
t.Logf("created user %q", testUsername)
|
||
|
||
// Verify the user exists in PocketBase.
|
||
rec, err := getAppUserByUsername(ctx, f, testUsername)
|
||
if err != nil {
|
||
t.Fatalf("getAppUserByUsername: %v", err)
|
||
}
|
||
if rec == nil {
|
||
t.Fatal("user not found in app_users after creation")
|
||
}
|
||
if rec["username"] != testUsername {
|
||
t.Errorf("username = %q, want %q", rec["username"], testUsername)
|
||
}
|
||
t.Logf("user verified in PocketBase: id=%v username=%v role=%v", rec["id"], rec["username"], rec["role"])
|
||
})
|
||
|
||
// ── Step 3: Scrape ranking page 1, capture first book ────────────────────
|
||
var firstBook scraper.BookMeta
|
||
t.Run("step3_scrape_ranking", func(t *testing.T) {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||
defer cancel()
|
||
|
||
entries, errs := f.sc.ScrapeRanking(ctx, 1) // maxPages=1 → only page 1
|
||
|
||
select {
|
||
case meta, ok := <-entries:
|
||
if !ok {
|
||
t.Fatal("ranking channel closed without any entry")
|
||
}
|
||
firstBook = meta
|
||
case err := <-errs:
|
||
t.Fatalf("ScrapeRanking error: %v", err)
|
||
case <-ctx.Done():
|
||
t.Fatal("ScrapeRanking timed out waiting for first entry")
|
||
}
|
||
|
||
// Drain remaining entries and errors.
|
||
for range entries {
|
||
}
|
||
for range errs {
|
||
}
|
||
|
||
if firstBook.Slug == "" {
|
||
t.Fatal("first book has empty slug")
|
||
}
|
||
if firstBook.Title == "" {
|
||
t.Fatal("first book has empty title")
|
||
}
|
||
if firstBook.SourceURL == "" {
|
||
t.Fatal("first book has empty SourceURL")
|
||
}
|
||
t.Logf("first ranked book: slug=%q title=%q rank=%d url=%s",
|
||
firstBook.Slug, firstBook.Title, firstBook.Ranking, firstBook.SourceURL)
|
||
})
|
||
|
||
if firstBook.Slug == "" || firstBook.SourceURL == "" {
|
||
t.Fatal("cannot continue: step3 did not produce a valid first book")
|
||
}
|
||
|
||
// Use a unique slug for the test to avoid colliding with real scraped data.
|
||
testSlug := fmt.Sprintf("%s-e2e-%d", firstBook.Slug, time.Now().UnixMilli()%100000)
|
||
t.Logf("using test slug: %q", testSlug)
|
||
|
||
// Register cleanup for all data written by subsequent steps.
|
||
t.Cleanup(func() {
|
||
cleanCtx, cleanCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||
defer cleanCancel()
|
||
cleanupTestData(t, f, cleanCtx, testSlug)
|
||
})
|
||
|
||
// ── Step 4: Scrape book metadata and persist ──────────────────────────────
|
||
var fullMeta scraper.BookMeta
|
||
t.Run("step4_scrape_metadata", func(t *testing.T) {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||
defer cancel()
|
||
|
||
meta, err := f.sc.ScrapeMetadata(ctx, firstBook.SourceURL)
|
||
if err != nil {
|
||
t.Fatalf("ScrapeMetadata: %v", err)
|
||
}
|
||
t.Logf("scraped metadata: title=%q author=%q totalChapters=%d",
|
||
meta.Title, meta.Author, meta.TotalChapters)
|
||
|
||
// Override slug so data lands under our test slug.
|
||
meta.Slug = testSlug
|
||
fullMeta = meta
|
||
|
||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||
defer storeCancel()
|
||
|
||
if err := f.hs.WriteMetadata(storeCtx, meta); err != nil {
|
||
t.Fatalf("WriteMetadata: %v", err)
|
||
}
|
||
|
||
// Verify in PocketBase.
|
||
got, found, err := f.hs.ReadMetadata(storeCtx, testSlug)
|
||
if err != nil {
|
||
t.Fatalf("ReadMetadata: %v", err)
|
||
}
|
||
if !found {
|
||
t.Fatal("book not found in PocketBase after WriteMetadata")
|
||
}
|
||
if got.Title == "" {
|
||
t.Error("book title is empty after round-trip")
|
||
}
|
||
if got.Author == "" {
|
||
t.Logf("WARNING: book author is empty after round-trip (site may not expose author for this book)")
|
||
}
|
||
t.Logf("PocketBase verified: title=%q author=%q totalChapters=%d", got.Title, got.Author, got.TotalChapters)
|
||
})
|
||
|
||
if fullMeta.SourceURL == "" {
|
||
fullMeta.SourceURL = firstBook.SourceURL
|
||
}
|
||
|
||
// ── Step 5: Scrape first 3 chapters and persist ───────────────────────────
|
||
var chapterRefs []scraper.ChapterRef
|
||
t.Run("step5_scrape_chapters", func(t *testing.T) {
|
||
// Fetch only page 1 of the chapter list from
|
||
// https://novelfire.net/book/{slug}/chapters?page=1
|
||
// to avoid paginating through hundreds of pages for popular books.
|
||
listCtx, listCancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||
defer listCancel()
|
||
|
||
chaptersPageURL := firstBook.SourceURL + "/chapters?page=1"
|
||
refs, err := scrapeChapterListPage1(listCtx, f, chaptersPageURL)
|
||
if err != nil {
|
||
t.Fatalf("scrapeChapterListPage1: %v", err)
|
||
}
|
||
if len(refs) == 0 {
|
||
t.Fatal("chapter list page 1 returned no chapters")
|
||
}
|
||
t.Logf("chapter list page 1: %d chapters found", len(refs))
|
||
|
||
// Take the first 3 (or fewer if page 1 has < 3 chapters).
|
||
n := 3
|
||
if len(refs) < n {
|
||
n = len(refs)
|
||
}
|
||
chapterRefs = refs[:n]
|
||
t.Logf("will scrape first %d chapters: %v", n, chapterNumbers(chapterRefs))
|
||
|
||
for _, ref := range chapterRefs {
|
||
ref := ref
|
||
t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) {
|
||
scrapeCtx, scrapeCancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||
defer scrapeCancel()
|
||
|
||
ch, err := f.sc.ScrapeChapterText(scrapeCtx, ref)
|
||
if err != nil {
|
||
t.Fatalf("ScrapeChapterText(%d): %v", ref.Number, err)
|
||
}
|
||
t.Logf("scraped chapter %d: %d bytes", ref.Number, len(ch.Text))
|
||
if len(ch.Text) < 50 {
|
||
t.Errorf("chapter %d text too short (%d bytes)", ref.Number, len(ch.Text))
|
||
}
|
||
|
||
// Override ref slug with our test slug.
|
||
ch.Ref.Number = ref.Number
|
||
ch.Ref.Title = ref.Title
|
||
|
||
storeCtx, storeCancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||
defer storeCancel()
|
||
|
||
if err := f.hs.WriteChapter(storeCtx, testSlug, ch); err != nil {
|
||
t.Fatalf("WriteChapter(%d): %v", ref.Number, err)
|
||
}
|
||
|
||
// Verify in MinIO via ReadChapter.
|
||
got, err := f.hs.ReadChapter(storeCtx, testSlug, ref.Number)
|
||
if err != nil {
|
||
t.Fatalf("ReadChapter(%d): %v", ref.Number, err)
|
||
}
|
||
if got == "" {
|
||
t.Errorf("chapter %d: ReadChapter returned empty content", ref.Number)
|
||
}
|
||
if !strings.HasPrefix(got, "# ") {
|
||
t.Errorf("chapter %d: stored content missing markdown header (got %q)", ref.Number, got[:min(len(got), 80)])
|
||
}
|
||
|
||
// Verify PocketBase chapters_idx entry.
|
||
idxCtx, idxCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||
defer idxCancel()
|
||
count := f.hs.CountChapters(idxCtx, testSlug)
|
||
if count == 0 {
|
||
t.Errorf("chapter %d: chapters_idx count = 0 after WriteChapter", ref.Number)
|
||
}
|
||
t.Logf("chapter %d stored; chapters_idx count=%d", ref.Number, count)
|
||
})
|
||
}
|
||
})
|
||
|
||
if len(chapterRefs) == 0 {
|
||
t.Fatal("cannot continue: step5 produced no chapter refs")
|
||
}
|
||
|
||
// ── Step 6: Generate TTS audio via scraper HTTP API ───────────────────────
|
||
t.Run("step6_tts_audio", func(t *testing.T) {
|
||
if os.Getenv("SCRAPER_URL") == "" {
|
||
t.Skip("SCRAPER_URL not set — skipping TTS step")
|
||
}
|
||
|
||
voice := envOr("KOKORO_VOICE", "af_bella")
|
||
|
||
for _, ref := range chapterRefs {
|
||
ref := ref
|
||
t.Run(fmt.Sprintf("audio-chapter-%d", ref.Number), func(t *testing.T) {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||
defer cancel()
|
||
|
||
audioURL := fmt.Sprintf("%s/api/audio/%s/%d", f.scraperURL, testSlug, ref.Number)
|
||
body, _ := json.Marshal(map[string]interface{}{
|
||
"voice": voice,
|
||
"speed": 1.0,
|
||
})
|
||
|
||
audioReq, _ := http.NewRequestWithContext(ctx, http.MethodPost, audioURL, bytes.NewReader(body))
|
||
audioReq.Header.Set("Content-Type", "application/json")
|
||
resp, err := http.DefaultClient.Do(audioReq)
|
||
if err != nil {
|
||
t.Fatalf("POST %s: %v", audioURL, err)
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
if resp.StatusCode != http.StatusOK {
|
||
raw, _ := io.ReadAll(resp.Body)
|
||
t.Fatalf("audio generation status=%d body=%s", resp.StatusCode, raw)
|
||
}
|
||
|
||
var audioResp struct {
|
||
URL string `json:"url"`
|
||
Filename string `json:"filename"`
|
||
}
|
||
if err := json.NewDecoder(resp.Body).Decode(&audioResp); err != nil {
|
||
t.Fatalf("decode audio response: %v", err)
|
||
}
|
||
if audioResp.URL == "" {
|
||
t.Error("audio response has empty url field")
|
||
}
|
||
if audioResp.Filename == "" {
|
||
t.Error("audio response has empty filename field")
|
||
}
|
||
t.Logf("chapter %d audio: url=%s filename=%s", ref.Number, audioResp.URL, audioResp.Filename)
|
||
|
||
// Verify audio_cache entry exists in PocketBase.
|
||
pbCtx, pbCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||
defer pbCancel()
|
||
|
||
cacheKey := fmt.Sprintf("%s/%d/%s/1.00", testSlug, ref.Number, voice)
|
||
filename, found := f.hs.GetAudioCache(pbCtx, cacheKey)
|
||
if !found {
|
||
t.Errorf("audio_cache entry not found for key=%q", cacheKey)
|
||
} else {
|
||
t.Logf("audio_cache[%q] = %q", cacheKey, filename)
|
||
}
|
||
})
|
||
}
|
||
})
|
||
|
||
// ── Step 7: Presigned URLs ────────────────────────────────────────────────
|
||
t.Run("step7_presigned_urls", func(t *testing.T) {
|
||
if os.Getenv("SCRAPER_URL") == "" {
|
||
t.Skip("SCRAPER_URL not set — skipping presign step")
|
||
}
|
||
|
||
// Give the background MinIO upload goroutines (launched by handleAudioGenerate)
|
||
// a moment to complete before we attempt to access the presigned URLs.
|
||
time.Sleep(5 * time.Second)
|
||
|
||
voice := envOr("KOKORO_VOICE", "af_bella")
|
||
|
||
for _, ref := range chapterRefs {
|
||
ref := ref
|
||
t.Run(fmt.Sprintf("presign-chapter-%d", ref.Number), func(t *testing.T) {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||
defer cancel()
|
||
|
||
// Chapter markdown presign.
|
||
chPresignURL := fmt.Sprintf("%s/api/presign/chapter/%s/%d",
|
||
f.scraperURL, testSlug, ref.Number)
|
||
chPresigned := fetchPresignedURL(t, ctx, chPresignURL, "chapter presign")
|
||
if chPresigned != "" {
|
||
assertURLAccessible(t, ctx, chPresigned, fmt.Sprintf("chapter %d presigned URL", ref.Number))
|
||
}
|
||
|
||
// Audio presign — poll with retries to allow background MinIO upload to finish.
|
||
auPresignURL := fmt.Sprintf("%s/api/presign/audio/%s/%d?voice=%s&speed=1.0",
|
||
f.scraperURL, testSlug, ref.Number, voice)
|
||
auPresigned := fetchPresignedURL(t, ctx, auPresignURL, "audio presign")
|
||
if auPresigned != "" {
|
||
assertURLAccessibleWithRetry(t, ctx, auPresigned, fmt.Sprintf("chapter %d audio presigned URL", ref.Number), 6, 5*time.Second)
|
||
}
|
||
})
|
||
}
|
||
})
|
||
}
|
||
|
||
// ─── PocketBase admin helpers ─────────────────────────────────────────────────
|
||
|
||
// pbAuthToken obtains a PocketBase superuser JWT.
|
||
func pbAuthToken(ctx context.Context, f *e2eFixture) (string, error) {
|
||
body, _ := json.Marshal(map[string]string{
|
||
"identity": f.pbEmail,
|
||
"password": f.pbPassword,
|
||
})
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||
f.pbBaseURL+"/api/collections/_superusers/auth-with-password",
|
||
bytes.NewReader(body))
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
return "", fmt.Errorf("pbAuthToken: %w", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode != http.StatusOK {
|
||
b, _ := io.ReadAll(resp.Body)
|
||
return "", fmt.Errorf("pbAuthToken status %d: %s", resp.StatusCode, b)
|
||
}
|
||
var result struct {
|
||
Token string `json:"token"`
|
||
}
|
||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||
return "", fmt.Errorf("pbAuthToken decode: %w", err)
|
||
}
|
||
return result.Token, nil
|
||
}
|
||
|
||
// createAppUser inserts a record into app_users via PocketBase admin API.
|
||
func createAppUser(ctx context.Context, f *e2eFixture, username, passwordHash, role string) error {
|
||
tok, err := pbAuthToken(ctx, f)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
payload, _ := json.Marshal(map[string]interface{}{
|
||
"username": username,
|
||
"password_hash": passwordHash,
|
||
"role": role,
|
||
"created": time.Now().UTC().Format(time.RFC3339),
|
||
})
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||
f.pbBaseURL+"/api/collections/app_users/records",
|
||
bytes.NewReader(payload))
|
||
if err != nil {
|
||
return err
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
req.Header.Set("Authorization", tok)
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
return fmt.Errorf("createAppUser: %w", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||
b, _ := io.ReadAll(resp.Body)
|
||
return fmt.Errorf("createAppUser status %d: %s", resp.StatusCode, b)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// getAppUserByUsername fetches an app_users record by username.
|
||
// Returns nil, nil when not found.
|
||
func getAppUserByUsername(ctx context.Context, f *e2eFixture, username string) (map[string]interface{}, error) {
|
||
tok, err := pbAuthToken(ctx, f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
url := fmt.Sprintf("%s/api/collections/app_users/records?filter=username%%3D%%22%s%%22&perPage=1",
|
||
f.pbBaseURL, username)
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("Authorization", tok)
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("getAppUserByUsername: %w", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
var result struct {
|
||
Items []map[string]interface{} `json:"items"`
|
||
}
|
||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||
return nil, fmt.Errorf("getAppUserByUsername decode: %w", err)
|
||
}
|
||
if len(result.Items) == 0 {
|
||
return nil, nil
|
||
}
|
||
return result.Items[0], nil
|
||
}
|
||
|
||
// deleteAppUser removes app_users records matching username.
|
||
func deleteAppUser(t *testing.T, f *e2eFixture, ctx context.Context, username string) {
|
||
t.Helper()
|
||
tok, err := pbAuthToken(ctx, f)
|
||
if err != nil {
|
||
t.Logf("deleteAppUser: pbAuthToken error: %v", err)
|
||
return
|
||
}
|
||
// List matching records.
|
||
url := fmt.Sprintf("%s/api/collections/app_users/records?filter=username%%3D%%22%s%%22&perPage=10",
|
||
f.pbBaseURL, username)
|
||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||
req.Header.Set("Authorization", tok)
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
t.Logf("deleteAppUser list error: %v", err)
|
||
return
|
||
}
|
||
defer resp.Body.Close()
|
||
var result struct {
|
||
Items []map[string]interface{} `json:"items"`
|
||
}
|
||
_ = json.NewDecoder(resp.Body).Decode(&result)
|
||
for _, item := range result.Items {
|
||
id, _ := item["id"].(string)
|
||
delURL := fmt.Sprintf("%s/api/collections/app_users/records/%s", f.pbBaseURL, id)
|
||
delReq, _ := http.NewRequestWithContext(ctx, http.MethodDelete, delURL, nil)
|
||
delReq.Header.Set("Authorization", tok)
|
||
delResp, _ := http.DefaultClient.Do(delReq)
|
||
if delResp != nil {
|
||
delResp.Body.Close()
|
||
}
|
||
}
|
||
}
|
||
|
||
// cleanupTestData removes all PocketBase + MinIO data for the given slug.
|
||
func cleanupTestData(t *testing.T, f *e2eFixture, ctx context.Context, slug string) {
|
||
t.Helper()
|
||
tok, err := pbAuthToken(ctx, f)
|
||
if err != nil {
|
||
t.Logf("cleanupTestData: pbAuthToken error: %v", err)
|
||
return
|
||
}
|
||
pbDelete := func(collection, filter string) {
|
||
listURL := fmt.Sprintf("%s/api/collections/%s/records?filter=%s&perPage=500",
|
||
f.pbBaseURL, collection, filter)
|
||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, listURL, nil)
|
||
req.Header.Set("Authorization", tok)
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
t.Logf("cleanupTestData list %s error: %v", collection, err)
|
||
return
|
||
}
|
||
defer resp.Body.Close()
|
||
var result struct {
|
||
Items []map[string]interface{} `json:"items"`
|
||
}
|
||
_ = json.NewDecoder(resp.Body).Decode(&result)
|
||
for _, item := range result.Items {
|
||
id, _ := item["id"].(string)
|
||
delURL := fmt.Sprintf("%s/api/collections/%s/records/%s", f.pbBaseURL, collection, id)
|
||
delReq, _ := http.NewRequestWithContext(ctx, http.MethodDelete, delURL, nil)
|
||
delReq.Header.Set("Authorization", tok)
|
||
delResp, _ := http.DefaultClient.Do(delReq)
|
||
if delResp != nil {
|
||
delResp.Body.Close()
|
||
}
|
||
}
|
||
}
|
||
slugFilter := fmt.Sprintf("slug%%3D%%22%s%%22", slug)
|
||
ckFilter := fmt.Sprintf("cache_key%%7E%%22%s%%2F%%22", slug) // cache_key ~ "slug/"
|
||
pbDelete("books", slugFilter)
|
||
pbDelete("chapters_idx", slugFilter)
|
||
pbDelete("audio_cache", ckFilter)
|
||
t.Logf("cleanup complete for slug=%q", slug)
|
||
}
|
||
|
||
// ─── HTTP assertion helpers ───────────────────────────────────────────────────
|
||
|
||
// checkHTTP asserts that a GET to url returns 2xx within the context deadline.
|
||
func checkHTTP(t *testing.T, ctx context.Context, url, name string) {
|
||
t.Helper()
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||
if err != nil {
|
||
t.Errorf("%s health check: build request: %v", name, err)
|
||
return
|
||
}
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
t.Errorf("%s health check failed: %v", name, err)
|
||
return
|
||
}
|
||
resp.Body.Close()
|
||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||
t.Errorf("%s health check: status %d, want 2xx", name, resp.StatusCode)
|
||
return
|
||
}
|
||
t.Logf("%s health OK (HTTP %d)", name, resp.StatusCode)
|
||
}
|
||
|
||
// waitForHTTP retries GET url until a 2xx is received or timeout is reached.
|
||
func waitForHTTP(t *testing.T, ctx context.Context, url, name string, timeout time.Duration) {
|
||
t.Helper()
|
||
deadline := time.Now().Add(timeout)
|
||
var lastErr error
|
||
for time.Now().Before(deadline) {
|
||
select {
|
||
case <-ctx.Done():
|
||
t.Errorf("%s: context cancelled while waiting for health", name)
|
||
return
|
||
default:
|
||
}
|
||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||
resp.Body.Close()
|
||
t.Logf("%s health OK (HTTP %d)", name, resp.StatusCode)
|
||
return
|
||
}
|
||
if resp != nil {
|
||
resp.Body.Close()
|
||
lastErr = fmt.Errorf("status %d", resp.StatusCode)
|
||
} else {
|
||
lastErr = err
|
||
}
|
||
time.Sleep(500 * time.Millisecond)
|
||
}
|
||
t.Errorf("%s not healthy after %s: %v", name, timeout, lastErr)
|
||
}
|
||
|
||
// fetchPresignedURL calls the presign endpoint and returns the presigned URL.
|
||
// It logs and returns "" on failure (non-fatal) so the caller can decide.
|
||
func fetchPresignedURL(t *testing.T, ctx context.Context, presignEndpoint, label string) string {
|
||
t.Helper()
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, presignEndpoint, nil)
|
||
if err != nil {
|
||
t.Errorf("fetchPresignedURL %s: %v", label, err)
|
||
return ""
|
||
}
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
t.Errorf("fetchPresignedURL %s: %v", label, err)
|
||
return ""
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode != http.StatusOK {
|
||
b, _ := io.ReadAll(resp.Body)
|
||
t.Errorf("fetchPresignedURL %s: status %d body=%s", label, resp.StatusCode, b)
|
||
return ""
|
||
}
|
||
var body struct {
|
||
URL string `json:"url"`
|
||
}
|
||
if err := json.NewDecoder(resp.Body).Decode(&body); err != nil {
|
||
t.Errorf("fetchPresignedURL %s decode: %v", label, err)
|
||
return ""
|
||
}
|
||
if body.URL == "" {
|
||
t.Errorf("fetchPresignedURL %s: empty url in response", label)
|
||
return ""
|
||
}
|
||
t.Logf("%s presigned URL: %s", label, body.URL)
|
||
return body.URL
|
||
}
|
||
|
||
// assertURLAccessible does a GET to url and asserts HTTP 200.
|
||
func assertURLAccessible(t *testing.T, ctx context.Context, url, label string) {
|
||
t.Helper()
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||
if err != nil {
|
||
t.Errorf("%s: build request: %v", label, err)
|
||
return
|
||
}
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
t.Errorf("%s: GET error: %v", label, err)
|
||
return
|
||
}
|
||
resp.Body.Close()
|
||
if resp.StatusCode != http.StatusOK {
|
||
t.Errorf("%s: status %d, want 200", label, resp.StatusCode)
|
||
return
|
||
}
|
||
t.Logf("%s: HTTP 200 OK", label)
|
||
}
|
||
|
||
// assertURLAccessibleWithRetry retries GET url up to maxAttempts times with
|
||
// interval between attempts, asserting HTTP 200 on any success.
|
||
func assertURLAccessibleWithRetry(t *testing.T, ctx context.Context, url, label string, maxAttempts int, interval time.Duration) {
|
||
t.Helper()
|
||
var lastStatus int
|
||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||
if err != nil {
|
||
t.Errorf("%s: build request: %v", label, err)
|
||
return
|
||
}
|
||
resp, err := http.DefaultClient.Do(req)
|
||
if err != nil {
|
||
t.Logf("%s: attempt %d GET error: %v", label, attempt, err)
|
||
} else {
|
||
resp.Body.Close()
|
||
lastStatus = resp.StatusCode
|
||
if resp.StatusCode == http.StatusOK {
|
||
t.Logf("%s: HTTP 200 OK (attempt %d)", label, attempt)
|
||
return
|
||
}
|
||
t.Logf("%s: attempt %d status %d", label, attempt, resp.StatusCode)
|
||
}
|
||
if attempt < maxAttempts {
|
||
select {
|
||
case <-ctx.Done():
|
||
t.Errorf("%s: context cancelled before success", label)
|
||
return
|
||
case <-time.After(interval):
|
||
}
|
||
}
|
||
}
|
||
t.Errorf("%s: status %d after %d attempts, want 200", label, lastStatus, maxAttempts)
|
||
}
|
||
|
||
// ─── stdlib helpers ───────────────────────────────────────────────────────────
|
||
|
||
func min(a, b int) int {
|
||
if a < b {
|
||
return a
|
||
}
|
||
return b
|
||
}
|
||
|
||
func chapterNumbers(refs []scraper.ChapterRef) []int {
|
||
ns := make([]int, len(refs))
|
||
for i, r := range refs {
|
||
ns[i] = r.Number
|
||
}
|
||
return ns
|
||
}
|
||
|
||
// scrapeChapterListPage1 fetches a single chapter-list page URL via Browserless
|
||
// and returns the chapter refs found on that page (no pagination).
|
||
// URL should be: https://novelfire.net/book/{slug}/chapters?page=1
|
||
func scrapeChapterListPage1(ctx context.Context, f *e2eFixture, pageURL string) ([]scraper.ChapterRef, error) {
|
||
return f.sc.ScrapeChapterListPage(ctx, pageURL)
|
||
}
|