test: add integration tests for storage, server, and scrape+store flows
- scraper/internal/storage/integration_test.go: MinioClient and PocketBaseStore integration tests covering chapter/audio round-trips, presign URLs, book metadata, chapter index, ranking, progress, and audio cache CRUD - scraper/internal/storage/hybrid_integration_test.go: HybridStore end-to-end tests for metadata, chapters, ranking, progress, presign, and audio cache - scraper/internal/storage/scrape_integration_test.go: live Browserless + storage tests that scrape book metadata and first 3 chapters, store them, and verify the round-trip via HybridStore - scraper/internal/server/integration_test.go: HTTP server functional tests for health, scrape status, presign chapter, reading progress, and chapter-text endpoints against real MinIO + PocketBase backends - justfile: task runner at repo root with recipes for build, test, lint, UI, docker-compose, and individual service management
This commit is contained in:
124
justfile
Normal file
124
justfile
Normal file
@@ -0,0 +1,124 @@
|
||||
# justfile — libnovel-v2 task runner
|
||||
# Install just: https://just.systems
|
||||
|
||||
scraper_dir := "scraper"
|
||||
ui_dir := "ui"
|
||||
|
||||
# ─── Build ────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Build the scraper binary
|
||||
build:
|
||||
cd {{scraper_dir}} && go build -o bin/scraper ./cmd/scraper
|
||||
|
||||
# Build and verify all Go packages compile cleanly
|
||||
build-all:
|
||||
cd {{scraper_dir}} && go build ./...
|
||||
|
||||
# ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Run unit tests only (no integration services required)
|
||||
test:
|
||||
cd {{scraper_dir}} && go test ./...
|
||||
|
||||
# Run integration tests (requires MinIO, PocketBase, optional Browserless)
|
||||
# Override env vars as needed, e.g.:
|
||||
# just test-integration MINIO_ENDPOINT=localhost:9000
|
||||
test-integration:
|
||||
cd {{scraper_dir}} && go test -v -tags integration -timeout 600s ./...
|
||||
|
||||
# Run unit + integration tests
|
||||
test-all: test test-integration
|
||||
|
||||
# Run a specific package's integration tests, e.g.:
|
||||
# just test-pkg internal/storage
|
||||
test-pkg pkg:
|
||||
cd {{scraper_dir}} && go test -v -tags integration -timeout 600s ./{{pkg}}/...
|
||||
|
||||
# ─── Code quality ─────────────────────────────────────────────────────────────
|
||||
|
||||
# Run go vet on all packages (including integration build tag)
|
||||
lint:
|
||||
cd {{scraper_dir}} && go vet ./...
|
||||
cd {{scraper_dir}} && go vet -tags integration ./...
|
||||
|
||||
# ─── UI ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Type-check the SvelteKit UI
|
||||
ui-check:
|
||||
cd {{ui_dir}} && npx svelte-check
|
||||
|
||||
# Start the SvelteKit dev server
|
||||
ui-dev:
|
||||
cd {{ui_dir}} && npm run dev
|
||||
|
||||
# Install UI dependencies
|
||||
ui-install:
|
||||
cd {{ui_dir}} && npm install
|
||||
|
||||
# Build the UI for production
|
||||
ui-build:
|
||||
cd {{ui_dir}} && npm run build
|
||||
|
||||
# ─── Docker Compose ───────────────────────────────────────────────────────────
|
||||
|
||||
# Start all services (browserless, kokoro, scraper, minio, pocketbase)
|
||||
up:
|
||||
docker compose up -d
|
||||
|
||||
# Stop all services
|
||||
down:
|
||||
docker compose down
|
||||
|
||||
# Tail logs for all services
|
||||
logs:
|
||||
docker compose logs -f
|
||||
|
||||
# Tail logs for a specific service, e.g.: just logs-service scraper
|
||||
logs-service service:
|
||||
docker compose logs -f {{service}}
|
||||
|
||||
# Rebuild and restart a specific service
|
||||
restart service:
|
||||
docker compose up -d --build {{service}}
|
||||
|
||||
# ─── Local dev: individual services ──────────────────────────────────────────
|
||||
|
||||
# Start only PocketBase (for local storage testing)
|
||||
pb-up:
|
||||
docker compose up -d pocketbase
|
||||
|
||||
# Start only MinIO (for local storage testing)
|
||||
minio-up:
|
||||
docker compose up -d minio
|
||||
|
||||
# Start only Browserless (for local scraping tests)
|
||||
browserless-up:
|
||||
docker compose up -d browserless
|
||||
|
||||
# Start storage backends only (MinIO + PocketBase)
|
||||
storage-up:
|
||||
docker compose up -d minio pocketbase
|
||||
|
||||
# ─── Convenience ─────────────────────────────────────────────────────────────
|
||||
|
||||
# Show status of all docker compose services
|
||||
status:
|
||||
docker compose ps
|
||||
|
||||
# Remove all stopped containers and unused images
|
||||
prune:
|
||||
docker compose down --remove-orphans
|
||||
docker image prune -f
|
||||
|
||||
# One-shot scrape of the full catalogue (requires services to be running)
|
||||
scrape-run: build
|
||||
cd {{scraper_dir}} && ./bin/scraper run
|
||||
|
||||
# One-shot scrape of a single book URL, e.g.:
|
||||
# just scrape-book https://novelfire.net/book/my-novel
|
||||
scrape-book url: build
|
||||
cd {{scraper_dir}} && ./bin/scraper run --url {{url}}
|
||||
|
||||
# Start the HTTP server
|
||||
serve: build
|
||||
cd {{scraper_dir}} && ./bin/scraper serve
|
||||
Reference in New Issue
Block a user