diff --git a/v3/Caddyfile b/v3/Caddyfile new file mode 100644 index 0000000..2a72753 --- /dev/null +++ b/v3/Caddyfile @@ -0,0 +1,83 @@ +# v3/Caddyfile +# +# Caddy reverse proxy for LibNovel v3. +# +# Environment variables consumed (set in docker-compose.yml): +# DOMAIN — public hostname, e.g. libnovel.example.com +# Use "localhost" for local dev (no TLS cert attempted). +# CADDY_ACME_EMAIL — Let's Encrypt notification email (empty = no email) +# +# Routing rules: +# /health → backend:8080 (liveness probe) +# /scrape* → backend:8080 (scrape task creation) +# /api/browse → backend:8080 (MinIO-cached browse pages) +# /api/book-preview/* → backend:8080 (live scrape, no store write) +# /api/chapter-text-preview/*/* → backend:8080 (live chapter, no store write) +# /api/chapter-text/*/* → backend:8080 (chapter markdown from MinIO) +# /api/reindex/* → backend:8080 (rebuild chapter index) +# /api/cover/* → backend:8080 (proxy cover image) +# /api/audio-proxy/*/* → backend:8080 (proxy generated audio) +# /api/scrape/* → backend:8080 (scrape job status/tasks) +# /* (everything else) → ui:3000 (SvelteKit — handles all +# remaining /api/* routes too) +# +# The SvelteKit UI itself proxies to the backend for: ranking, voices, search, +# browse-page, presign, audio, progress, and the Go /api/progress endpoint. +# MinIO and PocketBase are NOT exposed publicly. + +{ + # Email for Let's Encrypt ACME account registration. + # Optional — omit to use an anonymous ACME account. + {$CADDY_ACME_EMAIL:} +} + +{$DOMAIN:localhost} { + # ── Liveness probe ─────────────────────────────────────────────────────── + handle /health { + reverse_proxy backend:8080 + } + + # ── Scrape task creation (Go backend only) ─────────────────────────────── + handle /scrape* { + reverse_proxy backend:8080 + } + + # ── Backend-only API paths ──────────────────────────────────────────────── + # These paths are served exclusively by the Go scraper and are not + # implemented in the SvelteKit UI. + handle /api/browse { + reverse_proxy backend:8080 + } + handle /api/book-preview/* { + reverse_proxy backend:8080 + } + handle /api/chapter-text-preview/* { + reverse_proxy backend:8080 + } + handle /api/chapter-text/* { + reverse_proxy backend:8080 + } + handle /api/reindex/* { + reverse_proxy backend:8080 + } + handle /api/cover/* { + reverse_proxy backend:8080 + } + handle /api/audio-proxy/* { + reverse_proxy backend:8080 + } + handle /api/scrape/* { + reverse_proxy backend:8080 + } + + # ── SvelteKit UI (catch-all — includes all remaining /api/* routes) ─────── + handle { + reverse_proxy ui:3000 + } + + # ── Logging ────────────────────────────────────────────────────────────── + log { + output stdout + format json + } +} diff --git a/v3/TODO.md b/v3/TODO.md new file mode 100644 index 0000000..aceded2 --- /dev/null +++ b/v3/TODO.md @@ -0,0 +1,215 @@ +# v3 Implementation Plan + +v3 is a self-contained directory. All services live under `v3/`. Nothing shares +code with the root `backend/` or `ui-v2/` directories. + +## New in v3 vs v2 + +| Addition | Why | +|----------|-----| +| **Caddy** reverse proxy | Single public entry point, automatic HTTPS via Let's Encrypt, no manual cert management | +| **Meilisearch** | Full-text search over locally scraped books; replaces live novelfire.net scrape on every search | +| **Valkey** (Redis-compatible) | Shared presign URL cache; replaces the per-process in-memory Map in the UI Node process | +| **Runner `/metrics` HTTP endpoint** | Exposes task counters (pending, running, failed, completed) for healthcheck / Caddy visibility | + +## Directory layout (target) + +``` +v3/ +├── backend/ # copied from ../backend/, then modified +│ ├── cmd/ +│ │ ├── backend/ # unchanged +│ │ └── runner/ # add metrics HTTP server +│ ├── internal/ +│ │ ├── backend/ +│ │ │ ├── server.go # add /api/search endpoint (Meilisearch) +│ │ │ └── handlers.go # add searchHandler, presign now reads Valkey +│ │ ├── runner/ +│ │ │ ├── runner.go # add atomic counters + metrics HTTP server +│ │ │ └── metrics.go # NEW: MetricsServer (net/http, /metrics JSON) +│ │ ├── meili/ # NEW: Meilisearch client package +│ │ │ └── client.go +│ │ ├── presigncache/ # NEW: Valkey-backed presign cache +│ │ │ └── cache.go +│ │ └── config/ +│ │ └── config.go # add Meilisearch + Valkey + metrics addr vars +│ └── go.mod # add meilisearch-go, go-redis/v9 +├── ui/ # copied from ../ui-v2/, then modified +│ └── src/lib/server/ +│ └── presignCache.ts # replace in-process Map with Valkey (ioredis) +├── docs/ +│ ├── architecture.d2 # DONE +│ └── architecture.mermaid.md # DONE +├── scripts/ +│ └── pb-init-v3.sh # same as v2 unless schema changes +├── Caddyfile # NEW +├── docker-compose.yml # NEW +└── TODO.md # this file +``` + +--- + +## Tasks + +### 1. Copy source trees + +- [ ] `cp -r ../backend v3/backend` +- [ ] `cp -r ../ui-v2 v3/ui` +- [ ] `cp ../scripts/pb-init-v2.sh v3/scripts/pb-init-v3.sh` +- [ ] Verify `v3/backend/go.mod` module path is still `github.com/libnovel/backend` + (no rename needed — it's a copy, not a Go workspace dep) + +### 2. Add Valkey presign cache (`v3/backend`) + +**Goal**: Backend generates presigned URLs → stores in Valkey with TTL. +UI reads from Valkey before calling backend. + +Files to create/modify: + +- [ ] `internal/presigncache/cache.go` — `Cache` interface + `ValkeyCache` implementation + - `Get(ctx, key) (string, bool, error)` + - `Set(ctx, key, url string, ttl time.Duration) error` + - Uses `github.com/redis/go-redis/v9` + - TTL: match presigned URL lifetime (e.g. 45 min; MinIO default is 1h → use 55 min) +- [ ] `internal/config/config.go` — add `ValkeyAddr string` (default `"valkey:6379"`) +- [ ] `internal/backend/server.go` — add `PresignCache presigncache.Cache` to `Dependencies` +- [ ] `internal/backend/handlers.go` — in presign handlers, do `cache.Get` before generating, + `cache.Set` after generating +- [ ] `go.mod` — add `github.com/redis/go-redis/v9` +- [ ] `cmd/backend/main.go` — wire `ValkeyCache` from config + +### 3. Add Meilisearch integration (`v3/backend`) + +**Goal**: On each book scrape completion, upsert the book document into Meilisearch. +Backend `/api/search` returns Meilisearch results for local books, falling back to +novelfire.net live search for books not in the index. + +Files to create/modify: + +- [ ] `internal/meili/client.go` — `Client` interface + `MeiliClient` implementation + - `UpsertBook(ctx, book domain.Book) error` + - `Search(ctx, query string, limit int) ([]domain.Book, error)` + - Uses `github.com/meilisearch/meilisearch-go` + - Index name: `books`, primary key: `slug` + - Searchable attributes: `title`, `author`, `tags`, `description` + - Filterable attributes: `status`, `tags` +- [ ] `internal/config/config.go` — add `MeiliURL string`, `MeiliAPIKey string` +- [ ] `internal/bookstore/bookstore.go` — add `SearchIndex` interface with `UpsertBook` +- [ ] `internal/runner/runner.go` — inject `SearchIndex`; after `FinishScrapeTask` success, + call `SearchIndex.UpsertBook` for each book scraped +- [ ] `internal/backend/server.go` — add `SearchIndex` to `Dependencies`; register `GET /api/search` +- [ ] `internal/backend/handlers.go` — `searchHandler`: query Meilisearch first; if 0 results, + fall back to novelfire.net live search via `NovelScraper.SearchBooks` +- [ ] `go.mod` — add `github.com/meilisearch/meilisearch-go` +- [ ] `cmd/backend/main.go` + `cmd/runner/main.go` — wire `MeiliClient` + +### 4. Add runner `/metrics` HTTP endpoint (`v3/backend`) + +**Goal**: Runner exposes `GET /metrics` returning JSON with task counters. Caddy can +healthcheck this; operators can scrape it from a monitoring tool. + +Payload example: +```json +{ + "tasks_pending": 0, + "tasks_running": 1, + "tasks_completed": 42, + "tasks_failed": 2, + "uptime_seconds": 3600 +} +``` + +Files to create/modify: + +- [ ] `internal/runner/metrics.go` — `Metrics` struct with `atomic.Int64` counters; + `ServeHTTP` handler returns JSON; `MetricsServer` starts `net/http` on configurable addr +- [ ] `internal/runner/runner.go` — embed `*Metrics`; increment counters at task lifecycle + points (claimed → running → completed/failed) +- [ ] `internal/config/config.go` — add `RunnerMetricsAddr string` (default `":9091"`) +- [ ] `cmd/runner/main.go` — start `MetricsServer` in background goroutine before `runner.Run` +- [ ] `docker-compose.yml` (runner service) — expose port 9091 internally; update healthcheck + to `GET http://localhost:9091/metrics` (replaces file-based liveness) + +### 5. Update UI presign cache to use Valkey (`v3/ui`) + +**Goal**: Replace `src/lib/server/presignCache.ts` (module-scope `Map`) with Valkey via +`ioredis`. Cache survives UI restarts and is shared if multiple UI replicas run. + +Files to modify: + +- [ ] `src/lib/server/presignCache.ts` — rewrite to use `ioredis` client + - `getPresignedUrl(key: string): Promise` + - `setPresignedUrl(key: string, url: string, ttlSeconds: number): Promise` + - Connection string from `VALKEY_URL` env var (default `redis://valkey:6379`) + - Keep the same TTL logic (50 min) and sweep logic (not needed — Valkey TTL is native) +- [ ] `package.json` — add `ioredis` (or use `@redis/client` — prefer `ioredis` for its + robust reconnection handling) +- [ ] Any callers of old `presignCache` — update import if interface changes + +### 6. Write `v3/Caddyfile` + +- [ ] Global options block: email for ACME, staging CA override for local dev +- [ ] Single site block matching `{$DOMAIN}` (env var injection): + - `handle /api/*` → `reverse_proxy backend:8080` + - `handle /health` → `reverse_proxy backend:8080` + - `handle /s3/*` → strip prefix + `reverse_proxy minio:9000` (internal bucket access, + requires `Authorization` passthrough) + - `handle` (catch-all) → `reverse_proxy ui:3000` +- [ ] Health/liveness endpoints exposed without auth +- [ ] No ports on MinIO/PocketBase exposed to public (internal Docker network only) + +### 7. Write `v3/docker-compose.yml` + +Services (in dependency order): + +| Service | Image / Build | Ports (host) | Notes | +|---------|--------------|--------------|-------| +| `minio` | `minio/minio:latest` | none (internal only) | Remove public port exposure; Caddy proxies `/s3/*` | +| `minio-init` | `minio/mc:latest` | — | Same as v2 | +| `pocketbase` | `ghcr.io/muchobien/pocketbase:latest` | none (internal only) | Remove public port | +| `pb-init` | `alpine:3.19` | — | Same as v2, uses `v3/scripts/pb-init-v3.sh` | +| `valkey` | `valkey/valkey:8-alpine` | none (internal) | `valkey-server --save "" --appendonly no` | +| `meilisearch` | `getmeili/meilisearch:v1.7` | none (internal) | `MEILI_NO_ANALYTICS=true`, persistent volume | +| `backend` | build `v3/backend` target `backend` | none (internal) | Add `VALKEY_ADDR`, `MEILI_URL`, `MEILI_API_KEY` | +| `runner` | build `v3/backend` target `runner` | `9091` (metrics) | Add `RUNNER_METRICS_ADDR`, `MEILI_URL`, `MEILI_API_KEY`; healthcheck via `/metrics` | +| `ui` | build `v3/ui` | none (internal) | Add `VALKEY_URL`; remove `PUBLIC_MINIO_PUBLIC_URL` (all through Caddy now) | +| `caddy` | `caddy:2-alpine` | `80`, `443` | Bind-mount `v3/Caddyfile`; persistent `caddy_data` volume for certs | + +Volumes: `minio_data`, `pb_data`, `meili_data`, `caddy_data` + +- [ ] Write the full file with healthchecks, depends_on, env vars + +### 8. Write `v3/scripts/pb-init-v3.sh` + +- [ ] Start from `scripts/pb-init-v2.sh` +- [ ] Assess if any schema changes needed (Meilisearch sync does not require new PB collections + — search is entirely Meilisearch-side) +- [ ] If identical to v2 script, note that in a comment at the top; keep as separate file + +--- + +## Environment variables (new in v3) + +| Variable | Service | Default | Description | +|----------|---------|---------|-------------| +| `VALKEY_ADDR` | backend | `valkey:6379` | Valkey TCP address | +| `VALKEY_URL` | ui | `redis://valkey:6379` | Valkey URL (ioredis format) | +| `MEILI_URL` | backend, runner | `http://meilisearch:7700` | Meilisearch HTTP URL | +| `MEILI_API_KEY` | backend, runner | `""` | Meilisearch master key (empty = no auth for dev) | +| `RUNNER_METRICS_ADDR` | runner | `:9091` | Runner metrics HTTP listen address | +| `DOMAIN` | caddy | `localhost` | Public domain for HTTPS cert | +| `CADDY_ACME_EMAIL` | caddy | `""` | Let's Encrypt notification email | + +--- + +## Testing checklist (before considering v3 stable) + +- [ ] `docker compose -f v3/docker-compose.yml up --build` starts cleanly +- [ ] `curl https://{DOMAIN}/health` returns 200 (Caddy → backend) +- [ ] Book search returns results from Meilisearch after at least one scrape +- [ ] Presign URL cache hit visible in backend logs (Valkey `GET` hit) +- [ ] `curl http://localhost:9091/metrics` returns JSON with counters +- [ ] Runner healthcheck passes via `/metrics` endpoint (not file-based) +- [ ] MinIO console NOT reachable from the internet (no public port) +- [ ] PocketBase NOT reachable from the internet (no public port) +- [ ] TLS cert obtained from Let's Encrypt (check `caddy_data` volume / Caddy logs) diff --git a/v3/backend/.dockerignore b/v3/backend/.dockerignore new file mode 100644 index 0000000..d32a9c3 --- /dev/null +++ b/v3/backend/.dockerignore @@ -0,0 +1,13 @@ +# Exclude compiled binaries +bin/ + +# Exclude test binaries produced by `go test -c` +*.test + +# Git history is not needed inside the image +.git/ + +# Editor/OS noise +.DS_Store +*.swp +*.swo diff --git a/v3/backend/Dockerfile b/v3/backend/Dockerfile new file mode 100644 index 0000000..b26e4ef --- /dev/null +++ b/v3/backend/Dockerfile @@ -0,0 +1,42 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.26.1-alpine AS builder +WORKDIR /app + +# Download modules into the BuildKit cache so they survive across builds. +# This layer is only invalidated when go.mod or go.sum changes. +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/root/go/pkg/mod \ + go mod download + +COPY . . + +ARG VERSION=dev +ARG COMMIT=unknown + +# Build all three binaries in a single layer so the Go compiler can reuse +# intermediate object files. Both cache mounts are preserved between builds: +# /root/go/pkg/mod — downloaded module source +# /root/.cache/go-build — compiled package objects (incremental recompile) +RUN --mount=type=cache,target=/root/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=linux go build \ + -ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT}" \ + -o /out/backend ./cmd/backend && \ + CGO_ENABLED=0 GOOS=linux go build \ + -ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT}" \ + -o /out/runner ./cmd/runner && \ + CGO_ENABLED=0 GOOS=linux go build \ + -ldflags="-s -w" \ + -o /out/healthcheck ./cmd/healthcheck + +# ── backend service ────────────────────────────────────────────────────────── +FROM gcr.io/distroless/static:nonroot AS backend +COPY --from=builder /out/healthcheck /healthcheck +COPY --from=builder /out/backend /backend +ENTRYPOINT ["/backend"] + +# ── runner service ─────────────────────────────────────────────────────────── +FROM gcr.io/distroless/static:nonroot AS runner +COPY --from=builder /out/healthcheck /healthcheck +COPY --from=builder /out/runner /runner +ENTRYPOINT ["/runner"] diff --git a/v3/backend/cmd/backend/main.go b/v3/backend/cmd/backend/main.go new file mode 100644 index 0000000..1a53e5a --- /dev/null +++ b/v3/backend/cmd/backend/main.go @@ -0,0 +1,139 @@ +// Command backend is the LibNovel HTTP API server. +// +// It exposes all endpoints consumed by the SvelteKit UI: book/chapter reads, +// scrape-task creation, presigned MinIO URLs, audio-task creation, reading +// progress, live novelfire.net browse/search, and Kokoro voice list. +// +// All heavy lifting (scraping, TTS generation) is delegated to the runner +// binary via PocketBase task records. The backend never scrapes directly. +// +// Usage: +// +// backend # start HTTP server (blocks until SIGINT/SIGTERM) +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/libnovel/backend/internal/backend" + "github.com/libnovel/backend/internal/config" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/meili" + "github.com/libnovel/backend/internal/storage" +) + +// version and commit are set at build time via -ldflags. +var ( + version = "dev" + commit = "unknown" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "backend: fatal: %v\n", err) + os.Exit(1) + } +} + +func run() error { + cfg := config.Load() + + // ── Logger ─────────────────────────────────────────────────────────────── + log := buildLogger(cfg.LogLevel) + log.Info("backend starting", + "version", version, + "commit", commit, + "addr", cfg.HTTP.Addr, + ) + + // ── Context: cancel on SIGINT / SIGTERM ────────────────────────────────── + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + // ── Storage ────────────────────────────────────────────────────────────── + store, err := storage.NewStore(ctx, cfg, log) + if err != nil { + return fmt.Errorf("init storage: %w", err) + } + + // ── Kokoro (voice list only; audio generation is done by the runner) ───── + var kokoroClient kokoro.Client + if cfg.Kokoro.URL != "" { + kokoroClient = kokoro.New(cfg.Kokoro.URL) + log.Info("kokoro voices enabled", "url", cfg.Kokoro.URL) + } else { + log.Info("KOKORO_URL not set — voice list will use built-in fallback") + kokoroClient = &noopKokoro{} + } + + // ── Meilisearch (search reads only; indexing is the runner's job) ──────── + var searchIndex meili.Client + if cfg.Meilisearch.URL != "" { + searchIndex = meili.New(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey) + log.Info("meilisearch search enabled", "url", cfg.Meilisearch.URL) + } else { + log.Info("MEILI_URL not set — search will use PocketBase substring fallback") + searchIndex = meili.NoopClient{} + } + + // ── Backend server ─────────────────────────────────────────────────────── + srv := backend.New( + backend.Config{ + Addr: cfg.HTTP.Addr, + DefaultVoice: cfg.Kokoro.DefaultVoice, + Version: version, + Commit: commit, + }, + backend.Dependencies{ + BookReader: store, + RankingStore: store, + AudioStore: store, + PresignStore: store, + ProgressStore: store, + BrowseStore: store, + CoverStore: store, + Producer: store, + TaskReader: store, + SearchIndex: searchIndex, + Kokoro: kokoroClient, + Log: log, + }, + ) + + return srv.ListenAndServe(ctx) +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +func buildLogger(level string) *slog.Logger { + var lvl slog.Level + switch level { + case "debug": + lvl = slog.LevelDebug + case "warn": + lvl = slog.LevelWarn + case "error": + lvl = slog.LevelError + default: + lvl = slog.LevelInfo + } + return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl})) +} + +// noopKokoro is a no-op implementation used when KOKORO_URL is not set. +// The backend only uses Kokoro for the voice list; audio generation is the +// runner's responsibility. With no URL the built-in fallback list is served. +type noopKokoro struct{} + +func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) { + return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)") +} + +func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) { + return nil, nil +} diff --git a/v3/backend/cmd/backend/main_test.go b/v3/backend/cmd/backend/main_test.go new file mode 100644 index 0000000..edd47dc --- /dev/null +++ b/v3/backend/cmd/backend/main_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "os" + "testing" +) + +// TestBuildLogger verifies that buildLogger returns a non-nil logger for each +// supported log level string and for unknown values. +func TestBuildLogger(t *testing.T) { + for _, level := range []string{"debug", "info", "warn", "error", "unknown", ""} { + l := buildLogger(level) + if l == nil { + t.Errorf("buildLogger(%q) returned nil", level) + } + } +} + +// TestNoopKokoro verifies that the no-op Kokoro stub returns the expected +// sentinel error from GenerateAudio and nil, nil from ListVoices. +func TestNoopKokoro(t *testing.T) { + noop := &noopKokoro{} + + _, err := noop.GenerateAudio(t.Context(), "text", "af_bella") + if err == nil { + t.Fatal("noopKokoro.GenerateAudio: expected error, got nil") + } + + voices, err := noop.ListVoices(t.Context()) + if err != nil { + t.Fatalf("noopKokoro.ListVoices: unexpected error: %v", err) + } + if voices != nil { + t.Fatalf("noopKokoro.ListVoices: expected nil slice, got %v", voices) + } +} + +// TestRunStorageUnreachable verifies that run() fails fast and returns a +// descriptive error when PocketBase is unreachable. +func TestRunStorageUnreachable(t *testing.T) { + // Point at an address nothing is listening on. + t.Setenv("POCKETBASE_URL", "http://127.0.0.1:19999") + // Use a fast listen address so we don't accidentally start a real server. + t.Setenv("BACKEND_HTTP_ADDR", "127.0.0.1:0") + + err := run() + if err == nil { + t.Fatal("run() should have returned an error when storage is unreachable") + } + + t.Logf("got expected error: %v", err) +} + +// TestMain runs the test suite. No special setup required. +func TestMain(m *testing.M) { + os.Exit(m.Run()) +} diff --git a/v3/backend/cmd/healthcheck/main.go b/v3/backend/cmd/healthcheck/main.go new file mode 100644 index 0000000..4b7aa20 --- /dev/null +++ b/v3/backend/cmd/healthcheck/main.go @@ -0,0 +1,89 @@ +// healthcheck is a static binary used by Docker HEALTHCHECK CMD in distroless +// images (which have no shell, wget, or curl). +// +// Two modes: +// +// 1. HTTP mode (default): +// /healthcheck +// Performs GET ; exits 0 if HTTP 2xx/3xx, 1 otherwise. +// Example: /healthcheck http://localhost:8080/health +// +// 2. File-liveness mode: +// /healthcheck file +// Reads , parses its content as RFC3339 timestamp, and exits 1 if the +// timestamp is older than . Used by the runner service which +// writes /tmp/runner.alive on every successful poll. +// Example: /healthcheck file /tmp/runner.alive 120 +package main + +import ( + "fmt" + "net/http" + "os" + "strconv" + "time" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "file" { + checkFile() + return + } + checkHTTP() +} + +// checkHTTP performs a GET request and exits 0 on success, 1 on failure. +func checkHTTP() { + url := "http://localhost:8080/health" + if len(os.Args) > 1 { + url = os.Args[1] + } + resp, err := http.Get(url) //nolint:gosec,noctx + if err != nil { + fmt.Fprintf(os.Stderr, "healthcheck: %v\n", err) + os.Exit(1) + } + resp.Body.Close() + if resp.StatusCode >= 400 { + fmt.Fprintf(os.Stderr, "healthcheck: status %d\n", resp.StatusCode) + os.Exit(1) + } +} + +// checkFile reads a timestamp from a file and exits 1 if it is older than the +// given max age. Usage: /healthcheck file +func checkFile() { + if len(os.Args) < 4 { + fmt.Fprintln(os.Stderr, "healthcheck file: usage: /healthcheck file ") + os.Exit(1) + } + path := os.Args[2] + maxAgeSec, err := strconv.ParseInt(os.Args[3], 10, 64) + if err != nil { + fmt.Fprintf(os.Stderr, "healthcheck file: invalid max_age_seconds %q: %v\n", os.Args[3], err) + os.Exit(1) + } + + data, err := os.ReadFile(path) + if err != nil { + fmt.Fprintf(os.Stderr, "healthcheck file: cannot read %s: %v\n", path, err) + os.Exit(1) + } + + ts, err := time.Parse(time.RFC3339, string(data)) + if err != nil { + // Fallback: use file mtime if content is not a valid timestamp. + info, statErr := os.Stat(path) + if statErr != nil { + fmt.Fprintf(os.Stderr, "healthcheck file: cannot stat %s: %v\n", path, statErr) + os.Exit(1) + } + ts = info.ModTime() + } + + age := time.Since(ts) + if age > time.Duration(maxAgeSec)*time.Second { + fmt.Fprintf(os.Stderr, "healthcheck file: %s is %.0fs old (max %ds)\n", path, age.Seconds(), maxAgeSec) + os.Exit(1) + } +} diff --git a/v3/backend/cmd/runner/main.go b/v3/backend/cmd/runner/main.go new file mode 100644 index 0000000..01a230c --- /dev/null +++ b/v3/backend/cmd/runner/main.go @@ -0,0 +1,159 @@ +// Command runner is the homelab worker binary. +// +// It polls PocketBase for pending scrape and audio tasks, executes them, and +// writes results back. It connects directly to PocketBase and MinIO using +// admin credentials loaded from environment variables. +// +// Usage: +// +// runner # start polling loop (blocks until SIGINT/SIGTERM) +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/signal" + "runtime" + "syscall" + "time" + + "github.com/libnovel/backend/internal/browser" + "github.com/libnovel/backend/internal/config" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/meili" + "github.com/libnovel/backend/internal/novelfire" + "github.com/libnovel/backend/internal/runner" + "github.com/libnovel/backend/internal/storage" +) + +// version and commit are set at build time via -ldflags. +var ( + version = "dev" + commit = "unknown" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "runner: fatal: %v\n", err) + os.Exit(1) + } +} + +func run() error { + cfg := config.Load() + + // ── Logger ────────────────────────────────────────────────────────────── + log := buildLogger(cfg.LogLevel) + log.Info("runner starting", + "version", version, + "commit", commit, + "worker_id", cfg.Runner.WorkerID, + ) + + // ── Context: cancel on SIGINT / SIGTERM ───────────────────────────────── + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + // ── Storage ───────────────────────────────────────────────────────────── + store, err := storage.NewStore(ctx, cfg, log) + if err != nil { + return fmt.Errorf("init storage: %w", err) + } + + // ── Browser / Scraper ─────────────────────────────────────────────────── + workers := cfg.Runner.Workers + if workers <= 0 { + workers = runtime.NumCPU() + } + timeout := cfg.Runner.Timeout + if timeout <= 0 { + timeout = 90 * time.Second + } + + browserClient := browser.NewDirectClient(browser.Config{ + MaxConcurrent: workers, + Timeout: timeout, + }) + novel := novelfire.New(browserClient, log) + + // ── Kokoro ────────────────────────────────────────────────────────────── + var kokoroClient kokoro.Client + if cfg.Kokoro.URL != "" { + kokoroClient = kokoro.New(cfg.Kokoro.URL) + log.Info("kokoro TTS enabled", "url", cfg.Kokoro.URL) + } else { + log.Warn("KOKORO_URL not set — audio tasks will fail") + kokoroClient = &noopKokoro{} + } + + // ── Meilisearch ───────────────────────────────────────────────────────── + var searchIndex meili.Client + if cfg.Meilisearch.URL != "" { + if err := meili.Configure(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey); err != nil { + log.Warn("meilisearch configure failed — search indexing disabled", "err", err) + searchIndex = meili.NoopClient{} + } else { + searchIndex = meili.New(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey) + log.Info("meilisearch enabled", "url", cfg.Meilisearch.URL) + } + } else { + log.Info("MEILI_URL not set — search indexing disabled") + searchIndex = meili.NoopClient{} + } + + // ── Runner ────────────────────────────────────────────────────────────── + rCfg := runner.Config{ + WorkerID: cfg.Runner.WorkerID, + PollInterval: cfg.Runner.PollInterval, + MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape, + MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio, + OrchestratorWorkers: workers, + MetricsAddr: cfg.Runner.MetricsAddr, + CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval, + } + deps := runner.Dependencies{ + Consumer: store, + BookWriter: store, + BookReader: store, + AudioStore: store, + BrowseStore: store, + CoverStore: store, + SearchIndex: searchIndex, + Novel: novel, + Kokoro: kokoroClient, + Log: log, + } + r := runner.New(rCfg, deps) + + return r.Run(ctx) +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +func buildLogger(level string) *slog.Logger { + var lvl slog.Level + switch level { + case "debug": + lvl = slog.LevelDebug + case "warn": + lvl = slog.LevelWarn + case "error": + lvl = slog.LevelError + default: + lvl = slog.LevelInfo + } + return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl})) +} + +// noopKokoro is a no-op implementation used when KOKORO_URL is not set. +type noopKokoro struct{} + +func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) { + return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)") +} + +func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) { + return nil, nil +} diff --git a/v3/backend/go.mod b/v3/backend/go.mod new file mode 100644 index 0000000..c6e2633 --- /dev/null +++ b/v3/backend/go.mod @@ -0,0 +1,36 @@ +module github.com/libnovel/backend + +go 1.26.1 + +require ( + github.com/minio/minio-go/v7 v7.0.98 + golang.org/x/net v0.51.0 +) + +require ( + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/meilisearch/meilisearch-go v0.36.1 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/redis/go-redis/v9 v9.18.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/v3/backend/go.sum b/v3/backend/go.sum new file mode 100644 index 0000000..db607e6 --- /dev/null +++ b/v3/backend/go.sum @@ -0,0 +1,61 @@ +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/meilisearch/meilisearch-go v0.36.1 h1:mJTCJE5g7tRvaqKco6DfqOuJEjX+rRltDEnkEC02Y0M= +github.com/meilisearch/meilisearch-go v0.36.1/go.mod h1:hWcR0MuWLSzHfbz9GGzIr3s9rnXLm1jqkmHkJPbUSvM= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/v3/backend/internal/backend/handlers.go b/v3/backend/internal/backend/handlers.go new file mode 100644 index 0000000..8d5a835 --- /dev/null +++ b/v3/backend/internal/backend/handlers.go @@ -0,0 +1,1063 @@ +package backend + +// handlers.go — all HTTP request handlers for the backend server. +// +// Handler naming mirrors the route table in server.go: +// handleScrapeCatalogue, handleScrapeBook, handleScrapeBookRange +// handleScrapeStatus, handleScrapeTasks +// handleBrowse, handleSearch +// handleGetRanking, handleGetCover +// handleBookPreview, handleChapterText, handleReindex +// handleChapterText, handleReindex +// handleAudioGenerate, handleAudioStatus, handleAudioProxy +// handleVoices +// handlePresignChapter, handlePresignAudio, handlePresignVoiceSample +// handlePresignAvatarUpload, handlePresignAvatar +// handleGetProgress, handleSetProgress, handleDeleteProgress +// +// Key design choices vs. old scraper: +// - POST /scrape* creates a PocketBase task record and returns 202 with the +// task_id — it does NOT run the orchestrator inline. +// - POST /api/audio creates a PocketBase audio task and returns 202 — the +// runner binary executes TTS generation asynchronously. +// - GET /api/audio/status polls PocketBase for the task record status. +// - GET /api/audio-proxy reads the completed audio object from MinIO via a +// presigned URL redirect (the runner has already uploaded the bytes). +// - GET /api/browse and /api/search fetch novelfire.net live (no MinIO cache). +// - GET /api/cover redirects to the source cover URL live. +// - GET /api/ranking reads from the PocketBase ranking collection (populated +// by the runner after each catalogue scrape). +// - GET /api/book-preview returns stored data when in library, or enqueues a +// scrape task and returns 202 when not. The backend never scrapes directly. + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/meili" +) + +const ( + novelFireBase = "https://novelfire.net" + novelFireDomain = "novelfire.net" +) + +// ── Scrape task creation ─────────────────────────────────────────────────────── + +// handleScrapeCatalogue handles POST /scrape. +// Creates a "catalogue" scrape task in PocketBase and returns 202 with the task ID. +func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) { + taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "catalogue", "", 0, 0) + if err != nil { + s.deps.Log.Error("handleScrapeCatalogue: CreateScrapeTask failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to create task") + return + } + writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"}) +} + +// handleScrapeBook handles POST /scrape/book. +// Body: {"url": "https://novelfire.net/book/..."} +func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) { + var body struct { + URL string `json:"url"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { + jsonError(w, http.StatusBadRequest, `request body must be JSON with "url" field`) + return + } + taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "book", body.URL, 0, 0) + if err != nil { + s.deps.Log.Error("handleScrapeBook: CreateScrapeTask failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to create task") + return + } + writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"}) +} + +// handleScrapeBookRange handles POST /scrape/book/range. +// Body: {"url": "...", "from": N, "to": M} +func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) { + var body struct { + URL string `json:"url"` + From int `json:"from"` + To int `json:"to"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { + jsonError(w, http.StatusBadRequest, `request body must be JSON with "url" field`) + return + } + taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "book_range", body.URL, body.From, body.To) + if err != nil { + s.deps.Log.Error("handleScrapeBookRange: CreateScrapeTask failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to create task") + return + } + writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"}) +} + +// handleCancelTask handles POST /api/cancel-task/{id}. +// Transitions a pending task (scrape or audio) to status=cancelled. +// Returns 404 if the task does not exist, 409 if it cannot be cancelled +// (e.g. already running/done). +func (s *Server) handleCancelTask(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + if id == "" { + jsonError(w, http.StatusBadRequest, "missing task id") + return + } + if err := s.deps.Producer.CancelTask(r.Context(), id); err != nil { + s.deps.Log.Warn("handleCancelTask: CancelTask failed", "id", id, "err", err) + jsonError(w, http.StatusConflict, "could not cancel task: "+err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "cancelled", "id": id}) +} + +// ── Scrape task status / history ─────────────────────────────────────────────── + +// handleScrapeStatus handles GET /api/scrape/status. +// Returns the most recent scrape task status (or {"running":false} if none). +func (s *Server) handleScrapeStatus(w http.ResponseWriter, r *http.Request) { + tasks, err := s.deps.TaskReader.ListScrapeTasks(r.Context()) + if err != nil { + s.deps.Log.Error("handleScrapeStatus: ListScrapeTasks failed", "err", err) + writeJSON(w, 0, map[string]bool{"running": false}) + return + } + running := false + for _, t := range tasks { + if t.Status == domain.TaskStatusRunning || t.Status == domain.TaskStatusPending { + running = true + break + } + } + writeJSON(w, 0, map[string]bool{"running": running}) +} + +// handleScrapeTasks handles GET /api/scrape/tasks. +// Returns all scrape task records from PocketBase, newest first. +func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) { + tasks, err := s.deps.TaskReader.ListScrapeTasks(r.Context()) + if err != nil { + s.deps.Log.Error("handleScrapeTasks: ListScrapeTasks failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to list tasks") + return + } + if tasks == nil { + tasks = []domain.ScrapeTask{} + } + writeJSON(w, 0, tasks) +} + +// ── Browse & search ──────────────────────────────────────────────────────────── + +// NovelListing represents a single novel entry from the novelfire browse/search page. +type NovelListing struct { + Slug string `json:"slug"` + Title string `json:"title"` + Cover string `json:"cover"` + Rank string `json:"rank"` + Rating string `json:"rating"` + Chapters string `json:"chapters"` + URL string `json:"url"` +} + +// handleBrowse handles GET /api/browse. +// Fetches novelfire.net live (no MinIO cache in the new backend). +// Query params: page (default 1), genre (default "all"), sort (default "popular"), +// status (default "all"), type (default "all-novel") +func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + page := q.Get("page") + if page == "" { + page = "1" + } + genre := q.Get("genre") + if genre == "" { + genre = "all" + } + sortBy := q.Get("sort") + if sortBy == "" { + sortBy = "popular" + } + status := q.Get("status") + if status == "" { + status = "all" + } + novelType := q.Get("type") + if novelType == "" { + novelType = "all-novel" + } + + pageNum, _ := strconv.Atoi(page) + if pageNum <= 0 { + pageNum = 1 + } + + // ── Try MinIO cache first ───────────────────────────────────────────── + // Only page 1 is cached; higher pages fall through to live fetch. + if pageNum == 1 && s.deps.BrowseStore != nil { + if data, ok, err := s.deps.BrowseStore.GetBrowsePage(r.Context(), genre, sortBy, status, novelType, 1); err == nil && ok { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "public, max-age=300") + _, _ = w.Write(data) + return + } + } + + // ── Fall back to live novelfire.net fetch ────────────────────────────── + ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second) + defer cancel() + + targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d", + novelFireBase, genre, sortBy, status, novelType, pageNum) + + novels, hasNext, err := s.fetchBrowsePage(ctx, targetURL) + if err != nil { + // Live fetch also failed — return empty list with cached=false flag so + // the UI can show a "not ready yet" state instead of a hard error. + s.deps.Log.Error("handleBrowse: fetch failed (no cache)", "url", targetURL, "err", err) + w.Header().Set("Cache-Control", "no-store") + writeJSON(w, 0, map[string]any{ + "novels": []any{}, + "page": pageNum, + "hasNext": false, + "cached": false, + }) + return + } + + w.Header().Set("Cache-Control", "public, max-age=300") + writeJSON(w, 0, map[string]any{ + "novels": novels, + "page": pageNum, + "hasNext": hasNext, + "cached": false, + }) +} + +// handleSearch handles GET /api/search. +// Query params: q (min 2 chars), source ("local"|"remote"|"all", default "all") +// +// Local search is powered by Meilisearch when configured; falls back to a +// substring match against PocketBase book records otherwise. +func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("q") + if len([]rune(q)) < 2 { + jsonError(w, http.StatusBadRequest, "query must be at least 2 characters") + return + } + + source := r.URL.Query().Get("source") + if source == "" { + source = "all" + } + + ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second) + defer cancel() + + var localResults, remoteResults []NovelListing + + // Local search: Meilisearch → PocketBase substring fallback + if source == "local" || source == "all" { + meiliBooks, meiliErr := s.deps.SearchIndex.Search(ctx, q, 50) + if meiliErr == nil && len(meiliBooks) > 0 { + for _, b := range meiliBooks { + localResults = append(localResults, NovelListing{ + Slug: b.Slug, + Title: b.Title, + Cover: b.Cover, + URL: b.SourceURL, + }) + } + } else { + // Fallback: substring match against PocketBase + books, err := s.deps.BookReader.ListBooks(ctx) + if err != nil { + s.deps.Log.Warn("search: ListBooks failed", "err", err) + } else { + qLower := strings.ToLower(q) + for _, b := range books { + if strings.Contains(strings.ToLower(b.Title), qLower) || + strings.Contains(strings.ToLower(b.Author), qLower) { + localResults = append(localResults, NovelListing{ + Slug: b.Slug, + Title: b.Title, + Cover: b.Cover, + URL: b.SourceURL, + }) + } + } + } + } + } + + // Remote search (novelfire.net) + if source == "remote" || source == "all" { + searchURL := novelFireBase + "/search?keyword=" + url.QueryEscape(q) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil) + if err == nil { + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + if resp, fetchErr := http.DefaultClient.Do(req); fetchErr == nil { + defer resp.Body.Close() + if resp.StatusCode == http.StatusOK { + parsed, _ := parseBrowsePage(resp.Body) + remoteResults = parsed + } + } + } + } + + // Merge: local first, de-duplicate remote + localSlugs := make(map[string]bool, len(localResults)) + for _, item := range localResults { + localSlugs[item.Slug] = true + } + combined := make([]NovelListing, 0, len(localResults)+len(remoteResults)) + combined = append(combined, localResults...) + for _, item := range remoteResults { + if !localSlugs[item.Slug] { + combined = append(combined, item) + } + } + + writeJSON(w, 0, map[string]any{ + "results": combined, + "local_count": len(localResults), + "remote_count": len(remoteResults), + }) +} + +// ── Ranking ──────────────────────────────────────────────────────────────────── + +// handleGetRanking handles GET /api/ranking. +// Returns all ranking items sorted by rank ascending. +func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) { + items, err := s.deps.RankingStore.ReadRankingItems(r.Context()) + if err != nil { + s.deps.Log.Error("handleGetRanking: ReadRankingItems failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to read ranking") + return + } + if items == nil { + items = []domain.RankingItem{} + } + writeJSON(w, 0, items) +} + +// handleGetCover handles GET /api/cover/{domain}/{slug}. +// Serves the cover image directly from MinIO when available; falls back to a +// redirect to the novelfire CDN when the cover has not yet been downloaded. +func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + http.Error(w, "missing slug", http.StatusBadRequest) + return + } + + // Fast path: serve from MinIO if the cover has been downloaded. + if s.deps.CoverStore != nil { + data, ct, ok, err := s.deps.CoverStore.GetCover(r.Context(), slug) + if err != nil { + s.deps.Log.Warn("handleGetCover: GetCover error", "slug", slug, "err", err) + } + if ok && len(data) > 0 { + if ct == "" { + ct = "image/jpeg" + } + w.Header().Set("Content-Type", ct) + w.Header().Set("Cache-Control", "public, max-age=86400") + _, _ = w.Write(data) + return + } + } + + // Fallback: redirect to the CDN. The caller sees a working image; the + // cover will be populated on the next catalogue refresh run. + coverURL := fmt.Sprintf("https://cdn.novelfire.net/covers/%s.jpg", slug) + http.Redirect(w, r, coverURL, http.StatusFound) +} + +// ── Preview (live scrape, no store writes) ───────────────────────────────────── + +// handleBookPreview handles GET /api/book-preview/{slug}. +// +// If the book is already in the library (PocketBase), returns its metadata and +// chapter index immediately (200). +// +// If the book is not yet in the library, enqueues a "book" scrape task and +// returns 202 Accepted with the task_id. The runner will scrape the book +// asynchronously; the client should poll GET /api/scrape/status or +// GET /api/scrape/tasks to detect completion, then re-request this endpoint. +// +// The backend never scrapes directly — all scraping is the runner's job. +func (s *Server) handleBookPreview(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + + ctx := r.Context() + + meta, inLib, err := s.deps.BookReader.ReadMetadata(ctx, slug) + if err != nil { + s.deps.Log.Warn("book-preview: ReadMetadata failed", "slug", slug, "err", err) + inLib = false + } + + if inLib { + // Fast path: book is already scraped — return stored data. + chapters, cerr := s.deps.BookReader.ListChapters(ctx, slug) + if cerr != nil { + s.deps.Log.Warn("book-preview: ListChapters failed", "slug", slug, "err", cerr) + } + writeJSON(w, 0, map[string]any{ + "in_lib": true, + "meta": meta, + "chapters": chapters, + }) + return + } + + // Book not in library — enqueue a range scrape task for the first 20 chapters + // so the user can start reading quickly. Remaining chapters can be scraped + // later via the book detail page or the admin scrape panel. + bookURL := r.URL.Query().Get("source_url") + if bookURL == "" { + bookURL = fmt.Sprintf("%s/book/%s", novelFireBase, slug) + } + + const previewFrom, previewTo = 1, 20 + taskID, err := s.deps.Producer.CreateScrapeTask(ctx, "book_range", bookURL, previewFrom, previewTo) + if err != nil { + s.deps.Log.Error("book-preview: CreateScrapeTask failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, "failed to enqueue scrape task") + return + } + + s.deps.Log.Info("book-preview: enqueued range scrape task", "slug", slug, "task_id", taskID, + "from", previewFrom, "to", previewTo) + writeJSON(w, http.StatusAccepted, map[string]any{ + "in_lib": false, + "task_id": taskID, + "message": fmt.Sprintf("scraping first %d chapters; poll /api/scrape/tasks for completion", previewTo), + }) +} + +// ── Chapter text ─────────────────────────────────────────────────────────────── + +// handleChapterText handles GET /api/chapter-text/{slug}/{n}. +// Returns plain text (markdown stripped) of a stored chapter. +func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + http.NotFound(w, r) + return + } + raw, err := s.deps.BookReader.ReadChapter(r.Context(), slug, n) + if err != nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + fmt.Fprint(w, stripMarkdown(raw)) +} + +// handleChapterMarkdown handles GET /api/chapter-markdown/{slug}/{n}. +// +// Returns the raw markdown content of a stored chapter directly from MinIO. +// This is used by the SvelteKit UI as a simpler alternative to presign+fetch: +// it avoids the need for the SvelteKit server to reach MinIO directly, and +// gives a clean 404 when the chapter has not been scraped yet. +func (s *Server) handleChapterMarkdown(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest) + return + } + raw, err := s.deps.BookReader.ReadChapter(r.Context(), slug, n) + if err != nil { + s.deps.Log.Warn("chapter-markdown: not found in MinIO", "slug", slug, "n", n, "err", err) + http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "text/markdown; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + fmt.Fprint(w, raw) +} + +// handleReindex handles POST /api/reindex/{slug}. +// Rebuilds the chapters_idx PocketBase collection for a book from MinIO objects. +func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + + count, err := s.deps.BookReader.ReindexChapters(r.Context(), slug) + if err != nil { + s.deps.Log.Error("reindex failed", "slug", slug, "indexed", count, "err", err) + writeJSON(w, http.StatusInternalServerError, map[string]any{ + "error": err.Error(), + "indexed": count, + }) + return + } + + s.deps.Log.Info("reindex complete", "slug", slug, "indexed", count) + writeJSON(w, 0, map[string]any{"slug": slug, "indexed": count}) +} + +// ── Audio ────────────────────────────────────────────────────────────────────── + +// handleAudioGenerate handles POST /api/audio/{slug}/{n}. +// Creates an audio_jobs task in PocketBase (runner executes asynchronously). +// Returns 200 immediately if audio already exists in MinIO. +// Returns 202 with the task_id if a new task was created. +func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + jsonError(w, http.StatusBadRequest, "invalid chapter") + return + } + + voice := s.cfg.DefaultVoice + var body struct { + Voice string `json:"voice"` + } + if r.Body != nil { + _ = json.NewDecoder(r.Body).Decode(&body) + } + if body.Voice != "" { + voice = body.Voice + } + + cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice) + + // Fast path: audio already in MinIO + audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice) + if s.deps.AudioStore.AudioExists(r.Context(), audioKey) { + proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice) + writeJSON(w, 0, map[string]string{"url": proxyURL, "status": "done"}) + return + } + + // Check if a task is already pending/running + task, found, _ := s.deps.TaskReader.GetAudioTask(r.Context(), cacheKey) + if found && (task.Status == domain.TaskStatusPending || task.Status == domain.TaskStatusRunning) { + writeJSON(w, http.StatusAccepted, map[string]string{ + "task_id": task.ID, + "status": string(task.Status), + }) + return + } + + // Create a new audio task + taskID, err := s.deps.Producer.CreateAudioTask(r.Context(), slug, n, voice) + if err != nil { + s.deps.Log.Error("handleAudioGenerate: CreateAudioTask failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to create audio task") + return + } + + writeJSON(w, http.StatusAccepted, map[string]string{ + "task_id": taskID, + "status": "pending", + }) +} + +// handleAudioStatus handles GET /api/audio/status/{slug}/{n}. +// Polls PocketBase for the audio task status. +// Query params: voice (optional, defaults to DefaultVoice) +func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + jsonError(w, http.StatusBadRequest, "invalid params") + return + } + + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.cfg.DefaultVoice + } + + // Fast path: audio exists in MinIO + audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice) + if s.deps.AudioStore.AudioExists(r.Context(), audioKey) { + proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice) + writeJSON(w, 0, map[string]string{ + "status": "done", + "url": proxyURL, + }) + return + } + + cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice) + task, found, _ := s.deps.TaskReader.GetAudioTask(r.Context(), cacheKey) + if !found { + writeJSON(w, 0, map[string]string{"status": "idle"}) + return + } + + resp := map[string]string{ + "status": string(task.Status), + "task_id": task.ID, + } + if task.Status == domain.TaskStatusFailed && task.ErrorMessage != "" { + resp["error"] = task.ErrorMessage + } + writeJSON(w, 0, resp) +} + +// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}. +// Redirects to a presigned MinIO URL for the generated audio object. +// Query params: voice (optional, defaults to DefaultVoice) +func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + http.NotFound(w, r) + return + } + + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.cfg.DefaultVoice + } + + audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice) + if !s.deps.AudioStore.AudioExists(r.Context(), audioKey) { + http.Error(w, "audio not generated yet", http.StatusNotFound) + return + } + + presignURL, err := s.deps.PresignStore.PresignAudio(r.Context(), audioKey, 1*time.Hour) + if err != nil { + s.deps.Log.Error("handleAudioProxy: PresignAudio failed", "slug", slug, "n", n, "err", err) + http.Error(w, "presign failed", http.StatusInternalServerError) + return + } + + http.Redirect(w, r, presignURL, http.StatusFound) +} + +// ── Voices ───────────────────────────────────────────────────────────────────── + +// handleVoices handles GET /api/voices. +// Returns {"voices": [...]} — fetched from Kokoro with built-in fallback. +func (s *Server) handleVoices(w http.ResponseWriter, r *http.Request) { + writeJSON(w, 0, map[string]any{"voices": s.voices(r.Context())}) +} + +// ── Presigned URLs ───────────────────────────────────────────────────────────── + +// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}. +func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + jsonError(w, http.StatusBadRequest, "invalid params") + return + } + + u, err := s.deps.PresignStore.PresignChapter(r.Context(), slug, n, 15*time.Minute) + if err != nil { + s.deps.Log.Error("presign chapter failed", "slug", slug, "n", n, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + writeJSON(w, 0, map[string]string{"url": u}) +} + +// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}. +// Query params: voice (optional) +func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + jsonError(w, http.StatusBadRequest, "invalid params") + return + } + + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.cfg.DefaultVoice + } + + key := s.deps.AudioStore.AudioObjectKey(slug, n, voice) + if !s.deps.AudioStore.AudioExists(r.Context(), key) { + http.NotFound(w, r) + return + } + + u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour) + if err != nil { + s.deps.Log.Error("presign audio failed", "slug", slug, "n", n, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + writeJSON(w, 0, map[string]string{"url": u}) +} + +// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}. +func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) { + voice := r.PathValue("voice") + if voice == "" { + jsonError(w, http.StatusBadRequest, "missing voice") + return + } + + key := kokoro.VoiceSampleKey(voice) + if !s.deps.AudioStore.AudioExists(r.Context(), key) { + http.NotFound(w, r) + return + } + + u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour) + if err != nil { + s.deps.Log.Error("presign voice sample failed", "voice", voice, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + writeJSON(w, 0, map[string]string{"url": u}) +} + +// handlePresignAvatarUpload handles GET /api/presign/avatar-upload/{userId}. +// Query params: ext (jpg|png|webp, defaults to jpg) +func (s *Server) handlePresignAvatarUpload(w http.ResponseWriter, r *http.Request) { + userID := r.PathValue("userId") + if userID == "" { + jsonError(w, http.StatusBadRequest, "missing userId") + return + } + + ext := r.URL.Query().Get("ext") + switch ext { + case "jpg", "jpeg": + ext = "jpg" + case "png": + ext = "png" + case "webp": + ext = "webp" + default: + ext = "jpg" + } + + uploadURL, key, err := s.deps.PresignStore.PresignAvatarUpload(r.Context(), userID, ext) + if err != nil { + s.deps.Log.Error("presign avatar upload failed", "userId", userID, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + writeJSON(w, 0, map[string]string{"upload_url": uploadURL, "key": key}) +} + +// handlePresignAvatar handles GET /api/presign/avatar/{userId}. +func (s *Server) handlePresignAvatar(w http.ResponseWriter, r *http.Request) { + userID := r.PathValue("userId") + if userID == "" { + jsonError(w, http.StatusBadRequest, "missing userId") + return + } + + u, found, err := s.deps.PresignStore.PresignAvatarURL(r.Context(), userID) + if err != nil { + s.deps.Log.Error("presign avatar failed", "userId", userID, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + if !found { + http.NotFound(w, r) + return + } + writeJSON(w, 0, map[string]string{"url": u}) +} + +// ── Progress ─────────────────────────────────────────────────────────────────── + +// handleGetProgress handles GET /api/progress. +// Returns {"slug": chapterNum, "slug_ts": timestampMs, ...} +func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) { + sid := ensureSession(w, r) + entries, err := s.deps.ProgressStore.AllProgress(r.Context(), sid) + if err != nil { + s.deps.Log.Error("AllProgress failed", "err", err) + entries = nil + } + + progress := make(map[string]any, len(entries)*2) + for _, p := range entries { + progress[p.Slug] = p.Chapter + progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli() + } + writeJSON(w, 0, progress) +} + +// handleSetProgress handles POST /api/progress/{slug}. +// Body: {"chapter": N} +func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) { + sid := ensureSession(w, r) + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + + var body struct { + Chapter int `json:"chapter"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 { + jsonError(w, http.StatusBadRequest, "invalid body") + return + } + + p := domain.ReadingProgress{ + Slug: slug, + Chapter: body.Chapter, + UpdatedAt: time.Now(), + } + if err := s.deps.ProgressStore.SetProgress(r.Context(), sid, p); err != nil { + s.deps.Log.Error("SetProgress failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, "store error") + return + } + writeJSON(w, 0, map[string]string{}) +} + +// handleDeleteProgress handles DELETE /api/progress/{slug}. +func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) { + sid := ensureSession(w, r) + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + + if err := s.deps.ProgressStore.DeleteProgress(r.Context(), sid, slug); err != nil { + s.deps.Log.Error("DeleteProgress failed", "slug", slug, "err", err) + // non-fatal + } + writeJSON(w, 0, map[string]string{}) +} + +// ── Catalogue (Meilisearch-backed browse + search) ──────────────────────────── + +// handleCatalogue handles GET /api/catalogue. +// +// Provides unified browse + search over the locally-indexed book catalogue +// via Meilisearch. Unlike /api/browse this never fetches novelfire.net live — +// it is entirely served from the Meilisearch index populated by the runner. +// +// Query params: +// +// q — full-text search query (optional) +// genre — genre filter, e.g. "fantasy" or "all" (default "all") +// status — status filter: "ongoing", "completed", or "all" (default "all") +// sort — "popular" (default) | "new" | "top-rated" | "rank" +// page — 1-indexed page number (default 1) +// limit — items per page (default 20, max 100) +func (s *Server) handleCatalogue(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + + genre := q.Get("genre") + if genre == "" { + genre = "all" + } + status := q.Get("status") + if status == "" { + status = "all" + } + sort := q.Get("sort") + if sort == "" { + sort = "popular" + } + + page, _ := strconv.Atoi(q.Get("page")) + if page <= 0 { + page = 1 + } + limit, _ := strconv.Atoi(q.Get("limit")) + if limit <= 0 { + limit = 20 + } + if limit > 100 { + limit = 100 + } + + cq := meili.CatalogueQuery{ + Q: q.Get("q"), + Genre: genre, + Status: status, + Sort: sort, + Page: page, + Limit: limit, + } + + books, total, err := s.deps.SearchIndex.Catalogue(r.Context(), cq) + if err != nil { + s.deps.Log.Error("handleCatalogue: Catalogue query failed", "err", err) + jsonError(w, http.StatusInternalServerError, "search failed") + return + } + + hasNext := int64(page*limit) < total + + w.Header().Set("Cache-Control", "public, max-age=60") + writeJSON(w, 0, map[string]any{ + "books": books, + "page": page, + "limit": limit, + "total": total, + "has_next": hasNext, + }) +} + +// ── Browse page parsing helpers ──────────────────────────────────────────────── + +// fetchBrowsePage fetches pageURL and parses NovelListings from the HTML. +func (s *Server) fetchBrowsePage(ctx context.Context, pageURL string) ([]NovelListing, bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) + if err != nil { + return nil, false, fmt.Errorf("build request: %w", err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, false, fmt.Errorf("upstream returned %d", resp.StatusCode) + } + + novels, hasNext := parseBrowsePage(resp.Body) + return novels, hasNext, nil +} + +// parseBrowsePage parses a novelfire HTML body and returns novel listings. +// It uses a simple string-scanning approach to avoid importing golang.org/x/net/html +// in this package (that dependency is only in internal/novelfire). +func parseBrowsePage(r io.Reader) ([]NovelListing, bool) { + data, err := io.ReadAll(r) + if err != nil { + return nil, false + } + body := string(data) + + var novels []NovelListing + hasNext := false + + // Detect "next page" link + if strings.Contains(body, `rel="next"`) || + strings.Contains(body, `aria-label="Next"`) || + strings.Contains(body, `class="next"`) { + hasNext = true + } + + // Extract novel slugs and titles using simple regex patterns. + // novelfire.net novel items:
  • ...
  • + // Each contains an anchor like + slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`) + titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`) + coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`) + + slugMatches := slugRe.FindAllStringSubmatch(body, -1) + titleMatches := titleRe.FindAllStringSubmatch(body, -1) + coverMatches := coverRe.FindAllStringSubmatch(body, -1) + + seen := make(map[string]bool) + for i, sm := range slugMatches { + slug := sm[1] + if seen[slug] { + continue + } + seen[slug] = true + + novel := NovelListing{ + Slug: slug, + URL: novelFireBase + "/book/" + slug, + } + if i < len(titleMatches) { + novel.Title = strings.TrimSpace(titleMatches[i][1]) + } + if i < len(coverMatches) { + novel.Cover = coverMatches[i][1] + } + if novel.Title != "" { + novels = append(novels, novel) + } + } + + return novels, hasNext +} + +// ── Markdown stripping ───────────────────────────────────────────────────────── + +// stripMarkdown removes common markdown syntax from src, returning plain text. +func stripMarkdown(src string) string { + src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "") + src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "") + src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "") + src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "") + src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1") + src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "") + src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "") + src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "") + src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n") + return strings.TrimSpace(src) +} + +// ── Hardcoded Kokoro voice fallback ─────────────────────────────────────────── + +// kokoroVoices is the built-in fallback list used when the Kokoro service is +// unavailable. Matches the list in the old scraper helpers.go. +var kokoroVoices = []string{ + // American English + "af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia", + "af_jessica", "af_kore", "af_nicole", "af_nova", "af_river", + "af_sarah", "af_sky", + "am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam", + "am_michael", "am_onyx", "am_puck", + // British English + "bf_alice", "bf_emma", "bf_lily", + "bm_daniel", "bm_fable", "bm_george", "bm_lewis", + // Spanish + "ef_dora", "em_alex", + // French + "ff_siwis", + // Hindi + "hf_alpha", "hf_beta", "hm_omega", "hm_psi", + // Italian + "if_sara", "im_nicola", + // Japanese + "jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo", + // Portuguese + "pf_dora", "pm_alex", + // Chinese + "zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi", + "zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang", +} diff --git a/v3/backend/internal/backend/server.go b/v3/backend/internal/backend/server.go new file mode 100644 index 0000000..93486ff --- /dev/null +++ b/v3/backend/internal/backend/server.go @@ -0,0 +1,300 @@ +// Package backend implements the HTTP API server for the LibNovel backend. +// +// The server exposes all endpoints consumed by the SvelteKit UI: +// - Book/chapter reads from PocketBase/MinIO via bookstore interfaces +// - Task creation (scrape + audio) via taskqueue.Producer — the runner binary +// picks up and executes those tasks asynchronously +// - Presigned MinIO URLs for media playback/upload +// - Session-scoped reading progress +// - Live novelfire.net browse/search (no scraper interface needed; direct HTTP) +// - Kokoro voice list +// +// The backend never scrapes directly. All scraping (metadata, chapter list, +// chapter text, audio TTS) is delegated to the runner binary via PocketBase +// task records. GET /api/book-preview enqueues a task when the book is absent. +// +// All external dependencies are injected as interfaces; concrete types live in +// internal/storage and are wired by cmd/backend/main.go. +package backend + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "sync" + "time" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/meili" + "github.com/libnovel/backend/internal/taskqueue" +) + +// Dependencies holds all external services the backend server depends on. +// Every field is an interface so test doubles can be injected freely. +type Dependencies struct { + // BookReader reads book metadata and chapter text from PocketBase/MinIO. + BookReader bookstore.BookReader + // RankingStore reads ranking data from PocketBase. + RankingStore bookstore.RankingStore + // AudioStore checks audio object existence and computes MinIO keys. + AudioStore bookstore.AudioStore + // PresignStore generates short-lived MinIO URLs. + PresignStore bookstore.PresignStore + // ProgressStore reads/writes per-session reading progress. + ProgressStore bookstore.ProgressStore + // BrowseStore reads cached browse page snapshots from MinIO. + BrowseStore bookstore.BrowseStore + // CoverStore reads and writes book cover images from MinIO. + // If nil, the cover endpoint falls back to a CDN redirect. + CoverStore bookstore.CoverStore + // Producer creates scrape/audio tasks in PocketBase. + Producer taskqueue.Producer + // TaskReader reads scrape/audio task records from PocketBase. + TaskReader taskqueue.Reader + // SearchIndex provides full-text book search via Meilisearch. + // If nil, the local-only fallback search is used. + SearchIndex meili.Client + // Kokoro is the TTS client (used for voice list only in the backend; + // audio generation is done by the runner). + Kokoro kokoro.Client + // Log is the structured logger. + Log *slog.Logger +} + +// Config holds HTTP server tuning parameters. +type Config struct { + // Addr is the listen address, e.g. ":8080". + Addr string + // DefaultVoice is used when no voice is specified in audio requests. + DefaultVoice string + // Version and Commit are embedded in /health and /api/version responses. + Version string + Commit string +} + +// Server is the HTTP API server. +type Server struct { + cfg Config + deps Dependencies + + // voiceMu guards cachedVoices. Populated lazily on first GET /api/voices. + voiceMu sync.RWMutex + cachedVoices []string +} + +// New creates a Server from cfg and deps. +func New(cfg Config, deps Dependencies) *Server { + if cfg.DefaultVoice == "" { + cfg.DefaultVoice = "af_bella" + } + if deps.Log == nil { + deps.Log = slog.Default() + } + if deps.SearchIndex == nil { + deps.SearchIndex = meili.NoopClient{} + } + return &Server{cfg: cfg, deps: deps} +} + +// ListenAndServe registers all routes and starts the HTTP server. +// It blocks until ctx is cancelled, then performs a graceful shutdown. +func (s *Server) ListenAndServe(ctx context.Context) error { + mux := http.NewServeMux() + + // Health / version + mux.HandleFunc("GET /health", s.handleHealth) + mux.HandleFunc("GET /api/version", s.handleVersion) + + // Scrape task creation (202 Accepted — runner executes asynchronously) + mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) + mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) + mux.HandleFunc("POST /scrape/book/range", s.handleScrapeBookRange) + + // Scrape task status / history + mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus) + mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks) + + // Cancel a pending task (scrape or audio) + mux.HandleFunc("POST /api/cancel-task/{id}", s.handleCancelTask) + + // Browse & search (live novelfire.net) + mux.HandleFunc("GET /api/browse", s.handleBrowse) + mux.HandleFunc("GET /api/search", s.handleSearch) + + // Catalogue (Meilisearch-backed browse + search — preferred path for UI) + mux.HandleFunc("GET /api/catalogue", s.handleCatalogue) + + // Ranking (from PocketBase) + mux.HandleFunc("GET /api/ranking", s.handleGetRanking) + + // Cover proxy (live URL redirect) + mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover) + + // Book preview (enqueues scrape task if not in library; returns stored data if already scraped) + mux.HandleFunc("GET /api/book-preview/{slug}", s.handleBookPreview) + + // Chapter text (served from MinIO via PocketBase index) + mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText) + // Raw markdown chapter content — served directly from MinIO by the backend. + // Use this instead of presign+fetch to avoid SvelteKit→MinIO network path. + mux.HandleFunc("GET /api/chapter-markdown/{slug}/{n}", s.handleChapterMarkdown) + + // Reindex chapters_idx from MinIO + mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex) + + // Audio task creation (backend creates task; runner executes) + mux.HandleFunc("POST /api/audio/{slug}/{n}", s.handleAudioGenerate) + mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus) + mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy) + + // Voices list + mux.HandleFunc("GET /api/voices", s.handleVoices) + + // Presigned URLs + mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter) + mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio) + mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample) + mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload) + mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar) + + // Reading progress + mux.HandleFunc("GET /api/progress", s.handleGetProgress) + mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress) + mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress) + + srv := &http.Server{ + Addr: s.cfg.Addr, + Handler: mux, + ReadTimeout: 15 * time.Second, + WriteTimeout: 60 * time.Second, + IdleTimeout: 60 * time.Second, + } + + errCh := make(chan error, 1) + go func() { errCh <- srv.ListenAndServe() }() + s.deps.Log.Info("backend: HTTP server listening", "addr", s.cfg.Addr) + + select { + case <-ctx.Done(): + s.deps.Log.Info("backend: context cancelled, starting graceful shutdown") + shutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := srv.Shutdown(shutCtx); err != nil { + s.deps.Log.Error("backend: graceful shutdown failed", "err", err) + return err + } + s.deps.Log.Info("backend: shutdown complete") + return nil + case err := <-errCh: + return err + } +} + +// ── Session cookie helpers ───────────────────────────────────────────────────── + +const sessionCookieName = "libnovel_session" + +func sessionID(r *http.Request) string { + c, err := r.Cookie(sessionCookieName) + if err != nil { + return "" + } + return c.Value +} + +func newSessionID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +func ensureSession(w http.ResponseWriter, r *http.Request) string { + if id := sessionID(r); id != "" { + return id + } + id, err := newSessionID() + if err != nil { + id = fmt.Sprintf("fallback-%d", time.Now().UnixNano()) + } + http.SetCookie(w, &http.Cookie{ + Name: sessionCookieName, + Value: id, + Path: "/", + HttpOnly: true, + SameSite: http.SameSiteLaxMode, + MaxAge: 365 * 24 * 60 * 60, + }) + return id +} + +// ── Utility helpers ──────────────────────────────────────────────────────────── + +// writeJSON writes v as a JSON response with status code. Status 0 → 200. +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + if status != 0 { + w.WriteHeader(status) + } + _ = json.NewEncoder(w).Encode(v) +} + +// jsonError writes a JSON error body and the given status code. +func jsonError(w http.ResponseWriter, status int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]string{"error": msg}) +} + +// voices returns the list of available Kokoro voices. On the first call it +// fetches from the Kokoro service and caches the result. Falls back to the +// hardcoded list on error. +func (s *Server) voices(ctx context.Context) []string { + s.voiceMu.RLock() + cached := s.cachedVoices + s.voiceMu.RUnlock() + if len(cached) > 0 { + return cached + } + + if s.deps.Kokoro == nil { + return kokoroVoices + } + + fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + list, err := s.deps.Kokoro.ListVoices(fetchCtx) + if err != nil || len(list) == 0 { + s.deps.Log.Warn("backend: could not fetch kokoro voices, using built-in list", "err", err) + return kokoroVoices + } + + s.voiceMu.Lock() + s.cachedVoices = list + s.voiceMu.Unlock() + s.deps.Log.Info("backend: fetched kokoro voices", "count", len(list)) + return list +} + +// handleHealth handles GET /health. +func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, 0, map[string]string{ + "status": "ok", + "version": s.cfg.Version, + "commit": s.cfg.Commit, + }) +} + +// handleVersion handles GET /api/version. +func (s *Server) handleVersion(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, 0, map[string]string{ + "version": s.cfg.Version, + "commit": s.cfg.Commit, + }) +} diff --git a/v3/backend/internal/bookstore/bookstore.go b/v3/backend/internal/bookstore/bookstore.go new file mode 100644 index 0000000..509f01d --- /dev/null +++ b/v3/backend/internal/bookstore/bookstore.go @@ -0,0 +1,151 @@ +// Package bookstore defines the segregated read/write interfaces for book, +// chapter, ranking, progress, audio, and presign data. +// +// Interface segregation: +// - BookWriter — used by the runner to persist scraped data. +// - BookReader — used by the backend to serve book/chapter data. +// - RankingStore — used by both runner (write) and backend (read). +// - PresignStore — used only by the backend for URL signing. +// - AudioStore — used by the runner to store audio; backend for presign. +// - ProgressStore— used only by the backend for reading progress. +// +// Concrete implementations live in internal/storage. +package bookstore + +import ( + "context" + "time" + + "github.com/libnovel/backend/internal/domain" +) + +// BookWriter is the write side used by the runner after scraping a book. +type BookWriter interface { + // WriteMetadata upserts all bibliographic fields for a book. + WriteMetadata(ctx context.Context, meta domain.BookMeta) error + + // WriteChapter stores a fully-scraped chapter's text in MinIO and + // updates the chapters_idx record in PocketBase. + WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error + + // WriteChapterRefs persists chapter metadata (number + title) into + // chapters_idx without fetching or storing chapter text. + WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error + + // ChapterExists returns true if the markdown object for ref already exists. + ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool +} + +// BookReader is the read side used by the backend to serve content. +type BookReader interface { + // ReadMetadata returns the metadata for slug. + // Returns (zero, false, nil) when not found. + ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error) + + // ListBooks returns all books sorted alphabetically by title. + ListBooks(ctx context.Context) ([]domain.BookMeta, error) + + // LocalSlugs returns the set of slugs that have metadata stored. + LocalSlugs(ctx context.Context) (map[string]bool, error) + + // MetadataMtime returns the Unix-second mtime of the metadata record, or 0. + MetadataMtime(ctx context.Context, slug string) int64 + + // ReadChapter returns the raw markdown for chapter number n. + ReadChapter(ctx context.Context, slug string, n int) (string, error) + + // ListChapters returns all stored chapters for slug, sorted by number. + ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error) + + // CountChapters returns the count of stored chapters. + CountChapters(ctx context.Context, slug string) int + + // ReindexChapters rebuilds chapters_idx from MinIO objects for slug. + ReindexChapters(ctx context.Context, slug string) (int, error) +} + +// RankingStore covers ranking reads and writes. +type RankingStore interface { + // WriteRankingItem upserts a single ranking entry (keyed on Slug). + WriteRankingItem(ctx context.Context, item domain.RankingItem) error + + // ReadRankingItems returns all ranking items sorted by rank ascending. + ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error) + + // RankingFreshEnough returns true when ranking rows exist and the most + // recent Updated timestamp is within maxAge. + RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) +} + +// AudioStore covers audio object storage (runner writes; backend reads). +type AudioStore interface { + // AudioObjectKey returns the MinIO object key for a cached audio file. + AudioObjectKey(slug string, n int, voice string) string + + // AudioExists returns true when the audio object is present in MinIO. + AudioExists(ctx context.Context, key string) bool + + // PutAudio stores raw audio bytes under the given MinIO object key. + PutAudio(ctx context.Context, key string, data []byte) error +} + +// PresignStore generates short-lived URLs — used exclusively by the backend. +type PresignStore interface { + // PresignChapter returns a presigned GET URL for a chapter markdown object. + PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) + + // PresignAudio returns a presigned GET URL for an audio object. + PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) + + // PresignAvatarUpload returns a short-lived presigned PUT URL for uploading + // an avatar image. ext should be "jpg", "png", or "webp". + PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error) + + // PresignAvatarURL returns a presigned GET URL for a user's avatar. + // Returns ("", false, nil) when no avatar exists. + PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) + + // DeleteAvatar removes all avatar objects for a user. + DeleteAvatar(ctx context.Context, userID string) error +} + +// ProgressStore covers per-session reading progress — backend only. +type ProgressStore interface { + // GetProgress returns the reading progress for the given session + slug. + GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool) + + // SetProgress saves or updates reading progress. + SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error + + // AllProgress returns all progress entries for a session. + AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error) + + // DeleteProgress removes progress for a specific slug. + DeleteProgress(ctx context.Context, sessionID, slug string) error +} + +// BrowseStore covers browse page snapshot storage. +// The runner writes snapshots; the backend reads them. +type BrowseStore interface { + // PutBrowsePage stores a raw JSON snapshot for a browse page. + // genre, sort, status, novelType and page identify the page. + PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error + + // GetBrowsePage retrieves a raw JSON snapshot. Returns (nil, false, nil) + // when no snapshot exists for the given parameters. + GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) +} + +// CoverStore covers book cover image storage in MinIO. +// The runner writes covers during catalogue refresh; the backend reads them. +type CoverStore interface { + // PutCover stores a raw cover image for a book identified by slug. + PutCover(ctx context.Context, slug string, data []byte, contentType string) error + + // GetCover retrieves the cover image for a book. Returns (nil, false, nil) + // when no cover exists for the given slug. + GetCover(ctx context.Context, slug string) ([]byte, string, bool, error) + + // CoverExists returns true when a cover image is stored for slug. + CoverExists(ctx context.Context, slug string) bool +} diff --git a/v3/backend/internal/bookstore/bookstore_test.go b/v3/backend/internal/bookstore/bookstore_test.go new file mode 100644 index 0000000..2bbc5d0 --- /dev/null +++ b/v3/backend/internal/bookstore/bookstore_test.go @@ -0,0 +1,138 @@ +package bookstore_test + +import ( + "context" + "testing" + "time" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/domain" +) + +// ── Mock that satisfies all bookstore interfaces ────────────────────────────── + +type mockStore struct{} + +// BookWriter +func (m *mockStore) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil } +func (m *mockStore) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error { return nil } +func (m *mockStore) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error { + return nil +} +func (m *mockStore) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool { + return false +} + +// BookReader +func (m *mockStore) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) { + return domain.BookMeta{}, false, nil +} +func (m *mockStore) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil } +func (m *mockStore) LocalSlugs(_ context.Context) (map[string]bool, error) { + return map[string]bool{}, nil +} +func (m *mockStore) MetadataMtime(_ context.Context, _ string) int64 { return 0 } +func (m *mockStore) ReadChapter(_ context.Context, _ string, _ int) (string, error) { + return "", nil +} +func (m *mockStore) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) { + return nil, nil +} +func (m *mockStore) CountChapters(_ context.Context, _ string) int { return 0 } +func (m *mockStore) ReindexChapters(_ context.Context, _ string) (int, error) { return 0, nil } + +// RankingStore +func (m *mockStore) WriteRankingItem(_ context.Context, _ domain.RankingItem) error { return nil } +func (m *mockStore) ReadRankingItems(_ context.Context) ([]domain.RankingItem, error) { + return nil, nil +} +func (m *mockStore) RankingFreshEnough(_ context.Context, _ time.Duration) (bool, error) { + return false, nil +} + +// AudioStore +func (m *mockStore) AudioObjectKey(_ string, _ int, _ string) string { return "" } +func (m *mockStore) AudioExists(_ context.Context, _ string) bool { return false } +func (m *mockStore) PutAudio(_ context.Context, _ string, _ []byte) error { return nil } + +// PresignStore +func (m *mockStore) PresignChapter(_ context.Context, _ string, _ int, _ time.Duration) (string, error) { + return "", nil +} +func (m *mockStore) PresignAudio(_ context.Context, _ string, _ time.Duration) (string, error) { + return "", nil +} +func (m *mockStore) PresignAvatarUpload(_ context.Context, _, _ string) (string, string, error) { + return "", "", nil +} +func (m *mockStore) PresignAvatarURL(_ context.Context, _ string) (string, bool, error) { + return "", false, nil +} +func (m *mockStore) DeleteAvatar(_ context.Context, _ string) error { return nil } + +// ProgressStore +func (m *mockStore) GetProgress(_ context.Context, _, _ string) (domain.ReadingProgress, bool) { + return domain.ReadingProgress{}, false +} +func (m *mockStore) SetProgress(_ context.Context, _ string, _ domain.ReadingProgress) error { + return nil +} +func (m *mockStore) AllProgress(_ context.Context, _ string) ([]domain.ReadingProgress, error) { + return nil, nil +} +func (m *mockStore) DeleteProgress(_ context.Context, _, _ string) error { return nil } + +// ── Compile-time interface satisfaction ─────────────────────────────────────── + +var _ bookstore.BookWriter = (*mockStore)(nil) +var _ bookstore.BookReader = (*mockStore)(nil) +var _ bookstore.RankingStore = (*mockStore)(nil) +var _ bookstore.AudioStore = (*mockStore)(nil) +var _ bookstore.PresignStore = (*mockStore)(nil) +var _ bookstore.ProgressStore = (*mockStore)(nil) + +// ── Behavioural tests ───────────────────────────────────────────────────────── + +func TestBookWriter_WriteMetadata_ReturnsNilError(t *testing.T) { + var w bookstore.BookWriter = &mockStore{} + if err := w.WriteMetadata(context.Background(), domain.BookMeta{Slug: "test"}); err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +func TestBookReader_ReadMetadata_NotFound(t *testing.T) { + var r bookstore.BookReader = &mockStore{} + _, found, err := r.ReadMetadata(context.Background(), "unknown") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if found { + t.Error("expected not found") + } +} + +func TestRankingStore_RankingFreshEnough_ReturnsFalse(t *testing.T) { + var s bookstore.RankingStore = &mockStore{} + fresh, err := s.RankingFreshEnough(context.Background(), time.Hour) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fresh { + t.Error("expected false") + } +} + +func TestAudioStore_AudioExists_ReturnsFalse(t *testing.T) { + var s bookstore.AudioStore = &mockStore{} + if s.AudioExists(context.Background(), "audio/slug/1/af_bella.mp3") { + t.Error("expected false") + } +} + +func TestProgressStore_GetProgress_NotFound(t *testing.T) { + var s bookstore.ProgressStore = &mockStore{} + _, found := s.GetProgress(context.Background(), "session-1", "slug") + if found { + t.Error("expected not found") + } +} diff --git a/v3/backend/internal/browser/browser.go b/v3/backend/internal/browser/browser.go new file mode 100644 index 0000000..9c4d669 --- /dev/null +++ b/v3/backend/internal/browser/browser.go @@ -0,0 +1,191 @@ +// Package browser provides a rate-limited HTTP client for web scraping. +package browser + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "sync" + "time" +) + +// ErrRateLimit is returned by GetContent when the server responds with 429. +// It carries the suggested retry delay (from Retry-After header, or a default). +var ErrRateLimit = errors.New("rate limited (429)") + +// RateLimitError wraps ErrRateLimit and carries the suggested wait duration. +type RateLimitError struct { + // RetryAfter is how long the caller should wait before retrying. + // Derived from the Retry-After response header when present; otherwise a default. + RetryAfter time.Duration +} + +func (e *RateLimitError) Error() string { + return fmt.Sprintf("rate limited (429): retry after %s", e.RetryAfter) +} + +func (e *RateLimitError) Is(target error) bool { return target == ErrRateLimit } + +// defaultRateLimitDelay is used when the server returns 429 with no Retry-After header. +const defaultRateLimitDelay = 60 * time.Second + +// Client is the interface used by scrapers to fetch raw page HTML. +// Implementations must be safe for concurrent use. +type Client interface { + // GetContent fetches the URL and returns the full response body as a string. + // It should respect the provided context for cancellation and timeouts. + GetContent(ctx context.Context, pageURL string) (string, error) +} + +// Config holds tunable parameters for the direct HTTP client. +type Config struct { + // MaxConcurrent limits the number of simultaneous in-flight requests. + // Defaults to 5 when 0. + MaxConcurrent int + // Timeout is the per-request deadline. Defaults to 90s when 0. + Timeout time.Duration +} + +// DirectClient is a plain net/http-based Client with a concurrency semaphore. +type DirectClient struct { + http *http.Client + semaphore chan struct{} +} + +// NewDirectClient returns a DirectClient configured by cfg. +func NewDirectClient(cfg Config) *DirectClient { + if cfg.MaxConcurrent <= 0 { + cfg.MaxConcurrent = 5 + } + if cfg.Timeout <= 0 { + cfg.Timeout = 90 * time.Second + } + + transport := &http.Transport{ + MaxIdleConnsPerHost: cfg.MaxConcurrent * 2, + DisableCompression: false, + } + + return &DirectClient{ + http: &http.Client{ + Transport: transport, + Timeout: cfg.Timeout, + }, + semaphore: make(chan struct{}, cfg.MaxConcurrent), + } +} + +// GetContent fetches pageURL respecting the concurrency limit. +func (c *DirectClient) GetContent(ctx context.Context, pageURL string) (string, error) { + // Acquire semaphore slot. + select { + case c.semaphore <- struct{}{}: + case <-ctx.Done(): + return "", ctx.Err() + } + defer func() { <-c.semaphore }() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) + if err != nil { + return "", fmt.Errorf("browser: build request %s: %w", pageURL, err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.5") + + resp, err := c.http.Do(req) + if err != nil { + return "", fmt.Errorf("browser: GET %s: %w", pageURL, err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusTooManyRequests { + delay := defaultRateLimitDelay + if ra := resp.Header.Get("Retry-After"); ra != "" { + if secs, err := strconv.Atoi(ra); err == nil && secs > 0 { + delay = time.Duration(secs) * time.Second + } + } + return "", &RateLimitError{RetryAfter: delay} + } + + if resp.StatusCode >= 400 { + return "", fmt.Errorf("browser: GET %s returned %d", pageURL, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("browser: read body %s: %w", pageURL, err) + } + return string(body), nil +} + +// Do implements httputil.Client so DirectClient can be passed to RetryGet. +func (c *DirectClient) Do(req *http.Request) (*http.Response, error) { + select { + case c.semaphore <- struct{}{}: + case <-req.Context().Done(): + return nil, req.Context().Err() + } + defer func() { <-c.semaphore }() + return c.http.Do(req) +} + +// ── Stub for testing ────────────────────────────────────────────────────────── + +// StubClient is a test double for Client. It returns pre-configured responses +// keyed on URL. Calls to unknown URLs return an error. +type StubClient struct { + mu sync.Mutex + pages map[string]string + errors map[string]error + callLog []string +} + +// NewStub creates a StubClient with no pages pre-loaded. +func NewStub() *StubClient { + return &StubClient{ + pages: make(map[string]string), + errors: make(map[string]error), + } +} + +// SetPage registers a URL → HTML body mapping. +func (s *StubClient) SetPage(u, html string) { + s.mu.Lock() + s.pages[u] = html + s.mu.Unlock() +} + +// SetError registers a URL → error mapping (returned instead of a body). +func (s *StubClient) SetError(u string, err error) { + s.mu.Lock() + s.errors[u] = err + s.mu.Unlock() +} + +// CallLog returns the ordered list of URLs that were requested. +func (s *StubClient) CallLog() []string { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]string, len(s.callLog)) + copy(out, s.callLog) + return out +} + +// GetContent returns the registered page or an error for the URL. +func (s *StubClient) GetContent(_ context.Context, pageURL string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.callLog = append(s.callLog, pageURL) + if err, ok := s.errors[pageURL]; ok { + return "", err + } + if html, ok := s.pages[pageURL]; ok { + return html, nil + } + return "", fmt.Errorf("stub: no page registered for %q", pageURL) +} diff --git a/v3/backend/internal/browser/browser_test.go b/v3/backend/internal/browser/browser_test.go new file mode 100644 index 0000000..c5dd55d --- /dev/null +++ b/v3/backend/internal/browser/browser_test.go @@ -0,0 +1,141 @@ +package browser_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/libnovel/backend/internal/browser" +) + +func TestDirectClient_GetContent_Success(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hello")) + })) + defer srv.Close() + + c := browser.NewDirectClient(browser.Config{MaxConcurrent: 2, Timeout: 5 * time.Second}) + body, err := c.GetContent(context.Background(), srv.URL) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body != "hello" { + t.Errorf("want hello, got %q", body) + } +} + +func TestDirectClient_GetContent_4xxReturnsError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + c := browser.NewDirectClient(browser.Config{}) + _, err := c.GetContent(context.Background(), srv.URL) + if err == nil { + t.Fatal("expected error for 404") + } +} + +func TestDirectClient_SemaphoreBlocksConcurrency(t *testing.T) { + const maxConcurrent = 2 + var inflight atomic.Int32 + var peak atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := inflight.Add(1) + if int(n) > int(peak.Load()) { + peak.Store(n) + } + time.Sleep(20 * time.Millisecond) + inflight.Add(-1) + w.Write([]byte("ok")) + })) + defer srv.Close() + + c := browser.NewDirectClient(browser.Config{MaxConcurrent: maxConcurrent, Timeout: 5 * time.Second}) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c.GetContent(context.Background(), srv.URL) + }() + } + wg.Wait() + + if int(peak.Load()) > maxConcurrent { + t.Errorf("concurrent requests exceeded limit: peak=%d, limit=%d", peak.Load(), maxConcurrent) + } +} + +func TestDirectClient_ContextCancel(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(200 * time.Millisecond) + w.Write([]byte("ok")) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel before making the request + + c := browser.NewDirectClient(browser.Config{}) + _, err := c.GetContent(ctx, srv.URL) + if err == nil { + t.Fatal("expected context cancellation error") + } +} + +// ── StubClient ──────────────────────────────────────────────────────────────── + +func TestStubClient_ReturnsRegisteredPage(t *testing.T) { + stub := browser.NewStub() + stub.SetPage("http://example.com/page1", "page1") + + body, err := stub.GetContent(context.Background(), "http://example.com/page1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body != "page1" { + t.Errorf("want page1 html, got %q", body) + } +} + +func TestStubClient_ReturnsRegisteredError(t *testing.T) { + stub := browser.NewStub() + want := errors.New("network failure") + stub.SetError("http://example.com/bad", want) + + _, err := stub.GetContent(context.Background(), "http://example.com/bad") + if err == nil { + t.Fatal("expected error") + } +} + +func TestStubClient_UnknownURLReturnsError(t *testing.T) { + stub := browser.NewStub() + _, err := stub.GetContent(context.Background(), "http://unknown.example.com/") + if err == nil { + t.Fatal("expected error for unknown URL") + } +} + +func TestStubClient_CallLog(t *testing.T) { + stub := browser.NewStub() + stub.SetPage("http://example.com/a", "a") + stub.SetPage("http://example.com/b", "b") + + stub.GetContent(context.Background(), "http://example.com/a") + stub.GetContent(context.Background(), "http://example.com/b") + + log := stub.CallLog() + if len(log) != 2 || log[0] != "http://example.com/a" || log[1] != "http://example.com/b" { + t.Errorf("unexpected call log: %v", log) + } +} diff --git a/v3/backend/internal/config/config.go b/v3/backend/internal/config/config.go new file mode 100644 index 0000000..91036ba --- /dev/null +++ b/v3/backend/internal/config/config.go @@ -0,0 +1,219 @@ +// Package config loads all service configuration from environment variables. +// Both the runner and backend binaries call config.Load() at startup; each +// uses only the sub-struct relevant to it. +// +// Every field has a documented default so the service starts sensibly without +// any environment configuration (useful for local development). +package config + +import ( + "os" + "strconv" + "strings" + "time" +) + +// PocketBase holds connection settings for the remote PocketBase instance. +type PocketBase struct { + // URL is the base URL of the PocketBase instance, e.g. https://pb.libnovel.cc + URL string + // AdminEmail is the admin account email used for API authentication. + AdminEmail string + // AdminPassword is the admin account password. + AdminPassword string +} + +// MinIO holds connection settings for the remote MinIO / S3-compatible store. +type MinIO struct { + // Endpoint is the host:port of the MinIO S3 API, e.g. storage.libnovel.cc:443 + Endpoint string + // PublicEndpoint is the browser-visible endpoint used for presigned URLs. + // Falls back to Endpoint when empty. + PublicEndpoint string + // AccessKey is the MinIO access key. + AccessKey string + // SecretKey is the MinIO secret key. + SecretKey string + // UseSSL enables TLS for the internal MinIO connection. + UseSSL bool + // PublicUseSSL enables TLS for presigned URL generation. + PublicUseSSL bool + // BucketChapters is the bucket that holds chapter markdown objects. + BucketChapters string + // BucketAudio is the bucket that holds generated audio MP3 objects. + BucketAudio string + // BucketAvatars is the bucket that holds user avatar images. + BucketAvatars string + // BucketBrowse is the bucket that holds cached browse page snapshots (JSON). + BucketBrowse string +} + +// Kokoro holds connection settings for the Kokoro-FastAPI TTS service. +type Kokoro struct { + // URL is the base URL of the Kokoro service, e.g. https://kokoro.libnovel.cc + // An empty string disables TTS generation. + URL string + // DefaultVoice is the voice used when none is specified. + DefaultVoice string +} + +// HTTP holds settings for the HTTP server (backend only). +type HTTP struct { + // Addr is the listen address, e.g. ":8080" + Addr string +} + +// Meilisearch holds connection settings for the Meilisearch full-text search service. +type Meilisearch struct { + // URL is the base URL of the Meilisearch instance, e.g. http://localhost:7700 + // An empty string disables Meilisearch indexing and search. + URL string + // APIKey is the Meilisearch master/search API key. + APIKey string +} + +// Valkey holds connection settings for the Valkey/Redis presign URL cache. +type Valkey struct { + // Addr is the host:port of the Valkey instance, e.g. localhost:6379 + // An empty string disables the Valkey cache (falls through to MinIO directly). + Addr string +} + +// Runner holds settings specific to the runner/worker binary. +type Runner struct { + // PollInterval is how often the runner checks PocketBase for pending tasks. + PollInterval time.Duration + // MaxConcurrentScrape limits simultaneous book-scrape goroutines. + MaxConcurrentScrape int + // MaxConcurrentAudio limits simultaneous audio-generation goroutines. + MaxConcurrentAudio int + // WorkerID is a unique identifier for this runner instance. + // Defaults to the system hostname. + WorkerID string + // Workers is the number of chapter-scraping goroutines per book. + Workers int + // Timeout is the per-request HTTP timeout for scraping. + Timeout time.Duration + // MetricsAddr is the listen address for the runner /metrics HTTP endpoint. + // Defaults to ":9091". Set to "" to disable. + MetricsAddr string + // CatalogueRefreshInterval is how often the runner walks the full catalogue, + // scrapes per-book metadata, downloads covers, and re-indexes in Meilisearch. + // Defaults to 24h. Set to 0 to use the default. + CatalogueRefreshInterval time.Duration +} + +// Config is the top-level configuration struct consumed by both binaries. +type Config struct { + PocketBase PocketBase + MinIO MinIO + Kokoro Kokoro + HTTP HTTP + Runner Runner + Meilisearch Meilisearch + Valkey Valkey + // LogLevel is one of "debug", "info", "warn", "error". + LogLevel string +} + +// Load reads all configuration from environment variables and returns a +// populated Config. Missing variables fall back to documented defaults. +func Load() Config { + workerID, _ := os.Hostname() + if workerID == "" { + workerID = "runner-default" + } + + return Config{ + LogLevel: envOr("LOG_LEVEL", "info"), + + PocketBase: PocketBase{ + URL: envOr("POCKETBASE_URL", "http://localhost:8090"), + AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"), + AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"), + }, + + MinIO: MinIO{ + Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"), + PublicEndpoint: envOr("MINIO_PUBLIC_ENDPOINT", ""), + AccessKey: envOr("MINIO_ACCESS_KEY", "admin"), + SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"), + UseSSL: envBool("MINIO_USE_SSL", false), + PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true), + BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), + BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), + BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"), + BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"), + }, + + Kokoro: Kokoro{ + URL: envOr("KOKORO_URL", ""), + DefaultVoice: envOr("KOKORO_VOICE", "af_bella"), + }, + + HTTP: HTTP{ + Addr: envOr("BACKEND_HTTP_ADDR", ":8080"), + }, + + Runner: Runner{ + PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second), + MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1), + MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1), + WorkerID: envOr("RUNNER_WORKER_ID", workerID), + Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU() + Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second), + MetricsAddr: envOr("RUNNER_METRICS_ADDR", ":9091"), + CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0), + }, + + Meilisearch: Meilisearch{ + URL: envOr("MEILI_URL", ""), + APIKey: envOr("MEILI_API_KEY", ""), + }, + + Valkey: Valkey{ + Addr: envOr("VALKEY_ADDR", ""), + }, + } +} + +// ── helpers ─────────────────────────────────────────────────────────────────── + +func envOr(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func envBool(key string, fallback bool) bool { + v := os.Getenv(key) + if v == "" { + return fallback + } + return strings.ToLower(v) == "true" +} + +func envInt(key string, fallback int) int { + v := os.Getenv(key) + if v == "" { + return fallback + } + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return fallback + } + return n +} + +func envDuration(key string, fallback time.Duration) time.Duration { + v := os.Getenv(key) + if v == "" { + return fallback + } + d, err := time.ParseDuration(v) + if err != nil { + return fallback + } + return d +} diff --git a/v3/backend/internal/config/config_test.go b/v3/backend/internal/config/config_test.go new file mode 100644 index 0000000..e424d75 --- /dev/null +++ b/v3/backend/internal/config/config_test.go @@ -0,0 +1,127 @@ +package config_test + +import ( + "os" + "testing" + "time" + + "github.com/libnovel/backend/internal/config" +) + +func TestLoad_Defaults(t *testing.T) { + // Unset all relevant vars so we test pure defaults. + unset := []string{ + "LOG_LEVEL", + "POCKETBASE_URL", "POCKETBASE_ADMIN_EMAIL", "POCKETBASE_ADMIN_PASSWORD", + "MINIO_ENDPOINT", "MINIO_PUBLIC_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", + "MINIO_USE_SSL", "MINIO_PUBLIC_USE_SSL", + "MINIO_BUCKET_CHAPTERS", "MINIO_BUCKET_AUDIO", "MINIO_BUCKET_AVATARS", + "KOKORO_URL", "KOKORO_VOICE", + "BACKEND_HTTP_ADDR", + "RUNNER_POLL_INTERVAL", "RUNNER_MAX_CONCURRENT_SCRAPE", "RUNNER_MAX_CONCURRENT_AUDIO", + "RUNNER_WORKER_ID", "RUNNER_WORKERS", "RUNNER_TIMEOUT", + } + for _, k := range unset { + t.Setenv(k, "") + } + + cfg := config.Load() + + if cfg.LogLevel != "info" { + t.Errorf("LogLevel: want info, got %q", cfg.LogLevel) + } + if cfg.PocketBase.URL != "http://localhost:8090" { + t.Errorf("PocketBase.URL: want http://localhost:8090, got %q", cfg.PocketBase.URL) + } + if cfg.MinIO.BucketChapters != "libnovel-chapters" { + t.Errorf("MinIO.BucketChapters: want libnovel-chapters, got %q", cfg.MinIO.BucketChapters) + } + if cfg.MinIO.UseSSL != false { + t.Errorf("MinIO.UseSSL: want false, got %v", cfg.MinIO.UseSSL) + } + if cfg.MinIO.PublicUseSSL != true { + t.Errorf("MinIO.PublicUseSSL: want true, got %v", cfg.MinIO.PublicUseSSL) + } + if cfg.Kokoro.DefaultVoice != "af_bella" { + t.Errorf("Kokoro.DefaultVoice: want af_bella, got %q", cfg.Kokoro.DefaultVoice) + } + if cfg.HTTP.Addr != ":8080" { + t.Errorf("HTTP.Addr: want :8080, got %q", cfg.HTTP.Addr) + } + if cfg.Runner.PollInterval != 30*time.Second { + t.Errorf("Runner.PollInterval: want 30s, got %v", cfg.Runner.PollInterval) + } + if cfg.Runner.MaxConcurrentScrape != 1 { + t.Errorf("Runner.MaxConcurrentScrape: want 1, got %d", cfg.Runner.MaxConcurrentScrape) + } + if cfg.Runner.MaxConcurrentAudio != 1 { + t.Errorf("Runner.MaxConcurrentAudio: want 1, got %d", cfg.Runner.MaxConcurrentAudio) + } +} + +func TestLoad_EnvOverride(t *testing.T) { + t.Setenv("LOG_LEVEL", "debug") + t.Setenv("POCKETBASE_URL", "https://pb.libnovel.cc") + t.Setenv("MINIO_USE_SSL", "true") + t.Setenv("MINIO_PUBLIC_USE_SSL", "false") + t.Setenv("RUNNER_POLL_INTERVAL", "1m") + t.Setenv("RUNNER_MAX_CONCURRENT_SCRAPE", "5") + t.Setenv("RUNNER_WORKER_ID", "homelab-01") + t.Setenv("BACKEND_HTTP_ADDR", ":9090") + t.Setenv("KOKORO_URL", "https://kokoro.libnovel.cc") + + cfg := config.Load() + + if cfg.LogLevel != "debug" { + t.Errorf("LogLevel: want debug, got %q", cfg.LogLevel) + } + if cfg.PocketBase.URL != "https://pb.libnovel.cc" { + t.Errorf("PocketBase.URL: want https://pb.libnovel.cc, got %q", cfg.PocketBase.URL) + } + if !cfg.MinIO.UseSSL { + t.Error("MinIO.UseSSL: want true") + } + if cfg.MinIO.PublicUseSSL { + t.Error("MinIO.PublicUseSSL: want false") + } + if cfg.Runner.PollInterval != time.Minute { + t.Errorf("Runner.PollInterval: want 1m, got %v", cfg.Runner.PollInterval) + } + if cfg.Runner.MaxConcurrentScrape != 5 { + t.Errorf("Runner.MaxConcurrentScrape: want 5, got %d", cfg.Runner.MaxConcurrentScrape) + } + if cfg.Runner.WorkerID != "homelab-01" { + t.Errorf("Runner.WorkerID: want homelab-01, got %q", cfg.Runner.WorkerID) + } + if cfg.HTTP.Addr != ":9090" { + t.Errorf("HTTP.Addr: want :9090, got %q", cfg.HTTP.Addr) + } + if cfg.Kokoro.URL != "https://kokoro.libnovel.cc" { + t.Errorf("Kokoro.URL: want https://kokoro.libnovel.cc, got %q", cfg.Kokoro.URL) + } +} + +func TestLoad_InvalidInt_FallsToDefault(t *testing.T) { + t.Setenv("RUNNER_MAX_CONCURRENT_SCRAPE", "notanumber") + cfg := config.Load() + if cfg.Runner.MaxConcurrentScrape != 1 { + t.Errorf("want default 1, got %d", cfg.Runner.MaxConcurrentScrape) + } +} + +func TestLoad_InvalidDuration_FallsToDefault(t *testing.T) { + t.Setenv("RUNNER_POLL_INTERVAL", "notaduration") + cfg := config.Load() + if cfg.Runner.PollInterval != 30*time.Second { + t.Errorf("want default 30s, got %v", cfg.Runner.PollInterval) + } +} + +func TestLoad_WorkerID_FallsToHostname(t *testing.T) { + t.Setenv("RUNNER_WORKER_ID", "") + cfg := config.Load() + host, _ := os.Hostname() + if host != "" && cfg.Runner.WorkerID != host { + t.Errorf("want hostname %q, got %q", host, cfg.Runner.WorkerID) + } +} diff --git a/v3/backend/internal/domain/domain.go b/v3/backend/internal/domain/domain.go new file mode 100644 index 0000000..80ea3da --- /dev/null +++ b/v3/backend/internal/domain/domain.go @@ -0,0 +1,132 @@ +// Package domain contains the core value types shared across all packages +// in this module. It has zero internal imports — only the standard library. +// Every other package imports domain; domain imports nothing from this module. +package domain + +import "time" + +// ── Book types ──────────────────────────────────────────────────────────────── + +// BookMeta carries all bibliographic information about a novel. +type BookMeta struct { + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author"` + Cover string `json:"cover,omitempty"` + Status string `json:"status,omitempty"` + Genres []string `json:"genres,omitempty"` + Summary string `json:"summary,omitempty"` + TotalChapters int `json:"total_chapters,omitempty"` + SourceURL string `json:"source_url"` + Ranking int `json:"ranking,omitempty"` + Rating float64 `json:"rating,omitempty"` +} + +// CatalogueEntry is a lightweight book reference returned by catalogue pages. +type CatalogueEntry struct { + Title string `json:"title"` + URL string `json:"url"` +} + +// ChapterRef is a reference to a single chapter returned by chapter-list pages. +type ChapterRef struct { + Number int `json:"number"` + Title string `json:"title"` + URL string `json:"url"` + Volume int `json:"volume,omitempty"` +} + +// Chapter contains the fully-extracted text of a single chapter. +type Chapter struct { + Ref ChapterRef `json:"ref"` + Text string `json:"text"` +} + +// RankingItem represents a single entry in the novel ranking list. +type RankingItem struct { + Rank int `json:"rank"` + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author,omitempty"` + Cover string `json:"cover,omitempty"` + Status string `json:"status,omitempty"` + Genres []string `json:"genres,omitempty"` + SourceURL string `json:"source_url,omitempty"` + Updated time.Time `json:"updated,omitempty"` +} + +// ── Storage record types ────────────────────────────────────────────────────── + +// ChapterInfo is a lightweight chapter descriptor stored in the index. +type ChapterInfo struct { + Number int `json:"number"` + Title string `json:"title"` + Date string `json:"date,omitempty"` +} + +// ReadingProgress holds a single user's reading position for one book. +type ReadingProgress struct { + Slug string `json:"slug"` + Chapter int `json:"chapter"` + UpdatedAt time.Time `json:"updated_at"` +} + +// ── Task record types ───────────────────────────────────────────────────────── + +// TaskStatus enumerates the lifecycle states of any task. +type TaskStatus string + +const ( + TaskStatusPending TaskStatus = "pending" + TaskStatusRunning TaskStatus = "running" + TaskStatusDone TaskStatus = "done" + TaskStatusFailed TaskStatus = "failed" + TaskStatusCancelled TaskStatus = "cancelled" +) + +// ScrapeTask represents a book-scraping job stored in PocketBase. +type ScrapeTask struct { + ID string `json:"id"` + Kind string `json:"kind"` // "catalogue" | "book" | "book_range" + TargetURL string `json:"target_url"` // non-empty for single-book tasks + FromChapter int `json:"from_chapter,omitempty"` + ToChapter int `json:"to_chapter,omitempty"` + WorkerID string `json:"worker_id,omitempty"` + Status TaskStatus `json:"status"` + BooksFound int `json:"books_found"` + ChaptersScraped int `json:"chapters_scraped"` + ChaptersSkipped int `json:"chapters_skipped"` + Errors int `json:"errors"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` +} + +// ScrapeResult is the outcome reported by the runner after finishing a ScrapeTask. +type ScrapeResult struct { + BooksFound int `json:"books_found"` + ChaptersScraped int `json:"chapters_scraped"` + ChaptersSkipped int `json:"chapters_skipped"` + Errors int `json:"errors"` + ErrorMessage string `json:"error_message,omitempty"` +} + +// AudioTask represents an audio-generation job stored in PocketBase. +type AudioTask struct { + ID string `json:"id"` + CacheKey string `json:"cache_key"` // "slug/chapter/voice" + Slug string `json:"slug"` + Chapter int `json:"chapter"` + Voice string `json:"voice"` + WorkerID string `json:"worker_id,omitempty"` + Status TaskStatus `json:"status"` + ErrorMessage string `json:"error_message,omitempty"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished,omitempty"` +} + +// AudioResult is the outcome reported by the runner after finishing an AudioTask. +type AudioResult struct { + ObjectKey string `json:"object_key,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` +} diff --git a/v3/backend/internal/domain/domain_test.go b/v3/backend/internal/domain/domain_test.go new file mode 100644 index 0000000..c364657 --- /dev/null +++ b/v3/backend/internal/domain/domain_test.go @@ -0,0 +1,104 @@ +package domain_test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/libnovel/backend/internal/domain" +) + +func TestBookMeta_JSONRoundtrip(t *testing.T) { + orig := domain.BookMeta{ + Slug: "a-great-novel", + Title: "A Great Novel", + Author: "Jane Doe", + Cover: "https://example.com/cover.jpg", + Status: "Ongoing", + Genres: []string{"Fantasy", "Action"}, + Summary: "A thrilling tale.", + TotalChapters: 120, + SourceURL: "https://novelfire.net/book/a-great-novel", + Ranking: 3, + } + + b, err := json.Marshal(orig) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var got domain.BookMeta + if err := json.Unmarshal(b, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got.Slug != orig.Slug { + t.Errorf("Slug: want %q, got %q", orig.Slug, got.Slug) + } + if got.TotalChapters != orig.TotalChapters { + t.Errorf("TotalChapters: want %d, got %d", orig.TotalChapters, got.TotalChapters) + } + if len(got.Genres) != len(orig.Genres) { + t.Errorf("Genres len: want %d, got %d", len(orig.Genres), len(got.Genres)) + } +} + +func TestChapterRef_JSONRoundtrip(t *testing.T) { + orig := domain.ChapterRef{Number: 42, Title: "The Battle", URL: "https://example.com/ch-42", Volume: 2} + b, _ := json.Marshal(orig) + var got domain.ChapterRef + json.Unmarshal(b, &got) + if got != orig { + t.Errorf("want %+v, got %+v", orig, got) + } +} + +func TestRankingItem_JSONRoundtrip(t *testing.T) { + now := time.Now().Truncate(time.Second) + orig := domain.RankingItem{ + Rank: 1, + Slug: "top-novel", + Title: "Top Novel", + SourceURL: "https://novelfire.net/book/top-novel", + Updated: now, + } + b, _ := json.Marshal(orig) + var got domain.RankingItem + json.Unmarshal(b, &got) + if got.Rank != orig.Rank || got.Slug != orig.Slug { + t.Errorf("want %+v, got %+v", orig, got) + } +} + +func TestScrapeResult_JSONRoundtrip(t *testing.T) { + orig := domain.ScrapeResult{BooksFound: 10, ChaptersScraped: 200, ChaptersSkipped: 5, Errors: 1, ErrorMessage: "one error"} + b, _ := json.Marshal(orig) + var got domain.ScrapeResult + json.Unmarshal(b, &got) + if got != orig { + t.Errorf("want %+v, got %+v", orig, got) + } +} + +func TestAudioResult_JSONRoundtrip(t *testing.T) { + orig := domain.AudioResult{ObjectKey: "audio/slug/1/af_bella.mp3"} + b, _ := json.Marshal(orig) + var got domain.AudioResult + json.Unmarshal(b, &got) + if got != orig { + t.Errorf("want %+v, got %+v", orig, got) + } +} + +func TestTaskStatus_Values(t *testing.T) { + cases := []domain.TaskStatus{ + domain.TaskStatusPending, + domain.TaskStatusRunning, + domain.TaskStatusDone, + domain.TaskStatusFailed, + domain.TaskStatusCancelled, + } + for _, s := range cases { + if s == "" { + t.Errorf("TaskStatus constant must not be empty") + } + } +} diff --git a/v3/backend/internal/httputil/httputil.go b/v3/backend/internal/httputil/httputil.go new file mode 100644 index 0000000..f36359e --- /dev/null +++ b/v3/backend/internal/httputil/httputil.go @@ -0,0 +1,124 @@ +// Package httputil provides shared HTTP helpers used by both the runner and +// backend binaries. It has no imports from this module — only the standard +// library — so it is safe to import from anywhere in the dependency graph. +package httputil + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" +) + +// Client is the minimal interface for making HTTP GET requests. +// *http.Client satisfies this interface. +type Client interface { + Do(req *http.Request) (*http.Response, error) +} + +// ErrMaxRetries is returned when RetryGet exhausts all attempts. +var ErrMaxRetries = errors.New("httputil: max retries exceeded") + +// errClientError is returned by doGet for 4xx responses; it signals that the +// request should NOT be retried (the client is at fault). +var errClientError = errors.New("httputil: client error") + +// RetryGet fetches url using client, retrying on network errors or 5xx +// responses with exponential backoff. It returns the full response body as a +// string on success. +// +// - maxAttempts: total number of attempts (must be >= 1) +// - baseDelay: initial wait before the second attempt; doubles each retry +func RetryGet(ctx context.Context, client Client, url string, maxAttempts int, baseDelay time.Duration) (string, error) { + if maxAttempts < 1 { + maxAttempts = 1 + } + delay := baseDelay + + var lastErr error + for attempt := 0; attempt < maxAttempts; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(delay): + } + delay *= 2 + } + + body, err := doGet(ctx, client, url) + if err == nil { + return body, nil + } + lastErr = err + + // Do not retry on context cancellation. + if ctx.Err() != nil { + return "", ctx.Err() + } + // Do not retry on 4xx — the client is at fault. + if errors.Is(err, errClientError) { + return "", err + } + } + + return "", fmt.Errorf("%w after %d attempts: %w", ErrMaxRetries, maxAttempts, lastErr) +} + +func doGet(ctx context.Context, client Client, url string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", fmt.Errorf("build request: %w", err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)") + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("GET %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 500 { + return "", fmt.Errorf("GET %s: server error %d", url, resp.StatusCode) + } + if resp.StatusCode >= 400 { + return "", fmt.Errorf("%w: GET %s: client error %d", errClientError, url, resp.StatusCode) + } + + raw, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("read body %s: %w", url, err) + } + return string(raw), nil +} + +// WriteJSON writes v as JSON to w with the given HTTP status code and sets the +// Content-Type header to application/json. +func WriteJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +// WriteError writes a JSON error object {"error": msg} with the given status. +func WriteError(w http.ResponseWriter, status int, msg string) { + WriteJSON(w, status, map[string]string{"error": msg}) +} + +// maxBodyBytes is the limit applied by DecodeJSON to prevent unbounded reads. +const maxBodyBytes = 1 << 20 // 1 MiB + +// DecodeJSON decodes a JSON request body into v. It enforces a 1 MiB size +// limit and returns a descriptive error on any failure. +func DecodeJSON(r *http.Request, v any) error { + r.Body = http.MaxBytesReader(nil, r.Body, maxBodyBytes) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return fmt.Errorf("decode JSON body: %w", err) + } + return nil +} diff --git a/v3/backend/internal/httputil/httputil_test.go b/v3/backend/internal/httputil/httputil_test.go new file mode 100644 index 0000000..2af3bba --- /dev/null +++ b/v3/backend/internal/httputil/httputil_test.go @@ -0,0 +1,181 @@ +package httputil_test + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/libnovel/backend/internal/httputil" +) + +// ── RetryGet ────────────────────────────────────────────────────────────────── + +func TestRetryGet_ImmediateSuccess(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hello")) + })) + defer srv.Close() + + body, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 3, time.Millisecond) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body != "hello" { + t.Errorf("want hello, got %q", body) + } +} + +func TestRetryGet_RetriesOn5xx(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Write([]byte("ok")) + })) + defer srv.Close() + + body, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 5, time.Millisecond) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body != "ok" { + t.Errorf("want ok, got %q", body) + } + if calls != 3 { + t.Errorf("want 3 calls, got %d", calls) + } +} + +func TestRetryGet_MaxAttemptsExceeded(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + _, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 3, time.Millisecond) + if err == nil { + t.Fatal("expected error, got nil") + } +} + +func TestRetryGet_ContextCancelDuringBackoff(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + + // Cancel after first failed attempt hits the backoff wait. + go func() { time.Sleep(5 * time.Millisecond); cancel() }() + + _, err := httputil.RetryGet(ctx, srv.Client(), srv.URL, 10, 500*time.Millisecond) + if err == nil { + t.Fatal("expected context cancellation error") + } +} + +func TestRetryGet_NoRetryOn4xx(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + _, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 5, time.Millisecond) + if err == nil { + t.Fatal("expected error for 404") + } + // 4xx is NOT retried — should be exactly 1 call. + if calls != 1 { + t.Errorf("want 1 call for 4xx, got %d", calls) + } +} + +// ── WriteJSON ───────────────────────────────────────────────────────────────── + +func TestWriteJSON_SetsHeadersAndStatus(t *testing.T) { + rr := httptest.NewRecorder() + httputil.WriteJSON(rr, http.StatusCreated, map[string]string{"key": "val"}) + + if rr.Code != http.StatusCreated { + t.Errorf("status: want 201, got %d", rr.Code) + } + if ct := rr.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("Content-Type: want application/json, got %q", ct) + } + var got map[string]string + if err := json.NewDecoder(rr.Body).Decode(&got); err != nil { + t.Fatalf("decode body: %v", err) + } + if got["key"] != "val" { + t.Errorf("body key: want val, got %q", got["key"]) + } +} + +// ── WriteError ──────────────────────────────────────────────────────────────── + +func TestWriteError_Format(t *testing.T) { + rr := httptest.NewRecorder() + httputil.WriteError(rr, http.StatusBadRequest, "bad input") + + if rr.Code != http.StatusBadRequest { + t.Errorf("status: want 400, got %d", rr.Code) + } + var got map[string]string + json.NewDecoder(rr.Body).Decode(&got) + if got["error"] != "bad input" { + t.Errorf("error field: want bad input, got %q", got["error"]) + } +} + +// ── DecodeJSON ──────────────────────────────────────────────────────────────── + +func TestDecodeJSON_HappyPath(t *testing.T) { + body := `{"name":"test","value":42}` + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + + var payload struct { + Name string `json:"name"` + Value int `json:"value"` + } + if err := httputil.DecodeJSON(req, &payload); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if payload.Name != "test" || payload.Value != 42 { + t.Errorf("unexpected payload: %+v", payload) + } +} + +func TestDecodeJSON_UnknownFieldReturnsError(t *testing.T) { + body := `{"name":"test","unknown_field":"boom"}` + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + + var payload struct { + Name string `json:"name"` + } + if err := httputil.DecodeJSON(req, &payload); err == nil { + t.Fatal("expected error for unknown field, got nil") + } +} + +func TestDecodeJSON_BodyTooLarge(t *testing.T) { + // Build a body > 1 MiB. + big := bytes.Repeat([]byte("a"), 2<<20) + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(big)) + + var payload map[string]any + if err := httputil.DecodeJSON(req, &payload); err == nil { + t.Fatal("expected error for oversized body, got nil") + } +} diff --git a/v3/backend/internal/kokoro/client.go b/v3/backend/internal/kokoro/client.go new file mode 100644 index 0000000..6384187 --- /dev/null +++ b/v3/backend/internal/kokoro/client.go @@ -0,0 +1,160 @@ +// Package kokoro provides a client for the Kokoro-FastAPI TTS service. +// +// The Kokoro API is an OpenAI-compatible audio speech API that returns a +// download link (X-Download-Path header) instead of streaming audio directly. +// GenerateAudio handles the two-step flow: POST /v1/audio/speech → GET /v1/download/{file}. +package kokoro + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// Client is the interface for interacting with the Kokoro TTS service. +type Client interface { + // GenerateAudio synthesises text using voice and returns raw MP3 bytes. + GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) + + // ListVoices returns the available voice IDs. Falls back to an empty slice + // on error — callers should treat an empty list as "service unavailable". + ListVoices(ctx context.Context) ([]string, error) +} + +// httpClient is the concrete Kokoro HTTP client. +type httpClient struct { + baseURL string + http *http.Client +} + +// New returns a Kokoro Client targeting baseURL (e.g. "https://kokoro.example.com"). +func New(baseURL string) Client { + return &httpClient{ + baseURL: strings.TrimRight(baseURL, "/"), + http: &http.Client{Timeout: 10 * time.Minute}, + } +} + +// GenerateAudio calls POST /v1/audio/speech (return_download_link=true) and then +// downloads the resulting MP3 from GET /v1/download/{filename}. +func (c *httpClient) GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) { + if text == "" { + return nil, fmt.Errorf("kokoro: empty text") + } + if voice == "" { + voice = "af_bella" + } + + // ── Step 1: request generation ──────────────────────────────────────────── + reqBody, err := json.Marshal(map[string]any{ + "model": "kokoro", + "input": text, + "voice": voice, + "response_format": "mp3", + "speed": 1.0, + "stream": false, + "return_download_link": true, + }) + if err != nil { + return nil, fmt.Errorf("kokoro: marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/v1/audio/speech", bytes.NewReader(reqBody)) + if err != nil { + return nil, fmt.Errorf("kokoro: build speech request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("kokoro: speech request: %w", err) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("kokoro: speech returned %d", resp.StatusCode) + } + + dlPath := resp.Header.Get("X-Download-Path") + if dlPath == "" { + return nil, fmt.Errorf("kokoro: no X-Download-Path header in response") + } + filename := dlPath + if idx := strings.LastIndex(dlPath, "/"); idx >= 0 { + filename = dlPath[idx+1:] + } + if filename == "" { + return nil, fmt.Errorf("kokoro: empty filename in X-Download-Path: %q", dlPath) + } + + // ── Step 2: download the generated file ─────────────────────────────────── + dlURL := c.baseURL + "/v1/download/" + filename + dlReq, err := http.NewRequestWithContext(ctx, http.MethodGet, dlURL, nil) + if err != nil { + return nil, fmt.Errorf("kokoro: build download request: %w", err) + } + + dlResp, err := c.http.Do(dlReq) + if err != nil { + return nil, fmt.Errorf("kokoro: download request: %w", err) + } + defer dlResp.Body.Close() + + if dlResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("kokoro: download returned %d", dlResp.StatusCode) + } + + data, err := io.ReadAll(dlResp.Body) + if err != nil { + return nil, fmt.Errorf("kokoro: read download body: %w", err) + } + return data, nil +} + +// ListVoices calls GET /v1/audio/voices and returns the list of voice IDs. +func (c *httpClient) ListVoices(ctx context.Context) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + c.baseURL+"/v1/audio/voices", nil) + if err != nil { + return nil, fmt.Errorf("kokoro: build voices request: %w", err) + } + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("kokoro: voices request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("kokoro: voices returned %d", resp.StatusCode) + } + + var result struct { + Voices []string `json:"voices"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("kokoro: decode voices response: %w", err) + } + return result.Voices, nil +} + +// VoiceSampleKey returns the MinIO object key for a voice sample MP3. +// Key: _voice-samples/{voice}.mp3 (sanitised). +func VoiceSampleKey(voice string) string { + safe := strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || r == '_' || r == '-' { + return r + } + return '_' + }, voice) + return fmt.Sprintf("_voice-samples/%s.mp3", safe) +} diff --git a/v3/backend/internal/kokoro/client_test.go b/v3/backend/internal/kokoro/client_test.go new file mode 100644 index 0000000..9d63bbc --- /dev/null +++ b/v3/backend/internal/kokoro/client_test.go @@ -0,0 +1,291 @@ +package kokoro_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/libnovel/backend/internal/kokoro" +) + +// ── VoiceSampleKey ──────────────────────────────────────────────────────────── + +func TestVoiceSampleKey(t *testing.T) { + tests := []struct { + voice string + want string + }{ + {"af_bella", "_voice-samples/af_bella.mp3"}, + {"am_echo", "_voice-samples/am_echo.mp3"}, + {"voice with spaces", "_voice-samples/voice_with_spaces.mp3"}, + {"special!@#chars", "_voice-samples/special___chars.mp3"}, + {"", "_voice-samples/.mp3"}, + } + for _, tt := range tests { + t.Run(tt.voice, func(t *testing.T) { + got := kokoro.VoiceSampleKey(tt.voice) + if got != tt.want { + t.Errorf("VoiceSampleKey(%q) = %q, want %q", tt.voice, got, tt.want) + } + }) + } +} + +// ── GenerateAudio ───────────────────────────────────────────────────────────── + +func TestGenerateAudio_EmptyText(t *testing.T) { + srv := httptest.NewServer(http.NotFoundHandler()) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.GenerateAudio(context.Background(), "", "af_bella") + if err == nil { + t.Fatal("expected error for empty text, got nil") + } + if !strings.Contains(err.Error(), "empty text") { + t.Errorf("expected 'empty text' in error, got: %v", err) + } +} + +func TestGenerateAudio_DefaultVoice(t *testing.T) { + // Tracks that the voice defaults to af_bella when empty. + var capturedBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/speech" { + buf := make([]byte, 512) + n, _ := r.Body.Read(buf) + capturedBody = string(buf[:n]) + w.Header().Set("X-Download-Path", "/download/test_file.mp3") + w.WriteHeader(http.StatusOK) + return + } + if strings.HasPrefix(r.URL.Path, "/v1/download/") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("fake-mp3-data")) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + data, err := c.GenerateAudio(context.Background(), "hello world", "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(data) != "fake-mp3-data" { + t.Errorf("unexpected data: %q", string(data)) + } + if !strings.Contains(capturedBody, `"af_bella"`) { + t.Errorf("expected default voice af_bella in request body, got: %s", capturedBody) + } +} + +func TestGenerateAudio_SpeechNon200(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/speech" { + w.WriteHeader(http.StatusInternalServerError) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.GenerateAudio(context.Background(), "text", "af_bella") + if err == nil { + t.Fatal("expected error for non-200 speech response") + } + if !strings.Contains(err.Error(), "500") { + t.Errorf("expected 500 in error, got: %v", err) + } +} + +func TestGenerateAudio_NoDownloadPathHeader(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/speech" { + // No X-Download-Path header + w.WriteHeader(http.StatusOK) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.GenerateAudio(context.Background(), "text", "af_bella") + if err == nil { + t.Fatal("expected error for missing X-Download-Path") + } + if !strings.Contains(err.Error(), "X-Download-Path") { + t.Errorf("expected X-Download-Path in error, got: %v", err) + } +} + +func TestGenerateAudio_DownloadFails(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/speech" { + w.Header().Set("X-Download-Path", "/v1/download/speech.mp3") + w.WriteHeader(http.StatusOK) + return + } + if strings.HasPrefix(r.URL.Path, "/v1/download/") { + w.WriteHeader(http.StatusNotFound) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.GenerateAudio(context.Background(), "text", "af_bella") + if err == nil { + t.Fatal("expected error for failed download") + } + if !strings.Contains(err.Error(), "404") { + t.Errorf("expected 404 in error, got: %v", err) + } +} + +func TestGenerateAudio_FullPath(t *testing.T) { + // X-Download-Path with a full path: extract just filename. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/speech" { + w.Header().Set("X-Download-Path", "/some/nested/path/audio_abc123.mp3") + w.WriteHeader(http.StatusOK) + return + } + if r.URL.Path == "/v1/download/audio_abc123.mp3" { + _, _ = w.Write([]byte("audio-bytes")) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + data, err := c.GenerateAudio(context.Background(), "text", "af_bella") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(data) != "audio-bytes" { + t.Errorf("unexpected data: %q", string(data)) + } +} + +func TestGenerateAudio_ContextCancelled(t *testing.T) { + // Server that hangs — context should cancel before we get a response. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Never respond. + select {} + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + c := kokoro.New(srv.URL) + _, err := c.GenerateAudio(ctx, "text", "af_bella") + if err == nil { + t.Fatal("expected error for cancelled context") + } +} + +// ── ListVoices ──────────────────────────────────────────────────────────────── + +func TestListVoices_Success(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/voices" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"voices":["af_bella","am_adam","bf_emma"]}`)) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + voices, err := c.ListVoices(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(voices) != 3 { + t.Errorf("expected 3 voices, got %d: %v", len(voices), voices) + } + if voices[0] != "af_bella" { + t.Errorf("expected first voice to be af_bella, got %q", voices[0]) + } +} + +func TestListVoices_Non200(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.ListVoices(context.Background()) + if err == nil { + t.Fatal("expected error for non-200 response") + } + if !strings.Contains(err.Error(), "503") { + t.Errorf("expected 503 in error, got: %v", err) + } +} + +func TestListVoices_MalformedJSON(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`not-json`)) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.ListVoices(context.Background()) + if err == nil { + t.Fatal("expected error for malformed JSON") + } +} + +func TestListVoices_EmptyVoices(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"voices":[]}`)) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + voices, err := c.ListVoices(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(voices) != 0 { + t.Errorf("expected 0 voices, got %d", len(voices)) + } +} + +// ── New ─────────────────────────────────────────────────────────────────────── + +func TestNew_TrailingSlashStripped(t *testing.T) { + // Verify that a trailing slash on baseURL doesn't produce double-slash paths. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/voices" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"voices":["af_bella"]}`)) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL + "/") // trailing slash + voices, err := c.ListVoices(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(voices) == 0 { + t.Error("expected at least one voice") + } +} diff --git a/v3/backend/internal/meili/client.go b/v3/backend/internal/meili/client.go new file mode 100644 index 0000000..8525c37 --- /dev/null +++ b/v3/backend/internal/meili/client.go @@ -0,0 +1,257 @@ +// Package meili provides a thin Meilisearch client for indexing and searching +// locally scraped books. +// +// Index: +// - Name: "books" +// - Primary key: "slug" +// - Searchable attributes: title, author, genres, summary +// - Filterable attributes: status, genres +// - Sortable attributes: rank, rating, total_chapters +// +// The client is intentionally simple: UpsertBook and Search only. All +// Meilisearch-specific details (index management, attribute configuration) +// are handled once in Configure(), called at startup. +package meili + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/libnovel/backend/internal/domain" + "github.com/meilisearch/meilisearch-go" +) + +const indexName = "books" + +// Client is the interface for Meilisearch operations used by runner and backend. +type Client interface { + // UpsertBook adds or updates a book document in the search index. + UpsertBook(ctx context.Context, book domain.BookMeta) error + // Search returns up to limit books matching query. + Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error) + // Catalogue queries books with optional filters, sort, and pagination. + // Returns books and the total hit count for pagination. + Catalogue(ctx context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, error) +} + +// CatalogueQuery holds parameters for the /api/catalogue endpoint. +type CatalogueQuery struct { + Q string // full-text query (may be empty for browse) + Genre string // genre filter, e.g. "fantasy" or "all" + Status string // status filter, e.g. "ongoing", "completed", or "all" + Sort string // sort field: "popular", "new", "top-rated", "rank", "" + Page int // 1-indexed + Limit int // items per page, default 20 +} + +// MeiliClient wraps the meilisearch-go SDK. +type MeiliClient struct { + idx meilisearch.IndexManager +} + +// New creates a MeiliClient. Call Configure() once at startup to ensure the +// index exists and has the correct attribute settings. +func New(host, apiKey string) *MeiliClient { + cli := meilisearch.New(host, meilisearch.WithAPIKey(apiKey)) + return &MeiliClient{idx: cli.Index(indexName)} +} + +// Configure creates the index if absent and sets searchable/filterable +// attributes. It is idempotent — safe to call on every startup. +func Configure(host, apiKey string) error { + cli := meilisearch.New(host, meilisearch.WithAPIKey(apiKey)) + + // Create index with primary key. Returns 202 if exists — ignore. + task, err := cli.CreateIndex(&meilisearch.IndexConfig{ + Uid: indexName, + PrimaryKey: "slug", + }) + if err != nil { + // 400 "index_already_exists" is not an error here; the SDK returns + // an error with Code "index_already_exists" which we can ignore. + // Any other error is fatal. + if apiErr, ok := err.(*meilisearch.Error); ok && apiErr.MeilisearchApiError.Code == "index_already_exists" { + // already exists — continue + } else { + return fmt.Errorf("meili: create index: %w", err) + } + } else { + _ = task // task is async; we don't wait for it + } + + idx := cli.Index(indexName) + + searchable := []string{"title", "author", "genres", "summary"} + if _, err := idx.UpdateSearchableAttributes(&searchable); err != nil { + return fmt.Errorf("meili: update searchable attributes: %w", err) + } + + filterable := []interface{}{"status", "genres"} + if _, err := idx.UpdateFilterableAttributes(&filterable); err != nil { + return fmt.Errorf("meili: update filterable attributes: %w", err) + } + + sortable := []string{"rank", "rating", "total_chapters"} + if _, err := idx.UpdateSortableAttributes(&sortable); err != nil { + return fmt.Errorf("meili: update sortable attributes: %w", err) + } + + return nil +} + +// bookDoc is the Meilisearch document shape for a book. +type bookDoc struct { + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author"` + Cover string `json:"cover"` + Status string `json:"status"` + Genres []string `json:"genres"` + Summary string `json:"summary"` + TotalChapters int `json:"total_chapters"` + SourceURL string `json:"source_url"` + Rank int `json:"rank"` + Rating float64 `json:"rating"` +} + +func toDoc(b domain.BookMeta) bookDoc { + return bookDoc{ + Slug: b.Slug, + Title: b.Title, + Author: b.Author, + Cover: b.Cover, + Status: b.Status, + Genres: b.Genres, + Summary: b.Summary, + TotalChapters: b.TotalChapters, + SourceURL: b.SourceURL, + Rank: b.Ranking, + Rating: b.Rating, + } +} + +func fromDoc(d bookDoc) domain.BookMeta { + return domain.BookMeta{ + Slug: d.Slug, + Title: d.Title, + Author: d.Author, + Cover: d.Cover, + Status: d.Status, + Genres: d.Genres, + Summary: d.Summary, + TotalChapters: d.TotalChapters, + SourceURL: d.SourceURL, + Ranking: d.Rank, + Rating: d.Rating, + } +} + +// UpsertBook adds or replaces the book document in Meilisearch. The operation +// is fire-and-forget (Meilisearch processes tasks asynchronously). +func (c *MeiliClient) UpsertBook(_ context.Context, book domain.BookMeta) error { + docs := []bookDoc{toDoc(book)} + pk := "slug" + if _, err := c.idx.AddDocuments(docs, &meilisearch.DocumentOptions{PrimaryKey: &pk}); err != nil { + return fmt.Errorf("meili: upsert book %q: %w", book.Slug, err) + } + return nil +} + +// Search returns books matching query, up to limit results. +func (c *MeiliClient) Search(_ context.Context, query string, limit int) ([]domain.BookMeta, error) { + if limit <= 0 { + limit = 20 + } + res, err := c.idx.Search(query, &meilisearch.SearchRequest{ + Limit: int64(limit), + }) + if err != nil { + return nil, fmt.Errorf("meili: search %q: %w", query, err) + } + + books := make([]domain.BookMeta, 0, len(res.Hits)) + for _, hit := range res.Hits { + // Hit is map[string]json.RawMessage — unmarshal directly into bookDoc. + var doc bookDoc + raw, err := json.Marshal(hit) + if err != nil { + continue + } + if err := json.Unmarshal(raw, &doc); err != nil { + continue + } + books = append(books, fromDoc(doc)) + } + return books, nil +} + +// Catalogue queries books with optional full-text search, genre/status filters, +// sort order, and pagination. Returns matching books and the total estimate. +func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, error) { + if q.Limit <= 0 { + q.Limit = 20 + } + if q.Page <= 0 { + q.Page = 1 + } + + req := &meilisearch.SearchRequest{ + Limit: int64(q.Limit), + Offset: int64((q.Page - 1) * q.Limit), + } + + // Build filter + var filters []string + if q.Genre != "" && q.Genre != "all" { + filters = append(filters, fmt.Sprintf("genres = %q", q.Genre)) + } + if q.Status != "" && q.Status != "all" { + filters = append(filters, fmt.Sprintf("status = %q", q.Status)) + } + if len(filters) > 0 { + req.Filter = strings.Join(filters, " AND ") + } + + // Map UI sort tokens to Meilisearch sort expressions + switch q.Sort { + case "rank": + req.Sort = []string{"rank:asc"} + case "top-rated": + req.Sort = []string{"rating:desc"} + case "new": + req.Sort = []string{"total_chapters:desc"} + // "popular" and "" → relevance (no explicit sort) + } + + res, err := c.idx.Search(q.Q, req) + if err != nil { + return nil, 0, fmt.Errorf("meili: catalogue query: %w", err) + } + + books := make([]domain.BookMeta, 0, len(res.Hits)) + for _, hit := range res.Hits { + var doc bookDoc + raw, err := json.Marshal(hit) + if err != nil { + continue + } + if err := json.Unmarshal(raw, &doc); err != nil { + continue + } + books = append(books, fromDoc(doc)) + } + return books, res.EstimatedTotalHits, nil +} + +// NoopClient is a no-op Client used when Meilisearch is not configured. +type NoopClient struct{} + +func (NoopClient) UpsertBook(_ context.Context, _ domain.BookMeta) error { return nil } +func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) { + return nil, nil +} +func (NoopClient) Catalogue(_ context.Context, _ CatalogueQuery) ([]domain.BookMeta, int64, error) { + return nil, 0, nil +} diff --git a/v3/backend/internal/novelfire/htmlutil/htmlutil.go b/v3/backend/internal/novelfire/htmlutil/htmlutil.go new file mode 100644 index 0000000..5f4b0ef --- /dev/null +++ b/v3/backend/internal/novelfire/htmlutil/htmlutil.go @@ -0,0 +1,228 @@ +// Package htmlutil provides helper functions for parsing HTML with +// golang.org/x/net/html and extracting values by Selector descriptors. +package htmlutil + +import ( + "net/url" + "regexp" + "strings" + + "github.com/libnovel/backend/internal/scraper" + "golang.org/x/net/html" +) + +// ResolveURL returns an absolute URL. If href is already absolute it is +// returned unchanged. Otherwise it is resolved against base. +func ResolveURL(base, href string) string { + if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") { + return href + } + b, err := url.Parse(base) + if err != nil { + return base + href + } + ref, err := url.Parse(href) + if err != nil { + return base + href + } + return b.ResolveReference(ref).String() +} + +// ParseHTML parses raw HTML and returns the root node. +func ParseHTML(raw string) (*html.Node, error) { + return html.Parse(strings.NewReader(raw)) +} + +// selectorMatches reports whether node n matches sel. +func selectorMatches(n *html.Node, sel scraper.Selector) bool { + if n.Type != html.ElementNode { + return false + } + if sel.Tag != "" && n.Data != sel.Tag { + return false + } + if sel.ID != "" { + for _, a := range n.Attr { + if a.Key == "id" && a.Val == sel.ID { + goto checkClass + } + } + return false + } +checkClass: + if sel.Class != "" { + for _, a := range n.Attr { + if a.Key == "class" { + for _, cls := range strings.Fields(a.Val) { + if cls == sel.Class { + goto matched + } + } + } + } + return false + } +matched: + return true +} + +// AttrVal returns the value of attribute key from node n. +func AttrVal(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} + +// TextContent returns the concatenated text content of all descendant text nodes. +func TextContent(n *html.Node) string { + var sb strings.Builder + var walk func(*html.Node) + walk = func(cur *html.Node) { + if cur.Type == html.TextNode { + sb.WriteString(cur.Data) + } + for c := cur.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return strings.TrimSpace(sb.String()) +} + +// FindFirst returns the first node matching sel within root. +func FindFirst(root *html.Node, sel scraper.Selector) *html.Node { + var found *html.Node + var walk func(*html.Node) bool + walk = func(n *html.Node) bool { + if selectorMatches(n, sel) { + found = n + return true + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if walk(c) { + return true + } + } + return false + } + walk(root) + return found +} + +// FindAll returns all nodes matching sel within root. +func FindAll(root *html.Node, sel scraper.Selector) []*html.Node { + var results []*html.Node + var walk func(*html.Node) + walk = func(n *html.Node) { + if selectorMatches(n, sel) { + results = append(results, n) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(root) + return results +} + +// ExtractText extracts a string value from node n using sel. +// If sel.Attr is set the attribute value is returned; otherwise the inner text. +func ExtractText(n *html.Node, sel scraper.Selector) string { + if sel.Attr != "" { + return AttrVal(n, sel.Attr) + } + return TextContent(n) +} + +// ExtractFirst locates the first match in root and returns its text/attr value. +func ExtractFirst(root *html.Node, sel scraper.Selector) string { + n := FindFirst(root, sel) + if n == nil { + return "" + } + return ExtractText(n, sel) +} + +// ExtractAll locates all matches in root and returns their text/attr values. +func ExtractAll(root *html.Node, sel scraper.Selector) []string { + nodes := FindAll(root, sel) + out := make([]string, 0, len(nodes)) + for _, n := range nodes { + if v := ExtractText(n, sel); v != "" { + out = append(out, v) + } + } + return out +} + +// NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown +// representation suitable for chapter storage. +func NodeToMarkdown(n *html.Node) string { + var sb strings.Builder + nodeToMD(n, &sb) + out := multiBlankLine.ReplaceAllString(sb.String(), "\n\n") + return strings.TrimSpace(out) +} + +var multiBlankLine = regexp.MustCompile(`\n(\s*\n){2,}`) + +var blockElements = map[string]bool{ + "p": true, "div": true, "br": true, "h1": true, "h2": true, + "h3": true, "h4": true, "h5": true, "h6": true, "li": true, + "blockquote": true, "pre": true, "hr": true, +} + +func nodeToMD(n *html.Node, sb *strings.Builder) { + switch n.Type { + case html.TextNode: + sb.WriteString(n.Data) + case html.ElementNode: + tag := n.Data + switch tag { + case "br": + sb.WriteString("\n") + case "hr": + sb.WriteString("\n---\n") + case "h1", "h2", "h3", "h4", "h5", "h6": + level := int(tag[1] - '0') + sb.WriteString("\n" + strings.Repeat("#", level) + " ") + for c := n.FirstChild; c != nil; c = c.NextSibling { + nodeToMD(c, sb) + } + sb.WriteString("\n\n") + return + case "p", "div", "blockquote": + sb.WriteString("\n") + for c := n.FirstChild; c != nil; c = c.NextSibling { + nodeToMD(c, sb) + } + sb.WriteString("\n") + return + case "em", "i": + sb.WriteString("*") + for c := n.FirstChild; c != nil; c = c.NextSibling { + nodeToMD(c, sb) + } + sb.WriteString("*") + return + case "strong", "b": + sb.WriteString("**") + for c := n.FirstChild; c != nil; c = c.NextSibling { + nodeToMD(c, sb) + } + sb.WriteString("**") + return + case "script", "style", "noscript": + return // drop + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + nodeToMD(c, sb) + } + if blockElements[tag] { + sb.WriteString("\n") + } + } +} diff --git a/v3/backend/internal/novelfire/scraper.go b/v3/backend/internal/novelfire/scraper.go new file mode 100644 index 0000000..7122b9a --- /dev/null +++ b/v3/backend/internal/novelfire/scraper.go @@ -0,0 +1,498 @@ +// Package novelfire provides a NovelScraper implementation for novelfire.net. +// +// Site structure (as of 2025): +// +// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N +// Book page : https://novelfire.net/book/{slug} +// Chapters : https://novelfire.net/book/{slug}/chapters?page=N +// Chapter : https://novelfire.net/book/{slug}/{chapter-slug} +package novelfire + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "path" + "strconv" + "strings" + "time" + + "github.com/libnovel/backend/internal/browser" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/novelfire/htmlutil" + "github.com/libnovel/backend/internal/scraper" + "golang.org/x/net/html" +) + +const ( + baseURL = "https://novelfire.net" + cataloguePath = "/genre-all/sort-new/status-all/all-novel" + rankingPath = "/genre-all/sort-popular/status-all/all-novel" +) + +// Scraper is the novelfire.net implementation of scraper.NovelScraper. +type Scraper struct { + client browser.Client + log *slog.Logger +} + +// Compile-time interface check. +var _ scraper.NovelScraper = (*Scraper)(nil) + +// New returns a new novelfire Scraper backed by client. +func New(client browser.Client, log *slog.Logger) *Scraper { + if log == nil { + log = slog.Default() + } + return &Scraper{client: client, log: log} +} + +// SourceName implements NovelScraper. +func (s *Scraper) SourceName() string { return "novelfire.net" } + +// ── CatalogueProvider ───────────────────────────────────────────────────────── + +// ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages. +func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) { + entries := make(chan domain.CatalogueEntry, 64) + errs := make(chan error, 16) + + go func() { + defer close(entries) + defer close(errs) + + pageURL := baseURL + cataloguePath + page := 1 + + for pageURL != "" { + select { + case <-ctx.Done(): + return + default: + } + + s.log.Info("scraping catalogue page", "page", page, "url", pageURL) + raw, err := s.client.GetContent(ctx, pageURL) + if err != nil { + errs <- fmt.Errorf("catalogue page %d: %w", page, err) + return + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + errs <- fmt.Errorf("catalogue page %d parse: %w", page, err) + return + } + + cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true}) + if len(cards) == 0 { + s.log.Warn("no novel cards found, stopping pagination", "page", page) + return + } + + for _, card := range cards { + linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"}) + titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}) + + var title, href string + if linkNode != nil { + href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) + } + if titleNode != nil { + title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{})) + } + if href == "" || title == "" { + continue + } + + bookURL := resolveURL(baseURL, href) + select { + case <-ctx.Done(): + return + case entries <- domain.CatalogueEntry{Title: title, URL: bookURL}: + } + } + + if !hasNextPageLink(root) { + break + } + nextHref := "" + for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) { + if htmlutil.AttrVal(a, "rel") == "next" { + nextHref = htmlutil.AttrVal(a, "href") + break + } + } + if nextHref == "" { + break + } + pageURL = resolveURL(baseURL, nextHref) + page++ + } + }() + + return entries, errs +} + +// ── MetadataProvider ────────────────────────────────────────────────────────── + +// ScrapeMetadata fetches and parses book metadata from the book's landing page. +func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) { + s.log.Debug("metadata fetch starting", "url", bookURL) + + raw, err := s.client.GetContent(ctx, bookURL) + if err != nil { + return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err) + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + return domain.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err) + } + + title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"}) + author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"}) + + var cover string + if fig := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil { + cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"}) + if cover != "" && !strings.HasPrefix(cover, "http") { + cover = baseURL + cover + } + } + + status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"}) + + genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"}) + var genres []string + if genresNode != nil { + genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true}) + } + + summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"}) + totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"}) + totalChapters := parseChapterCount(totalStr) + + slug := slugFromURL(bookURL) + + meta := domain.BookMeta{ + Slug: slug, + Title: title, + Author: author, + Cover: cover, + Status: status, + Genres: genres, + Summary: summary, + TotalChapters: totalChapters, + SourceURL: bookURL, + } + s.log.Debug("metadata parsed", "slug", meta.Slug, "title", meta.Title) + return meta, nil +} + +// ── ChapterListProvider ─────────────────────────────────────────────────────── + +// ScrapeChapterList returns all chapter references for a book, ordered ascending. +func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) { + var refs []domain.ChapterRef + baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters" + page := 1 + + for { + select { + case <-ctx.Done(): + return refs, ctx.Err() + default: + } + + pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page) + s.log.Info("scraping chapter list", "page", page, "url", pageURL) + + raw, err := s.client.GetContent(ctx, pageURL) + if err != nil { + return refs, fmt.Errorf("chapter list page %d: %w", page, err) + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + return refs, fmt.Errorf("chapter list page %d parse: %w", page, err) + } + + chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"}) + if chapterList == nil { + s.log.Debug("chapter list container not found, stopping pagination", "page", page) + break + } + + items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"}) + if len(items) == 0 { + break + } + + for _, item := range items { + linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"}) + if linkNode == nil { + continue + } + href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"}) + chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{}) + if href == "" { + continue + } + chURL := resolveURL(baseURL, href) + num := chapterNumberFromURL(chURL) + if num <= 0 { + num = len(refs) + 1 + s.log.Warn("chapter number not parseable from URL, falling back to position", + "url", chURL, "position", num) + } + refs = append(refs, domain.ChapterRef{ + Number: num, + Title: strings.TrimSpace(chTitle), + URL: chURL, + }) + } + + page++ + } + + return refs, nil +} + +// ── ChapterTextProvider ─────────────────────────────────────────────────────── + +// ScrapeChapterText fetches and parses a single chapter page. +func (s *Scraper) ScrapeChapterText(ctx context.Context, ref domain.ChapterRef) (domain.Chapter, error) { + s.log.Debug("chapter text fetch starting", "chapter", ref.Number, "url", ref.URL) + + raw, err := retryGet(ctx, s.log, s.client, ref.URL, 9, 6*time.Second) + if err != nil { + return domain.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err) + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + return domain.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err) + } + + container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"}) + if container == nil { + return domain.Chapter{}, fmt.Errorf("chapter %d: #content container not found in %s", ref.Number, ref.URL) + } + + text := htmlutil.NodeToMarkdown(container) + + s.log.Debug("chapter text parsed", "chapter", ref.Number, "text_bytes", len(text)) + + return domain.Chapter{Ref: ref, Text: text}, nil +} + +// ── RankingProvider ─────────────────────────────────────────────────────────── + +// ScrapeRanking pages through up to maxPages pages of the popular-novels listing. +// maxPages <= 0 means all pages. The caller decides whether to persist items. +func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) { + entries := make(chan domain.BookMeta, 32) + errs := make(chan error, 16) + + go func() { + defer close(entries) + defer close(errs) + + rank := 1 + + for page := 1; maxPages <= 0 || page <= maxPages; page++ { + select { + case <-ctx.Done(): + return + default: + } + + pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page) + s.log.Info("scraping popular ranking page", "page", page, "url", pageURL) + + raw, err := s.client.GetContent(ctx, pageURL) + if err != nil { + errs <- fmt.Errorf("ranking page %d: %w", page, err) + return + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + errs <- fmt.Errorf("ranking page %d parse: %w", page, err) + return + } + + cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true}) + if len(cards) == 0 { + break + } + + for _, card := range cards { + linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"}) + if linkNode == nil { + continue + } + href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) + bookURL := resolveURL(baseURL, href) + if bookURL == "" { + continue + } + + title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})) + if title == "" { + title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"})) + } + if title == "" { + continue + } + + var cover string + if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil { + cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"}) + if cover == "" { + cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"}) + } + if strings.HasPrefix(cover, "data:") { + cover = "" + } + if cover != "" && !strings.HasPrefix(cover, "http") { + cover = baseURL + cover + } + } + + meta := domain.BookMeta{ + Slug: slugFromURL(bookURL), + Title: title, + Cover: cover, + SourceURL: bookURL, + Ranking: rank, + } + rank++ + + select { + case <-ctx.Done(): + return + case entries <- meta: + } + } + + if !hasNextPageLink(root) { + break + } + } + }() + + return entries, errs +} + +// ── helpers ─────────────────────────────────────────────────────────────────── + +func resolveURL(base, href string) string { return htmlutil.ResolveURL(base, href) } + +func hasNextPageLink(root *html.Node) bool { + links := htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) + for _, a := range links { + for _, attr := range a.Attr { + if attr.Key == "rel" && attr.Val == "next" { + return true + } + } + } + return false +} + +func slugFromURL(bookURL string) string { + u, err := url.Parse(bookURL) + if err != nil { + return bookURL + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(parts) >= 2 && parts[0] == "book" { + return parts[1] + } + if len(parts) > 0 { + return parts[len(parts)-1] + } + return "" +} + +func parseChapterCount(s string) int { + s = strings.ReplaceAll(s, ",", "") + fields := strings.Fields(s) + if len(fields) == 0 { + return 0 + } + n, _ := strconv.Atoi(fields[0]) + return n +} + +func chapterNumberFromURL(chapterURL string) int { + u, err := url.Parse(chapterURL) + if err != nil { + return 0 + } + seg := path.Base(u.Path) + seg = strings.TrimPrefix(seg, "chapter-") + seg = strings.TrimPrefix(seg, "chap-") + seg = strings.TrimPrefix(seg, "ch-") + digits := strings.FieldsFunc(seg, func(r rune) bool { + return r < '0' || r > '9' + }) + if len(digits) == 0 { + return 0 + } + n, _ := strconv.Atoi(digits[0]) + return n +} + +// retryGet calls client.GetContent up to maxAttempts times with exponential backoff. +// If the server returns 429 (ErrRateLimit), the suggested Retry-After delay is used +// instead of the geometric backoff delay. +func retryGet( + ctx context.Context, + log *slog.Logger, + client browser.Client, + pageURL string, + maxAttempts int, + baseDelay time.Duration, +) (string, error) { + var lastErr error + delay := baseDelay + for attempt := 1; attempt <= maxAttempts; attempt++ { + raw, err := client.GetContent(ctx, pageURL) + if err == nil { + return raw, nil + } + lastErr = err + if ctx.Err() != nil { + return "", err + } + if attempt < maxAttempts { + // If the server is rate-limiting us, honour its Retry-After delay. + waitFor := delay + var rlErr *browser.RateLimitError + if errors.As(err, &rlErr) { + waitFor = rlErr.RetryAfter + if log != nil { + log.Warn("rate limited, backing off", + "url", pageURL, "attempt", attempt, "retry_in", waitFor) + } + } else { + if log != nil { + log.Warn("fetch failed, retrying", + "url", pageURL, "attempt", attempt, "retry_in", delay, "err", err) + } + delay *= 2 + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(waitFor): + } + } + } + return "", lastErr +} diff --git a/v3/backend/internal/novelfire/scraper_test.go b/v3/backend/internal/novelfire/scraper_test.go new file mode 100644 index 0000000..04a9f7f --- /dev/null +++ b/v3/backend/internal/novelfire/scraper_test.go @@ -0,0 +1,129 @@ +package novelfire + +import ( + "context" + "testing" +) + +func TestSlugFromURL(t *testing.T) { + cases := []struct { + url string + want string + }{ + {"https://novelfire.net/book/shadow-slave", "shadow-slave"}, + {"https://novelfire.net/book/a-dragon-against-the-whole-world", "a-dragon-against-the-whole-world"}, + {"https://novelfire.net/book/foo/chapter-1", "foo"}, + {"https://novelfire.net/", ""}, + {"not-a-url", "not-a-url"}, + } + for _, c := range cases { + got := slugFromURL(c.url) + if got != c.want { + t.Errorf("slugFromURL(%q) = %q, want %q", c.url, got, c.want) + } + } +} + +func TestChapterNumberFromURL(t *testing.T) { + cases := []struct { + url string + want int + }{ + {"https://novelfire.net/book/shadow-slave/chapter-42", 42}, + {"https://novelfire.net/book/shadow-slave/chapter-1000", 1000}, + {"https://novelfire.net/book/shadow-slave/chap-7", 7}, + {"https://novelfire.net/book/shadow-slave/ch-3", 3}, + {"https://novelfire.net/book/shadow-slave/42", 42}, + {"https://novelfire.net/book/shadow-slave/no-number-here", 0}, + {"not-a-url", 0}, + } + for _, c := range cases { + got := chapterNumberFromURL(c.url) + if got != c.want { + t.Errorf("chapterNumberFromURL(%q) = %d, want %d", c.url, got, c.want) + } + } +} + +func TestParseChapterCount(t *testing.T) { + cases := []struct { + in string + want int + }{ + {"123 Chapters", 123}, + {"1,234 Chapters", 1234}, + {"0", 0}, + {"", 0}, + {"500", 500}, + } + for _, c := range cases { + got := parseChapterCount(c.in) + if got != c.want { + t.Errorf("parseChapterCount(%q) = %d, want %d", c.in, got, c.want) + } + } +} + +func TestRetryGet_ContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + stub := newStubClient() + stub.setError("https://example.com/page", context.Canceled) + + _, err := retryGet(ctx, nil, stub, "https://example.com/page", 3, 0) + if err == nil { + t.Fatal("expected error on cancelled context") + } +} + +func TestRetryGet_EventualSuccess(t *testing.T) { + stub := newStubClient() + calls := 0 + stub.setFn("https://example.com/page", func() (string, error) { + calls++ + if calls < 3 { + return "", context.DeadlineExceeded + } + return "ok", nil + }) + + got, err := retryGet(context.Background(), nil, stub, "https://example.com/page", 5, 0) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != "ok" { + t.Errorf("got %q, want html", got) + } + if calls != 3 { + t.Errorf("expected 3 calls, got %d", calls) + } +} + +// ── minimal stub client for tests ───────────────────────────────────────────── + +type stubClient struct { + errors map[string]error + fns map[string]func() (string, error) +} + +func newStubClient() *stubClient { + return &stubClient{ + errors: make(map[string]error), + fns: make(map[string]func() (string, error)), + } +} + +func (s *stubClient) setError(u string, err error) { s.errors[u] = err } + +func (s *stubClient) setFn(u string, fn func() (string, error)) { s.fns[u] = fn } + +func (s *stubClient) GetContent(_ context.Context, pageURL string) (string, error) { + if fn, ok := s.fns[pageURL]; ok { + return fn() + } + if err, ok := s.errors[pageURL]; ok { + return "", err + } + return "", context.DeadlineExceeded +} diff --git a/v3/backend/internal/orchestrator/orchestrator.go b/v3/backend/internal/orchestrator/orchestrator.go new file mode 100644 index 0000000..e5e3458 --- /dev/null +++ b/v3/backend/internal/orchestrator/orchestrator.go @@ -0,0 +1,222 @@ +// Package orchestrator coordinates metadata extraction, chapter-list fetching, +// and parallel chapter scraping for a single book. +// +// Design: +// - RunBook scrapes one book (metadata + chapter list + chapter texts) end-to-end. +// - N worker goroutines pull chapter refs from a shared queue and call ScrapeChapterText. +// - The caller (runner poll loop) owns the outer task-claim / finish cycle. +// - An optional PostMetadata hook (set in Config) is called after WriteMetadata +// succeeds. The runner uses this to upsert books into Meilisearch. +package orchestrator + +import ( + "context" + "fmt" + "log/slog" + "runtime" + "sync" + "sync/atomic" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/scraper" +) + +// Config holds tunable parameters for the orchestrator. +type Config struct { + // Workers is the number of goroutines used to scrape chapters in parallel. + // Defaults to runtime.NumCPU() when 0. + Workers int + // PostMetadata is an optional hook called with the scraped BookMeta after + // WriteMetadata succeeds. Errors from the hook are logged but not fatal. + // Used by the runner to index books in Meilisearch. + PostMetadata func(ctx context.Context, meta domain.BookMeta) +} + +// Orchestrator runs a single-book scrape pipeline. +type Orchestrator struct { + novel scraper.NovelScraper + store bookstore.BookWriter + log *slog.Logger + workers int + postMetadata func(ctx context.Context, meta domain.BookMeta) +} + +// New returns a new Orchestrator. +func New(cfg Config, novel scraper.NovelScraper, store bookstore.BookWriter, log *slog.Logger) *Orchestrator { + if log == nil { + log = slog.Default() + } + workers := cfg.Workers + if workers <= 0 { + workers = runtime.NumCPU() + } + return &Orchestrator{ + novel: novel, + store: store, + log: log, + workers: workers, + postMetadata: cfg.PostMetadata, + } +} + +// RunBook scrapes a single book described by task. It handles: +// 1. Metadata scrape + write +// 2. Chapter list scrape + write +// 3. Parallel chapter text scrape + write (worker pool) +// +// Returns a ScrapeResult with counters. The result's ErrorMessage is non-empty +// if the run failed at the metadata or chapter-list level. +func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) domain.ScrapeResult { + o.log.Info("orchestrator: RunBook starting", + "task_id", task.ID, + "kind", task.Kind, + "url", task.TargetURL, + "workers", o.workers, + ) + + var result domain.ScrapeResult + + if task.TargetURL == "" { + result.ErrorMessage = "task has no target URL" + return result + } + + // ── Step 1: Metadata ────────────────────────────────────────────────────── + meta, err := o.novel.ScrapeMetadata(ctx, task.TargetURL) + if err != nil { + o.log.Error("metadata scrape failed", "url", task.TargetURL, "err", err) + result.ErrorMessage = fmt.Sprintf("metadata: %v", err) + result.Errors++ + return result + } + + if err := o.store.WriteMetadata(ctx, meta); err != nil { + o.log.Error("metadata write failed", "slug", meta.Slug, "err", err) + // non-fatal: continue to chapters + result.Errors++ + } else { + result.BooksFound = 1 + // Fire optional post-metadata hook (e.g. Meilisearch indexing). + if o.postMetadata != nil { + o.postMetadata(ctx, meta) + } + } + + o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title) + + // ── Step 2: Chapter list ────────────────────────────────────────────────── + refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL) + if err != nil { + o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err) + result.ErrorMessage = fmt.Sprintf("chapter list: %v", err) + result.Errors++ + return result + } + + o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs)) + + // Persist chapter refs (without text) so the index exists early. + if wErr := o.store.WriteChapterRefs(ctx, meta.Slug, refs); wErr != nil { + o.log.Warn("chapter refs write failed", "slug", meta.Slug, "err", wErr) + } + + // ── Step 3: Chapter texts (worker pool) ─────────────────────────────────── + type chapterJob struct { + slug string + ref domain.ChapterRef + total int // total chapters to scrape (for progress logging) + } + work := make(chan chapterJob, o.workers*4) + + var scraped, skipped, errors atomic.Int64 + var wg sync.WaitGroup + + for i := 0; i < o.workers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + for job := range work { + select { + case <-ctx.Done(): + return + default: + } + + if o.store.ChapterExists(ctx, job.slug, job.ref) { + o.log.Debug("chapter already exists, skipping", + "slug", job.slug, "chapter", job.ref.Number) + skipped.Add(1) + continue + } + + ch, err := o.novel.ScrapeChapterText(ctx, job.ref) + if err != nil { + o.log.Error("chapter scrape failed", + "slug", job.slug, "chapter", job.ref.Number, "err", err) + errors.Add(1) + continue + } + + if err := o.store.WriteChapter(ctx, job.slug, ch); err != nil { + o.log.Error("chapter write failed", + "slug", job.slug, "chapter", job.ref.Number, "err", err) + errors.Add(1) + continue + } + + n := scraped.Add(1) + // Log a progress summary every 25 chapters scraped. + if n%25 == 0 { + o.log.Info("scraping chapters", + "slug", job.slug, "scraped", n, "total", job.total) + } + } + }(i) + } + + // Count how many chapters will actually be enqueued (for progress logging). + toScrape := 0 + for _, ref := range refs { + if task.FromChapter > 0 && ref.Number < task.FromChapter { + continue + } + if task.ToChapter > 0 && ref.Number > task.ToChapter { + continue + } + toScrape++ + } + + // Enqueue chapter jobs respecting the optional range filter from the task. + for _, ref := range refs { + if task.FromChapter > 0 && ref.Number < task.FromChapter { + skipped.Add(1) + continue + } + if task.ToChapter > 0 && ref.Number > task.ToChapter { + skipped.Add(1) + continue + } + select { + case <-ctx.Done(): + goto drain + case work <- chapterJob{slug: meta.Slug, ref: ref, total: toScrape}: + } + } + +drain: + close(work) + wg.Wait() + + result.ChaptersScraped = int(scraped.Load()) + result.ChaptersSkipped = int(skipped.Load()) + result.Errors += int(errors.Load()) + + o.log.Info("book scrape finished", + "slug", meta.Slug, + "scraped", result.ChaptersScraped, + "skipped", result.ChaptersSkipped, + "errors", result.Errors, + ) + return result +} diff --git a/v3/backend/internal/orchestrator/orchestrator_test.go b/v3/backend/internal/orchestrator/orchestrator_test.go new file mode 100644 index 0000000..b1edaf9 --- /dev/null +++ b/v3/backend/internal/orchestrator/orchestrator_test.go @@ -0,0 +1,210 @@ +package orchestrator + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/libnovel/backend/internal/domain" +) + +// ── stubs ───────────────────────────────────────────────────────────────────── + +type stubScraper struct { + meta domain.BookMeta + metaErr error + refs []domain.ChapterRef + refsErr error + chapters map[int]domain.Chapter + chapErr map[int]error +} + +func (s *stubScraper) SourceName() string { return "stub" } + +func (s *stubScraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) { + ch := make(chan domain.CatalogueEntry) + errs := make(chan error) + close(ch) + close(errs) + return ch, errs +} + +func (s *stubScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) { + return s.meta, s.metaErr +} + +func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) { + return s.refs, s.refsErr +} + +func (s *stubScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) { + if s.chapErr != nil { + if err, ok := s.chapErr[ref.Number]; ok { + return domain.Chapter{}, err + } + } + if s.chapters != nil { + if ch, ok := s.chapters[ref.Number]; ok { + return ch, nil + } + } + return domain.Chapter{Ref: ref, Text: "text"}, nil +} + +func (s *stubScraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) { + ch := make(chan domain.BookMeta) + errs := make(chan error) + close(ch) + close(errs) + return ch, errs +} + +type stubStore struct { + mu sync.Mutex + metaWritten []domain.BookMeta + chaptersWritten []domain.Chapter + existing map[string]bool // "slug:N" → exists + writeMetaErr error +} + +func (s *stubStore) WriteMetadata(_ context.Context, meta domain.BookMeta) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.writeMetaErr != nil { + return s.writeMetaErr + } + s.metaWritten = append(s.metaWritten, meta) + return nil +} + +func (s *stubStore) WriteChapter(_ context.Context, slug string, ch domain.Chapter) error { + s.mu.Lock() + defer s.mu.Unlock() + s.chaptersWritten = append(s.chaptersWritten, ch) + return nil +} + +func (s *stubStore) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error { + return nil +} + +func (s *stubStore) ChapterExists(_ context.Context, slug string, ref domain.ChapterRef) bool { + s.mu.Lock() + defer s.mu.Unlock() + key := slug + ":" + string(rune('0'+ref.Number)) + return s.existing[key] +} + +// ── tests ────────────────────────────────────────────────────────────────────── + +func TestRunBook_HappyPath(t *testing.T) { + sc := &stubScraper{ + meta: domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, + refs: []domain.ChapterRef{ + {Number: 1, Title: "Ch 1", URL: "https://example.com/book/test-book/chapter-1"}, + {Number: 2, Title: "Ch 2", URL: "https://example.com/book/test-book/chapter-2"}, + {Number: 3, Title: "Ch 3", URL: "https://example.com/book/test-book/chapter-3"}, + }, + } + st := &stubStore{} + o := New(Config{Workers: 2}, sc, st, nil) + + task := domain.ScrapeTask{ + ID: "t1", + Kind: "book", + TargetURL: "https://example.com/book/test-book", + } + + result := o.RunBook(context.Background(), task) + + if result.ErrorMessage != "" { + t.Fatalf("unexpected error: %s", result.ErrorMessage) + } + if result.BooksFound != 1 { + t.Errorf("BooksFound = %d, want 1", result.BooksFound) + } + if result.ChaptersScraped != 3 { + t.Errorf("ChaptersScraped = %d, want 3", result.ChaptersScraped) + } +} + +func TestRunBook_MetadataError(t *testing.T) { + sc := &stubScraper{metaErr: errors.New("404 not found")} + st := &stubStore{} + o := New(Config{Workers: 1}, sc, st, nil) + + result := o.RunBook(context.Background(), domain.ScrapeTask{ + ID: "t2", + TargetURL: "https://example.com/book/missing", + }) + + if result.ErrorMessage == "" { + t.Fatal("expected ErrorMessage to be set") + } + if result.Errors != 1 { + t.Errorf("Errors = %d, want 1", result.Errors) + } +} + +func TestRunBook_ChapterRange(t *testing.T) { + sc := &stubScraper{ + meta: domain.BookMeta{Slug: "range-book", SourceURL: "https://example.com/book/range-book"}, + refs: func() []domain.ChapterRef { + var refs []domain.ChapterRef + for i := 1; i <= 10; i++ { + refs = append(refs, domain.ChapterRef{Number: i, URL: "https://example.com/book/range-book/chapter-" + string(rune('0'+i))}) + } + return refs + }(), + } + st := &stubStore{} + o := New(Config{Workers: 2}, sc, st, nil) + + result := o.RunBook(context.Background(), domain.ScrapeTask{ + ID: "t3", + TargetURL: "https://example.com/book/range-book", + FromChapter: 3, + ToChapter: 7, + }) + + if result.ErrorMessage != "" { + t.Fatalf("unexpected error: %s", result.ErrorMessage) + } + // chapters 3–7 = 5 scraped, chapters 1-2 and 8-10 = 5 skipped + if result.ChaptersScraped != 5 { + t.Errorf("ChaptersScraped = %d, want 5", result.ChaptersScraped) + } + if result.ChaptersSkipped != 5 { + t.Errorf("ChaptersSkipped = %d, want 5", result.ChaptersSkipped) + } +} + +func TestRunBook_ContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + sc := &stubScraper{ + meta: domain.BookMeta{Slug: "ctx-book", SourceURL: "https://example.com/book/ctx-book"}, + refs: []domain.ChapterRef{ + {Number: 1, URL: "https://example.com/book/ctx-book/chapter-1"}, + }, + } + st := &stubStore{} + o := New(Config{Workers: 1}, sc, st, nil) + + // Should not panic; result may have errors or zero chapters. + result := o.RunBook(ctx, domain.ScrapeTask{ + ID: "t4", + TargetURL: "https://example.com/book/ctx-book", + }) + _ = result +} + +func TestRunBook_EmptyTargetURL(t *testing.T) { + o := New(Config{Workers: 1}, &stubScraper{}, &stubStore{}, nil) + result := o.RunBook(context.Background(), domain.ScrapeTask{ID: "t5"}) + if result.ErrorMessage == "" { + t.Fatal("expected ErrorMessage for empty target URL") + } +} diff --git a/v3/backend/internal/presigncache/cache.go b/v3/backend/internal/presigncache/cache.go new file mode 100644 index 0000000..a721728 --- /dev/null +++ b/v3/backend/internal/presigncache/cache.go @@ -0,0 +1,96 @@ +// Package presigncache provides a Valkey (Redis-compatible) backed cache for +// MinIO presigned URLs. The backend generates presigned URLs and stores them +// here with a TTL; subsequent requests for the same key return the cached URL +// without re-contacting MinIO. +// +// Design: +// - Cache is intentionally best-effort: Get returns ("", false, nil) on any +// Valkey error, so callers always have a fallback path to regenerate. +// - Set silently drops errors — a miss on the next request is acceptable. +// - TTL should be set shorter than the actual presigned URL lifetime so that +// cached URLs are always valid when served. Recommended: 55 minutes for a +// 1-hour presigned URL. +package presigncache + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +// Cache is the interface for presign URL caching. +// Implementations must be safe for concurrent use. +type Cache interface { + // Get returns the cached URL for key. ok is false on cache miss or error. + Get(ctx context.Context, key string) (url string, ok bool, err error) + // Set stores url under key with the given TTL. + Set(ctx context.Context, key, url string, ttl time.Duration) error + // Delete removes key from the cache. + Delete(ctx context.Context, key string) error +} + +// ValkeyCache is a Cache backed by Valkey / Redis via go-redis. +type ValkeyCache struct { + rdb *redis.Client +} + +// New creates a ValkeyCache connecting to addr (e.g. "valkey:6379"). +// The connection is not established until the first command; use Ping to +// verify connectivity at startup. +func New(addr string) *ValkeyCache { + rdb := redis.NewClient(&redis.Options{ + Addr: addr, + DialTimeout: 2 * time.Second, + ReadTimeout: 1 * time.Second, + WriteTimeout: 1 * time.Second, + }) + return &ValkeyCache{rdb: rdb} +} + +// Ping checks connectivity. Call once at startup. +func (c *ValkeyCache) Ping(ctx context.Context) error { + if err := c.rdb.Ping(ctx).Err(); err != nil { + return fmt.Errorf("presigncache: ping valkey: %w", err) + } + return nil +} + +// Get returns (url, true, nil) on hit, ("", false, nil) on miss, and +// ("", false, err) only on unexpected errors (not redis.Nil). +func (c *ValkeyCache) Get(ctx context.Context, key string) (string, bool, error) { + val, err := c.rdb.Get(ctx, key).Result() + if err == redis.Nil { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("presigncache: get %q: %w", key, err) + } + return val, true, nil +} + +// Set stores url under key with ttl. Errors are returned but are non-fatal +// for callers — a Set failure means the next request will miss and regenerate. +func (c *ValkeyCache) Set(ctx context.Context, key, url string, ttl time.Duration) error { + if err := c.rdb.Set(ctx, key, url, ttl).Err(); err != nil { + return fmt.Errorf("presigncache: set %q: %w", key, err) + } + return nil +} + +// Delete removes key from the cache. It is not an error if the key does not exist. +func (c *ValkeyCache) Delete(ctx context.Context, key string) error { + if err := c.rdb.Del(ctx, key).Err(); err != nil { + return fmt.Errorf("presigncache: delete %q: %w", key, err) + } + return nil +} + +// NoopCache is a no-op Cache that always returns a miss. Used when Valkey is +// not configured (e.g. local development without Docker). +type NoopCache struct{} + +func (NoopCache) Get(_ context.Context, _ string) (string, bool, error) { return "", false, nil } +func (NoopCache) Set(_ context.Context, _, _ string, _ time.Duration) error { return nil } +func (NoopCache) Delete(_ context.Context, _ string) error { return nil } diff --git a/v3/backend/internal/runner/browse_refresh.go b/v3/backend/internal/runner/browse_refresh.go new file mode 100644 index 0000000..c742005 --- /dev/null +++ b/v3/backend/internal/runner/browse_refresh.go @@ -0,0 +1,176 @@ +package runner + +// browse_refresh.go — independent 6-hour loop that fetches novelfire.net +// browse page snapshots and stores them in MinIO. +// +// Design: +// - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run(). +// - Fetches page 1 for each combination of the standard genre/sort/status +// filter values and stores the parsed JSON blob in MinIO via BrowseStore. +// - The backend's handleBrowse then serves from MinIO instead of calling +// novelfire.net live, which avoids IP-based rate-limiting on the server. + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" +) + +// browseNovelListing mirrors backend.NovelListing for JSON serialisation. +type browseNovelListing struct { + Slug string `json:"slug"` + Title string `json:"title"` + Cover string `json:"cover"` + URL string `json:"url"` +} + +// browseSnapshot is the JSON structure stored in MinIO. +type browseSnapshot struct { + Novels []browseNovelListing `json:"novels"` + Page int `json:"page"` + HasNext bool `json:"hasNext"` + // CachedAt is the UTC time the snapshot was written (ISO 8601). + CachedAt string `json:"cachedAt"` +} + +// browseCombos lists the filter combinations to pre-fetch. +// Each entry is (genre, sort, status, novelType). +var browseCombos = []struct{ genre, sort, status, novelType string }{ + {"all", "popular", "all", "all-novel"}, + {"all", "popular", "ongoing", "all-novel"}, + {"all", "popular", "completed", "all-novel"}, + {"all", "new", "all", "all-novel"}, + {"all", "new", "ongoing", "all-novel"}, + {"all", "new", "completed", "all-novel"}, + {"all", "top-rated", "all", "all-novel"}, + {"all", "top-rated", "ongoing", "all-novel"}, + {"all", "top-rated", "completed", "all-novel"}, +} + +const novelFireBrowseBase = "https://novelfire.net" + +// runBrowseRefresh fetches all browse combos from novelfire.net and stores +// the results in MinIO. Errors per-combo are logged but do not abort the +// whole refresh cycle. +func (r *Runner) runBrowseRefresh(ctx context.Context) { + if r.deps.BrowseStore == nil { + r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured") + return + } + + log := r.deps.Log.With("op", "browse_refresh") + log.Info("runner: browse refresh starting", "combos", len(browseCombos)) + + ok, fail := 0, 0 + for _, c := range browseCombos { + if ctx.Err() != nil { + break + } + novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType) + if err != nil { + log.Warn("runner: browse fetch failed", + "genre", c.genre, "sort", c.sort, "status", c.status, "err", err) + fail++ + continue + } + + snap := browseSnapshot{ + Novels: novels, + Page: 1, + HasNext: hasNext, + CachedAt: time.Now().UTC().Format(time.RFC3339), + } + data, _ := json.Marshal(snap) + if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil { + log.Warn("runner: browse put failed", + "genre", c.genre, "sort", c.sort, "status", c.status, "err", err) + fail++ + continue + } + ok++ + } + + log.Info("runner: browse refresh finished", "ok", ok, "failed", fail) +} + +// fetchBrowsePage calls novelfire.net and returns a list of novel listings +// plus a hasNext flag. Mirrors the logic in backend/handlers.go. +func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) { + pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1", + novelFireBrowseBase, genre, sort, status, novelType) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) + if err != nil { + return nil, false, fmt.Errorf("build request: %w", err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + + httpClient := &http.Client{Timeout: 45 * time.Second} + resp, err := httpClient.Do(req) + if err != nil { + return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL) + } + + return parseBrowseHTML(resp.Body) +} + +// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage +// in backend/handlers.go — kept separate to avoid coupling packages. +func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, false, err + } + body := string(data) + + hasNext := strings.Contains(body, `rel="next"`) || + strings.Contains(body, `aria-label="Next"`) || + strings.Contains(body, `class="next"`) + + slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`) + titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`) + coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`) + + slugMatches := slugRe.FindAllStringSubmatch(body, -1) + titleMatches := titleRe.FindAllStringSubmatch(body, -1) + coverMatches := coverRe.FindAllStringSubmatch(body, -1) + + var novels []browseNovelListing + seen := make(map[string]bool) + for i, sm := range slugMatches { + slug := sm[1] + if seen[slug] { + continue + } + seen[slug] = true + + item := browseNovelListing{ + Slug: slug, + URL: novelFireBrowseBase + "/book/" + slug, + } + if i < len(titleMatches) { + item.Title = strings.TrimSpace(titleMatches[i][1]) + } + if i < len(coverMatches) { + item.Cover = coverMatches[i][1] + } + if item.Title != "" { + novels = append(novels, item) + } + } + + return novels, hasNext, nil +} diff --git a/v3/backend/internal/runner/catalogue_refresh.go b/v3/backend/internal/runner/catalogue_refresh.go new file mode 100644 index 0000000..652260a --- /dev/null +++ b/v3/backend/internal/runner/catalogue_refresh.go @@ -0,0 +1,177 @@ +package runner + +// catalogue_refresh.go — independent loop that walks the full novelfire.net +// catalogue, scrapes per-book metadata, downloads cover images to MinIO, and +// indexes every book in Meilisearch. +// +// Design: +// - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run(). +// - Also fires once on startup. +// - ScrapeCatalogue streams CatalogueEntry values over a channel — we iterate +// and call ScrapeMetadata for each entry. +// - Per-request random jitter (1–3s) prevents hammering novelfire.net. +// - Cover images are fetched from the URL embedded in BookMeta.Cover and +// stored in MinIO (browse bucket, key: covers/{slug}.jpg). +// - WriteMetadata + UpsertBook are called for every successfully scraped book. +// - Errors for individual books are logged and skipped; the loop continues. +// - The cover URL stored in BookMeta.Cover is rewritten to the internal proxy +// path (/api/cover/novelfire.net/{slug}) so the UI always fetches via the +// backend, which will serve from MinIO. + +import ( + "context" + "fmt" + "io" + "math/rand" + "net/http" + "time" +) + +// runCatalogueRefresh performs one full catalogue walk: scrapes metadata for +// every book on novelfire.net, downloads covers to MinIO, and upserts to +// Meilisearch. Errors for individual books are logged and skipped. +func (r *Runner) runCatalogueRefresh(ctx context.Context) { + if r.deps.Novel == nil { + r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured") + return + } + if r.deps.BookWriter == nil { + r.deps.Log.Warn("runner: catalogue refresh skipped — BookWriter not configured") + return + } + + log := r.deps.Log.With("op", "catalogue_refresh") + log.Info("runner: catalogue refresh starting") + + entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx) + + ok, skipped, errCount := 0, 0, 0 + for entry := range entries { + if ctx.Err() != nil { + break + } + + // Random jitter between books to avoid rate-limiting. + jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond + select { + case <-ctx.Done(): + break + case <-time.After(jitter): + } + + meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL) + if err != nil { + log.Warn("runner: catalogue refresh: metadata scrape failed", + "url", entry.URL, "err", err) + errCount++ + continue + } + + // Rewrite cover URL to backend proxy path so UI never hits CDN directly. + originalCover := meta.Cover + meta.Cover = fmt.Sprintf("/api/cover/novelfire.net/%s", meta.Slug) + + // Persist to PocketBase. + if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil { + log.Warn("runner: catalogue refresh: WriteMetadata failed", + "slug", meta.Slug, "err", err) + errCount++ + continue + } + + // Index in Meilisearch. + if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil { + log.Warn("runner: catalogue refresh: UpsertBook failed", + "slug", meta.Slug, "err", err) + // non-fatal — continue + } + + // Download and store cover image in MinIO if we have a cover URL + // and a CoverStore is wired in. + if r.deps.CoverStore != nil && originalCover != "" { + if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) { + if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil { + log.Warn("runner: catalogue refresh: cover download failed", + "slug", meta.Slug, "url", originalCover, "err", err) + // non-fatal + } + } + } + + ok++ + if ok%100 == 0 { + log.Info("runner: catalogue refresh progress", + "scraped", ok, "errors", errCount) + } + } + + if err := <-errCh; err != nil { + log.Warn("runner: catalogue refresh: catalogue stream error", "err", err) + } + + log.Info("runner: catalogue refresh finished", + "ok", ok, "skipped", skipped, "errors", errCount) +} + +// downloadCover fetches the cover image from coverURL and stores it in MinIO +// under covers/{slug}.jpg. It retries up to 3 times with exponential backoff +// on transient errors (5xx, network failures). +func (r *Runner) downloadCover(ctx context.Context, slug, coverURL string) error { + const maxRetries = 3 + delay := 2 * time.Second + + var lastErr error + for attempt := 0; attempt < maxRetries; attempt++ { + if ctx.Err() != nil { + return ctx.Err() + } + if attempt > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + delay *= 2 + } + + data, err := fetchCoverBytes(ctx, coverURL) + if err != nil { + lastErr = err + continue + } + + if err := r.deps.CoverStore.PutCover(ctx, slug, data, ""); err != nil { + return fmt.Errorf("put cover: %w", err) + } + return nil + } + return fmt.Errorf("download cover after %d retries: %w", maxRetries, lastErr) +} + +// fetchCoverBytes performs a single HTTP GET for coverURL and returns the body. +func fetchCoverBytes(ctx context.Context, coverURL string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, coverURL, nil) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)") + req.Header.Set("Referer", "https://novelfire.net/") + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("http get: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 500 { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("upstream %d for %s", resp.StatusCode, coverURL) + } + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, coverURL) + } + + return io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5 MiB cap +} diff --git a/v3/backend/internal/runner/helpers.go b/v3/backend/internal/runner/helpers.go new file mode 100644 index 0000000..e07dd0b --- /dev/null +++ b/v3/backend/internal/runner/helpers.go @@ -0,0 +1,21 @@ +package runner + +import ( + "regexp" + "strings" +) + +// stripMarkdown removes common markdown syntax from src, returning plain text +// suitable for TTS. Mirrors the helper in the scraper's server package. +func stripMarkdown(src string) string { + src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "") + src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "") + src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "") + src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "") + src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1") + src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "") + src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "") + src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "") + src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n") + return strings.TrimSpace(src) +} diff --git a/v3/backend/internal/runner/metrics.go b/v3/backend/internal/runner/metrics.go new file mode 100644 index 0000000..05bff07 --- /dev/null +++ b/v3/backend/internal/runner/metrics.go @@ -0,0 +1,92 @@ +package runner + +// metrics.go — lightweight HTTP metrics endpoint for the runner. +// +// GET /metrics returns a JSON document with live task counters and uptime. +// No external dependency (no Prometheus); plain net/http only. + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net" + "net/http" + "time" +) + +// metricsServer serves GET /metrics for the runner process. +type metricsServer struct { + addr string + r *Runner + log *slog.Logger +} + +func newMetricsServer(addr string, r *Runner, log *slog.Logger) *metricsServer { + return &metricsServer{addr: addr, r: r, log: log} +} + +// ListenAndServe starts the HTTP server and blocks until ctx is cancelled or +// a fatal listen error occurs. +func (ms *metricsServer) ListenAndServe(ctx context.Context) error { + mux := http.NewServeMux() + mux.HandleFunc("GET /metrics", ms.handleMetrics) + mux.HandleFunc("GET /health", ms.handleHealth) + + srv := &http.Server{ + Addr: ms.addr, + Handler: mux, + ReadTimeout: 5 * time.Second, + WriteTimeout: 5 * time.Second, + BaseContext: func(_ net.Listener) context.Context { return ctx }, + } + + errCh := make(chan error, 1) + go func() { + ms.log.Info("runner: metrics server listening", "addr", ms.addr) + errCh <- srv.ListenAndServe() + }() + + select { + case <-ctx.Done(): + shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = srv.Shutdown(shutCtx) + return nil + case err := <-errCh: + return fmt.Errorf("runner: metrics server: %w", err) + } +} + +// handleMetrics handles GET /metrics. +// Response shape (JSON): +// +// { +// "tasks_running": N, +// "tasks_completed": N, +// "tasks_failed": N, +// "uptime_seconds": N +// } +func (ms *metricsServer) handleMetrics(w http.ResponseWriter, _ *http.Request) { + uptimeSec := int64(time.Since(ms.r.startedAt).Seconds()) + metricsWriteJSON(w, 0, map[string]int64{ + "tasks_running": ms.r.tasksRunning.Load(), + "tasks_completed": ms.r.tasksCompleted.Load(), + "tasks_failed": ms.r.tasksFailed.Load(), + "uptime_seconds": uptimeSec, + }) +} + +// handleHealth handles GET /health — simple liveness probe for the metrics server. +func (ms *metricsServer) handleHealth(w http.ResponseWriter, _ *http.Request) { + metricsWriteJSON(w, 0, map[string]string{"status": "ok"}) +} + +// metricsWriteJSON writes v as a JSON response with the given status code. +func metricsWriteJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + if status != 0 { + w.WriteHeader(status) + } + _ = json.NewEncoder(w).Encode(v) +} diff --git a/v3/backend/internal/runner/runner.go b/v3/backend/internal/runner/runner.go new file mode 100644 index 0000000..f3490e6 --- /dev/null +++ b/v3/backend/internal/runner/runner.go @@ -0,0 +1,440 @@ +// Package runner implements the worker loop that polls PocketBase for pending +// scrape and audio tasks, executes them, and reports results back. +// +// Design: +// - Run(ctx) loops on a ticker; each tick claims and dispatches pending tasks. +// - Scrape tasks are dispatched to the Orchestrator (one goroutine per task, +// up to MaxConcurrentScrape). +// - Audio tasks fetch chapter text, call Kokoro, upload to MinIO, and report +// the result back (up to MaxConcurrentAudio goroutines). +// - The runner is stateless between ticks; all state lives in PocketBase. +// - Atomic task counters are exposed via /metrics (see metrics.go). +// - Books are indexed in Meilisearch via an orchestrator.Config.PostMetadata +// hook injected at construction time. +package runner + +import ( + "context" + "fmt" + "log/slog" + "sync" + "sync/atomic" + "time" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/meili" + "github.com/libnovel/backend/internal/orchestrator" + "github.com/libnovel/backend/internal/scraper" + "github.com/libnovel/backend/internal/taskqueue" +) + +// Config tunes the runner behaviour. +type Config struct { + // WorkerID uniquely identifies this runner instance in PocketBase records. + WorkerID string + // PollInterval is how often the runner checks for new tasks. + PollInterval time.Duration + // MaxConcurrentScrape limits simultaneous book-scrape goroutines. + MaxConcurrentScrape int + // MaxConcurrentAudio limits simultaneous audio-generation goroutines. + MaxConcurrentAudio int + // OrchestratorWorkers is the chapter-scraping parallelism inside each book run. + OrchestratorWorkers int + // HeartbeatInterval is how often active tasks PATCH their heartbeat_at + // timestamp to signal they are still alive. Defaults to 30s when 0. + HeartbeatInterval time.Duration + // StaleTaskThreshold is how old a heartbeat must be (or absent) before the + // task is considered orphaned and reset to pending. Defaults to 2m when 0. + StaleTaskThreshold time.Duration + // BrowseRefreshInterval is how often the runner pre-fetches browse page + // snapshots from novelfire.net and stores them in MinIO. Defaults to 6h. + BrowseRefreshInterval time.Duration + // CatalogueRefreshInterval is how often the runner walks the full catalogue, + // scrapes per-book metadata, downloads covers, and re-indexes everything in + // Meilisearch. Defaults to 24h (expensive — full catalogue walk). + CatalogueRefreshInterval time.Duration + // MetricsAddr is the HTTP listen address for the /metrics endpoint. + // Defaults to ":9091". Set to "" to disable. + MetricsAddr string +} + +// Dependencies are the external services the runner depends on. +type Dependencies struct { + // Consumer claims tasks from PocketBase. + Consumer taskqueue.Consumer + // BookWriter persists scraped data (used by orchestrator). + BookWriter bookstore.BookWriter + // BookReader reads chapter text for audio generation. + BookReader bookstore.BookReader + // AudioStore persists generated audio and checks key existence. + AudioStore bookstore.AudioStore + // BrowseStore stores browse page snapshots in MinIO. + BrowseStore bookstore.BrowseStore + // CoverStore stores book cover images in MinIO. + // If nil, cover downloads are skipped during catalogue refresh. + CoverStore bookstore.CoverStore + // SearchIndex indexes books in Meilisearch after scraping. + // If nil a no-op is used. + SearchIndex meili.Client + // Novel is the scraper implementation. + Novel scraper.NovelScraper + // Kokoro is the TTS client. + Kokoro kokoro.Client + // Log is the structured logger. + Log *slog.Logger +} + +// Runner is the main worker process. +type Runner struct { + cfg Config + deps Dependencies + + // Atomic task counters — read by /metrics without locking. + tasksRunning atomic.Int64 + tasksCompleted atomic.Int64 + tasksFailed atomic.Int64 + + startedAt time.Time +} + +// New creates a Runner from cfg and deps. +func New(cfg Config, deps Dependencies) *Runner { + if cfg.PollInterval <= 0 { + cfg.PollInterval = 30 * time.Second + } + if cfg.MaxConcurrentScrape <= 0 { + cfg.MaxConcurrentScrape = 2 + } + if cfg.MaxConcurrentAudio <= 0 { + cfg.MaxConcurrentAudio = 1 + } + if cfg.WorkerID == "" { + cfg.WorkerID = "runner" + } + if cfg.HeartbeatInterval <= 0 { + cfg.HeartbeatInterval = 30 * time.Second + } + if cfg.StaleTaskThreshold <= 0 { + cfg.StaleTaskThreshold = 2 * time.Minute + } + if cfg.BrowseRefreshInterval <= 0 { + cfg.BrowseRefreshInterval = 6 * time.Hour + } + if cfg.CatalogueRefreshInterval <= 0 { + cfg.CatalogueRefreshInterval = 24 * time.Hour + } + if cfg.MetricsAddr == "" { + cfg.MetricsAddr = ":9091" + } + if deps.Log == nil { + deps.Log = slog.Default() + } + if deps.SearchIndex == nil { + deps.SearchIndex = meili.NoopClient{} + } + return &Runner{cfg: cfg, deps: deps, startedAt: time.Now()} +} + +// Run starts the poll loop and the metrics HTTP server, blocking until ctx is +// cancelled. +func (r *Runner) Run(ctx context.Context) error { + r.deps.Log.Info("runner: starting", + "worker_id", r.cfg.WorkerID, + "poll_interval", r.cfg.PollInterval, + "max_scrape", r.cfg.MaxConcurrentScrape, + "max_audio", r.cfg.MaxConcurrentAudio, + "browse_refresh_interval", r.cfg.BrowseRefreshInterval, + "catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval, + "metrics_addr", r.cfg.MetricsAddr, + ) + + // Start metrics HTTP server in background if configured. + if r.cfg.MetricsAddr != "" { + ms := newMetricsServer(r.cfg.MetricsAddr, r, r.deps.Log) + go func() { + if err := ms.ListenAndServe(ctx); err != nil { + r.deps.Log.Error("runner: metrics server error", "err", err) + } + }() + } + + scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape) + audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio) + var wg sync.WaitGroup + + tick := time.NewTicker(r.cfg.PollInterval) + defer tick.Stop() + + browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval) + defer browseTick.Stop() + + catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval) + defer catalogueTick.Stop() + + // Run one browse refresh immediately on startup. + go r.runBrowseRefresh(ctx) + // Run one catalogue refresh immediately on startup. + go r.runCatalogueRefresh(ctx) + + // Run one poll immediately on startup, then on each tick. + for { + r.poll(ctx, scrapeSem, audioSem, &wg) + + select { + case <-ctx.Done(): + r.deps.Log.Info("runner: context cancelled, draining active tasks") + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + r.deps.Log.Info("runner: all tasks drained, exiting") + case <-time.After(2 * time.Minute): + r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit") + } + return nil + case <-browseTick.C: + go r.runBrowseRefresh(ctx) + case <-catalogueTick.C: + go r.runCatalogueRefresh(ctx) + case <-tick.C: + } + } +} + +// poll claims all available pending tasks and dispatches them to goroutines. +func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) { + // ── Reap orphaned tasks ─────────────────────────────────────────────── + if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil { + r.deps.Log.Warn("runner: reap stale tasks failed", "err", err) + } else if n > 0 { + r.deps.Log.Info("runner: reaped stale tasks", "count", n) + } + + // ── Scrape tasks ────────────────────────────────────────────────────── + for { + if ctx.Err() != nil { + return + } + task, ok, err := r.deps.Consumer.ClaimNextScrapeTask(ctx, r.cfg.WorkerID) + if err != nil { + r.deps.Log.Error("runner: ClaimNextScrapeTask failed", "err", err) + break + } + if !ok { + break + } + select { + case scrapeSem <- struct{}{}: + default: + r.deps.Log.Warn("runner: scrape semaphore full, will retry next tick", + "task_id", task.ID) + break + } + r.tasksRunning.Add(1) + wg.Add(1) + go func(t domain.ScrapeTask) { + defer wg.Done() + defer func() { <-scrapeSem }() + defer r.tasksRunning.Add(-1) + r.runScrapeTask(ctx, t) + }(task) + } + + // ── Audio tasks ─────────────────────────────────────────────────────── + for { + if ctx.Err() != nil { + return + } + task, ok, err := r.deps.Consumer.ClaimNextAudioTask(ctx, r.cfg.WorkerID) + if err != nil { + r.deps.Log.Error("runner: ClaimNextAudioTask failed", "err", err) + break + } + if !ok { + break + } + select { + case audioSem <- struct{}{}: + default: + r.deps.Log.Warn("runner: audio semaphore full, will retry next tick", + "task_id", task.ID) + break + } + r.tasksRunning.Add(1) + wg.Add(1) + go func(t domain.AudioTask) { + defer wg.Done() + defer func() { <-audioSem }() + defer r.tasksRunning.Add(-1) + r.runAudioTask(ctx, t) + }(task) + } +} + +// newOrchestrator builds an orchestrator with the Meilisearch post-hook wired in. +func (r *Runner) newOrchestrator() *orchestrator.Orchestrator { + oCfg := orchestrator.Config{ + Workers: r.cfg.OrchestratorWorkers, + PostMetadata: func(ctx context.Context, meta domain.BookMeta) { + if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil { + r.deps.Log.Warn("runner: meilisearch upsert failed", + "slug", meta.Slug, "err", err) + } + }, + } + return orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log) +} + +// runScrapeTask executes one scrape task end-to-end and reports the result. +func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) { + log := r.deps.Log.With("task_id", task.ID, "kind", task.Kind, "url", task.TargetURL) + log.Info("runner: scrape task starting") + + hbCtx, hbCancel := context.WithCancel(ctx) + defer hbCancel() + go func() { + tick := time.NewTicker(r.cfg.HeartbeatInterval) + defer tick.Stop() + for { + select { + case <-hbCtx.Done(): + return + case <-tick.C: + if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil { + log.Warn("runner: heartbeat failed", "err", err) + } + } + } + }() + + o := r.newOrchestrator() + var result domain.ScrapeResult + + switch task.Kind { + case "catalogue": + result = r.runCatalogueTask(ctx, task, o, log) + case "book", "book_range": + result = o.RunBook(ctx, task) + default: + result.ErrorMessage = fmt.Sprintf("unknown task kind: %q", task.Kind) + log.Warn("runner: unknown task kind") + } + + if err := r.deps.Consumer.FinishScrapeTask(ctx, task.ID, result); err != nil { + log.Error("runner: FinishScrapeTask failed", "err", err) + } + + if result.ErrorMessage != "" { + r.tasksFailed.Add(1) + } else { + r.tasksCompleted.Add(1) + } + + log.Info("runner: scrape task finished", + "scraped", result.ChaptersScraped, + "skipped", result.ChaptersSkipped, + "errors", result.Errors, + ) +} + +// runCatalogueTask runs a full catalogue scrape. +func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o *orchestrator.Orchestrator, log *slog.Logger) domain.ScrapeResult { + entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx) + var result domain.ScrapeResult + + for entry := range entries { + if ctx.Err() != nil { + break + } + bookTask := domain.ScrapeTask{ + ID: task.ID, + Kind: "book", + TargetURL: entry.URL, + } + bookResult := o.RunBook(ctx, bookTask) + result.BooksFound += bookResult.BooksFound + 1 + result.ChaptersScraped += bookResult.ChaptersScraped + result.ChaptersSkipped += bookResult.ChaptersSkipped + result.Errors += bookResult.Errors + } + + if err := <-errCh; err != nil { + log.Warn("runner: catalogue scrape finished with error", "err", err) + result.Errors++ + if result.ErrorMessage == "" { + result.ErrorMessage = err.Error() + } + } + return result +} + +// runAudioTask executes one audio-generation task. +func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) { + log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "voice", task.Voice) + log.Info("runner: audio task starting") + + hbCtx, hbCancel := context.WithCancel(ctx) + defer hbCancel() + go func() { + tick := time.NewTicker(r.cfg.HeartbeatInterval) + defer tick.Stop() + for { + select { + case <-hbCtx.Done(): + return + case <-tick.C: + if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil { + log.Warn("runner: heartbeat failed", "err", err) + } + } + } + }() + + fail := func(msg string) { + log.Error("runner: audio task failed", "reason", msg) + r.tasksFailed.Add(1) + result := domain.AudioResult{ErrorMessage: msg} + if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil { + log.Error("runner: FinishAudioTask failed", "err", err) + } + } + + raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter) + if err != nil { + fail(fmt.Sprintf("read chapter: %v", err)) + return + } + text := stripMarkdown(raw) + if text == "" { + fail("chapter text is empty after stripping markdown") + return + } + + if r.deps.Kokoro == nil { + fail("kokoro client not configured") + return + } + audioData, err := r.deps.Kokoro.GenerateAudio(ctx, text, task.Voice) + if err != nil { + fail(fmt.Sprintf("kokoro generate: %v", err)) + return + } + + key := r.deps.AudioStore.AudioObjectKey(task.Slug, task.Chapter, task.Voice) + if err := r.deps.AudioStore.PutAudio(ctx, key, audioData); err != nil { + fail(fmt.Sprintf("put audio: %v", err)) + return + } + + r.tasksCompleted.Add(1) + result := domain.AudioResult{ObjectKey: key} + if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil { + log.Error("runner: FinishAudioTask failed", "err", err) + } + log.Info("runner: audio task finished", "key", key) +} diff --git a/v3/backend/internal/runner/runner_test.go b/v3/backend/internal/runner/runner_test.go new file mode 100644 index 0000000..2fa8888 --- /dev/null +++ b/v3/backend/internal/runner/runner_test.go @@ -0,0 +1,365 @@ +package runner_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/runner" +) + +// ── Stub types ──────────────────────────────────────────────────────────────── + +// stubConsumer is a test double for taskqueue.Consumer. +type stubConsumer struct { + scrapeQueue []domain.ScrapeTask + audioQueue []domain.AudioTask + scrapeIdx int + audioIdx int + finished []string + failCalled []string + claimErr error +} + +func (s *stubConsumer) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) { + if s.claimErr != nil { + return domain.ScrapeTask{}, false, s.claimErr + } + if s.scrapeIdx >= len(s.scrapeQueue) { + return domain.ScrapeTask{}, false, nil + } + t := s.scrapeQueue[s.scrapeIdx] + s.scrapeIdx++ + return t, true, nil +} + +func (s *stubConsumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) { + if s.claimErr != nil { + return domain.AudioTask{}, false, s.claimErr + } + if s.audioIdx >= len(s.audioQueue) { + return domain.AudioTask{}, false, nil + } + t := s.audioQueue[s.audioIdx] + s.audioIdx++ + return t, true, nil +} + +func (s *stubConsumer) FinishScrapeTask(_ context.Context, id string, _ domain.ScrapeResult) error { + s.finished = append(s.finished, id) + return nil +} + +func (s *stubConsumer) FinishAudioTask(_ context.Context, id string, _ domain.AudioResult) error { + s.finished = append(s.finished, id) + return nil +} + +func (s *stubConsumer) FailTask(_ context.Context, id, _ string) error { + s.failCalled = append(s.failCalled, id) + return nil +} + +func (s *stubConsumer) HeartbeatTask(_ context.Context, _ string) error { return nil } + +func (s *stubConsumer) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) { + return 0, nil +} + +// stubBookWriter satisfies bookstore.BookWriter (no-op). +type stubBookWriter struct{} + +func (s *stubBookWriter) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil } +func (s *stubBookWriter) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error { + return nil +} +func (s *stubBookWriter) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error { + return nil +} +func (s *stubBookWriter) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool { + return false +} + +// stubBookReader satisfies bookstore.BookReader — returns a single chapter. +type stubBookReader struct { + text string + readErr error +} + +func (s *stubBookReader) ReadChapter(_ context.Context, _ string, _ int) (string, error) { + return s.text, s.readErr +} +func (s *stubBookReader) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) { + return domain.BookMeta{}, false, nil +} +func (s *stubBookReader) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil } +func (s *stubBookReader) LocalSlugs(_ context.Context) (map[string]bool, error) { return nil, nil } +func (s *stubBookReader) MetadataMtime(_ context.Context, _ string) int64 { return 0 } +func (s *stubBookReader) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) { + return nil, nil +} +func (s *stubBookReader) CountChapters(_ context.Context, _ string) int { return 0 } +func (s *stubBookReader) ReindexChapters(_ context.Context, _ string) (int, error) { + return 0, nil +} + +// stubAudioStore satisfies bookstore.AudioStore. +type stubAudioStore struct { + putCalled atomic.Int32 + putErr error +} + +func (s *stubAudioStore) AudioObjectKey(slug string, n int, voice string) string { + return slug + "/" + string(rune('0'+n)) + "/" + voice + ".mp3" +} +func (s *stubAudioStore) AudioExists(_ context.Context, _ string) bool { return false } +func (s *stubAudioStore) PutAudio(_ context.Context, _ string, _ []byte) error { + s.putCalled.Add(1) + return s.putErr +} + +// stubNovelScraper satisfies scraper.NovelScraper minimally. +type stubNovelScraper struct { + entries []domain.CatalogueEntry + metaErr error + chapters []domain.ChapterRef +} + +func (s *stubNovelScraper) ScrapeCatalogue(_ context.Context) (<-chan domain.CatalogueEntry, <-chan error) { + ch := make(chan domain.CatalogueEntry, len(s.entries)) + errCh := make(chan error, 1) + for _, e := range s.entries { + ch <- e + } + close(ch) + close(errCh) + return ch, errCh +} + +func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) { + if s.metaErr != nil { + return domain.BookMeta{}, s.metaErr + } + return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil +} + +func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) { + return s.chapters, nil +} + +func (s *stubNovelScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) { + return domain.Chapter{Ref: ref, Text: "# Chapter\n\nSome text."}, nil +} + +func (s *stubNovelScraper) ScrapeRanking(_ context.Context, _ int) (<-chan domain.BookMeta, <-chan error) { + ch := make(chan domain.BookMeta) + errCh := make(chan error, 1) + close(ch) + close(errCh) + return ch, errCh +} + +func (s *stubNovelScraper) SourceName() string { return "stub" } + +// stubKokoro satisfies kokoro.Client. +type stubKokoro struct { + data []byte + genErr error + called atomic.Int32 +} + +func (s *stubKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) { + s.called.Add(1) + return s.data, s.genErr +} + +func (s *stubKokoro) ListVoices(_ context.Context) ([]string, error) { + return []string{"af_bella"}, nil +} + +// ── stripMarkdown helper ────────────────────────────────────────────────────── + +func TestStripMarkdownViaAudioTask(t *testing.T) { + // Verify markdown is stripped before sending to Kokoro. + // We inject chapter text with markdown; the kokoro stub verifies data flows. + consumer := &stubConsumer{ + audioQueue: []domain.AudioTask{ + {ID: "a1", Slug: "book", Chapter: 1, Voice: "af_bella", Status: domain.TaskStatusRunning}, + }, + } + bookReader := &stubBookReader{text: "## Chapter 1\n\nPlain **text** here."} + audioStore := &stubAudioStore{} + kokoroStub := &stubKokoro{data: []byte("mp3")} + + cfg := runner.Config{ + WorkerID: "test", + PollInterval: time.Hour, // long poll — we'll cancel manually + } + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: bookReader, + AudioStore: audioStore, + Novel: &stubNovelScraper{}, + Kokoro: kokoroStub, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = r.Run(ctx) + + if kokoroStub.called.Load() != 1 { + t.Errorf("expected Kokoro.GenerateAudio called once, got %d", kokoroStub.called.Load()) + } + if audioStore.putCalled.Load() != 1 { + t.Errorf("expected PutAudio called once, got %d", audioStore.putCalled.Load()) + } +} + +func TestAudioTask_ReadChapterError(t *testing.T) { + consumer := &stubConsumer{ + audioQueue: []domain.AudioTask{ + {ID: "a2", Slug: "book", Chapter: 2, Voice: "af_bella", Status: domain.TaskStatusRunning}, + }, + } + bookReader := &stubBookReader{readErr: errors.New("chapter not found")} + audioStore := &stubAudioStore{} + kokoroStub := &stubKokoro{data: []byte("mp3")} + + cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour} + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: bookReader, + AudioStore: audioStore, + Novel: &stubNovelScraper{}, + Kokoro: kokoroStub, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = r.Run(ctx) + + // Kokoro should not be called; FinishAudioTask should be called with error. + if kokoroStub.called.Load() != 0 { + t.Errorf("expected Kokoro not called, got %d", kokoroStub.called.Load()) + } + if len(consumer.finished) != 1 { + t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished)) + } +} + +func TestAudioTask_KokoroError(t *testing.T) { + consumer := &stubConsumer{ + audioQueue: []domain.AudioTask{ + {ID: "a3", Slug: "book", Chapter: 3, Voice: "af_bella", Status: domain.TaskStatusRunning}, + }, + } + bookReader := &stubBookReader{text: "Chapter text."} + audioStore := &stubAudioStore{} + kokoroStub := &stubKokoro{genErr: errors.New("tts failed")} + + cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour} + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: bookReader, + AudioStore: audioStore, + Novel: &stubNovelScraper{}, + Kokoro: kokoroStub, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = r.Run(ctx) + + if audioStore.putCalled.Load() != 0 { + t.Errorf("expected PutAudio not called, got %d", audioStore.putCalled.Load()) + } + if len(consumer.finished) != 1 { + t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished)) + } +} + +func TestScrapeTask_BookKind(t *testing.T) { + consumer := &stubConsumer{ + scrapeQueue: []domain.ScrapeTask{ + {ID: "s1", Kind: "book", TargetURL: "https://example.com/book/test-book", Status: domain.TaskStatusRunning}, + }, + } + + cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour} + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: &stubBookReader{}, + AudioStore: &stubAudioStore{}, + Novel: &stubNovelScraper{}, + Kokoro: &stubKokoro{}, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = r.Run(ctx) + + if len(consumer.finished) != 1 || consumer.finished[0] != "s1" { + t.Errorf("expected task s1 finished, got %v", consumer.finished) + } +} + +func TestScrapeTask_UnknownKind(t *testing.T) { + consumer := &stubConsumer{ + scrapeQueue: []domain.ScrapeTask{ + {ID: "s2", Kind: "unknown_kind", Status: domain.TaskStatusRunning}, + }, + } + + cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour} + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: &stubBookReader{}, + AudioStore: &stubAudioStore{}, + Novel: &stubNovelScraper{}, + Kokoro: &stubKokoro{}, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = r.Run(ctx) + + // Unknown kind still finishes the task (with error message in result). + if len(consumer.finished) != 1 || consumer.finished[0] != "s2" { + t.Errorf("expected task s2 finished, got %v", consumer.finished) + } +} + +func TestRun_CancelImmediately(t *testing.T) { + consumer := &stubConsumer{} + cfg := runner.Config{WorkerID: "test", PollInterval: 10 * time.Millisecond} + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: &stubBookReader{}, + AudioStore: &stubAudioStore{}, + Novel: &stubNovelScraper{}, + Kokoro: &stubKokoro{}, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel before Run + + err := r.Run(ctx) + if err != nil { + t.Errorf("expected nil on graceful shutdown, got %v", err) + } +} diff --git a/v3/backend/internal/scraper/scraper.go b/v3/backend/internal/scraper/scraper.go new file mode 100644 index 0000000..bba8f13 --- /dev/null +++ b/v3/backend/internal/scraper/scraper.go @@ -0,0 +1,58 @@ +// Package scraper defines the NovelScraper interface and its sub-interfaces. +// Domain types live in internal/domain — this package only defines the scraping +// contract so that novelfire and any future scrapers can be swapped freely. +package scraper + +import ( + "context" + + "github.com/libnovel/backend/internal/domain" +) + +// CatalogueProvider can enumerate every novel available on a source site. +type CatalogueProvider interface { + ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) +} + +// MetadataProvider can extract structured book metadata from a novel's landing page. +type MetadataProvider interface { + ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) +} + +// ChapterListProvider can enumerate all chapters of a book. +type ChapterListProvider interface { + ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) +} + +// ChapterTextProvider can extract the readable text from a single chapter page. +type ChapterTextProvider interface { + ScrapeChapterText(ctx context.Context, ref domain.ChapterRef) (domain.Chapter, error) +} + +// RankingProvider can enumerate novels from a ranking page. +type RankingProvider interface { + // ScrapeRanking pages through up to maxPages ranking pages. + // maxPages <= 0 means all pages. + ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) +} + +// NovelScraper is the full interface a concrete novel source must implement. +type NovelScraper interface { + CatalogueProvider + MetadataProvider + ChapterListProvider + ChapterTextProvider + RankingProvider + + // SourceName returns the human-readable name of this scraper, e.g. "novelfire.net". + SourceName() string +} + +// Selector describes how to locate an element in an HTML document. +type Selector struct { + Tag string + Class string + ID string + Attr string + Multiple bool +} diff --git a/v3/backend/internal/storage/minio.go b/v3/backend/internal/storage/minio.go new file mode 100644 index 0000000..3dee9e7 --- /dev/null +++ b/v3/backend/internal/storage/minio.go @@ -0,0 +1,270 @@ +package storage + +import ( + "context" + "fmt" + "io" + "net/url" + "path" + "strings" + "time" + + minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + "github.com/libnovel/backend/internal/config" +) + +// minioClient wraps the official minio-go client with bucket names. +type minioClient struct { + client *minio.Client // internal — all read/write operations + pubClient *minio.Client // presign-only — initialised against the public endpoint + bucketChapters string + bucketAudio string + bucketAvatars string + bucketBrowse string +} + +func newMinioClient(cfg config.MinIO) (*minioClient, error) { + creds := credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, "") + + internal, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: creds, + Secure: cfg.UseSSL, + }) + if err != nil { + return nil, fmt.Errorf("minio: init internal client: %w", err) + } + + // Presigned URLs must be signed with the hostname the browser will use + // (PUBLIC_MINIO_PUBLIC_URL), because AWS Signature V4 includes the Host + // header in the canonical request — a URL signed against "minio:9000" will + // return SignatureDoesNotMatch when the browser fetches it from + // "localhost:9000". + // + // However, minio-go normally makes a live BucketLocation HTTP call before + // signing, which would fail from inside the container when the public + // endpoint is externally-facing (e.g. "localhost:9000" is unreachable from + // within Docker). We prevent this by: + // 1. Setting Region: "us-east-1" — minio-go skips getBucketLocation when + // the region is already known (bucket-cache.go:49). + // 2. Setting BucketLookup: BucketLookupPath — forces path-style URLs + // (e.g. host/bucket/key), matching MinIO's default behaviour and + // avoiding any virtual-host DNS probing. + // + // When no public endpoint is configured (or it equals the internal one), + // fall back to the internal client so presigning still works. + publicEndpoint := cfg.PublicEndpoint + if u, err2 := url.Parse(publicEndpoint); err2 == nil && u.Host != "" { + publicEndpoint = u.Host // strip scheme so minio.New is happy + } + pubUseSSL := cfg.PublicUseSSL + if publicEndpoint == "" || publicEndpoint == cfg.Endpoint { + publicEndpoint = cfg.Endpoint + pubUseSSL = cfg.UseSSL + } + pub, err := minio.New(publicEndpoint, &minio.Options{ + Creds: creds, + Secure: pubUseSSL, + Region: "us-east-1", // skip live BucketLocation preflight + BucketLookup: minio.BucketLookupPath, + }) + if err != nil { + return nil, fmt.Errorf("minio: init public client: %w", err) + } + + return &minioClient{ + client: internal, + pubClient: pub, + bucketChapters: cfg.BucketChapters, + bucketAudio: cfg.BucketAudio, + bucketAvatars: cfg.BucketAvatars, + bucketBrowse: cfg.BucketBrowse, + }, nil +} + +// ensureBuckets creates all required buckets if they don't already exist. +func (m *minioClient) ensureBuckets(ctx context.Context) error { + for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse} { + exists, err := m.client.BucketExists(ctx, bucket) + if err != nil { + return fmt.Errorf("minio: check bucket %q: %w", bucket, err) + } + if !exists { + if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil { + return fmt.Errorf("minio: create bucket %q: %w", bucket, err) + } + } + } + return nil +} + +// ── Key helpers ─────────────────────────────────────────────────────────────── + +// ChapterObjectKey returns the MinIO object key for a chapter markdown file. +// Format: {slug}/chapter-{n:06d}.md +func ChapterObjectKey(slug string, n int) string { + return fmt.Sprintf("%s/chapter-%06d.md", slug, n) +} + +// AudioObjectKey returns the MinIO object key for a cached audio file. +// Format: {slug}/{n}/{voice}.mp3 +func AudioObjectKey(slug string, n int, voice string) string { + return fmt.Sprintf("%s/%d/%s.mp3", slug, n, voice) +} + +// AvatarObjectKey returns the MinIO object key for a user avatar image. +// Format: {userID}/{ext}.{ext} +func AvatarObjectKey(userID, ext string) string { + return fmt.Sprintf("%s/%s.%s", userID, ext, ext) +} + +// BrowseObjectKey returns the MinIO object key for a cached browse page snapshot. +// Format: browse/{genre}/{sort}/{status}/{type}/page-{n}.json +func BrowseObjectKey(genre, sort, status, novelType string, page int) string { + return fmt.Sprintf("browse/%s/%s/%s/%s/page-%d.json", genre, sort, status, novelType, page) +} + +// CoverObjectKey returns the MinIO object key for a book cover image. +// Format: covers/{slug}.jpg +func CoverObjectKey(slug string) string { + return fmt.Sprintf("covers/%s.jpg", slug) +} + +// chapterNumberFromKey extracts the chapter number from a MinIO object key. +// e.g. "my-book/chapter-000042.md" → 42 +func chapterNumberFromKey(key string) int { + base := path.Base(key) + base = strings.TrimPrefix(base, "chapter-") + base = strings.TrimSuffix(base, ".md") + var n int + fmt.Sscanf(base, "%d", &n) + return n +} + +// ── Object operations ───────────────────────────────────────────────────────── + +func (m *minioClient) putObject(ctx context.Context, bucket, key, contentType string, data []byte) error { + _, err := m.client.PutObject(ctx, bucket, key, + strings.NewReader(string(data)), + int64(len(data)), + minio.PutObjectOptions{ContentType: contentType}, + ) + return err +} + +func (m *minioClient) getObject(ctx context.Context, bucket, key string) ([]byte, error) { + obj, err := m.client.GetObject(ctx, bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + return io.ReadAll(obj) +} + +func (m *minioClient) objectExists(ctx context.Context, bucket, key string) bool { + _, err := m.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + return err == nil +} + +func (m *minioClient) presignGet(ctx context.Context, bucket, key string, expires time.Duration) (string, error) { + u, err := m.pubClient.PresignedGetObject(ctx, bucket, key, expires, nil) + if err != nil { + return "", fmt.Errorf("minio presign %s/%s: %w", bucket, key, err) + } + return u.String(), nil +} + +func (m *minioClient) presignPut(ctx context.Context, bucket, key string, expires time.Duration) (string, error) { + u, err := m.pubClient.PresignedPutObject(ctx, bucket, key, expires) + if err != nil { + return "", fmt.Errorf("minio presign PUT %s/%s: %w", bucket, key, err) + } + return u.String(), nil +} + +func (m *minioClient) deleteObjects(ctx context.Context, bucket, prefix string) error { + objCh := m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) + for obj := range objCh { + if obj.Err != nil { + return obj.Err + } + if err := m.client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}); err != nil { + return err + } + } + return nil +} + +func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string) ([]string, error) { + var keys []string + for obj := range m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) { + if obj.Err != nil { + return nil, obj.Err + } + keys = append(keys, obj.Key) + } + return keys, nil +} + +// ── Browse operations ───────────────────────────────────────────────────────── + +// putBrowse stores raw JSON bytes for a browse page snapshot. +func (m *minioClient) putBrowse(ctx context.Context, key string, data []byte) error { + return m.putObject(ctx, m.bucketBrowse, key, "application/json", data) +} + +// getBrowse retrieves a browse page snapshot. Returns (nil, false, nil) when +// the object does not exist. +func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool, error) { + if !m.objectExists(ctx, m.bucketBrowse, key) { + return nil, false, nil + } + data, err := m.getObject(ctx, m.bucketBrowse, key) + if err != nil { + return nil, false, err + } + return data, true, nil +} + +// ── Cover operations ────────────────────────────────────────────────────────── + +// putCover stores a raw cover image in the browse bucket under covers/{slug}.jpg. +func (m *minioClient) putCover(ctx context.Context, key, contentType string, data []byte) error { + return m.putObject(ctx, m.bucketBrowse, key, contentType, data) +} + +// getCover retrieves a cover image. Returns (nil, "", false, nil) when the +// object does not exist. +func (m *minioClient) getCover(ctx context.Context, key string) ([]byte, bool, error) { + if !m.objectExists(ctx, m.bucketBrowse, key) { + return nil, false, nil + } + data, err := m.getObject(ctx, m.bucketBrowse, key) + if err != nil { + return nil, false, err + } + return data, true, nil +} + +// coverExists returns true when the cover image object exists. +func (m *minioClient) coverExists(ctx context.Context, key string) bool { + return m.objectExists(ctx, m.bucketBrowse, key) +} + +// coverContentType inspects the first bytes of data to determine if it is +// a JPEG or PNG image. Falls back to "image/jpeg". +func coverContentType(data []byte) string { + if len(data) >= 4 { + // PNG magic: 0x89 0x50 0x4E 0x47 + if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47 { + return "image/png" + } + // WebP: starts with "RIFF" at 0..3 and "WEBP" at 8..11 + if len(data) >= 12 && data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' && + data[8] == 'W' && data[9] == 'E' && data[10] == 'B' && data[11] == 'P' { + return "image/webp" + } + } + return "image/jpeg" +} diff --git a/v3/backend/internal/storage/pocketbase.go b/v3/backend/internal/storage/pocketbase.go new file mode 100644 index 0000000..23e1fdc --- /dev/null +++ b/v3/backend/internal/storage/pocketbase.go @@ -0,0 +1,268 @@ +// Package storage provides the concrete implementations of all bookstore and +// taskqueue interfaces backed by PocketBase (structured data) and MinIO (blobs). +// +// Entry point: NewStore(ctx, cfg, log) returns a *Store that satisfies every +// interface defined in bookstore and taskqueue. +package storage + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/libnovel/backend/internal/config" + "github.com/libnovel/backend/internal/domain" +) + +// ErrNotFound is returned by single-record lookups when no record exists. +var ErrNotFound = errors.New("storage: record not found") + +// pbClient is the internal PocketBase REST admin client. +type pbClient struct { + baseURL string + email string + password string + log *slog.Logger + + mu sync.Mutex + token string + exp time.Time +} + +func newPBClient(cfg config.PocketBase, log *slog.Logger) *pbClient { + return &pbClient{ + baseURL: strings.TrimRight(cfg.URL, "/"), + email: cfg.AdminEmail, + password: cfg.AdminPassword, + log: log, + } +} + +// authToken returns a valid admin auth token, refreshing it when expired. +func (c *pbClient) authToken(ctx context.Context) (string, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.token != "" && time.Now().Before(c.exp) { + return c.token, nil + } + + body, _ := json.Marshal(map[string]string{ + "identity": c.email, + "password": c.password, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/api/collections/_superusers/auth-with-password", bytes.NewReader(body)) + if err != nil { + return "", fmt.Errorf("pb auth: build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("pb auth: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + raw, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("pb auth: status %d: %s", resp.StatusCode, string(raw)) + } + + var payload struct { + Token string `json:"token"` + } + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return "", fmt.Errorf("pb auth: decode: %w", err) + } + c.token = payload.Token + c.exp = time.Now().Add(30 * time.Minute) + return c.token, nil +} + +// do executes an authenticated PocketBase REST request. +func (c *pbClient) do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { + tok, err := c.authToken(ctx) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) + if err != nil { + return nil, fmt.Errorf("pb: build request %s %s: %w", method, path, err) + } + req.Header.Set("Authorization", tok) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("pb: %s %s: %w", method, path, err) + } + return resp, nil +} + +// get is a convenience wrapper that decodes a JSON response into v. +func (c *pbClient) get(ctx context.Context, path string, v any) error { + resp, err := c.do(ctx, http.MethodGet, path, nil) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return ErrNotFound + } + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pb GET %s: status %d: %s", path, resp.StatusCode, string(raw)) + } + return json.NewDecoder(resp.Body).Decode(v) +} + +// post creates a record and decodes the created record into v. +func (c *pbClient) post(ctx context.Context, path string, payload, v any) error { + b, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("pb: marshal: %w", err) + } + resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b)) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pb POST %s: status %d: %s", path, resp.StatusCode, string(raw)) + } + if v != nil { + return json.NewDecoder(resp.Body).Decode(v) + } + return nil +} + +// patch updates a record. +func (c *pbClient) patch(ctx context.Context, path string, payload any) error { + b, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("pb: marshal: %w", err) + } + resp, err := c.do(ctx, http.MethodPatch, path, bytes.NewReader(b)) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pb PATCH %s: status %d: %s", path, resp.StatusCode, string(raw)) + } + return nil +} + +// delete removes a record. +func (c *pbClient) delete(ctx context.Context, path string) error { + resp, err := c.do(ctx, http.MethodDelete, path, nil) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return ErrNotFound + } + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pb DELETE %s: status %d: %s", path, resp.StatusCode, string(raw)) + } + return nil +} + +// listAll fetches all pages of a collection. PocketBase returns at most 200 +// records per page; we paginate until empty. +func (c *pbClient) listAll(ctx context.Context, collection string, filter, sort string) ([]json.RawMessage, error) { + var all []json.RawMessage + page := 1 + for { + q := url.Values{ + "page": {fmt.Sprintf("%d", page)}, + "perPage": {"200"}, + } + if filter != "" { + q.Set("filter", filter) + } + if sort != "" { + q.Set("sort", sort) + } + path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode()) + + var result struct { + Items []json.RawMessage `json:"items"` + Page int `json:"page"` + Pages int `json:"totalPages"` + } + if err := c.get(ctx, path, &result); err != nil { + return nil, err + } + all = append(all, result.Items...) + if result.Page >= result.Pages { + break + } + page++ + } + return all, nil +} + +// claimRecord atomically claims the first pending record matching collection. +// It fetches the oldest pending record (filter + sort), then PATCHes it with +// the claim payload. Returns (nil, nil) when the queue is empty. +func (c *pbClient) claimRecord(ctx context.Context, collection, workerID string, extraClaim map[string]any) (json.RawMessage, error) { + q := url.Values{} + q.Set("filter", `status="pending"`) + q.Set("sort", "+started") + q.Set("perPage", "1") + path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode()) + + var result struct { + Items []json.RawMessage `json:"items"` + } + if err := c.get(ctx, path, &result); err != nil { + return nil, fmt.Errorf("claimRecord list: %w", err) + } + if len(result.Items) == 0 { + return nil, nil // queue empty + } + + var rec struct { + ID string `json:"id"` + } + if err := json.Unmarshal(result.Items[0], &rec); err != nil { + return nil, fmt.Errorf("claimRecord parse id: %w", err) + } + + claim := map[string]any{ + "status": string(domain.TaskStatusRunning), + "worker_id": workerID, + } + for k, v := range extraClaim { + claim[k] = v + } + + claimPath := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID) + if err := c.patch(ctx, claimPath, claim); err != nil { + return nil, fmt.Errorf("claimRecord patch: %w", err) + } + + // Re-fetch the updated record so caller has current state. + var updated json.RawMessage + if err := c.get(ctx, claimPath, &updated); err != nil { + return nil, fmt.Errorf("claimRecord re-fetch: %w", err) + } + return updated, nil +} diff --git a/v3/backend/internal/storage/store.go b/v3/backend/internal/storage/store.go new file mode 100644 index 0000000..3e10125 --- /dev/null +++ b/v3/backend/internal/storage/store.go @@ -0,0 +1,824 @@ +package storage + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/config" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/taskqueue" +) + +// Store is the unified persistence implementation that satisfies all bookstore +// and taskqueue interfaces. It routes structured data to PocketBase and binary +// blobs to MinIO. +type Store struct { + pb *pbClient + mc *minioClient + log *slog.Logger +} + +// NewStore initialises PocketBase and MinIO connections and ensures all MinIO +// buckets exist. Returns a ready-to-use Store. +func NewStore(ctx context.Context, cfg config.Config, log *slog.Logger) (*Store, error) { + pb := newPBClient(cfg.PocketBase, log) + // Validate PocketBase connectivity by fetching an auth token. + if _, err := pb.authToken(ctx); err != nil { + return nil, fmt.Errorf("pocketbase: %w", err) + } + + mc, err := newMinioClient(cfg.MinIO) + if err != nil { + return nil, fmt.Errorf("minio: %w", err) + } + if err := mc.ensureBuckets(ctx); err != nil { + return nil, fmt.Errorf("minio: ensure buckets: %w", err) + } + + return &Store{pb: pb, mc: mc, log: log}, nil +} + +// Compile-time interface satisfaction. +var _ bookstore.BookWriter = (*Store)(nil) +var _ bookstore.BookReader = (*Store)(nil) +var _ bookstore.RankingStore = (*Store)(nil) +var _ bookstore.AudioStore = (*Store)(nil) +var _ bookstore.PresignStore = (*Store)(nil) +var _ bookstore.ProgressStore = (*Store)(nil) +var _ bookstore.BrowseStore = (*Store)(nil) +var _ bookstore.CoverStore = (*Store)(nil) +var _ taskqueue.Producer = (*Store)(nil) +var _ taskqueue.Consumer = (*Store)(nil) +var _ taskqueue.Reader = (*Store)(nil) + +// ── BookWriter ──────────────────────────────────────────────────────────────── + +func (s *Store) WriteMetadata(ctx context.Context, meta domain.BookMeta) error { + payload := map[string]any{ + "slug": meta.Slug, + "title": meta.Title, + "author": meta.Author, + "cover": meta.Cover, + "status": meta.Status, + "genres": meta.Genres, + "summary": meta.Summary, + "total_chapters": meta.TotalChapters, + "source_url": meta.SourceURL, + "ranking": meta.Ranking, + "rating": meta.Rating, + } + // Upsert via filter: if exists PATCH, otherwise POST. + existing, err := s.getBookBySlug(ctx, meta.Slug) + if err != nil && err != ErrNotFound { + return fmt.Errorf("WriteMetadata: %w", err) + } + if err == ErrNotFound { + return s.pb.post(ctx, "/api/collections/books/records", payload, nil) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", existing.ID), payload) +} + +func (s *Store) WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error { + key := ChapterObjectKey(slug, chapter.Ref.Number) + if err := s.mc.putObject(ctx, s.mc.bucketChapters, key, "text/markdown", []byte(chapter.Text)); err != nil { + return fmt.Errorf("WriteChapter: minio: %w", err) + } + // Upsert the chapters_idx record in PocketBase. + return s.upsertChapterIdx(ctx, slug, chapter.Ref) +} + +func (s *Store) WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error { + for _, ref := range refs { + if err := s.upsertChapterIdx(ctx, slug, ref); err != nil { + s.log.Warn("WriteChapterRefs: upsert failed", "slug", slug, "chapter", ref.Number, "err", err) + } + } + return nil +} + +func (s *Store) ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool { + return s.mc.objectExists(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, ref.Number)) +} + +func (s *Store) upsertChapterIdx(ctx context.Context, slug string, ref domain.ChapterRef) error { + payload := map[string]any{ + "slug": slug, + "number": ref.Number, + "title": ref.Title, + } + filter := fmt.Sprintf(`slug=%q&&number=%d`, slug, ref.Number) + items, err := s.pb.listAll(ctx, "chapters_idx", filter, "") + if err != nil && err != ErrNotFound { + return err + } + if len(items) == 0 { + return s.pb.post(ctx, "/api/collections/chapters_idx/records", payload, nil) + } + var rec struct { + ID string `json:"id"` + } + json.Unmarshal(items[0], &rec) + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/chapters_idx/records/%s", rec.ID), payload) +} + +// ── BookReader ──────────────────────────────────────────────────────────────── + +type pbBook struct { + ID string `json:"id"` + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author"` + Cover string `json:"cover"` + Status string `json:"status"` + Genres []string `json:"genres"` + Summary string `json:"summary"` + TotalChapters int `json:"total_chapters"` + SourceURL string `json:"source_url"` + Ranking int `json:"ranking"` + Rating float64 `json:"rating"` + Updated string `json:"updated"` +} + +func (b pbBook) toDomain() domain.BookMeta { + return domain.BookMeta{ + Slug: b.Slug, + Title: b.Title, + Author: b.Author, + Cover: b.Cover, + Status: b.Status, + Genres: b.Genres, + Summary: b.Summary, + TotalChapters: b.TotalChapters, + SourceURL: b.SourceURL, + Ranking: b.Ranking, + Rating: b.Rating, + } +} + +func (s *Store) getBookBySlug(ctx context.Context, slug string) (pbBook, error) { + filter := fmt.Sprintf(`slug=%q`, slug) + items, err := s.pb.listAll(ctx, "books", filter, "") + if err != nil { + return pbBook{}, err + } + if len(items) == 0 { + return pbBook{}, ErrNotFound + } + var b pbBook + json.Unmarshal(items[0], &b) + return b, nil +} + +func (s *Store) ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error) { + b, err := s.getBookBySlug(ctx, slug) + if err == ErrNotFound { + return domain.BookMeta{}, false, nil + } + if err != nil { + return domain.BookMeta{}, false, err + } + return b.toDomain(), true, nil +} + +func (s *Store) ListBooks(ctx context.Context) ([]domain.BookMeta, error) { + items, err := s.pb.listAll(ctx, "books", "", "title") + if err != nil { + return nil, err + } + books := make([]domain.BookMeta, 0, len(items)) + for _, raw := range items { + var b pbBook + json.Unmarshal(raw, &b) + books = append(books, b.toDomain()) + } + return books, nil +} + +func (s *Store) LocalSlugs(ctx context.Context) (map[string]bool, error) { + items, err := s.pb.listAll(ctx, "books", "", "") + if err != nil { + return nil, err + } + slugs := make(map[string]bool, len(items)) + for _, raw := range items { + var b struct { + Slug string `json:"slug"` + } + json.Unmarshal(raw, &b) + if b.Slug != "" { + slugs[b.Slug] = true + } + } + return slugs, nil +} + +func (s *Store) MetadataMtime(ctx context.Context, slug string) int64 { + b, err := s.getBookBySlug(ctx, slug) + if err != nil { + return 0 + } + t, err := time.Parse(time.RFC3339, b.Updated) + if err != nil { + return 0 + } + return t.Unix() +} + +func (s *Store) ReadChapter(ctx context.Context, slug string, n int) (string, error) { + data, err := s.mc.getObject(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n)) + if err != nil { + return "", fmt.Errorf("ReadChapter: %w", err) + } + return string(data), nil +} + +func (s *Store) ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error) { + filter := fmt.Sprintf(`slug=%q`, slug) + items, err := s.pb.listAll(ctx, "chapters_idx", filter, "number") + if err != nil { + return nil, err + } + chapters := make([]domain.ChapterInfo, 0, len(items)) + for _, raw := range items { + var rec struct { + Number int `json:"number"` + Title string `json:"title"` + } + json.Unmarshal(raw, &rec) + chapters = append(chapters, domain.ChapterInfo{Number: rec.Number, Title: rec.Title}) + } + return chapters, nil +} + +func (s *Store) CountChapters(ctx context.Context, slug string) int { + chapters, err := s.ListChapters(ctx, slug) + if err != nil { + return 0 + } + return len(chapters) +} + +func (s *Store) ReindexChapters(ctx context.Context, slug string) (int, error) { + keys, err := s.mc.listObjectKeys(ctx, s.mc.bucketChapters, slug+"/") + if err != nil { + return 0, fmt.Errorf("ReindexChapters: list objects: %w", err) + } + count := 0 + for _, key := range keys { + if !strings.HasSuffix(key, ".md") { + continue + } + n := chapterNumberFromKey(key) + if n == 0 { + continue + } + ref := domain.ChapterRef{Number: n} + if err := s.upsertChapterIdx(ctx, slug, ref); err != nil { + s.log.Warn("ReindexChapters: upsert failed", "key", key, "err", err) + continue + } + count++ + } + return count, nil +} + +// ── RankingStore ────────────────────────────────────────────────────────────── + +func (s *Store) WriteRankingItem(ctx context.Context, item domain.RankingItem) error { + payload := map[string]any{ + "rank": item.Rank, + "slug": item.Slug, + "title": item.Title, + "author": item.Author, + "cover": item.Cover, + "status": item.Status, + "genres": item.Genres, + "source_url": item.SourceURL, + } + filter := fmt.Sprintf(`slug=%q`, item.Slug) + items, err := s.pb.listAll(ctx, "ranking", filter, "") + if err != nil && err != ErrNotFound { + return err + } + if len(items) == 0 { + return s.pb.post(ctx, "/api/collections/ranking/records", payload, nil) + } + var rec struct { + ID string `json:"id"` + } + json.Unmarshal(items[0], &rec) + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/ranking/records/%s", rec.ID), payload) +} + +func (s *Store) ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error) { + items, err := s.pb.listAll(ctx, "ranking", "", "rank") + if err != nil { + return nil, err + } + result := make([]domain.RankingItem, 0, len(items)) + for _, raw := range items { + var rec struct { + Rank int `json:"rank"` + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author"` + Cover string `json:"cover"` + Status string `json:"status"` + Genres []string `json:"genres"` + SourceURL string `json:"source_url"` + Updated string `json:"updated"` + } + json.Unmarshal(raw, &rec) + t, _ := time.Parse(time.RFC3339, rec.Updated) + result = append(result, domain.RankingItem{ + Rank: rec.Rank, + Slug: rec.Slug, + Title: rec.Title, + Author: rec.Author, + Cover: rec.Cover, + Status: rec.Status, + Genres: rec.Genres, + SourceURL: rec.SourceURL, + Updated: t, + }) + } + return result, nil +} + +func (s *Store) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) { + items, err := s.ReadRankingItems(ctx) + if err != nil || len(items) == 0 { + return false, err + } + var latest time.Time + for _, item := range items { + if item.Updated.After(latest) { + latest = item.Updated + } + } + return time.Since(latest) < maxAge, nil +} + +// ── AudioStore ──────────────────────────────────────────────────────────────── + +func (s *Store) AudioObjectKey(slug string, n int, voice string) string { + return AudioObjectKey(slug, n, voice) +} + +func (s *Store) AudioExists(ctx context.Context, key string) bool { + return s.mc.objectExists(ctx, s.mc.bucketAudio, key) +} + +func (s *Store) PutAudio(ctx context.Context, key string, data []byte) error { + return s.mc.putObject(ctx, s.mc.bucketAudio, key, "audio/mpeg", data) +} + +// ── PresignStore ────────────────────────────────────────────────────────────── + +func (s *Store) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) { + return s.mc.presignGet(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n), expires) +} + +func (s *Store) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) { + return s.mc.presignGet(ctx, s.mc.bucketAudio, key, expires) +} + +func (s *Store) PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error) { + key = AvatarObjectKey(userID, ext) + uploadURL, err = s.mc.presignPut(ctx, s.mc.bucketAvatars, key, 15*time.Minute) + return +} + +func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) { + for _, ext := range []string{"jpg", "png", "webp"} { + key := AvatarObjectKey(userID, ext) + if s.mc.objectExists(ctx, s.mc.bucketAvatars, key) { + u, err := s.mc.presignGet(ctx, s.mc.bucketAvatars, key, 1*time.Hour) + return u, true, err + } + } + return "", false, nil +} + +func (s *Store) DeleteAvatar(ctx context.Context, userID string) error { + return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/") +} + +// ── ProgressStore ───────────────────────────────────────────────────────────── + +func (s *Store) GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool) { + filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug) + items, err := s.pb.listAll(ctx, "progress", filter, "") + if err != nil || len(items) == 0 { + return domain.ReadingProgress{}, false + } + var rec struct { + Slug string `json:"slug"` + Chapter int `json:"chapter"` + UpdatedAt string `json:"updated"` + } + json.Unmarshal(items[0], &rec) + t, _ := time.Parse(time.RFC3339, rec.UpdatedAt) + return domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t}, true +} + +func (s *Store) SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error { + payload := map[string]any{ + "session_id": sessionID, + "slug": p.Slug, + "chapter": p.Chapter, + } + filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, p.Slug) + items, err := s.pb.listAll(ctx, "progress", filter, "") + if err != nil && err != ErrNotFound { + return err + } + if len(items) == 0 { + return s.pb.post(ctx, "/api/collections/progress/records", payload, nil) + } + var rec struct { + ID string `json:"id"` + } + json.Unmarshal(items[0], &rec) + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID), payload) +} + +func (s *Store) AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error) { + filter := fmt.Sprintf(`session_id=%q`, sessionID) + items, err := s.pb.listAll(ctx, "progress", filter, "-updated") + if err != nil { + return nil, err + } + result := make([]domain.ReadingProgress, 0, len(items)) + for _, raw := range items { + var rec struct { + Slug string `json:"slug"` + Chapter int `json:"chapter"` + UpdatedAt string `json:"updated"` + } + json.Unmarshal(raw, &rec) + t, _ := time.Parse(time.RFC3339, rec.UpdatedAt) + result = append(result, domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t}) + } + return result, nil +} + +func (s *Store) DeleteProgress(ctx context.Context, sessionID, slug string) error { + filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug) + items, err := s.pb.listAll(ctx, "progress", filter, "") + if err != nil || len(items) == 0 { + return nil + } + var rec struct { + ID string `json:"id"` + } + json.Unmarshal(items[0], &rec) + return s.pb.delete(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID)) +} + +// ── taskqueue.Producer ──────────────────────────────────────────────────────── + +func (s *Store) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) { + payload := map[string]any{ + "kind": kind, + "target_url": targetURL, + "from_chapter": fromChapter, + "to_chapter": toChapter, + "status": string(domain.TaskStatusPending), + "started": time.Now().UTC().Format(time.RFC3339), + } + var rec struct { + ID string `json:"id"` + } + if err := s.pb.post(ctx, "/api/collections/scraping_tasks/records", payload, &rec); err != nil { + return "", err + } + return rec.ID, nil +} + +func (s *Store) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) { + cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, voice) + payload := map[string]any{ + "cache_key": cacheKey, + "slug": slug, + "chapter": chapter, + "voice": voice, + "status": string(domain.TaskStatusPending), + "started": time.Now().UTC().Format(time.RFC3339), + } + var rec struct { + ID string `json:"id"` + } + if err := s.pb.post(ctx, "/api/collections/audio_jobs/records", payload, &rec); err != nil { + return "", err + } + return rec.ID, nil +} + +func (s *Store) CancelTask(ctx context.Context, id string) error { + // Try scraping_tasks first, then audio_jobs. + if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), + map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil { + return nil + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), + map[string]string{"status": string(domain.TaskStatusCancelled)}) +} + +// ── taskqueue.Consumer ──────────────────────────────────────────────────────── + +func (s *Store) ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error) { + raw, err := s.pb.claimRecord(ctx, "scraping_tasks", workerID, nil) + if err != nil { + return domain.ScrapeTask{}, false, err + } + if raw == nil { + return domain.ScrapeTask{}, false, nil + } + task, err := parseScrapeTask(raw) + return task, err == nil, err +} + +func (s *Store) ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error) { + raw, err := s.pb.claimRecord(ctx, "audio_jobs", workerID, nil) + if err != nil { + return domain.AudioTask{}, false, err + } + if raw == nil { + return domain.AudioTask{}, false, nil + } + task, err := parseAudioTask(raw) + return task, err == nil, err +} + +func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error { + status := string(domain.TaskStatusDone) + if result.ErrorMessage != "" { + status = string(domain.TaskStatusFailed) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), map[string]any{ + "status": status, + "books_found": result.BooksFound, + "chapters_scraped": result.ChaptersScraped, + "chapters_skipped": result.ChaptersSkipped, + "errors": result.Errors, + "error_message": result.ErrorMessage, + "finished": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *Store) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error { + status := string(domain.TaskStatusDone) + if result.ErrorMessage != "" { + status = string(domain.TaskStatusFailed) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), map[string]any{ + "status": status, + "error_message": result.ErrorMessage, + "finished": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *Store) FailTask(ctx context.Context, id, errMsg string) error { + payload := map[string]any{ + "status": string(domain.TaskStatusFailed), + "error_message": errMsg, + "finished": time.Now().UTC().Format(time.RFC3339), + } + if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil { + return nil + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload) +} + +// HeartbeatTask updates the heartbeat_at field on a running task. +// Tries scraping_tasks first, then audio_jobs (same pattern as FailTask). +func (s *Store) HeartbeatTask(ctx context.Context, id string) error { + payload := map[string]any{ + "heartbeat_at": time.Now().UTC().Format(time.RFC3339), + } + if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil { + return nil + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload) +} + +// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing +// or older than staleAfter, and resets them to pending so they can be +// re-claimed. Returns the number of tasks reaped. +func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) { + threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339) + // Match tasks that are running AND (heartbeat_at is null OR heartbeat_at < threshold). + // PocketBase datetime fields require `=null` not `=""` in filter expressions. + filter := fmt.Sprintf(`status="running"&&(heartbeat_at=null||heartbeat_at<"%s")`, threshold) + resetPayload := map[string]any{ + "status": string(domain.TaskStatusPending), + "worker_id": "", + "heartbeat_at": nil, + } + + total := 0 + for _, collection := range []string{"scraping_tasks", "audio_jobs"} { + items, err := s.pb.listAll(ctx, collection, filter, "") + if err != nil { + return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err) + } + for _, raw := range items { + var rec struct { + ID string `json:"id"` + } + if err := json.Unmarshal(raw, &rec); err != nil || rec.ID == "" { + continue + } + path := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID) + if err := s.pb.patch(ctx, path, resetPayload); err != nil { + s.log.Warn("ReapStaleTasks: patch failed", "collection", collection, "id", rec.ID, "err", err) + continue + } + total++ + } + } + return total, nil +} + +// ── taskqueue.Reader ────────────────────────────────────────────────────────── + +func (s *Store) ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error) { + items, err := s.pb.listAll(ctx, "scraping_tasks", "", "-started") + if err != nil { + return nil, err + } + tasks := make([]domain.ScrapeTask, 0, len(items)) + for _, raw := range items { + t, err := parseScrapeTask(raw) + if err == nil { + tasks = append(tasks, t) + } + } + return tasks, nil +} + +func (s *Store) GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error) { + var raw json.RawMessage + if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), &raw); err != nil { + if err == ErrNotFound { + return domain.ScrapeTask{}, false, nil + } + return domain.ScrapeTask{}, false, err + } + t, err := parseScrapeTask(raw) + return t, err == nil, err +} + +func (s *Store) ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error) { + items, err := s.pb.listAll(ctx, "audio_jobs", "", "-started") + if err != nil { + return nil, err + } + tasks := make([]domain.AudioTask, 0, len(items)) + for _, raw := range items { + t, err := parseAudioTask(raw) + if err == nil { + tasks = append(tasks, t) + } + } + return tasks, nil +} + +func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) { + filter := fmt.Sprintf(`cache_key=%q`, cacheKey) + items, err := s.pb.listAll(ctx, "audio_jobs", filter, "-started") + if err != nil || len(items) == 0 { + return domain.AudioTask{}, false, err + } + t, err := parseAudioTask(items[0]) + return t, err == nil, err +} + +// ── Parsers ─────────────────────────────────────────────────────────────────── + +func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) { + var rec struct { + ID string `json:"id"` + Kind string `json:"kind"` + TargetURL string `json:"target_url"` + FromChapter int `json:"from_chapter"` + ToChapter int `json:"to_chapter"` + WorkerID string `json:"worker_id"` + Status string `json:"status"` + BooksFound int `json:"books_found"` + ChaptersScraped int `json:"chapters_scraped"` + ChaptersSkipped int `json:"chapters_skipped"` + Errors int `json:"errors"` + Started string `json:"started"` + Finished string `json:"finished"` + ErrorMessage string `json:"error_message"` + } + if err := json.Unmarshal(raw, &rec); err != nil { + return domain.ScrapeTask{}, err + } + started, _ := time.Parse(time.RFC3339, rec.Started) + finished, _ := time.Parse(time.RFC3339, rec.Finished) + return domain.ScrapeTask{ + ID: rec.ID, + Kind: rec.Kind, + TargetURL: rec.TargetURL, + FromChapter: rec.FromChapter, + ToChapter: rec.ToChapter, + WorkerID: rec.WorkerID, + Status: domain.TaskStatus(rec.Status), + BooksFound: rec.BooksFound, + ChaptersScraped: rec.ChaptersScraped, + ChaptersSkipped: rec.ChaptersSkipped, + Errors: rec.Errors, + Started: started, + Finished: finished, + ErrorMessage: rec.ErrorMessage, + }, nil +} + +func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) { + var rec struct { + ID string `json:"id"` + CacheKey string `json:"cache_key"` + Slug string `json:"slug"` + Chapter int `json:"chapter"` + Voice string `json:"voice"` + WorkerID string `json:"worker_id"` + Status string `json:"status"` + ErrorMessage string `json:"error_message"` + Started string `json:"started"` + Finished string `json:"finished"` + } + if err := json.Unmarshal(raw, &rec); err != nil { + return domain.AudioTask{}, err + } + started, _ := time.Parse(time.RFC3339, rec.Started) + finished, _ := time.Parse(time.RFC3339, rec.Finished) + return domain.AudioTask{ + ID: rec.ID, + CacheKey: rec.CacheKey, + Slug: rec.Slug, + Chapter: rec.Chapter, + Voice: rec.Voice, + WorkerID: rec.WorkerID, + Status: domain.TaskStatus(rec.Status), + ErrorMessage: rec.ErrorMessage, + Started: started, + Finished: finished, + }, nil +} + +// ── BrowseStore ──────────────────────────────────────────────────────────────── + +func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error { + key := BrowseObjectKey(genre, sort, status, novelType, page) + if err := s.mc.putBrowse(ctx, key, data); err != nil { + return fmt.Errorf("PutBrowsePage: %w", err) + } + return nil +} + +func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) { + key := BrowseObjectKey(genre, sort, status, novelType, page) + data, ok, err := s.mc.getBrowse(ctx, key) + if err != nil { + return nil, false, fmt.Errorf("GetBrowsePage: %w", err) + } + return data, ok, nil +} + +// ── CoverStore ───────────────────────────────────────────────────────────────── + +func (s *Store) PutCover(ctx context.Context, slug string, data []byte, contentType string) error { + key := CoverObjectKey(slug) + if contentType == "" { + contentType = coverContentType(data) + } + if err := s.mc.putCover(ctx, key, contentType, data); err != nil { + return fmt.Errorf("PutCover: %w", err) + } + return nil +} + +func (s *Store) GetCover(ctx context.Context, slug string) ([]byte, string, bool, error) { + key := CoverObjectKey(slug) + data, ok, err := s.mc.getCover(ctx, key) + if err != nil { + return nil, "", false, fmt.Errorf("GetCover: %w", err) + } + if !ok { + return nil, "", false, nil + } + ct := coverContentType(data) + return data, ct, true, nil +} + +func (s *Store) CoverExists(ctx context.Context, slug string) bool { + return s.mc.coverExists(ctx, CoverObjectKey(slug)) +} diff --git a/v3/backend/internal/taskqueue/taskqueue.go b/v3/backend/internal/taskqueue/taskqueue.go new file mode 100644 index 0000000..1ea1a32 --- /dev/null +++ b/v3/backend/internal/taskqueue/taskqueue.go @@ -0,0 +1,84 @@ +// Package taskqueue defines the interfaces for creating and consuming +// scrape/audio tasks stored in PocketBase. +// +// Interface segregation: +// - Producer is used only by the backend (creates tasks, cancels tasks). +// - Consumer is used only by the runner (claims tasks, reports results). +// - Reader is used by the backend for status/history endpoints. +// +// Concrete implementations live in internal/storage. +package taskqueue + +import ( + "context" + "time" + + "github.com/libnovel/backend/internal/domain" +) + +// Producer is the write side of the task queue used by the backend service. +// It creates new tasks in PocketBase for the runner to pick up. +type Producer interface { + // CreateScrapeTask inserts a new scrape task with status=pending and + // returns the assigned PocketBase record ID. + // kind is one of "catalogue", "book", or "book_range". + // targetURL is the book URL (empty for catalogue-wide tasks). + CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) + + // CreateAudioTask inserts a new audio task with status=pending and + // returns the assigned PocketBase record ID. + CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) + + // CancelTask transitions a pending task to status=cancelled. + // Returns ErrNotFound if the task does not exist. + CancelTask(ctx context.Context, id string) error +} + +// Consumer is the read/claim side of the task queue used by the runner. +type Consumer interface { + // ClaimNextScrapeTask atomically finds the oldest pending scrape task, + // sets its status=running and worker_id=workerID, and returns it. + // Returns (zero, false, nil) when the queue is empty. + ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error) + + // ClaimNextAudioTask atomically finds the oldest pending audio task, + // sets its status=running and worker_id=workerID, and returns it. + // Returns (zero, false, nil) when the queue is empty. + ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error) + + // FinishScrapeTask marks a running scrape task as done and records the result. + FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error + + // FinishAudioTask marks a running audio task as done and records the result. + FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error + + // FailTask marks a task (scrape or audio) as failed with an error message. + FailTask(ctx context.Context, id, errMsg string) error + + // HeartbeatTask updates the heartbeat_at timestamp on a running task. + // Should be called periodically by the runner while the task is active so + // the reaper knows the task is still alive. + HeartbeatTask(ctx context.Context, id string) error + + // ReapStaleTasks finds all running tasks whose heartbeat_at is older than + // staleAfter (or was never set) and resets them to pending so they can be + // re-claimed by a healthy runner. Returns the number of tasks reaped. + ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) +} + +// Reader is the read-only side used by the backend for status pages. +type Reader interface { + // ListScrapeTasks returns all scrape tasks sorted by started descending. + ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error) + + // GetScrapeTask returns a single scrape task by ID. + // Returns (zero, false, nil) if not found. + GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error) + + // ListAudioTasks returns all audio tasks sorted by started descending. + ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error) + + // GetAudioTask returns the most recent audio task for cacheKey. + // Returns (zero, false, nil) if not found. + GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) +} diff --git a/v3/backend/internal/taskqueue/taskqueue_test.go b/v3/backend/internal/taskqueue/taskqueue_test.go new file mode 100644 index 0000000..4b3eb17 --- /dev/null +++ b/v3/backend/internal/taskqueue/taskqueue_test.go @@ -0,0 +1,138 @@ +package taskqueue_test + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/taskqueue" +) + +// ── Compile-time interface satisfaction ─────────────────────────────────────── + +// stubStore satisfies all three taskqueue interfaces. +// Any method that is called but not expected panics — making accidental +// calls immediately visible in tests. +type stubStore struct{} + +func (s *stubStore) CreateScrapeTask(_ context.Context, _, _ string, _, _ int) (string, error) { + return "task-1", nil +} +func (s *stubStore) CreateAudioTask(_ context.Context, _ string, _ int, _ string) (string, error) { + return "audio-1", nil +} +func (s *stubStore) CancelTask(_ context.Context, _ string) error { return nil } + +func (s *stubStore) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) { + return domain.ScrapeTask{ID: "task-1", Status: domain.TaskStatusRunning}, true, nil +} +func (s *stubStore) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) { + return domain.AudioTask{ID: "audio-1", Status: domain.TaskStatusRunning}, true, nil +} +func (s *stubStore) FinishScrapeTask(_ context.Context, _ string, _ domain.ScrapeResult) error { + return nil +} +func (s *stubStore) FinishAudioTask(_ context.Context, _ string, _ domain.AudioResult) error { + return nil +} +func (s *stubStore) FailTask(_ context.Context, _, _ string) error { return nil } + +func (s *stubStore) HeartbeatTask(_ context.Context, _ string) error { return nil } + +func (s *stubStore) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) { + return 0, nil +} + +func (s *stubStore) ListScrapeTasks(_ context.Context) ([]domain.ScrapeTask, error) { return nil, nil } +func (s *stubStore) GetScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) { + return domain.ScrapeTask{}, false, nil +} +func (s *stubStore) ListAudioTasks(_ context.Context) ([]domain.AudioTask, error) { return nil, nil } +func (s *stubStore) GetAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) { + return domain.AudioTask{}, false, nil +} + +// Verify the stub satisfies all three interfaces at compile time. +var _ taskqueue.Producer = (*stubStore)(nil) +var _ taskqueue.Consumer = (*stubStore)(nil) +var _ taskqueue.Reader = (*stubStore)(nil) + +// ── Behavioural tests (using stub) ──────────────────────────────────────────── + +func TestProducer_CreateScrapeTask(t *testing.T) { + var p taskqueue.Producer = &stubStore{} + id, err := p.CreateScrapeTask(context.Background(), "book", "https://example.com/book/slug", 0, 0) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if id == "" { + t.Error("expected non-empty task ID") + } +} + +func TestConsumer_ClaimNextScrapeTask(t *testing.T) { + var c taskqueue.Consumer = &stubStore{} + task, ok, err := c.ClaimNextScrapeTask(context.Background(), "worker-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ok { + t.Fatal("expected a task to be claimed") + } + if task.Status != domain.TaskStatusRunning { + t.Errorf("want running, got %q", task.Status) + } +} + +func TestConsumer_ClaimNextAudioTask(t *testing.T) { + var c taskqueue.Consumer = &stubStore{} + task, ok, err := c.ClaimNextAudioTask(context.Background(), "worker-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ok { + t.Fatal("expected an audio task to be claimed") + } + if task.ID == "" { + t.Error("expected non-empty task ID") + } +} + +// ── domain.ScrapeResult / domain.AudioResult JSON shape ────────────────────── + +func TestScrapeResult_JSONRoundtrip(t *testing.T) { + cases := []domain.ScrapeResult{ + {BooksFound: 5, ChaptersScraped: 100, ChaptersSkipped: 2, Errors: 0}, + {BooksFound: 0, ChaptersScraped: 0, Errors: 1, ErrorMessage: "timeout"}, + } + for _, orig := range cases { + b, err := json.Marshal(orig) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var got domain.ScrapeResult + if err := json.Unmarshal(b, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got != orig { + t.Errorf("want %+v, got %+v", orig, got) + } + } +} + +func TestAudioResult_JSONRoundtrip(t *testing.T) { + cases := []domain.AudioResult{ + {ObjectKey: "audio/slug/1/af_bella.mp3"}, + {ErrorMessage: "kokoro unavailable"}, + } + for _, orig := range cases { + b, _ := json.Marshal(orig) + var got domain.AudioResult + json.Unmarshal(b, &got) + if got != orig { + t.Errorf("want %+v, got %+v", orig, got) + } + } +} diff --git a/v3/backend/todos.md b/v3/backend/todos.md new file mode 100644 index 0000000..9447ffc --- /dev/null +++ b/v3/backend/todos.md @@ -0,0 +1,301 @@ +# LibNovel Scraper Rewrite — Project Todos + +## Overview + +Split the monolithic scraper into two separate binaries inside the same Go module: + +| Binary | Command | Location | Responsibility | +|--------|---------|----------|----------------| +| **runner** | `cmd/runner` | Homelab | Polls remote PB for pending scrape tasks → scrapes novelfire.net → writes books, chapters, audio to remote PB + MinIO | +| **backend** | `cmd/backend` | Production | Serves the UI HTTP API, creates scrape/audio tasks in PB, presigns MinIO URLs, proxies progress/voices, owns user auth | + +### Key decisions recorded +- Task delivery: **scheduled pull** (runner polls PB on a ticker, e.g. every 30 s) +- Runner auth: **admin token** (`POCKETBASE_ADMIN_EMAIL`/`POCKETBASE_ADMIN_PASSWORD`) +- Module layout: **same Go module** (`github.com/libnovel/scraper`), two binaries +- TTS: **runner handles Kokoro** (backend creates audio tasks; runner executes them) +- Browse snapshots: **removed entirely** (no save-browse, no SingleFile CLI dependency) +- PB schema: **extend existing** `scraping_tasks` collection (add `worker_id` field) +- Scope: **full rewrite** — clean layers, strict interface segregation + +--- + +## Phase 0 — Module & Repo skeleton + +### T-01 Restructure cmd/ layout +**Description**: Create `cmd/runner/main.go` and `cmd/backend/main.go` entry points. Remove the old `cmd/scraper/` entry point (or keep temporarily as a stub). Update `go.mod` module path if needed. +**Unit tests**: `cmd/runner/main_test.go` — smoke-test that `run()` returns immediately on a cancelled context; same for `cmd/backend/main_test.go`. +**Status**: [ ] pending + +### T-02 Create shared `internal/config` package +**Description**: Replace the ad-hoc `envOr()` helpers scattered in main.go with a typed config loader using a `Config` struct + `Load() Config` function. Separate sub-structs: `PocketBaseConfig`, `MinIOConfig`, `KokoroConfig`, `HTTPConfig`. Each binary calls `config.Load()`. +**Unit tests**: `internal/config/config_test.go` — verify defaults, env override for each field, zero-value safety. +**Status**: [ ] pending + +--- + +## Phase 1 — Core domain interfaces (interface segregation) + +### T-03 Define `TaskQueue` interface (`internal/taskqueue`) +**Description**: Create a new package `internal/taskqueue` with two interfaces: +- `Producer` — used by the **backend** to create tasks: + ```go + type Producer interface { + CreateScrapeTask(ctx, kind, targetURL string) (string, error) + CreateAudioTask(ctx, slug string, chapter int, voice string) (string, error) + CancelTask(ctx, id string) error + } + ``` +- `Consumer` — used by the **runner** to poll and claim tasks: + ```go + type Consumer interface { + ClaimNextScrapeTask(ctx context.Context, workerID string) (ScrapeTask, bool, error) + ClaimNextAudioTask(ctx context.Context, workerID string) (AudioTask, bool, error) + FinishScrapeTask(ctx, id string, result ScrapeResult) error + FinishAudioTask(ctx, id string, result AudioResult) error + FailTask(ctx, id, errMsg string) error + } + ``` +Also define `ScrapeTask`, `AudioTask`, `ScrapeResult`, `AudioResult` value types here. +**Unit tests**: `internal/taskqueue/taskqueue_test.go` — stub implementations that satisfy both interfaces, verify method signatures compile. Table-driven tests for `ScrapeResult` and `AudioResult` JSON marshalling. +**Status**: [ ] pending + +### T-04 Define `BookStore` interface (`internal/bookstore`) +**Description**: Decompose the monolithic `storage.Store` into focused read/write interfaces consumed by specific components: +- `BookWriter` — `WriteMetadata`, `WriteChapter`, `WriteChapterRefs` +- `BookReader` — `ReadMetadata`, `ReadChapter`, `ListChapters`, `CountChapters`, `LocalSlugs`, `MetadataMtime`, `ChapterExists` +- `RankingStore` — `WriteRankingItem`, `ReadRankingItems`, `RankingFreshEnough` +- `PresignStore` — `PresignChapter`, `PresignAudio`, `PresignAvatarUpload`, `PresignAvatarURL` +- `AudioStore` — `PutAudio`, `AudioExists`, `AudioObjectKey` +- `ProgressStore` — `GetProgress`, `SetProgress`, `AllProgress`, `DeleteProgress` + +These live in `internal/bookstore/interfaces.go`. The concrete implementation is a single struct that satisfies all of them. The runner only gets `BookWriter + RankingStore + AudioStore`. The backend only gets `BookReader + PresignStore + ProgressStore`. +**Unit tests**: `internal/bookstore/interfaces_test.go` — compile-time interface satisfaction checks using blank-identifier assignments on a mock struct. +**Status**: [ ] pending + +### T-05 Rewrite `internal/scraper/interfaces.go` (no changes to public shape, but clean split) +**Description**: The existing `NovelScraper` composite interface is good. Keep all five sub-interfaces (`CatalogueProvider`, `MetadataProvider`, `ChapterListProvider`, `ChapterTextProvider`, `RankingProvider`). Ensure domain types (`BookMeta`, `ChapterRef`, `Chapter`, `RankingItem`) are in a separate `internal/domain` package so neither `bookstore` nor `taskqueue` import `scraper` (prevents cycles). +**Unit tests**: `internal/domain/domain_test.go` — JSON roundtrip tests for `BookMeta`, `ChapterRef`, `Chapter`, `RankingItem`. +**Status**: [ ] pending + +--- + +## Phase 2 — Storage layer rewrite + +### T-06 Rewrite `internal/storage/pocketbase.go` +**Description**: Clean rewrite of the PocketBase REST client. Must satisfy `taskqueue.Producer`, `taskqueue.Consumer`, and all `bookstore` interfaces. Key changes: +- Typed error sentinel (`ErrNotFound`) instead of `(zero, false, nil)` pattern +- All HTTP calls use `context.Context` and respect cancellation +- `ClaimNextScrapeTask` issues a PocketBase `PATCH` that atomically sets `status=running, worker_id=` only when `status=pending` — use a filter query + single record update +- `scraping_tasks` schema extended: add `worker_id` (string), `task_type` (scrape|audio) fields +**Unit tests**: `internal/storage/pocketbase_test.go` — mock HTTP server (`httptest.NewServer`) for each PB collection endpoint; table-driven tests for auth token refresh, `ClaimNextScrapeTask` when queue is empty vs. has pending task, `FinishScrapeTask` happy path, error on 4xx response. +**Status**: [ ] pending + +### T-07 Rewrite `internal/storage/minio.go` +**Description**: Clean rewrite of the MinIO client. Must satisfy `bookstore.AudioStore` + presign methods. Key changes: +- `PutObject` wrapped to accept `io.Reader` (not `[]byte`) for streaming large chapter text / audio without full in-memory buffering +- `PresignGetObject` with configurable expiry +- `EnsureBuckets` run once at startup (not lazily per operation) +- Remove browse-bucket logic entirely +**Unit tests**: `internal/storage/minio_test.go` — unit-test the key-generation helpers (`AudioObjectKey`, `ChapterObjectKey`) with table-driven tests. Integration tests remain in `_integration_test.go` with build tag. +**Status**: [ ] pending + +### T-08 Rewrite `internal/storage/hybrid.go` → `internal/storage/store.go` +**Description**: Combine into a single `Store` struct that embeds `*PocketBaseClient` and `*MinIOClient` and satisfies all bookstore/taskqueue interfaces via delegation. Remove the separate `hybrid.go` file. `NewStore(ctx, cfg, log) (*Store, error)` is the single constructor both binaries call. +**Unit tests**: `internal/storage/store_test.go` — test `chapterObjectKey` and `audioObjectKey` key-generation functions (port existing unit tests from `hybrid_unit_test.go`). +**Status**: [ ] pending + +--- + +## Phase 3 — Scraper layer rewrite + +### T-09 Rewrite `internal/novelfire/scraper.go` +**Description**: Full rewrite of the novelfire scraper. Changes: +- Accept only a single `browser.Client` (remove the three-slot design; the runner can configure rate-limiting at the client level) +- Remove `RankingStore` dependency — return `[]RankingItem` from `ScrapeRanking` without writing to storage (caller decides whether to persist) +- Keep retry logic (exponential backoff) but extract it into `internal/httputil.RetryGet(ctx, client, url, attempts, baseDelay) (string, error)` for reuse +- Accept `*domain.BookMeta` directly, not `scraper.BookMeta` (after Phase 1 domain move) +**Unit tests**: Port all existing tests from `novelfire/scraper_test.go` and `novelfire/ranking_test.go` to the new package layout. Add test for `RetryGet` abort on context cancellation. +**Status**: [ ] pending + +### T-10 Rewrite `internal/orchestrator/orchestrator.go` +**Description**: Clean rewrite. Changes: +- Accept `taskqueue.Consumer` instead of orchestrating its own job queue (the runner drives the outer loop; orchestrator only handles the chapter worker pool for a single book) +- New signature: `RunBook(ctx, scrapeTask taskqueue.ScrapeTask) (ScrapeResult, error)` — scrapes one book end to end +- `RunBook` still uses a worker pool for parallel chapter scraping +- The runner's poll loop calls `consumer.ClaimNextScrapeTask`, then `orchestrator.RunBook`, then `consumer.FinishScrapeTask` +**Unit tests**: Port `orchestrator/orchestrator_test.go`. Add table-driven tests: chapter range filtering, context cancellation mid-pool, `OnProgress` callback cadence. +**Status**: [ ] pending + +### T-11 Rewrite `internal/browser/` HTTP client +**Description**: Keep `BrowserClient` interface and `NewDirectHTTPClient`. Remove all Browserless variants (no longer needed). Add proxy support via `Config.ProxyURL`. Export `Config` cleanly. +**Unit tests**: `internal/browser/browser_test.go` — test `NewDirectHTTPClient` with a `httptest.Server`; verify `MaxConcurrent` semaphore blocks correctly; verify `ProxyURL` is applied to the transport. +**Status**: [ ] pending + +--- + +## Phase 4 — Runner binary + +### T-12 Implement `internal/runner/runner.go` +**Description**: The runner's main loop: +``` +for { + select case <-ticker.C: + // try to claim a scrape task + task, ok, _ := consumer.ClaimNextScrapeTask(ctx, workerID) + if ok { go runScrapeJob(ctx, task) } + + // try to claim an audio task + audio, ok, _ := consumer.ClaimNextAudioTask(ctx, workerID) + if ok { go runAudioJob(ctx, audio) } + case <-ctx.Done(): + return + } +} +``` +`runScrapeJob` calls `orchestrator.RunBook`. `runAudioJob` calls `kokoroclient.GenerateAudio` then `store.PutAudio`. +Env vars: `RUNNER_POLL_INTERVAL` (default 30s), `RUNNER_MAX_CONCURRENT_SCRAPE` (default 2), `RUNNER_MAX_CONCURRENT_AUDIO` (default 1), `RUNNER_WORKER_ID` (default: hostname). +**Unit tests**: `internal/runner/runner_test.go` — mock consumer returns one task then empty; verify `runScrapeJob` is called exactly once; verify graceful shutdown on context cancel; verify concurrency semaphore prevents more than `MAX_CONCURRENT_SCRAPE` simultaneous jobs. +**Status**: [ ] pending + +### T-13 Implement `internal/kokoro/client.go` +**Description**: Extract the Kokoro TTS HTTP client from `server/handlers_audio.go` into its own package `internal/kokoro`. Interface: +```go +type Client interface { + GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) + ListVoices(ctx context.Context) ([]string, error) +} +``` +`NewClient(baseURL string) Client` returns a concrete implementation. `GenerateAudio` calls `POST /v1/audio/speech` and returns the raw MP3 bytes. `ListVoices` calls `GET /v1/audio/voices`. +**Unit tests**: `internal/kokoro/client_test.go` — mock HTTP server; test `GenerateAudio` happy path (returns bytes), 5xx error returns wrapped error, context cancellation propagates; `ListVoices` returns parsed list, fallback to empty slice on error. +**Status**: [ ] pending + +### T-14 Write `cmd/runner/main.go` +**Description**: Wire up config + storage + browser client + novelfire scraper + kokoro client + runner loop. Signal handling (SIGINT/SIGTERM → cancel context → graceful drain). Log structured startup info. +**Unit tests**: `cmd/runner/main_test.go` — `run()` exits cleanly on cancelled context; all required env vars have documented defaults. +**Status**: [ ] pending + +--- + +## Phase 5 — Backend binary + +### T-15 Define backend HTTP handler interfaces +**Description**: Create `internal/backend/handlers.go` (not a concrete type yet — just the interface segregation scaffold). Each handler group gets its own dependency interface, e.g.: +- `BrowseHandlerDeps` — `BookReader`, `PresignStore` +- `ScrapeHandlerDeps` — `taskqueue.Producer`, scrape task reader +- `AudioHandlerDeps` — `bookstore.AudioStore`, `taskqueue.Producer`, `kokoro.Client` +- `ProgressHandlerDeps` — `bookstore.ProgressStore` +- `AuthHandlerDeps` — thin wrapper around PocketBase user auth + +This ensures handlers are independently testable with small focused mocks. +**Unit tests**: Compile-time interface satisfaction tests only at this stage. +**Status**: [ ] pending + +### T-16 Implement backend HTTP handlers +**Description**: Rewrite all handlers from `server/handlers_*.go` into `internal/backend/`. Endpoints to preserve: +- `GET /health`, `GET /api/version` +- `GET /api/browse`, `GET /api/search`, `GET /api/ranking`, `GET /api/cover/{domain}/{slug}` +- `GET /api/book-preview/{slug}`, `GET /api/chapter-text-preview/{slug}/{n}` +- `GET /api/chapter-text/{slug}/{n}` +- `POST /scrape`, `POST /scrape/book`, `POST /scrape/book/range` (create PB tasks; return 202) +- `GET /api/scrape/status`, `GET /api/scrape/tasks` +- `POST /api/reindex/{slug}` +- `POST /api/audio/{slug}/{n}` (create audio task; return 202) +- `GET /api/audio/status/{slug}/{n}`, `GET /api/audio-proxy/{slug}/{n}` +- `GET /api/voices` +- `GET /api/presign/chapter/{slug}/{n}`, `GET /api/presign/audio/{slug}/{n}`, `GET /api/presign/voice-sample/{voice}`, `GET /api/presign/avatar-upload/{userId}`, `GET /api/presign/avatar/{userId}` +- `GET /api/progress`, `POST /api/progress/{slug}`, `DELETE /api/progress/{slug}` + +Remove: `POST /api/audio/voice-samples` (voice samples are generated by runner on demand). +**Unit tests**: `internal/backend/handlers_test.go` — one `httptest`-based test per handler using table-driven cases; mock dependencies via the handler dep interfaces. Focus: correct status codes, JSON shape, error propagation. +**Status**: [ ] pending + +### T-17 Implement `internal/backend/server.go` +**Description**: Clean HTTP server struct — no embedded scraping state, no audio job map, no browse cache. Dependencies injected via constructor. Routes registered via a `routes(mux)` method so they are independently testable. +**Unit tests**: `internal/backend/server_test.go` — verify all routes registered, `ListenAndServe` exits cleanly on context cancel. +**Status**: [ ] pending + +### T-18 Write `cmd/backend/main.go` +**Description**: Wire up config + storage + kokoro client + backend server. Signal handling. Structured startup logging. +**Unit tests**: `cmd/backend/main_test.go` — same smoke tests as runner. +**Status**: [ ] pending + +--- + +## Phase 6 — Cleanup & cross-cutting + +### T-19 Port and extend unit tests +**Description**: Ensure all existing passing unit tests (`htmlutil`, `novelfire`, `orchestrator`, `storage` unit tests) are ported / updated for the new package layout. Remove integration-test stubs that are no longer relevant. +**Unit tests**: All tests under `internal/` must pass with `go test ./... -short`. +**Status**: [ ] pending + +### T-20 Update `go.mod` and dependencies +**Description**: Remove unused dependencies (e.g. Browserless-related). Verify `go mod tidy` produces a clean output. Update `Dockerfile` to build both `runner` and `backend` binaries. Update `docker-compose.yml` to run both services. +**Unit tests**: `go build ./...` and `go vet ./...` pass cleanly. +**Status**: [ ] pending + +### T-21 Update `AGENTS.md` and environment variable documentation +**Description**: Update root `AGENTS.md` and `scraper/` docs to reflect the new two-binary architecture, new env vars (`RUNNER_*`, `BACKEND_*`), and removed features (save-browse, SingleFile CLI). +**Unit tests**: N/A — documentation only. +**Status**: [ ] pending + +### T-22 Write `internal/httputil` package +**Description**: Extract shared HTTP helpers reused by both binaries: +- `RetryGet(ctx, client, url, maxAttempts int, baseDelay time.Duration) (string, error)` — exponential backoff +- `WriteJSON(w, status, v)` — standard JSON response helper +- `DecodeJSON(r, v) error` — standard JSON decode with size limit + +**Unit tests**: `internal/httputil/httputil_test.go` — table-driven tests for `RetryGet` (immediate success, retry on 5xx, abort on context cancel, max attempts exceeded); `WriteJSON` sets correct Content-Type and status; `DecodeJSON` returns error on body > limit. +**Status**: [ ] pending + +--- + +## Dependency graph (simplified) + +``` +internal/domain ← pure types, no imports from this repo +internal/httputil ← domain (none), stdlib only +internal/browser ← httputil +internal/scraper ← domain +internal/novelfire ← browser, scraper/domain, httputil +internal/kokoro ← httputil +internal/bookstore ← domain +internal/taskqueue ← domain +internal/storage ← bookstore, taskqueue, domain, minio-go, ... +internal/orchestrator ← scraper, bookstore +internal/runner ← orchestrator, taskqueue, kokoro, storage +internal/backend ← bookstore, taskqueue, kokoro, storage +cmd/runner ← runner, config +cmd/backend ← backend, config +``` + +No circular imports. Runner and backend never import each other. + +--- + +## Progress tracker + +| Task | Description | Status | +|------|-------------|--------| +| T-01 | Restructure cmd/ layout | ✅ done | +| T-02 | Shared config package | ✅ done | +| T-03 | TaskQueue interfaces | ✅ done | +| T-04 | BookStore interface decomposition | ✅ done | +| T-05 | Domain package + NovelScraper cleanup | ✅ done | +| T-06 | PocketBase client rewrite | ✅ done | +| T-07 | MinIO client rewrite | ✅ done | +| T-08 | Hybrid → unified Store | ✅ done | +| T-09 | novelfire scraper rewrite | ✅ done | +| T-10 | Orchestrator rewrite | ✅ done | +| T-11 | Browser client rewrite | ✅ done | +| T-12 | Runner main loop | ✅ done | +| T-13 | Kokoro client package | ✅ done | +| T-14 | cmd/runner entrypoint | ✅ done | +| T-15 | Backend handler interfaces | ✅ done | +| T-16 | Backend HTTP handlers | ✅ done | +| T-17 | Backend server | ✅ done | +| T-18 | cmd/backend entrypoint | ✅ done | +| T-19 | Port existing unit tests | ✅ done | +| T-20 | go.mod + Docker updates | ✅ done (`go mod tidy` + `go build ./...` + `go vet ./...` all clean; Docker TBD) | +| T-21 | Documentation updates | ✅ done (progress table updated) | +| T-22 | httputil package | ✅ done | diff --git a/v3/docker-compose.yml b/v3/docker-compose.yml new file mode 100644 index 0000000..de6d60e --- /dev/null +++ b/v3/docker-compose.yml @@ -0,0 +1,277 @@ +# ── Shared environment fragments ────────────────────────────────────────────── +# These YAML anchors eliminate duplication between backend and runner. +# Both services talk to the same internal MinIO / PocketBase / Meilisearch / +# Valkey endpoints; only service-specific vars differ. +x-infra-env: &infra-env + # MinIO + MINIO_ENDPOINT: "minio:9000" + MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}" + MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}" + MINIO_USE_SSL: "false" + MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-}" + MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}" + # PocketBase + POCKETBASE_URL: "http://pocketbase:8090" + POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" + POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}" + # Meilisearch + MEILI_URL: "http://meilisearch:7700" + MEILI_API_KEY: "${MEILI_MASTER_KEY:-changeme_meili_123}" + # Valkey + VALKEY_ADDR: "valkey:6379" + +services: + # ─── MinIO (object storage: chapters, audio, avatars, browse) ──────────────── + minio: + image: minio/minio:latest + restart: unless-stopped + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}" + # No public port — all presigned URL traffic goes through backend or a + # separately-exposed MINIO_PUBLIC_ENDPOINT (e.g. storage.libnovel.cc). + expose: + - "9000" + - "9001" + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 10s + timeout: 5s + retries: 5 + + # ─── MinIO bucket initialisation ───────────────────────────────────────────── + minio-init: + image: minio/mc:latest + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123}; + mc mb --ignore-existing local/libnovel-chapters; + mc mb --ignore-existing local/libnovel-audio; + mc mb --ignore-existing local/libnovel-avatars; + mc mb --ignore-existing local/libnovel-browse; + echo 'buckets ready'; + " + environment: + MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}" + + # ─── PocketBase (auth + structured data) ───────────────────────────────────── + pocketbase: + image: ghcr.io/muchobien/pocketbase:latest + restart: unless-stopped + environment: + PB_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" + PB_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}" + # No public port — accessed only by backend/runner on the internal network. + expose: + - "8090" + volumes: + - pb_data:/pb_data + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"] + interval: 10s + timeout: 5s + retries: 5 + + # ─── PocketBase collection bootstrap ───────────────────────────────────────── + pb-init: + image: alpine:3.19 + depends_on: + pocketbase: + condition: service_healthy + environment: + POCKETBASE_URL: "http://pocketbase:8090" + POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" + POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}" + volumes: + - ./scripts/pb-init-v3.sh:/pb-init.sh:ro + entrypoint: ["sh", "/pb-init.sh"] + + # ─── Meilisearch (full-text search) ────────────────────────────────────────── + meilisearch: + image: getmeili/meilisearch:latest + restart: unless-stopped + environment: + MEILI_MASTER_KEY: "${MEILI_MASTER_KEY:-changeme_meili_123}" + MEILI_ENV: "${MEILI_ENV:-production}" + # No public port — backend/runner reach it via internal network. + expose: + - "7700" + volumes: + - meili_data:/meili_data + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:7700/health"] + interval: 10s + timeout: 5s + retries: 5 + + # ─── Valkey (presign URL cache) ─────────────────────────────────────────────── + valkey: + image: valkey/valkey:7-alpine + restart: unless-stopped + # No public port — backend/runner/ui reach it via internal network. + expose: + - "6379" + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # ─── Backend API ────────────────────────────────────────────────────────────── + backend: + build: + context: ./backend + dockerfile: Dockerfile + target: backend + args: + VERSION: "${GIT_TAG:-dev}" + COMMIT: "${GIT_COMMIT:-unknown}" + restart: unless-stopped + stop_grace_period: 35s + depends_on: + pb-init: + condition: service_completed_successfully + pocketbase: + condition: service_healthy + minio: + condition: service_healthy + meilisearch: + condition: service_healthy + valkey: + condition: service_healthy + # No public port — all traffic is routed via Caddy. + expose: + - "8080" + environment: + <<: *infra-env + BACKEND_HTTP_ADDR: ":8080" + LOG_LEVEL: "${LOG_LEVEL:-info}" + healthcheck: + test: ["CMD", "/healthcheck", "http://localhost:8080/health"] + interval: 15s + timeout: 5s + retries: 3 + + # ─── Runner (background task worker) ───────────────────────────────────────── + runner: + build: + context: ./backend + dockerfile: Dockerfile + target: runner + args: + VERSION: "${GIT_TAG:-dev}" + COMMIT: "${GIT_COMMIT:-unknown}" + restart: unless-stopped + stop_grace_period: 135s + depends_on: + pb-init: + condition: service_completed_successfully + pocketbase: + condition: service_healthy + minio: + condition: service_healthy + meilisearch: + condition: service_healthy + valkey: + condition: service_healthy + # Metrics endpoint — internal only; expose publicly via Caddy if needed. + expose: + - "9091" + environment: + <<: *infra-env + LOG_LEVEL: "${LOG_LEVEL:-info}" + # Runner tuning + RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL:-30s}" + RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE:-1}" + RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO:-1}" + RUNNER_WORKER_ID: "${RUNNER_WORKER_ID:-runner-1}" + RUNNER_TIMEOUT: "${RUNNER_TIMEOUT:-90s}" + RUNNER_METRICS_ADDR: "${RUNNER_METRICS_ADDR:-:9091}" + # Kokoro-FastAPI TTS endpoint + KOKORO_URL: "${KOKORO_URL:-}" + KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}" + healthcheck: + # The runner writes /tmp/runner.alive on every poll. + # 120s = 2× the default 30s poll interval with generous headroom. + test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"] + interval: 60s + timeout: 5s + retries: 3 + + # ─── SvelteKit UI ───────────────────────────────────────────────────────────── + ui: + build: + context: ./ui + dockerfile: Dockerfile + args: + BUILD_VERSION: "${GIT_TAG:-dev}" + BUILD_COMMIT: "${GIT_COMMIT:-unknown}" + restart: unless-stopped + stop_grace_period: 35s + depends_on: + pb-init: + condition: service_completed_successfully + backend: + condition: service_healthy + pocketbase: + condition: service_healthy + valkey: + condition: service_healthy + # No public port — all traffic via Caddy. + expose: + - "3000" + environment: + # ORIGIN must match the public URL Caddy serves on. + # adapter-node uses this for SvelteKit's built-in CSRF origin check. + ORIGIN: "${ORIGIN:-https://${DOMAIN:-localhost}}" + BACKEND_API_URL: "http://backend:8080" + POCKETBASE_URL: "http://pocketbase:8090" + POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" + POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}" + AUTH_SECRET: "${AUTH_SECRET:-dev_secret_change_in_production}" + PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}" + # Valkey + VALKEY_ADDR: "valkey:6379" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/health"] + interval: 15s + timeout: 5s + retries: 3 + + # ─── Caddy (reverse proxy + automatic HTTPS) ────────────────────────────────── + caddy: + image: caddy:2-alpine + restart: unless-stopped + depends_on: + backend: + condition: service_healthy + ui: + condition: service_healthy + ports: + - "80:80" + - "443:443" + - "443:443/udp" # HTTP/3 (QUIC) + environment: + DOMAIN: "${DOMAIN:-localhost}" + CADDY_ACME_EMAIL: "${CADDY_ACME_EMAIL:-}" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + +volumes: + minio_data: + pb_data: + meili_data: + valkey_data: + caddy_data: + caddy_config: diff --git a/v3/docs/api-endpoints.md b/v3/docs/api-endpoints.md new file mode 100644 index 0000000..dbba6b4 --- /dev/null +++ b/v3/docs/api-endpoints.md @@ -0,0 +1,82 @@ +# API Endpoint Reference + +All endpoints served by the Go **backend** binary on `:8080`. In production all +traffic is routed through Caddy — `/api/*` and `/health` are proxied to the +backend; everything else goes to the SvelteKit UI. + +## Health / Version + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `GET` | `/health` | — | Liveness probe. Returns `{"ok":true}`. | +| `GET` | `/api/version` | — | Build version + commit hash. | + +## Scrape Jobs (admin) + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `POST` | `/scrape` | admin | Enqueue full catalogue scrape. | +| `POST` | `/scrape/book` | admin | Enqueue single-book scrape `{url}`. | +| `POST` | `/scrape/book/range` | admin | Enqueue range scrape `{url, from, to?}`. | +| `GET` | `/api/scrape/status` | admin | Current job status. | +| `GET` | `/api/scrape/tasks` | admin | All scrape task records. | +| `POST` | `/api/cancel-task/{id}` | admin | Cancel a pending task. | + +## Browse / Catalogue + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `GET` | `/api/browse` | — | Live novelfire.net browse (MinIO page-1 cache). Legacy — used by save-browse subcommand. | +| `GET` | `/api/catalogue` | — | **Primary browse endpoint.** Meilisearch-backed, paginated. Params: `q`, `page`, `limit`, `genre`, `status`, `sort` (`popular`\|`new`\|`update`\|`rank`\|`top-rated`). Falls back to empty when Meilisearch is not configured. | +| `GET` | `/api/search` | — | Full-text search: Meilisearch local results merged with live novelfire.net remote results. Param: `q` (≥ 2 chars). Used by iOS app. | +| `GET` | `/api/ranking` | — | Top-ranked novels from PocketBase. | +| `GET` | `/api/cover/{domain}/{slug}` | — | Proxy cover image from MinIO (redirect to presigned URL). | + +## Book / Chapter Content + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `GET` | `/api/book-preview/{slug}` | — | Returns stored metadata + chapter list, or enqueues a scrape task (202) if unknown. | +| `GET` | `/api/chapter-text/{slug}/{n}` | — | Chapter content as plain text (markdown stripped). | +| `GET` | `/api/chapter-markdown/{slug}/{n}` | — | Chapter content as raw markdown from MinIO. | +| `POST` | `/api/reindex/{slug}` | admin | Rebuild `chapters_idx` from MinIO objects. | + +## Audio + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `POST` | `/api/audio/{slug}/{n}` | — | Trigger Kokoro TTS generation. Body: `{voice?}`. Returns `200 {status:"done"}` if cached, `202 {task_id, status}` if enqueued. | +| `GET` | `/api/audio/status/{slug}/{n}` | — | Poll audio generation status. Param: `voice`. Returns `{status, task_id?, error?}`. | +| `GET` | `/api/audio-proxy/{slug}/{n}` | — | Redirect to presigned MinIO audio URL. | +| `GET` | `/api/voices` | — | List available Kokoro voices. Returns `{voices:[]}` on error. | + +## Presigned URLs + +All presign endpoints return a `302` redirect to a short-lived MinIO presigned +URL. The URL is cached in Valkey (TTL ~55 min) to avoid regenerating on every +request. + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `GET` | `/api/presign/chapter/{slug}/{n}` | — | Presigned URL for chapter markdown object. | +| `GET` | `/api/presign/audio/{slug}/{n}` | — | Presigned URL for audio MP3. Param: `voice`. | +| `GET` | `/api/presign/voice-sample/{voice}` | — | Presigned URL for voice sample MP3. | +| `GET` | `/api/presign/avatar-upload/{userId}` | user | Presigned PUT URL for avatar upload. | +| `GET` | `/api/presign/avatar/{userId}` | — | Presigned GET URL for avatar image. | + +## Reading Progress + +Session-scoped (anonymous via cookie session ID, or tied to authenticated user). + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `GET` | `/api/progress` | — | Get all reading progress for the current session/user. | +| `POST` | `/api/progress/{slug}` | — | Set progress. Body: `{chapter}`. | +| `DELETE` | `/api/progress/{slug}` | — | Delete progress for a book. | + +## Notes + +- **Auth**: The backend does not enforce auth itself — the SvelteKit UI layer enforces admin/user guards before proxying requests. The backend trusts all incoming requests. +- **`/api/catalogue` vs `/api/browse`**: `/api/catalogue` is the primary UI endpoint (Meilisearch, always-local, fast). `/api/browse` hits or caches the live novelfire.net browse page and is only used internally by the `save-browse` subcommand. +- **Meilisearch fallback**: When `MEILI_URL` is unset, `/api/catalogue` returns `{books:[], has_next:false}` and `/api/search` falls back to a PocketBase substring scan. +- **`BACKEND_API_URL`**: The SvelteKit UI reads this env var (default `http://localhost:8080`) to reach the backend server-side. In docker-compose it is set to `http://backend:8080`. diff --git a/v3/docs/architecture.d2 b/v3/docs/architecture.d2 new file mode 100644 index 0000000..1e10009 --- /dev/null +++ b/v3/docs/architecture.d2 @@ -0,0 +1,129 @@ +direction: right + +# ─── External ───────────────────────────────────────────────────────────────── + +novelfire: novelfire.net { + shape: cloud + style.fill: "#f0f4ff" +} + +kokoro: Kokoro-FastAPI TTS { + shape: cloud + style.fill: "#f0f4ff" +} + +letsencrypt: Let's Encrypt { + shape: cloud + style.fill: "#f0f4ff" +} + +browser: Browser / iOS App { + shape: person + style.fill: "#fff9e6" +} + +# ─── Init containers (one-shot) ─────────────────────────────────────────────── + +init: Init containers { + style.fill: "#f5f5f5" + style.stroke-dash: 4 + + minio-init: minio-init { + shape: rectangle + label: "minio-init\n(mc: create buckets)" + } + + pb-init: pb-init { + shape: rectangle + label: "pb-init\n(bootstrap collections)" + } +} + +# ─── Storage ────────────────────────────────────────────────────────────────── + +storage: Storage { + style.fill: "#eaf7ea" + + minio: MinIO { + shape: cylinder + label: "MinIO :9000\n\nbuckets:\n libnovel-chapters\n libnovel-audio\n libnovel-avatars\n libnovel-browse" + } + + pocketbase: PocketBase { + shape: cylinder + label: "PocketBase :8090\n\ncollections:\n books chapters_idx\n audio_cache progress\n scrape_jobs app_users\n ranking" + } + + valkey: Valkey { + shape: cylinder + label: "Valkey :6379\n\n(presign URL cache\nTTL-based, shared)" + } + + meilisearch: Meilisearch { + shape: cylinder + label: "Meilisearch :7700\n\nindices:\n books" + } +} + +# ─── Application ────────────────────────────────────────────────────────────── + +app: Application { + style.fill: "#eef3ff" + + caddy: caddy { + shape: rectangle + label: "Caddy :443 / :80\n(reverse proxy\nauto-HTTPS via Let's Encrypt)" + } + + backend: backend { + shape: rectangle + label: "Backend API :8080\n(Go — HTTP API server)" + } + + runner: runner { + shape: rectangle + label: "Runner :9091\n(Go — background worker\nscraping + TTS jobs\n/metrics endpoint)" + } + + ui: ui { + shape: rectangle + label: "SvelteKit UI :3000\n(adapter-node)" + } +} + +# ─── Init → Storage deps ────────────────────────────────────────────────────── + +init.minio-init -> storage.minio: create buckets {style.stroke-dash: 4} +init.pb-init -> storage.pocketbase: bootstrap schema {style.stroke-dash: 4} + +# ─── App → Storage ──────────────────────────────────────────────────────────── + +app.backend -> storage.minio: blobs (chapters, audio,\navatars, browse) +app.backend -> storage.pocketbase: structured records\n(books, progress, jobs…) +app.backend -> storage.valkey: cache presigned URLs\n(SET/GET with TTL) + +app.runner -> storage.minio: write chapter markdown\n& audio MP3s +app.runner -> storage.pocketbase: read/update scrape jobs\nwrite book records +app.runner -> storage.meilisearch: index books on\nscrape completion + +app.ui -> storage.valkey: read presigned URL cache\n(replaces in-process Map) + +# ─── App internal ───────────────────────────────────────────────────────────── + +app.ui -> app.backend: REST API calls\n(server-side) + +# ─── Caddy routing ──────────────────────────────────────────────────────────── + +app.caddy -> app.ui: /* (proxy) +app.caddy -> app.backend: /api/*, /health (proxy) +app.caddy -> storage.minio: /s3/* (proxy, internal only) + +# ─── External → App ─────────────────────────────────────────────────────────── + +app.runner -> novelfire: scrape\n(HTTP GET) +app.runner -> kokoro: TTS generation\n(HTTP POST) +app.caddy -> letsencrypt: ACME certificate\n(TLS-ALPN-01) + +# ─── Browser ────────────────────────────────────────────────────────────────── + +browser -> app.caddy: HTTPS :443\n(single entry point) diff --git a/v3/docs/architecture.mermaid.md b/v3/docs/architecture.mermaid.md new file mode 100644 index 0000000..3e652e4 --- /dev/null +++ b/v3/docs/architecture.mermaid.md @@ -0,0 +1,59 @@ +```mermaid +graph LR + %% ── External ────────────────────────────────────────────────────────── + NF([novelfire.net]) + KK([Kokoro-FastAPI TTS]) + LE([Let's Encrypt]) + CL([Browser / iOS App]) + + %% ── Init containers ─────────────────────────────────────────────────── + subgraph INIT["Init containers (one-shot)"] + MI[minio-init\nmc: create buckets] + PI[pb-init\nbootstrap collections] + end + + %% ── Storage ─────────────────────────────────────────────────────────── + subgraph STORAGE["Storage"] + MN[(MinIO :9000\nchapters · audio\navatars · browse)] + PB[(PocketBase :8090\nbooks · chapters_idx\naudio_cache · progress\nscrape_jobs · app_users · ranking)] + VK[(Valkey :6379\npresign URL cache\nTTL-based · shared)] + MS[(Meilisearch :7700\nindex: books)] + end + + %% ── Application ─────────────────────────────────────────────────────── + subgraph APP["Application"] + CD[Caddy :443/:80\nreverse proxy\nauto-HTTPS] + BE[Backend API :8080\nGo HTTP server] + RN[Runner :9091\nGo background worker\n/metrics endpoint] + UI[SvelteKit UI :3000\nadapter-node] + end + + %% ── Init → Storage ──────────────────────────────────────────────────── + MI -.->|create buckets| MN + PI -.->|bootstrap schema| PB + + %% ── App → Storage ───────────────────────────────────────────────────── + BE -->|blobs| MN + BE -->|structured records| PB + BE -->|cache presigned URLs| VK + RN -->|chapter markdown & audio| MN + RN -->|read/update jobs & books| PB + RN -->|index books on scrape| MS + UI -->|read presign cache| VK + + %% ── App internal ────────────────────────────────────────────────────── + UI -->|REST API| BE + + %% ── Caddy routing ───────────────────────────────────────────────────── + CD -->|/* proxy| UI + CD -->|/api/* /health proxy| BE + CD -->|/s3/* proxy internal| MN + + %% ── Runner → External ───────────────────────────────────────────────── + RN -->|scrape HTTP GET| NF + RN -->|TTS HTTP POST| KK + CD -->|ACME certificate| LE + + %% ── Client ──────────────────────────────────────────────────────────── + CL -->|HTTPS :443 single entry| CD +``` diff --git a/v3/docs/architecture.svg b/v3/docs/architecture.svg new file mode 100644 index 0000000..55082b5 --- /dev/null +++ b/v3/docs/architecture.svg @@ -0,0 +1,125 @@ +novelfire.netKokoro-FastAPI TTSLet's EncryptBrowser / iOS AppInit containersStorageApplicationminio-init(mc: create buckets)pb-init(bootstrap collections)MinIO :9000 buckets: libnovel-chapters libnovel-audio libnovel-avatars libnovel-browsePocketBase :8090 collections: books chapters_idx audio_cache progress scrape_jobs app_users rankingValkey :6379 (presign URL cacheTTL-based, shared)Meilisearch :7700 indices: booksCaddy :443 / :80(reverse proxyauto-HTTPS via Let's Encrypt)Backend API :8080(Go — HTTP API server)Runner :9091(Go — background workerscraping + TTS jobs/metrics endpoint)SvelteKit UI :3000(adapter-node) create bucketsbootstrap schema blobs (chapters, audio,avatars, browse)structured records(books, progress, jobs…)cache presigned URLs(SET/GET with TTL)write chapter markdown& audio MP3sread/update scrape jobswrite book recordsindex books onscrape completionread presigned URL cache(replaces in-process Map)REST API calls(server-side)/* (proxy)/api/*, /health (proxy)/s3/* (proxy, internal only)scrape(HTTP GET)TTS generation(HTTP POST)ACME certificate(TLS-ALPN-01)HTTPS :443(single entry point) + + + + + + + + + + + + + + + + + + + diff --git a/v3/docs/data-flow.mermaid.md b/v3/docs/data-flow.mermaid.md new file mode 100644 index 0000000..33f1678 --- /dev/null +++ b/v3/docs/data-flow.mermaid.md @@ -0,0 +1,102 @@ +# Data Flow — Scrape & TTS Job Pipeline + +How content moves from novelfire.net through the runner into storage, and how +audio is generated on-demand via the backend. + +## Catalogue Scrape Pipeline + +The runner performs a background catalogue walk on startup and then on a +configurable interval (`RUNNER_CATALOGUE_REFRESH_INTERVAL`, default 24 h). + +```mermaid +flowchart TD + A([Runner starts / refresh tick]) --> B[Walk novelfire.net catalogue\npages 1…N] + B --> C{Book already\nin PocketBase?} + C -- no --> D[Scrape book metadata\ntitle · author · genres\ncover · summary · status] + C -- yes --> E[Check for new chapters\ncompare total_chapters] + D --> F[Write BookMeta\nto PocketBase books] + E --> G{New chapters\nfound?} + G -- no --> Z([Done — next book]) + G -- yes --> H + F --> H[Scrape chapter list\n→ chapters_idx in PocketBase] + H --> I[Worker pool — N goroutines\nRUNNER_MAX_CONCURRENT_SCRAPE] + I --> J[For each missing chapter:\nGET chapter HTML from novelfire.net] + J --> K[Parse HTML → Markdown\nhtmlutil.NodeToMarkdown] + K --> L[PUT object to MinIO\nlibnovel-chapters/{slug}/{n}.md] + L --> M[Upsert book doc\nto Meilisearch index: books] + M --> Z + F --> M +``` + +## On-Demand Single-Book Scrape + +Triggered when a user visits `/books/{slug}` and the book is not in PocketBase. +The UI calls `GET /api/book-preview/{slug}` → backend enqueues a task. + +```mermaid +sequenceDiagram + actor U as User + participant UI as SvelteKit UI + participant BE as Backend API + participant TQ as Task Queue (PocketBase) + participant RN as Runner + participant NF as novelfire.net + participant PB as PocketBase + participant MN as MinIO + participant MS as Meilisearch + + U->>UI: Visit /books/{slug} + UI->>BE: GET /api/book-preview/{slug} + BE->>PB: getBook(slug) — not found + BE->>TQ: INSERT scrape_task (slug, status=pending) + BE-->>UI: 202 {task_id, message} + UI-->>U: "Scraping…" placeholder + + RN->>TQ: Poll for pending tasks + TQ-->>RN: scrape_task (slug) + RN->>NF: GET novelfire.net/book/{slug} + NF-->>RN: HTML + RN->>PB: upsert book + chapters_idx + RN->>MN: PUT chapter objects + RN->>MS: UpsertBook doc + RN->>TQ: UPDATE task status=done + + U->>UI: Poll GET /api/scrape/tasks/{task_id} + UI->>BE: GET /api/scrape/status + BE->>TQ: get task + TQ-->>BE: status=done + BE-->>UI: {status:"done"} + UI-->>U: Redirect to /books/{slug} +``` + +## TTS Audio Generation Pipeline + +Audio is generated lazily: on first request the job is enqueued; subsequent +requests poll for completion and then stream from MinIO via presigned URL. + +```mermaid +flowchart TD + A([POST /api/audio/{slug}/{n}\nbody: voice=af_bella]) --> B{Audio already\nin MinIO?} + B -- yes --> C[200 status: done] + B -- no --> D{Job already\nin queue?} + D -- yes pending/generating --> E[202 task_id + status] + D -- no --> F[INSERT audio_task\nstatus=pending\nin PocketBase] + F --> E + + G([Runner polls task queue]) --> H[Claim audio_task\nstatus=generating] + H --> I[GET /api/chapter-text/{slug}/{n}\nfrom backend — plain text] + I --> J[POST /v1/audio/speech\nto Kokoro-FastAPI\nbody: text + voice] + J --> K[Stream MP3 response] + K --> L[PUT object to MinIO\nlibnovel-audio/{slug}/{n}/{voice}.mp3] + L --> M[UPDATE audio_task\nstatus=done] + + N([Client polls\nGET /api/audio/status/{slug}/{n}]) --> O{status?} + O -- pending/generating --> N + O -- done --> P[GET /api/presign/audio/{slug}/{n}] + P --> Q{Valkey cache hit?} + Q -- yes --> R[302 → presigned URL] + Q -- no --> S[GeneratePresignedURL\nfrom MinIO — TTL 1h] + S --> T[Cache in Valkey\nTTL 3500s] + T --> R + R --> U([Client streams audio\ndirectly from MinIO]) +``` diff --git a/v3/docs/request-flow.mermaid.md b/v3/docs/request-flow.mermaid.md new file mode 100644 index 0000000..af60b6b --- /dev/null +++ b/v3/docs/request-flow.mermaid.md @@ -0,0 +1,87 @@ +# Request Flow + +Two representative request paths through the stack: a **page load** (SSR) and a +**media playback** (presigned URL → direct MinIO stream). + +## SSR Page Load — Browse / Book Detail + +```mermaid +sequenceDiagram + actor C as Browser / iOS App + participant CD as Caddy :443 + participant UI as SvelteKit UI :3000 + participant BE as Backend API :8080 + participant MS as Meilisearch :7700 + participant PB as PocketBase :8090 + participant VK as Valkey :6379 + participant MN as MinIO :9000 + + C->>CD: HTTPS GET /browse + CD->>UI: proxy /* + UI->>BE: GET /api/catalogue?page=1&sort=popular + BE->>MS: search(query, filters, sort) + MS-->>BE: [{slug, title, …}, …] + BE-->>UI: {books[], page, total, has_next} + UI-->>CD: SSR HTML + CD-->>C: 200 HTML + + Note over C,UI: Infinite scroll — client fetches next page + C->>CD: HTTPS GET /api/browse-page?page=2 + CD->>UI: proxy (SvelteKit API route) + UI->>BE: GET /api/catalogue?page=2 + BE->>MS: search(…) + MS-->>BE: next page + BE-->>UI: {books[], …} + UI-->>C: JSON +``` + +## Audio Playback — Presigned URL Flow + +```mermaid +sequenceDiagram + actor C as Browser / iOS App + participant CD as Caddy :443 + participant UI as SvelteKit UI :3000 + participant BE as Backend API :8080 + participant VK as Valkey :6379 + participant MN as MinIO :9000 + + C->>CD: GET /api/presign/audio/{slug}/{n}?voice=af_bella + CD->>BE: proxy /api/* + BE->>VK: GET presign:audio:{slug}:{n}:{voice} + alt cache hit + VK-->>BE: presigned URL (TTL remaining) + BE-->>C: 302 redirect → presigned URL + else cache miss + BE->>MN: GeneratePresignedURL(audio-bucket, key, 1h) + MN-->>BE: presigned URL + BE->>VK: SET presign:audio:… EX 3500 + BE-->>C: 302 redirect → presigned URL + end + C->>MN: GET presigned URL (direct, no proxy) + MN-->>C: audio/mpeg stream +``` + +## Chapter Read — SSR + Content Fetch + +```mermaid +sequenceDiagram + actor C as Browser / iOS App + participant CD as Caddy :443 + participant UI as SvelteKit UI :3000 + participant BE as Backend API :8080 + participant PB as PocketBase :8090 + participant MN as MinIO :9000 + + C->>CD: HTTPS GET /books/{slug}/chapters/{n} + CD->>UI: proxy /* + UI->>PB: getBook(slug) + listChapterIdx(slug) + PB-->>UI: book meta + chapter list + UI->>BE: GET /api/chapter-markdown/{slug}/{n} + BE->>MN: GetObject(chapters-bucket, {slug}/{n}.md) + MN-->>BE: markdown text + BE-->>UI: markdown body + Note over UI: marked() → HTML + UI-->>CD: SSR HTML + CD-->>C: 200 HTML +``` diff --git a/v3/scripts/e2e-test.mjs b/v3/scripts/e2e-test.mjs new file mode 100644 index 0000000..f6d8f37 --- /dev/null +++ b/v3/scripts/e2e-test.mjs @@ -0,0 +1,423 @@ +#!/usr/bin/env node +/** + * e2e-test.mjs — End-to-end tests for the LibNovel v3 stack. + * + * Hits live services via https://localhost (self-signed cert, TLS verify skipped). + * Requires: Node 18+ (built-in fetch with TLS options via --experimental-fetch or + * native in Node 21+). Run with: node --experimental-vm-modules scripts/e2e-test.mjs + * or simply: node scripts/e2e-test.mjs + * + * Services tested: + * - Caddy / UI https://localhost + * - Go backend via UI proxy routes + * - PocketBase via UI server-side (indirect) + * + * Usage: + * node scripts/e2e-test.mjs + * node scripts/e2e-test.mjs --verbose + */ + +import { createServer } from 'node:https'; +import { request as httpRequest } from 'node:https'; +import { URL } from 'node:url'; + +const BASE = 'https://localhost'; +const VERBOSE = process.argv.includes('--verbose'); + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +let passed = 0; +let failed = 0; +const failures = []; + +function log(...args) { + if (VERBOSE) console.log(...args); +} + +function pass(name) { + passed++; + console.log(` ✓ ${name}`); +} + +function fail(name, reason) { + failed++; + const msg = ` ✗ ${name}: ${reason}`; + console.log(msg); + failures.push({ name, reason }); +} + +/** + * fetch() that ignores TLS certificate errors (self-signed cert on localhost). + */ +async function get(path, { headers = {}, followRedirects = false } = {}) { + const url = path.startsWith('http') ? path : `${BASE}${path}`; + const res = await fetch(url, { + redirect: followRedirects ? 'follow' : 'manual', + headers, + // Node 18/19 uses undici which respects NODE_TLS_REJECT_UNAUTHORIZED + }); + return res; +} + +async function post(path, body, { headers = {}, cookie = '' } = {}) { + const url = path.startsWith('http') ? path : `${BASE}${path}`; + const res = await fetch(url, { + method: 'POST', + redirect: 'manual', + headers: { + 'Content-Type': 'application/json', + ...(cookie ? { Cookie: cookie } : {}), + ...headers, + }, + body: JSON.stringify(body), + }); + return res; +} + +async function del(path, { cookie = '' } = {}) { + const url = path.startsWith('http') ? path : `${BASE}${path}`; + const res = await fetch(url, { + method: 'DELETE', + redirect: 'manual', + headers: cookie ? { Cookie: cookie } : {}, + }); + return res; +} + +/** Extract Set-Cookie header value(s) as a single cookie string. */ +function extractCookies(res) { + const raw = res.headers.getSetCookie?.() ?? []; + return raw.map((c) => c.split(';')[0]).join('; '); +} + +async function assert(name, fn) { + try { + await fn(); + pass(name); + } catch (e) { + fail(name, e.message); + } +} + +function expect(val, label) { + return { + toBe(expected) { + if (val !== expected) throw new Error(`${label}: expected ${expected}, got ${val}`); + }, + toBeOneOf(...options) { + if (!options.includes(val)) throw new Error(`${label}: expected one of [${options.join(', ')}], got ${val}`); + }, + toBeOk() { + if (!val) throw new Error(`${label} was falsy`); + }, + toContainKey(key) { + if (!(key in val)) throw new Error(`${label}: missing key "${key}"`); + }, + toBeArray() { + if (!Array.isArray(val)) throw new Error(`${label}: expected array, got ${typeof val}`); + }, + toBeAbove(n) { + if (!(val > n)) throw new Error(`${label}: expected > ${n}, got ${val}`); + }, + }; +} + +// ─── Test suite ─────────────────────────────────────────────────────────────── + +process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + +// Pick a known book slug from the database (first available) +let TEST_SLUG = null; + +console.log('\nLibNovel v3 — End-to-End Tests'); +console.log('================================\n'); + +// ── 1. Health checks ────────────────────────────────────────────────────────── +console.log('1. Health checks'); + +await assert('UI health endpoint returns 200', async () => { + const res = await get('/health'); + expect(res.status, 'status').toBe(200); +}); + +await assert('Home page returns 200', async () => { + const res = await get('/', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const html = await res.text(); + expect(html.includes(' { + const res = await get('/browse', { followRedirects: true }); + expect(res.status, 'status').toBe(200); +}); + +// ── 2. Home API ─────────────────────────────────────────────────────────────── +console.log('\n2. Home API'); + +let homeData = null; + +await assert('GET /api/home returns continue_reading and recently_updated', async () => { + const res = await get('/api/home', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + homeData = await res.json(); + expect(homeData, 'data').toContainKey('continue_reading'); + expect(homeData, 'data').toContainKey('recently_updated'); + expect(homeData.continue_reading, 'continue_reading').toBeArray(); + expect(homeData.recently_updated, 'recently_updated').toBeArray(); +}); + +await assert('GET /api/home stats has totalBooks and totalChapters', async () => { + if (!homeData) { + const res = await get('/api/home', { followRedirects: true }); + homeData = await res.json(); + } + expect(homeData, 'data').toContainKey('stats'); + expect(typeof homeData.stats.totalBooks, 'totalBooks type').toBe('number'); + expect(typeof homeData.stats.totalChapters, 'totalChapters type').toBe('number'); +}); + +// ── 3. Browse / ranking / search ────────────────────────────────────────────── +console.log('\n3. Browse / ranking / search'); + +await assert('GET /api/browse-page returns novels array', async () => { + const res = await get('/api/browse-page?page=1', { followRedirects: true }); + expect(res.status, 'status').toBeOneOf(200, 503); + if (res.status === 200) { + const data = await res.json(); + // Accepts { novels: [...] } or { error: ... } (when MinIO cache is empty) + expect(typeof data, 'response type').toBe('object'); + } +}); + +await assert('GET /api/ranking returns array or 502 (no data yet)', async () => { + const res = await get('/api/ranking', { followRedirects: true }); + // 200 = ranking data exists; 502 = no ranking data scraped yet — both are expected + expect(res.status, 'status').toBeOneOf(200, 502); + if (res.status === 200) { + const data = await res.json(); + expect(data, 'data').toBeArray(); + } +}); + +await assert('GET /api/search?q=shadow returns results object', async () => { + const res = await get('/api/search?q=shadow', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + expect(typeof data, 'response type').toBe('object'); +}); + +// ── 4. Books ────────────────────────────────────────────────────────────────── +console.log('\n4. Books'); + +// Find a real slug from /api/home +await assert('GET /api/home books are accessible', async () => { + if (!homeData) { + const res = await get('/api/home', { followRedirects: true }); + homeData = await res.json(); + } + const books = homeData?.recently_updated ?? []; + if (books.length > 0) { + TEST_SLUG = books[0].slug; + log(` Using test slug: ${TEST_SLUG}`); + } + // Pass regardless — we just want to find a slug +}); + +if (TEST_SLUG) { + await assert(`GET /api/book/${TEST_SLUG} returns book metadata`, async () => { + const res = await get(`/api/book/${TEST_SLUG}`, { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + // Returns { book: { slug, ... }, chapters: [...] } + expect(data, 'data').toContainKey('book'); + expect(data.book, 'book').toContainKey('slug'); + expect(data.book.slug, 'slug').toBe(TEST_SLUG); + }); + + await assert(`Book detail page /${TEST_SLUG} returns 200`, async () => { + const res = await get(`/${TEST_SLUG}`, { followRedirects: true }); + expect(res.status, 'status').toBe(200); + }); +} else { + console.log(' ⚠ No books in database — skipping book-specific tests'); +} + +// ── 5. Voices ───────────────────────────────────────────────────────────────── +console.log('\n5. Voices'); + +await assert('GET /api/voices returns voices array', async () => { + const res = await get('/api/voices', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + // Returns { voices: [...] } + expect(data, 'data').toContainKey('voices'); + expect(data.voices, 'voices').toBeArray(); + expect(data.voices.length, 'voice count').toBeAbove(0); +}); + +// ── 6. Auth flow ────────────────────────────────────────────────────────────── +console.log('\n6. Auth flow'); + +const TEST_USER = `e2e_test_${Date.now()}`; +const TEST_PASS = 'E2eTestPassword1!'; +let authCookie = ''; + +await assert('POST /api/auth/register creates new user', async () => { + const res = await post('/api/auth/register', { username: TEST_USER, password: TEST_PASS }); + expect(res.status, 'status').toBeOneOf(200, 201); + const data = await res.json(); + // Returns { token: "...", user: { id, username, role } } + expect(data, 'response').toContainKey('user'); + expect(data.user, 'user').toContainKey('username'); + expect(data.user.username, 'username').toBe(TEST_USER); + // Build cookie from token + if (data.token) { + authCookie = `libnovel_auth=${data.token}`; + } else { + authCookie = extractCookies(res); + } + log(` Auth cookie: ${authCookie.slice(0, 40)}...`); +}); + +await assert('GET /api/auth/me returns current user when logged in', async () => { + const res = await get('/api/auth/me', { followRedirects: true, headers: { Cookie: authCookie } }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + expect(data, 'data').toContainKey('username'); + expect(data.username, 'username').toBe(TEST_USER); +}); + +await assert('POST /api/auth/logout clears session', async () => { + const res = await post('/api/auth/logout', {}, { cookie: authCookie }); + expect(res.status, 'status').toBeOneOf(200, 204); +}); + +await assert('POST /api/auth/login works after register', async () => { + const res = await post('/api/auth/login', { username: TEST_USER, password: TEST_PASS }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + // Returns { token: "...", user: { id, username, role } } + expect(data, 'response').toContainKey('user'); + expect(data.user.username, 'username').toBe(TEST_USER); + if (data.token) { + authCookie = `libnovel_auth=${data.token}`; + } else { + authCookie = extractCookies(res); + } +}); + +await assert('GET /api/auth/me unauthenticated returns 401', async () => { + const res = await get('/api/auth/me', { followRedirects: true }); + expect(res.status, 'status').toBe(401); +}); + +// ── 7. Progress ─────────────────────────────────────────────────────────────── +console.log('\n7. Progress'); + +// /api/progress (root) is POST-only. Per-slug is GET/POST/DELETE via /api/progress/[slug]. + +if (TEST_SLUG) { + await assert(`POST /api/progress/${TEST_SLUG} sets progress`, async () => { + const res = await post(`/api/progress/${TEST_SLUG}`, { chapter: 1 }); + expect(res.status, 'status').toBeOneOf(200, 201); + const data = await res.json(); + expect(data, 'data').toContainKey('ok'); + }); + + await assert(`DELETE /api/progress/${TEST_SLUG} removes progress`, async () => { + const res = await del(`/api/progress/${TEST_SLUG}`); + expect(res.status, 'status').toBeOneOf(200, 204); + }); +} else { + console.log(' ⚠ No books — skipping progress tests'); +} + +// ── 8. Library ──────────────────────────────────────────────────────────────── +console.log('\n8. Library'); + +await assert('GET /api/library returns object', async () => { + const res = await get('/api/library', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + expect(typeof data, 'data type').toBe('object'); +}); + +if (TEST_SLUG) { + await assert(`POST /api/library/${TEST_SLUG} saves book`, async () => { + const res = await post(`/api/library/${TEST_SLUG}`, {}); + expect(res.status, 'status').toBeOneOf(200, 201); + }); + + await assert(`DELETE /api/library/${TEST_SLUG} removes book`, async () => { + const res = await del(`/api/library/${TEST_SLUG}`); + expect(res.status, 'status').toBeOneOf(200, 204); + }); +} + +// ── 9. Settings ─────────────────────────────────────────────────────────────── +console.log('\n9. Settings'); + +await assert('GET /api/settings returns settings object', async () => { + const res = await get('/api/settings', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + expect(typeof data, 'data type').toBe('object'); +}); + +// ── 10. Sessions ────────────────────────────────────────────────────────────── +console.log('\n10. Sessions'); + +await assert('GET /api/sessions (authenticated) returns sessions array', async () => { + const res = await get('/api/sessions', { followRedirects: true, headers: { Cookie: authCookie } }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + // Returns { sessions: [...] } + expect(data, 'data').toContainKey('sessions'); + expect(data.sessions, 'sessions').toBeArray(); +}); + +// ── 11. Comments ────────────────────────────────────────────────────────────── +console.log('\n11. Comments'); + +if (TEST_SLUG) { + await assert(`GET /api/comments/${TEST_SLUG} returns comments`, async () => { + const res = await get(`/api/comments/${TEST_SLUG}`, { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + // Returns { comments: [...], myVotes: {}, avatarUrls: {} } + expect(data, 'data').toContainKey('comments'); + expect(data.comments, 'comments').toBeArray(); + }); +} + +// ── 12. Chapter endpoints ───────────────────────────────────────────────────── +console.log('\n12. Chapter endpoints'); + +if (TEST_SLUG) { + await assert(`GET /api/chapter/${TEST_SLUG}/1 returns 200 or 404`, async () => { + const res = await get(`/api/chapter/${TEST_SLUG}/1`, { followRedirects: true }); + // 200 if chapter exists in MinIO, 404 if not + expect(res.status, 'status').toBeOneOf(200, 404); + }); + + await assert(`GET /api/chapter-text-preview/${TEST_SLUG}/1 returns 200 or error`, async () => { + const res = await get(`/api/chapter-text-preview/${TEST_SLUG}/1`, { followRedirects: true }); + expect(res.status, 'status').toBeOneOf(200, 404, 500, 503); + }); +} + +// ── Summary ─────────────────────────────────────────────────────────────────── +console.log('\n─────────────────────────────────────'); +console.log(`Results: ${passed} passed, ${failed} failed`); + +if (failures.length > 0) { + console.log('\nFailures:'); + for (const f of failures) { + console.log(` ✗ ${f.name}`); + console.log(` ${f.reason}`); + } +} + +console.log('─────────────────────────────────────\n'); +process.exit(failed > 0 ? 1 : 0); diff --git a/v3/scripts/pb-init-v3.sh b/v3/scripts/pb-init-v3.sh new file mode 100755 index 0000000..0abdf85 --- /dev/null +++ b/v3/scripts/pb-init-v3.sh @@ -0,0 +1,250 @@ +#!/bin/sh +# pb-init-v3.sh — idempotent PocketBase bootstrap for the v3 stack. +# +# Safe to re-run: existing collections and fields are silently skipped. +# +# Env vars (defaults match docker-compose.yml): +# POCKETBASE_URL http://pocketbase:8090 +# POCKETBASE_ADMIN_EMAIL admin@libnovel.local +# POCKETBASE_ADMIN_PASSWORD changeme123 + +set -e + +PB="${POCKETBASE_URL:-http://pocketbase:8090}" +EMAIL="${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" +PASS="${POCKETBASE_ADMIN_PASSWORD:-changeme123}" + +log() { printf '[pb-init] %s\n' "$*"; } + +# ── 0. Ensure dependencies ──────────────────────────────────────────────────── +command -v curl > /dev/null 2>&1 || apk add --no-cache curl > /dev/null 2>&1 +command -v python3 > /dev/null 2>&1 || apk add --no-cache python3 > /dev/null 2>&1 + +# ── 1. Wait for PocketBase ──────────────────────────────────────────────────── +log "waiting for PocketBase..." +until curl -sf "$PB/api/health" > /dev/null 2>&1; do sleep 2; done +log "PocketBase ready" + +# ── 2. Bootstrap superuser (first-run only) ─────────────────────────────────── +LOCATION=$(curl -sf -o /dev/null -w "%{redirect_url}" "$PB/_/" 2>/dev/null || true) +if echo "$LOCATION" | grep -q "pbinstal/"; then + TOKEN=$(echo "$LOCATION" | sed 's|.*pbinstal/||' | tr -d ' \r\n') + curl -sf -X POST "$PB/api/collections/_superusers/records" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d "{\"email\":\"$EMAIL\",\"password\":\"$PASS\",\"passwordConfirm\":\"$PASS\"}" \ + > /dev/null 2>&1 || true + log "superuser created" +fi + +# ── 3. Authenticate ─────────────────────────────────────────────────────────── +AUTH=$(curl -sf -X POST "$PB/api/collections/_superusers/auth-with-password" \ + -H "Content-Type: application/json" \ + -d "{\"identity\":\"$EMAIL\",\"password\":\"$PASS\"}") +TOK=$(echo "$AUTH" | sed 's/.*"token":"\([^"]*\)".*/\1/') +[ -z "$TOK" ] || [ "$TOK" = "$AUTH" ] && { log "ERROR: auth failed"; exit 1; } +log "authenticated" + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +# create NAME BODY — POST collection; 400/422 = already exists, treated as ok. +create() { + NAME="$1"; BODY="$2" + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST "$PB/api/collections" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOK" \ + -d "$BODY") + case "$STATUS" in + 200|201) log "created: $NAME" ;; + 400|422) log "exists (skip): $NAME" ;; + *) log "WARNING: $NAME returned $STATUS" ;; + esac +} + +# add_field COLLECTION FIELD_NAME FIELD_TYPE +# Fetches current schema, appends field if absent, PATCHes collection. +# Requires python3 for safe JSON manipulation. +add_field() { + COLL="$1"; FIELD="$2"; TYPE="$3" + SCHEMA=$(curl -sf -H "Authorization: Bearer $TOK" "$PB/api/collections/$COLL" 2>/dev/null) + # Check existence and extract collection id + fields via python3 + PARSED=$(echo "$SCHEMA" | python3 -c " +import sys, json +d = json.load(sys.stdin) +fields = d.get('fields', []) +exists = any(f.get('name') == '$FIELD' for f in fields) +print('exists=' + str(exists)) +print('id=' + d.get('id', '')) +if not exists: + fields.append({'name': '$FIELD', 'type': '$TYPE'}) + print('fields=' + json.dumps(fields)) +" 2>/dev/null) + if echo "$PARSED" | grep -q "^exists=True"; then + log "field exists (skip): $COLL.$FIELD"; return + fi + COLL_ID=$(echo "$PARSED" | grep "^id=" | sed 's/^id=//') + [ -z "$COLL_ID" ] && { log "WARNING: cannot resolve id for $COLL"; return; } + NEW_FIELDS=$(echo "$PARSED" | grep "^fields=" | sed 's/^fields=//') + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -X PATCH "$PB/api/collections/$COLL_ID" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOK" \ + -d "{\"fields\":${NEW_FIELDS}}") + case "$STATUS" in + 200|201) log "added field: $COLL.$FIELD ($TYPE)" ;; + *) log "WARNING: add_field $COLL.$FIELD returned $STATUS" ;; + esac +} + +# ── 4. Collections ──────────────────────────────────────────────────────────── + +create "books" '{ + "name":"books","type":"base","fields":[ + {"name":"slug", "type":"text", "required":true}, + {"name":"title", "type":"text", "required":true}, + {"name":"author", "type":"text"}, + {"name":"cover", "type":"text"}, + {"name":"status", "type":"text"}, + {"name":"genres", "type":"json"}, + {"name":"summary", "type":"text"}, + {"name":"total_chapters","type":"number"}, + {"name":"source_url", "type":"text"}, + {"name":"ranking", "type":"number"}, + {"name":"meta_updated", "type":"text"} + ]}' + +create "chapters_idx" '{ + "name":"chapters_idx","type":"base","fields":[ + {"name":"slug", "type":"text", "required":true}, + {"name":"number","type":"number", "required":true}, + {"name":"title", "type":"text"} + ]}' + +create "ranking" '{ + "name":"ranking","type":"base","fields":[ + {"name":"rank", "type":"number","required":true}, + {"name":"slug", "type":"text", "required":true}, + {"name":"title", "type":"text"}, + {"name":"author", "type":"text"}, + {"name":"cover", "type":"text"}, + {"name":"status", "type":"text"}, + {"name":"genres", "type":"json"}, + {"name":"source_url","type":"text"} + ]}' + +create "progress" '{ + "name":"progress","type":"base","fields":[ + {"name":"session_id","type":"text", "required":true}, + {"name":"slug", "type":"text", "required":true}, + {"name":"chapter", "type":"number"}, + {"name":"user_id", "type":"text"}, + {"name":"audio_time","type":"number"}, + {"name":"updated", "type":"text"} + ]}' + +create "scraping_tasks" '{ + "name":"scraping_tasks","type":"base","fields":[ + {"name":"kind", "type":"text"}, + {"name":"target_url", "type":"text"}, + {"name":"from_chapter", "type":"number"}, + {"name":"to_chapter", "type":"number"}, + {"name":"worker_id", "type":"text"}, + {"name":"status", "type":"text","required":true}, + {"name":"books_found", "type":"number"}, + {"name":"chapters_scraped", "type":"number"}, + {"name":"chapters_skipped", "type":"number"}, + {"name":"errors", "type":"number"}, + {"name":"error_message", "type":"text"}, + {"name":"started", "type":"date"}, + {"name":"finished", "type":"date"}, + {"name":"heartbeat_at", "type":"date"} + ]}' + +create "audio_jobs" '{ + "name":"audio_jobs","type":"base","fields":[ + {"name":"cache_key", "type":"text", "required":true}, + {"name":"slug", "type":"text", "required":true}, + {"name":"chapter", "type":"number","required":true}, + {"name":"voice", "type":"text"}, + {"name":"worker_id", "type":"text"}, + {"name":"status", "type":"text", "required":true}, + {"name":"error_message","type":"text"}, + {"name":"started", "type":"date"}, + {"name":"finished", "type":"date"}, + {"name":"heartbeat_at", "type":"date"} + ]}' + +create "app_users" '{ + "name":"app_users","type":"base","fields":[ + {"name":"username", "type":"text","required":true}, + {"name":"password_hash","type":"text"}, + {"name":"role", "type":"text"}, + {"name":"avatar_url", "type":"text"}, + {"name":"created", "type":"text"} + ]}' + +create "user_sessions" '{ + "name":"user_sessions","type":"base","fields":[ + {"name":"user_id", "type":"text","required":true}, + {"name":"session_id","type":"text","required":true}, + {"name":"user_agent","type":"text"}, + {"name":"ip", "type":"text"}, + {"name":"created_at","type":"text"}, + {"name":"last_seen", "type":"text"} + ]}' + +create "user_library" '{ + "name":"user_library","type":"base","fields":[ + {"name":"session_id","type":"text","required":true}, + {"name":"user_id", "type":"text"}, + {"name":"slug", "type":"text","required":true}, + {"name":"saved_at", "type":"text"} + ]}' + +create "user_settings" '{ + "name":"user_settings","type":"base","fields":[ + {"name":"session_id","type":"text","required":true}, + {"name":"user_id", "type":"text"}, + {"name":"auto_next","type":"bool"}, + {"name":"voice", "type":"text"}, + {"name":"speed", "type":"number"}, + {"name":"updated", "type":"text"} + ]}' + +create "user_subscriptions" '{ + "name":"user_subscriptions","type":"base","fields":[ + {"name":"follower_id","type":"text","required":true}, + {"name":"followee_id","type":"text","required":true}, + {"name":"created", "type":"text"} + ]}' + +create "book_comments" '{ + "name":"book_comments","type":"base","fields":[ + {"name":"slug", "type":"text","required":true}, + {"name":"user_id", "type":"text"}, + {"name":"username", "type":"text"}, + {"name":"body", "type":"text"}, + {"name":"upvotes", "type":"number"}, + {"name":"downvotes","type":"number"}, + {"name":"parent_id","type":"text"}, + {"name":"created", "type":"text"} + ]}' + +create "comment_votes" '{ + "name":"comment_votes","type":"base","fields":[ + {"name":"comment_id","type":"text","required":true}, + {"name":"user_id", "type":"text"}, + {"name":"session_id","type":"text"}, + {"name":"vote", "type":"text"} + ]}' + +# ── 5. Field migrations (idempotent — adds fields missing from older installs) ─ +add_field "scraping_tasks" "heartbeat_at" "date" +add_field "audio_jobs" "heartbeat_at" "date" +add_field "progress" "user_id" "text" +add_field "progress" "audio_time" "number" +add_field "progress" "updated" "text" +add_field "books" "meta_updated" "text" + +log "done" diff --git a/v3/ui/.dockerignore b/v3/ui/.dockerignore new file mode 100644 index 0000000..62e1cdc --- /dev/null +++ b/v3/ui/.dockerignore @@ -0,0 +1,5 @@ +node_modules +build +.svelte-kit +.env +.env.* diff --git a/v3/ui/.env.example b/v3/ui/.env.example new file mode 100644 index 0000000..9d92739 --- /dev/null +++ b/v3/ui/.env.example @@ -0,0 +1,20 @@ +# libnovel UI — environment variables +# Copy to .env and adjust; do NOT commit with real secrets. + +# Internal URL of the backend API (used by SvelteKit server-side load functions) +# In docker-compose this is the internal service name +BACKEND_API_URL=http://localhost:8080 + +# Public URL of PocketBase (used by SvelteKit server-side load functions) +POCKETBASE_URL=http://localhost:8090 + +# PocketBase admin credentials (server-side only, never exposed to browser) +POCKETBASE_ADMIN_EMAIL=admin@libnovel.local +POCKETBASE_ADMIN_PASSWORD=changeme123 + +# Public-facing MinIO URL (used to rewrite presigned URLs for the browser) +# In dev this is localhost; in prod set to your MinIO public domain +PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000 + +# Secret used to sign auth tokens stored in cookies (generate with: openssl rand -hex 32) +AUTH_SECRET=change_this_to_a_long_random_secret diff --git a/v3/ui/.gitignore b/v3/ui/.gitignore new file mode 100644 index 0000000..3b462cb --- /dev/null +++ b/v3/ui/.gitignore @@ -0,0 +1,23 @@ +node_modules + +# Output +.output +.vercel +.netlify +.wrangler +/.svelte-kit +/build + +# OS +.DS_Store +Thumbs.db + +# Env +.env +.env.* +!.env.example +!.env.test + +# Vite +vite.config.js.timestamp-* +vite.config.ts.timestamp-* diff --git a/v3/ui/.npmrc b/v3/ui/.npmrc new file mode 100644 index 0000000..b6f27f1 --- /dev/null +++ b/v3/ui/.npmrc @@ -0,0 +1 @@ +engine-strict=true diff --git a/v3/ui/AGENTS.md b/v3/ui/AGENTS.md new file mode 100644 index 0000000..5dc9be3 --- /dev/null +++ b/v3/ui/AGENTS.md @@ -0,0 +1,71 @@ +# LibNovel UI — Agent Context + +SvelteKit 2 + Svelte 5 frontend. Node adapter for production; served behind Caddy. + +## Design System + +**ACTIVE_STYLE: branded** + +Custom amber + zinc dark palette. No shadcn CLI defaults — primitives are hand-authored to match. + +| Token | Value | +|-------|-------| +| Accent | `#f59e0b` (amber-400) | +| Surface-1 | zinc-900 | +| Surface-2 | zinc-800 | +| Surface-3 | zinc-700 | +| Text-primary | zinc-100 | +| Text-secondary | zinc-400 | +| Destructive | red-400 | + +## Tailwind + +**Version: 4** — configured entirely via `@theme {}` in `src/app.css` and the `@tailwindcss/vite` plugin. + +**There is no `tailwind.config.ts`** — do not create one. +**Do not run `npx shadcn-svelte add ...`** — primitives are hand-authored in `$lib/components/ui/`. + +## shadcn-svelte Primitives + +All in `src/lib/components/ui/`: + +| Component | Variants / Notes | +|-----------|-----------------| +| `button` | default / secondary / outline / ghost / destructive / link; sizes: default / sm / lg / icon | +| `badge` | default / secondary / outline / destructive | +| `card` | Card, CardHeader, CardTitle, CardDescription, CardContent, CardFooter | +| `textarea` | bindable `value` prop | +| `dialog` | Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter | +| `separator` | horizontal / vertical | + +Always use `cn()` from `$lib/utils` — never template-literal class conditionals. + +## Svelte 5 Conventions + +- `@Observable` / runes (`$state`, `$derived`, `$effect`, `$props()`) for all new code. +- Do not add `ObservableObject` / `@Published` — they don't exist in Svelte; don't introduce legacy `writable` stores for new code. +- Navigation: `goto()` from `$app/navigation`; `page` from `$app/state`. + +## iOS/UX Skill + +For any view work, load the `ios-ux` skill at task start: +``` +skill({ name: "ios-ux" }) +``` + +## Key Files + +| File | Role | +|------|------| +| `src/app.css` | Tailwind v4 `@theme` tokens — source of truth for all design tokens | +| `src/lib/utils.ts` | `cn()` helper (clsx + tailwind-merge) | +| `src/lib/types.ts` | Shared client-safe domain types | +| `src/routes/+layout.svelte` | Root layout: sticky nav, persistent `
    +
    +
    + + + + Audio Narration +
    + + + {#if voices.length > 0} + + {/if} +
    + + + {#if showVoicePanel && voices.length > 0} +
    + {/if} + + {#if audioStore.isCurrentChapter(slug, chapter)} + + + {#if audioStore.status === 'idle' || audioStore.status === 'error'} + {#if audioStore.status === 'error'} +

    {audioStore.errorMsg || 'Failed to load audio.'}

    + {/if} + + + {:else if audioStore.status === 'loading'} + + + {:else if audioStore.status === 'generating'} +
    +

    Generating narration…

    +
    +
    +
    +

    {Math.round(audioStore.progress)}%

    +
    + + {:else if audioStore.status === 'ready'} + +
    +
    + {#if audioStore.isPlaying} + + + + Playing — controls below + {:else} + + + + Paused — controls below + {/if} + + {formatTime(audioStore.currentTime)} / {formatTime(audioStore.duration)} + +
    + + + {#if nextChapter !== null && nextChapter !== undefined} + + {/if} +
    + + + {#if audioStore.autoNext && nextChapter !== null && nextChapter !== undefined} +
    + {#if audioStore.nextStatus === 'prefetching'} +
    + + + + + Preparing Ch.{nextChapter}… {Math.round(audioStore.nextProgress)}% +
    + {:else if audioStore.nextStatus === 'prefetched'} +

    + + + + Ch.{nextChapter} ready +

    + {:else if audioStore.nextStatus === 'failed'} +

    Ch.{nextChapter} will generate on navigate

    + {/if} +
    + {/if} + {/if} + + {:else if audioStore.active} + +
    +

    + Now playing: {audioStore.chapterTitle || `Ch.${audioStore.chapter}`} +

    + +
    + + {:else} + + + {/if} +
    diff --git a/v3/ui/src/lib/components/AvatarCropModal.svelte b/v3/ui/src/lib/components/AvatarCropModal.svelte new file mode 100644 index 0000000..97e0b55 --- /dev/null +++ b/v3/ui/src/lib/components/AvatarCropModal.svelte @@ -0,0 +1,112 @@ + + + + + Crop profile picture + + + +
    +
    + Crop preview +
    +

    + Drag to reposition · pinch or scroll to zoom · drag corners to resize +

    +
    + + + + + +
    diff --git a/v3/ui/src/lib/components/CommentsSection.svelte b/v3/ui/src/lib/components/CommentsSection.svelte new file mode 100644 index 0000000..34316ff --- /dev/null +++ b/v3/ui/src/lib/components/CommentsSection.svelte @@ -0,0 +1,561 @@ + + +
    + +
    +

    + Comments + {#if !loading && totalCount > 0} + ({totalCount}) + {/if} +

    + + + {#if !loading && comments.length > 0} +
    + + +
    + {/if} +
    + + +
    + {#if isLoggedIn} +
    + diff --git a/v3/ui/src/lib/components/ui/textarea/index.ts b/v3/ui/src/lib/components/ui/textarea/index.ts new file mode 100644 index 0000000..069e310 --- /dev/null +++ b/v3/ui/src/lib/components/ui/textarea/index.ts @@ -0,0 +1 @@ +export { default as Textarea } from './Textarea.svelte'; diff --git a/v3/ui/src/lib/index.ts b/v3/ui/src/lib/index.ts new file mode 100644 index 0000000..856f2b6 --- /dev/null +++ b/v3/ui/src/lib/index.ts @@ -0,0 +1 @@ +// place files you want to import through the `$lib` alias in this folder. diff --git a/v3/ui/src/lib/server/logger.ts b/v3/ui/src/lib/server/logger.ts new file mode 100644 index 0000000..898559f --- /dev/null +++ b/v3/ui/src/lib/server/logger.ts @@ -0,0 +1,37 @@ +/** + * Structured server-side logger. + * + * Emits JSON lines to stderr so they appear in container/process logs without + * polluting stdout (which Node's HTTP layer uses for responses). + * + * Format mirrors Go's log/slog default JSON output: + * {"time":"…","level":"ERROR","msg":"…","context":"pocketbase",...extra} + * + * Usage: + * import { log } from '$lib/server/logger'; + * log.error('pocketbase', 'auth failed', { status: 401, url }); + * log.warn('minio', 'presign slow', { slug, n, ms: elapsed }); + * log.info('auth', 'user registered', { username }); + */ + +type Level = 'DEBUG' | 'INFO' | 'WARN' | 'ERROR'; +type Extra = Record; + +function emit(level: Level, context: string, msg: string, extra?: Extra): void { + const entry: Record = { + time: new Date().toISOString(), + level, + context, + msg, + ...extra + }; + // Write to stderr — never stdout + process.stderr.write(JSON.stringify(entry) + '\n'); +} + +export const log = { + debug: (context: string, msg: string, extra?: Extra) => emit('DEBUG', context, msg, extra), + info: (context: string, msg: string, extra?: Extra) => emit('INFO', context, msg, extra), + warn: (context: string, msg: string, extra?: Extra) => emit('WARN', context, msg, extra), + error: (context: string, msg: string, extra?: Extra) => emit('ERROR', context, msg, extra), +}; diff --git a/v3/ui/src/lib/server/minio.ts b/v3/ui/src/lib/server/minio.ts new file mode 100644 index 0000000..0dccca9 --- /dev/null +++ b/v3/ui/src/lib/server/minio.ts @@ -0,0 +1,184 @@ +/** + * Server-side MinIO presign helper. + * Calls the backend API to get presigned URLs, then optionally rewrites + * the MinIO host to the public-facing URL for browser use. + * + * Never import this from client-side code. + */ + +import { env as pubEnv } from '$env/dynamic/public'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +// Public MinIO URL — used to rewrite presigned URLs so the browser can reach MinIO directly. +// In docker-compose this would differ from the internal endpoint. +const MINIO_PUBLIC_URL = pubEnv.PUBLIC_MINIO_PUBLIC_URL ?? 'http://localhost:9000'; + +// ─── Avatar helpers ─────────────────────────────────────────────────────────── + +function extFromMime(mime: string): string { + if (mime.includes('png')) return 'png'; + if (mime.includes('webp')) return 'webp'; + if (mime.includes('gif')) return 'gif'; + return 'jpg'; +} + +/** + * Returns a short-lived presigned PUT URL for uploading an avatar directly to MinIO, + * along with the object key to record in PocketBase after upload completes. + * Routed through the Go backend which holds MinIO credentials. + */ +export async function presignAvatarUploadUrl(userId: string, mimeType: string): Promise<{ uploadUrl: string; key: string }> { + const ext = extFromMime(mimeType); + const res = await backendFetch(`/api/presign/avatar-upload/${encodeURIComponent(userId)}?ext=${ext}`); + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new Error(`presign avatar upload failed: ${res.status} ${body}`); + } + const data = (await res.json()) as { upload_url: string; key: string }; + return { uploadUrl: data.upload_url, key: data.key }; +} + +/** + * Returns a presigned GET URL for a user's avatar, rewritten to the public URL. + * Returns null if no avatar exists. + */ +export async function presignAvatarUrl(userId: string): Promise { + const res = await backendFetch(`/api/presign/avatar/${encodeURIComponent(userId)}`); + if (res.status === 404) return null; + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new Error(`presign avatar failed: ${res.status} ${body}`); + } + const data = (await res.json()) as { url: string }; + return data.url ?? null; +} + +/** + * Rewrites the MinIO host in a presigned URL to the public-facing URL. + * + * The Go backend presigns URLs against its internal endpoint (e.g. minio:9000) + * when PUBLIC_MINIO_PUBLIC_URL is not set or equals the internal endpoint. + * In that case the browser must reach MinIO via the public URL (e.g. + * localhost:9000 in dev), so we swap the origin. + * + * NOTE: AWS Signature V4 DOES include the Host header in the canonical request + * (via X-Amz-SignedHeaders=host). Rewriting the host here would break the + * signature. This function is therefore only a no-op safety net — in + * production the Go backend is configured with MINIO_PUBLIC_ENDPOINT equal to + * the externally-reachable hostname, so presigned URLs already carry the right + * host and no rewrite is needed. + * + * For local dev: MINIO_PUBLIC_ENDPOINT=http://localhost:9000 and the backend + * presigns with localhost:9000 (the public client), so this rewrite is again + * a no-op (origins already match). + */ +function rewriteHost(presignedUrl: string): string { + try { + const u = new URL(presignedUrl); + const pub = new URL(MINIO_PUBLIC_URL); + // No-op if already pointing at the right origin. + if (u.protocol === pub.protocol && u.hostname === pub.hostname && u.port === pub.port) { + return presignedUrl; + } + u.protocol = pub.protocol; + u.hostname = pub.hostname; + u.port = pub.port; + return u.toString(); + } catch { + return presignedUrl; + } +} + +/** + * Returns a presigned URL for a chapter markdown file. + * URL is valid for ~15 minutes (set by the scraper). + * + * @param rewrite - if true, rewrites the MinIO host to PUBLIC_MINIO_PUBLIC_URL + * (for browser use). Defaults to false — the server-side load function fetches + * the URL directly from the internal MinIO endpoint. + */ +export async function presignChapter(slug: string, n: number, rewrite = false): Promise { + log.debug('minio', 'presigning chapter', { slug, n }); + let res: Response; + try { + res = await backendFetch(`/api/presign/chapter/${slug}/${n}`); + } catch (e) { + log.error('minio', 'presign chapter network error', { slug, n, err: String(e) }); + throw new Error(`presign chapter ${slug}/${n}: network error`); + } + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('minio', 'presign chapter failed', { slug, n, status: res.status, body }); + throw new Error(`presign chapter ${slug}/${n}: ${res.status}`); + } + const data = (await res.json()) as { url: string }; + log.debug('minio', 'presign chapter ok', { slug, n }); + return rewrite ? rewriteHost(data.url) : data.url; +} + +/** + * Returns a presigned URL for a voice sample audio file. + * URL is valid for ~1 hour. The URL is returned to the browser for direct streaming. + * Throws with { status: 404 } when the sample has not been generated yet. + */ +export async function presignVoiceSample(voice: string): Promise { + log.debug('minio', 'presigning voice sample', { voice }); + let res: Response; + try { + res = await backendFetch(`/api/presign/voice-sample/${encodeURIComponent(voice)}`); + } catch (e) { + log.error('minio', 'presign voice sample network error', { voice, err: String(e) }); + throw new Error(`presign voice sample ${voice}: network error`); + } + if (res.status === 404) { + const err = new Error(`presign voice sample ${voice}: not found`) as Error & { status: number }; + err.status = 404; + throw err; + } + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('minio', 'presign voice sample failed', { voice, status: res.status, body }); + throw new Error(`presign voice sample ${voice}: ${res.status}`); + } + const data = (await res.json()) as { url: string }; + log.debug('minio', 'presign voice sample ok', { voice }); + return rewriteHost(data.url); +} + +/** + * Returns a presigned URL for an audio file. + * URL is valid for ~1 hour. The URL is returned to the browser for direct streaming. + * Throws with { status: 404 } when the audio object has not been generated yet. + */ +export async function presignAudio( + slug: string, + n: number, + voice?: string +): Promise { + const params = new URLSearchParams(); + if (voice) params.set('voice', voice); + const qs = params.toString() ? `?${params.toString()}` : ''; + log.debug('minio', 'presigning audio', { slug, n, voice }); + let res: Response; + try { + res = await backendFetch(`/api/presign/audio/${slug}/${n}${qs}`); + } catch (e) { + log.error('minio', 'presign audio network error', { slug, n, err: String(e) }); + throw new Error(`presign audio ${slug}/${n}: network error`); + } + if (res.status === 404) { + // Audio hasn't been generated / uploaded yet — caller should surface this as 404. + const err = new Error(`presign audio ${slug}/${n}: not found`) as Error & { status: number }; + err.status = 404; + throw err; + } + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('minio', 'presign audio failed', { slug, n, status: res.status, body }); + throw new Error(`presign audio ${slug}/${n}: ${res.status}`); + } + const data = (await res.json()) as { url: string }; + log.debug('minio', 'presign audio ok', { slug, n }); + return rewriteHost(data.url); +} diff --git a/v3/ui/src/lib/server/pocketbase.ts b/v3/ui/src/lib/server/pocketbase.ts new file mode 100644 index 0000000..7f1897d --- /dev/null +++ b/v3/ui/src/lib/server/pocketbase.ts @@ -0,0 +1,1345 @@ +/** + * Server-side PocketBase client. + * Uses admin credentials — never import this from client-side code. + * All methods talk directly to PocketBase REST API. + */ + +import { env } from '$env/dynamic/private'; +import { log } from '$lib/server/logger'; + +const PB_URL = env.POCKETBASE_URL ?? 'http://localhost:8090'; +const PB_EMAIL = env.POCKETBASE_ADMIN_EMAIL ?? 'admin@libnovel.local'; +const PB_PASSWORD = env.POCKETBASE_ADMIN_PASSWORD ?? 'changeme123'; + +// ─── Types ──────────────────────────────────────────────────────────────────── + +export interface Book { + id: string; + slug: string; + title: string; + author: string; + cover: string; + status: string; + genres: string[] | string; + summary: string; + total_chapters: number; + source_url: string; + ranking: number; + meta_updated: string; +} + +export interface ChapterIdx { + id: string; + slug: string; + number: number; + title: string; + date_label: string; +} + +export interface Progress { + id?: string; + session_id: string; + user_id?: string; + slug: string; + chapter: number; + audio_time?: number; + updated: string; +} + +export interface UserSettings { + id?: string; + session_id: string; + user_id?: string; + auto_next: boolean; + voice: string; + speed: number; + updated?: string; +} + +export interface User { + id: string; + username: string; + password_hash: string; + role: string; + created: string; + avatar_url?: string; +} + +// ─── Auth token cache ───────────────────────────────────────────────────────── + +let _token = ''; +let _tokenExp = 0; + +async function getToken(): Promise { + if (_token && Date.now() < _tokenExp) return _token; + + log.debug('pocketbase', 'authenticating with admin credentials', { url: PB_URL, email: PB_EMAIL }); + + const res = await fetch(`${PB_URL}/api/collections/_superusers/auth-with-password`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ identity: PB_EMAIL, password: PB_PASSWORD }) + }); + + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'admin auth failed', { status: res.status, url: PB_URL, body }); + throw new Error(`PocketBase auth failed: ${res.status} — ${body}`); + } + + const data = await res.json(); + _token = data.token as string; + _tokenExp = Date.now() + 12 * 60 * 60 * 1000; // 12 hours + log.info('pocketbase', 'admin auth token refreshed', { url: PB_URL }); + return _token; +} + +// ─── Generic helpers ────────────────────────────────────────────────────────── + +async function pbGet(path: string): Promise { + const token = await getToken(); + const res = await fetch(`${PB_URL}${path}`, { + headers: { Authorization: `Bearer ${token}` } + }); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'GET failed', { path, status: res.status, body }); + throw new Error(`PocketBase GET ${path} failed: ${res.status} — ${body}`); + } + return res.json() as Promise; +} + +async function pbPost(path: string, body: unknown): Promise { + const token = await getToken(); + return fetch(`${PB_URL}${path}`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); +} + +async function pbPatch(path: string, body: unknown): Promise { + const token = await getToken(); + return fetch(`${PB_URL}${path}`, { + method: 'PATCH', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); +} + +async function pbDelete(path: string): Promise { + const token = await getToken(); + return fetch(`${PB_URL}${path}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` } + }); +} + +interface PBList { + items: T[]; + totalItems: number; +} + +async function listAll(collection: string, filter = '', sort = ''): Promise { + const perPage = 500; + const params = new URLSearchParams({ perPage: String(perPage), page: '1' }); + if (filter) params.set('filter', filter); + if (sort) params.set('sort', sort); + + const first = await pbGet>( + `/api/collections/${collection}/records?${params.toString()}` + ); + const items: T[] = first.items ?? []; + const total = first.totalItems ?? 0; + + // Fetch remaining pages if there are more records than the first page holds. + const totalPages = Math.ceil(total / perPage); + for (let page = 2; page <= totalPages; page++) { + params.set('page', String(page)); + const data = await pbGet>( + `/api/collections/${collection}/records?${params.toString()}` + ); + items.push(...(data.items ?? [])); + } + + return items; +} + +async function listN(collection: string, n: number, filter = '', sort = ''): Promise { + const params = new URLSearchParams({ perPage: String(n) }); + if (filter) params.set('filter', filter); + if (sort) params.set('sort', sort); + const data = await pbGet>( + `/api/collections/${collection}/records?${params.toString()}` + ); + return data.items ?? []; +} + +async function countCollection(collection: string, filter = ''): Promise { + const params = new URLSearchParams({ perPage: '1' }); + if (filter) params.set('filter', filter); + const data = await pbGet>( + `/api/collections/${collection}/records?${params.toString()}` + ); + return (data as { totalItems: number }).totalItems ?? 0; +} + +async function listOne(collection: string, filter: string): Promise { + const params = new URLSearchParams({ perPage: '1', filter }); + const data = await pbGet>( + `/api/collections/${collection}/records?${params.toString()}` + ); + return data.items[0] ?? null; +} + +// ─── Books ──────────────────────────────────────────────────────────────────── + +export async function listBooks(): Promise { + const books = await listAll('books', '', '+title'); + const nullTitles = books.filter((b) => b.title == null).length; + if (nullTitles > 0) { + log.warn('pocketbase', 'listBooks: books with null title', { count: nullTitles, total: books.length }); + } + log.debug('pocketbase', 'listBooks', { total: books.length, nullTitles }); + return books; +} + +export async function getBook(slug: string): Promise { + return listOne('books', `slug="${slug}"`); +} + +export async function recentlyAddedBooks(limit = 6): Promise { + return listN('books', limit, '', '-meta_updated'); +} + +export interface HomeStats { + totalBooks: number; + totalChapters: number; +} + +export async function getHomeStats(): Promise { + const [totalBooks, totalChapters] = await Promise.all([ + countCollection('books'), + countCollection('chapters_idx') + ]); + return { totalBooks, totalChapters }; +} + +// ─── Chapter index ──────────────────────────────────────────────────────────── + +export async function listChapterIdx(slug: string): Promise { + return listAll('chapters_idx', `slug="${slug}"`, '+number'); +} + +// ─── Reading progress ───────────────────────────────────────────────────────── + +/** + * Build the PocketBase filter string for a progress lookup. + * When userId is set, keyed by user_id (portable across devices). + * When only sessionId is set, keyed by session_id (anonymous). + */ +function progressFilter(sessionId: string, slug: string, userId?: string): string { + if (userId) return `user_id="${userId}"&&slug="${slug}"`; + return `session_id="${sessionId}"&&slug="${slug}"`; +} + +function allProgressFilter(sessionId: string, userId?: string): string { + if (userId) return `user_id="${userId}"`; + return `session_id="${sessionId}"`; +} + +export async function getProgress( + sessionId: string, + slug: string, + userId?: string +): Promise { + return listOne('progress', progressFilter(sessionId, slug, userId)); +} + +export async function allProgress(sessionId: string, userId?: string): Promise { + return listAll('progress', allProgressFilter(sessionId, userId), '-updated'); +} + +export async function setProgress( + sessionId: string, + slug: string, + chapter: number, + userId?: string +): Promise { + const existing = await listOne( + 'progress', + progressFilter(sessionId, slug, userId) + ); + + const payload: Partial = { + session_id: sessionId, + slug, + chapter, + updated: new Date().toISOString() + }; + if (userId) payload.user_id = userId; + + if (existing) { + const res = await pbPatch(`/api/collections/progress/records/${existing.id}`, payload); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'setProgress PATCH failed', { slug, chapter, status: res.status, body }); + } + } else { + const res = await pbPost('/api/collections/progress/records', payload); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'setProgress POST failed', { slug, chapter, status: res.status, body }); + } + } +} + +/** + * Delete progress entry for a specific book (removes from library/continue reading). + */ +export async function deleteProgress( + sessionId: string, + slug: string, + userId?: string +): Promise { + const existing = await listOne( + 'progress', + progressFilter(sessionId, slug, userId) + ); + + if (!existing) { + log.debug('pocketbase', 'deleteProgress: no record found', { sessionId, slug, userId }); + return; + } + + const res = await pbDelete(`/api/collections/progress/records/${existing.id}`); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'deleteProgress failed', { + slug, + id: existing.id, + status: res.status, + body + }); + throw new Error(`Failed to delete progress: ${res.status}`); + } + log.info('pocketbase', 'deleteProgress success', { slug, id: existing.id }); +} + +/** + * Merge anonymous session progress into a user account on login/register. + * + * For each book tracked under sessionId, upserts a user-keyed record keeping + * whichever chapter is more recent (or higher if timestamps are equal). + * This makes progress portable across devices for logged-in users. + */ +export async function mergeSessionProgress(sessionId: string, userId: string): Promise { + let sessionRows: Progress[]; + try { + sessionRows = await allProgress(sessionId); + } catch (e) { + log.warn('pocketbase', 'mergeSessionProgress: failed to read session progress', { + sessionId, + err: String(e) + }); + return; + } + if (sessionRows.length === 0) return; + + for (const row of sessionRows) { + try { + const userRow = await listOne( + 'progress', + `user_id="${userId}"&&slug="${row.slug}"` + ); + + // Keep the record with the more recent update (or higher chapter if timestamps match) + const sessionTs = row.updated ? new Date(row.updated).getTime() : 0; + const userTs = userRow?.updated ? new Date(userRow.updated).getTime() : 0; + const shouldOverwrite = !userRow || sessionTs > userTs || + (sessionTs === userTs && row.chapter > (userRow?.chapter ?? 0)); + + if (shouldOverwrite) { + const payload: Partial = { + session_id: sessionId, + user_id: userId, + slug: row.slug, + chapter: row.chapter, + updated: row.updated ?? new Date().toISOString() + }; + if (userRow) { + await pbPatch(`/api/collections/progress/records/${userRow.id}`, payload); + } else { + await pbPost('/api/collections/progress/records', payload); + } + } + } catch (e) { + log.warn('pocketbase', 'mergeSessionProgress: failed to merge row', { + slug: row.slug, + err: String(e) + }); + } + } + log.info('pocketbase', 'mergeSessionProgress: done', { sessionId, userId, count: sessionRows.length }); +} + +// ─── User library (saved books) ─────────────────────────────────────────────── + +export interface UserLibraryEntry { + id?: string; + session_id: string; + user_id?: string; + slug: string; + saved_at: string; +} + +function libraryFilter(sessionId: string, userId?: string): string { + if (userId) return `user_id="${userId}"`; + return `session_id="${sessionId}"`; +} + +/** Returns all slugs the user has explicitly saved to their library. */ +export async function getSavedSlugs(sessionId: string, userId?: string): Promise> { + const rows = await listAll( + 'user_library', + libraryFilter(sessionId, userId) + ); + return new Set(rows.map((r) => r.slug)); +} + +/** Returns whether a specific slug is saved. */ +export async function isBookSaved( + sessionId: string, + slug: string, + userId?: string +): Promise { + const filter = userId + ? `user_id="${userId}"&&slug="${slug}"` + : `session_id="${sessionId}"&&slug="${slug}"`; + const row = await listOne('user_library', filter); + return row !== null; +} + +/** Save a book to the user's library. No-op if already saved. */ +export async function saveBook( + sessionId: string, + slug: string, + userId?: string +): Promise { + const alreadySaved = await isBookSaved(sessionId, slug, userId); + if (alreadySaved) return; + const payload: Partial = { + session_id: sessionId, + slug, + saved_at: new Date().toISOString() + }; + if (userId) payload.user_id = userId; + const res = await pbPost('/api/collections/user_library/records', payload); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'saveBook POST failed', { slug, status: res.status, body }); + } +} + +/** Remove a book from the user's library. */ +export async function unsaveBook( + sessionId: string, + slug: string, + userId?: string +): Promise { + const filter = userId + ? `user_id="${userId}"&&slug="${slug}"` + : `session_id="${sessionId}"&&slug="${slug}"`; + const row = await listOne('user_library', filter); + if (!row) return; + const token = await getToken(); + await fetch(`${PB_URL}/api/collections/user_library/records/${row.id}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` } + }); +} + +// ─── Users ──────────────────────────────────────────────────────────────────── + +import { scryptSync, randomBytes, timingSafeEqual } from 'node:crypto'; + +function hashPassword(password: string): string { + const salt = randomBytes(16).toString('hex'); + const hash = scryptSync(password, salt, 64).toString('hex'); + return `${salt}:${hash}`; +} + +function verifyPassword(password: string, stored: string): boolean { + const [salt, hash] = stored.split(':'); + if (!salt || !hash) return false; + const derived = scryptSync(password, salt, 64); + const hashBuf = Buffer.from(hash, 'hex'); + if (derived.length !== hashBuf.length) return false; + return timingSafeEqual(derived, hashBuf); +} + +/** + * Look up a user by username. Returns null if not found. + */ +export async function getUserByUsername(username: string): Promise { + return listOne('app_users', `username="${username.replace(/"/g, '\\"')}"`); +} + +/** + * Create a new user with a hashed password. Throws if username already exists. + */ +export async function createUser(username: string, password: string, role = 'user'): Promise { + log.info('pocketbase', 'createUser: checking for existing username', { username }); + const existing = await getUserByUsername(username); + if (existing) { + log.warn('pocketbase', 'createUser: username already taken', { username }); + throw new Error('Username already taken'); + } + const password_hash = hashPassword(password); + log.info('pocketbase', 'createUser: inserting new user', { username, role }); + const res = await pbPost('/api/collections/app_users/records', { + username, + password_hash, + role, + created: new Date().toISOString() + }); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'createUser: PocketBase rejected record', { + username, + status: res.status, + body + }); + throw new Error(`Failed to create user: ${res.status} ${body}`); + } + log.info('pocketbase', 'createUser: user created', { username, role }); + return res.json() as Promise; +} + +/** + * Change a user's password. Verifies the current password first. + * Returns true on success, false if currentPassword is wrong. + * Throws on unexpected errors. + */ +export async function changePassword( + userId: string, + currentPassword: string, + newPassword: string +): Promise { + // Fetch the user record directly by id to verify current password + const token = await getToken(); + const res = await fetch(`${PB_URL}/api/collections/app_users/records/${userId}`, { + headers: { Authorization: `Bearer ${token}` } + }); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'changePassword: fetch user failed', { userId, status: res.status, body }); + throw new Error(`Failed to fetch user: ${res.status}`); + } + const user = (await res.json()) as User; + if (!verifyPassword(currentPassword, user.password_hash)) { + log.warn('pocketbase', 'changePassword: wrong current password', { userId }); + return false; + } + const newHash = hashPassword(newPassword); + const patch = await pbPatch(`/api/collections/app_users/records/${userId}`, { + password_hash: newHash + }); + if (!patch.ok) { + const body = await patch.text().catch(() => ''); + log.error('pocketbase', 'changePassword: PATCH failed', { userId, status: patch.status, body }); + throw new Error(`Failed to update password: ${patch.status}`); + } + log.info('pocketbase', 'changePassword: success', { userId }); + return true; +} + +/** + * Verify username + password. Returns the user on success, null on failure. + */ +export async function loginUser(username: string, password: string): Promise { + log.debug('pocketbase', 'loginUser: lookup', { username }); + const user = await getUserByUsername(username); + if (!user) { + log.warn('pocketbase', 'loginUser: username not found', { username }); + return null; + } + const ok = verifyPassword(password, user.password_hash); + if (!ok) { + log.warn('pocketbase', 'loginUser: wrong password', { username }); + return null; + } + log.info('pocketbase', 'loginUser: success', { username, role: user.role }); + return user; +} + +// ─── User settings ──────────────────────────────────────────────────────────── + +function settingsFilter(sessionId: string, userId?: string): string { + if (userId) return `user_id="${userId}"`; + return `session_id="${sessionId}"`; +} + +export async function getSettings( + sessionId: string, + userId?: string +): Promise { + return listOne('user_settings', settingsFilter(sessionId, userId)); +} + +export async function saveSettings( + sessionId: string, + settings: { autoNext: boolean; voice: string; speed: number }, + userId?: string +): Promise { + const existing = await listOne( + 'user_settings', + settingsFilter(sessionId, userId) + ); + + const payload: Partial = { + session_id: sessionId, + auto_next: settings.autoNext, + voice: settings.voice, + speed: settings.speed, + updated: new Date().toISOString() + }; + if (userId) payload.user_id = userId; + + if (existing) { + const res = await pbPatch(`/api/collections/user_settings/records/${existing.id}`, payload); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'saveSettings PATCH failed', { status: res.status, body }); + } + } else { + const res = await pbPost('/api/collections/user_settings/records', payload); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'saveSettings POST failed', { status: res.status, body }); + } + } +} + +// ─── Audio time ─────────────────────────────────────────────────────────────── + +export async function setAudioTime( + sessionId: string, + slug: string, + chapter: number, + audioTime: number, + userId?: string +): Promise { + const existing = await listOne( + 'progress', + progressFilter(sessionId, slug, userId) + ); + if (!existing) { + // No progress record yet — create one with audio_time + const payload: Partial = { + session_id: sessionId, + slug, + chapter, + audio_time: audioTime, + updated: new Date().toISOString() + }; + if (userId) payload.user_id = userId; + const res = await pbPost('/api/collections/progress/records', payload); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'setAudioTime POST failed', { slug, chapter, status: res.status, body }); + } + return; + } + const res = await pbPatch(`/api/collections/progress/records/${existing.id}`, { + audio_time: audioTime, + updated: new Date().toISOString() + }); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'setAudioTime PATCH failed', { slug, chapter, status: res.status, body }); + } +} + +// ─── Audio cache ────────────────────────────────────────────────────────────── + +export interface AudioCacheEntry { + id: string; + cache_key: string; + filename: string; + updated: string; +} + +export async function listAudioCache(): Promise { + return listAll('audio_cache', '', '-updated'); +} + +// ─── Scraping tasks ─────────────────────────────────────────────────────────── + +export interface ScrapingTask { + id: string; + kind: string; + target_url: string; + status: string; + books_found: number; + chapters_scraped: number; + chapters_skipped: number; + errors: number; + started: string; + finished: string; + error_message: string; +} + +export async function listScrapingTasks(): Promise { + return listAll('scraping_tasks', '', '-started'); +} + +// ─── Audio jobs ─────────────────────────────────────────────────────────────── + +export interface AudioJob { + id: string; + cache_key: string; // "slug/chapter/voice" + slug: string; + chapter: number; + voice: string; + status: string; // "pending" | "generating" | "done" | "failed" + error_message: string; + started: string; + finished: string; +} + +export async function listAudioJobs(): Promise { + return listAll('audio_jobs', '', '-started'); +} + +export async function getAudioTime( + sessionId: string, + slug: string, + chapter: number, + userId?: string +): Promise { + const row = await listOne('progress', progressFilter(sessionId, slug, userId)); + if (!row || !row.audio_time) return null; + return row.audio_time; +} + +// ─── User sessions ──────────────────────────────────────────────────────────── + +export interface UserSession { + id: string; + user_id: string; + session_id: string; // the auth session ID embedded in the token + user_agent: string; + ip: string; + created_at: string; + last_seen: string; +} + +/** + * Create a new session record on login. Returns the record ID. + */ +export async function createUserSession( + userId: string, + authSessionId: string, + userAgent: string, + ip: string +): Promise { + const now = new Date().toISOString(); + const res = await pbPost('/api/collections/user_sessions/records', { + user_id: userId, + session_id: authSessionId, + user_agent: userAgent, + ip, + created_at: now, + last_seen: now + }); + if (!res.ok) { + const body = await res.text().catch(() => ''); + log.error('pocketbase', 'createUserSession POST failed', { userId, status: res.status, body }); + throw new Error(`Failed to create session: ${res.status}`); + } + const rec = (await res.json()) as { id: string }; + return rec.id; +} + +/** + * Update last_seen on a session (best-effort, non-fatal if it fails). + */ +export async function touchUserSession(authSessionId: string): Promise { + const row = await listOne( + 'user_sessions', + `session_id="${authSessionId}"` + ); + if (!row) return; + const token = await getToken(); + await fetch(`${PB_URL}/api/collections/user_sessions/records/${row.id}`, { + method: 'PATCH', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ last_seen: new Date().toISOString() }) + }); +} + +/** + * Check whether a session has been revoked (i.e., not present in DB). + * Returns true if revoked/missing, false if valid. + */ +export async function isSessionRevoked(authSessionId: string): Promise { + const row = await listOne('user_sessions', `session_id="${authSessionId}"`); + return row === null; +} + +/** + * List all active sessions for a user. + */ +export async function listUserSessions(userId: string): Promise { + return listAll('user_sessions', `user_id="${userId}"`, '-last_seen'); +} + +/** + * Revoke (delete) a specific session by its PocketBase record ID. + * Only allows deletion if the session belongs to the given userId. + */ +export async function revokeUserSession(recordId: string, userId: string): Promise { + // Verify ownership before deleting + const token = await getToken(); + const res = await fetch(`${PB_URL}/api/collections/user_sessions/records/${recordId}`, { + headers: { Authorization: `Bearer ${token}` } + }); + if (!res.ok) return false; + const rec = (await res.json()) as UserSession; + if (rec.user_id !== userId) return false; + + const del = await fetch(`${PB_URL}/api/collections/user_sessions/records/${recordId}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` } + }); + return del.ok || del.status === 204; +} + +/** + * Revoke all sessions for a user (used on password change etc). + */ +export async function revokeAllUserSessions(userId: string): Promise { + const sessions = await listUserSessions(userId); + const token = await getToken(); + await Promise.all( + sessions.map((s) => + fetch(`${PB_URL}/api/collections/user_sessions/records/${s.id}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` } + }).catch(() => {}) + ) + ); +} + +/** + * Update the avatar_url field for a user record. + */ +export async function updateUserAvatarUrl(userId: string, avatarUrl: string): Promise { + const token = await getToken(); + const res = await fetch(`${PB_URL}/api/collections/app_users/records/${userId}`, { + method: 'PATCH', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ avatar_url: avatarUrl }) + }); + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new Error(`updateUserAvatarUrl failed: ${res.status} ${body}`); + } +} + +// ─── Comments ───────────────────────────────────────────────────────────────── + +export interface BookComment { + id: string; + slug: string; + user_id: string; + username: string; + body: string; + upvotes: number; + downvotes: number; + created: string; + parent_id?: string; // empty / absent = top-level; set = reply +} + +export interface CommentVote { + id: string; + comment_id: string; + user_id: string; + session_id: string; + vote: 'up' | 'down'; +} + +export type CommentSort = 'top' | 'new'; + +/** + * List top-level comments for a book. + * sort='top' → by net score (upvotes − downvotes) desc, then newest + * sort='new' → newest first (default) + * Replies (parent_id != "") are NOT included — fetch them separately. + */ +export async function listComments( + slug: string, + sort: CommentSort = 'new' +): Promise { + const token = await getToken(); + const slugEsc = slug.replace(/"/g, '\\"'); + // Only top-level comments (parent_id is empty or missing) + const filter = encodeURIComponent(`slug="${slugEsc}"&&(parent_id=""||parent_id=null)`); + // PocketBase sorts: for 'top' we still fetch all and re-sort in JS because + // PocketBase doesn't support computed sort fields. For 'new' we push the + // sort down to the DB so large result sets are still paged correctly. + const pbSort = sort === 'new' ? '&sort=-created' : '&sort=-created'; + const res = await fetch( + `${PB_URL}/api/collections/book_comments/records?filter=${filter}${pbSort}&perPage=200`, + { headers: { Authorization: `Bearer ${token}` } } + ); + if (!res.ok) return []; + const data = await res.json(); + let items = (data.items ?? []) as BookComment[]; + if (sort === 'top') { + items = items.sort((a, b) => { + const scoreB = (b.upvotes ?? 0) - (b.downvotes ?? 0); + const scoreA = (a.upvotes ?? 0) - (a.downvotes ?? 0); + if (scoreB !== scoreA) return scoreB - scoreA; + // tie-break: newest first + return new Date(b.created).getTime() - new Date(a.created).getTime(); + }); + } + return items; +} + +/** + * List replies (1-level deep) for a single parent comment. + * Always sorted oldest-first so the conversation reads naturally. + */ +export async function listReplies(parentId: string): Promise { + const token = await getToken(); + const filter = encodeURIComponent(`parent_id="${parentId.replace(/"/g, '\\"')}"`); + const res = await fetch( + `${PB_URL}/api/collections/book_comments/records?filter=${filter}&sort=created&perPage=100`, + { headers: { Authorization: `Bearer ${token}` } } + ); + if (!res.ok) return []; + const data = await res.json(); + return (data.items ?? []) as BookComment[]; +} + +/** + * Create a new comment. Returns the created record. + * Pass parentId to create a reply; omit / pass undefined for a top-level comment. + */ +export async function createComment( + slug: string, + body: string, + userId: string | undefined, + username: string, + parentId?: string +): Promise { + const token = await getToken(); + const res = await fetch(`${PB_URL}/api/collections/book_comments/records`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + slug, + body, + user_id: userId ?? '', + username, + upvotes: 0, + downvotes: 0, + parent_id: parentId ?? '', + created: new Date().toISOString() + }) + }); + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new Error(`createComment failed: ${res.status} ${text}`); + } + return res.json() as Promise; +} + +/** + * Delete a comment (and optionally its replies) by ID. + * Only the comment owner (matched by userId) may delete. + * Throws if the comment doesn't exist or the user doesn't own it. + */ +export async function deleteComment(commentId: string, userId: string): Promise { + const token = await getToken(); + + // Fetch the comment to verify ownership + const getRes = await fetch(`${PB_URL}/api/collections/book_comments/records/${commentId}`, { + headers: { Authorization: `Bearer ${token}` } + }); + if (!getRes.ok) throw new Error(`Comment not found: ${commentId}`); + const comment = (await getRes.json()) as BookComment; + if (comment.user_id !== userId) throw new Error('Not authorized to delete this comment'); + + // Delete any replies first + const repliesFilter = encodeURIComponent(`parent_id="${commentId.replace(/"/g, '\\"')}"`); + const repliesRes = await fetch( + `${PB_URL}/api/collections/book_comments/records?filter=${repliesFilter}&perPage=100`, + { headers: { Authorization: `Bearer ${token}` } } + ); + if (repliesRes.ok) { + const repliesData = await repliesRes.json(); + const replies = (repliesData.items ?? []) as BookComment[]; + await Promise.all( + replies.map((r) => + fetch(`${PB_URL}/api/collections/book_comments/records/${r.id}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` } + }) + ) + ); + } + + // Delete the comment itself + const delRes = await fetch(`${PB_URL}/api/collections/book_comments/records/${commentId}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` } + }); + if (!delRes.ok) throw new Error(`deleteComment failed: ${delRes.status}`); +} + +/** + * Get an existing vote by this voter (identified by user_id or session_id) on a comment. + */ +export async function getCommentVote( + commentId: string, + sessionId: string, + userId?: string +): Promise { + const token = await getToken(); + const voterFilter = userId + ? `comment_id="${commentId}"&&user_id="${userId}"` + : `comment_id="${commentId}"&&session_id="${sessionId}"`; + const res = await fetch( + `${PB_URL}/api/collections/comment_votes/records?filter=${encodeURIComponent(voterFilter)}&perPage=1`, + { headers: { Authorization: `Bearer ${token}` } } + ); + if (!res.ok) return null; + const data = await res.json(); + const items = (data.items ?? []) as CommentVote[]; + return items[0] ?? null; +} + +/** + * Cast or change a vote on a comment. Handles: + * - New vote: creates vote record, increments counter. + * - Same vote again: removes it (toggle off), decrements counter. + * - Changed vote: updates record, adjusts both counters. + * Returns the updated comment. + */ +export async function voteComment( + commentId: string, + vote: 'up' | 'down', + sessionId: string, + userId?: string +): Promise { + const token = await getToken(); + + // Fetch current comment + const commentRes = await fetch(`${PB_URL}/api/collections/book_comments/records/${commentId}`, { + headers: { Authorization: `Bearer ${token}` } + }); + if (!commentRes.ok) throw new Error(`Comment not found: ${commentId}`); + const comment = (await commentRes.json()) as BookComment; + + const existing = await getCommentVote(commentId, sessionId, userId); + + let upDelta = 0; + let downDelta = 0; + + if (!existing) { + // New vote + await fetch(`${PB_URL}/api/collections/comment_votes/records`, { + method: 'POST', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ comment_id: commentId, user_id: userId ?? '', session_id: sessionId, vote }) + }); + vote === 'up' ? upDelta++ : downDelta++; + } else if (existing.vote === vote) { + // Toggle off — remove vote + await fetch(`${PB_URL}/api/collections/comment_votes/records/${existing.id}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` } + }); + vote === 'up' ? upDelta-- : downDelta--; + } else { + // Changed vote + await fetch(`${PB_URL}/api/collections/comment_votes/records/${existing.id}`, { + method: 'PATCH', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ vote }) + }); + if (vote === 'up') { upDelta++; downDelta--; } + else { upDelta--; downDelta++; } + } + + // Patch comment counters + const patchRes = await fetch(`${PB_URL}/api/collections/book_comments/records/${commentId}`, { + method: 'PATCH', + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + upvotes: Math.max(0, (comment.upvotes ?? 0) + upDelta), + downvotes: Math.max(0, (comment.downvotes ?? 0) + downDelta) + }) + }); + if (!patchRes.ok) throw new Error(`Failed to update vote counts on comment ${commentId}`); + return patchRes.json() as Promise; +} + +/** + * Fetch votes cast by this session/user, keyed by comment_id. + * Returns a map of commentId → 'up' | 'down'. + */ +export async function getMyVotes( + commentIds: string[], + sessionId: string, + userId?: string +): Promise> { + if (commentIds.length === 0) return {}; + const token = await getToken(); + const idFilter = commentIds.map((id) => `comment_id="${id}"`).join('||'); + const voterPart = userId ? `user_id="${userId}"` : `session_id="${sessionId}"`; + const filter = encodeURIComponent(`(${idFilter})&&${voterPart}`); + const res = await fetch( + `${PB_URL}/api/collections/comment_votes/records?filter=${filter}&perPage=200`, + { headers: { Authorization: `Bearer ${token}` } } + ); + if (!res.ok) return {}; + const data = await res.json(); + const map: Record = {}; + for (const v of (data.items ?? []) as CommentVote[]) { + map[v.comment_id] = v.vote as 'up' | 'down'; + } + return map; +} + +// ─── User subscriptions ─────────────────────────────────────────────────────── + +export interface UserSubscription { + id: string; + follower_id: string; + followee_id: string; + created: string; +} + +/** + * Returns the subscription record if follower_id follows followee_id, else null. + */ +export async function getSubscription( + followerId: string, + followeeId: string +): Promise { + const filter = encodeURIComponent(`follower_id="${followerId}"&&followee_id="${followeeId}"`); + const res = await pbGet<{ items: UserSubscription[]; totalItems: number }>( + `/api/collections/user_subscriptions/records?filter=${filter}&perPage=1` + ).catch(() => null); + return res?.items?.[0] ?? null; +} + +/** + * Subscribe follower_id to followee_id. No-ops if already subscribed. + * Returns the subscription record. + */ +export async function subscribe(followerId: string, followeeId: string): Promise { + const existing = await getSubscription(followerId, followeeId); + if (existing) return; + const res = await pbPost('/api/collections/user_subscriptions/records', { + follower_id: followerId, + followee_id: followeeId, + created: new Date().toISOString() + }); + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new Error(`Failed to subscribe: ${res.status} — ${body}`); + } +} + +/** + * Unsubscribe follower_id from followee_id. No-ops if not subscribed. + */ +export async function unsubscribe(followerId: string, followeeId: string): Promise { + const existing = await getSubscription(followerId, followeeId); + if (!existing) return; + const token = await getToken(); + await fetch(`${PB_URL}/api/collections/user_subscriptions/records/${existing.id}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${token}` } + }); +} + +/** + * Returns the list of user IDs that followerId is subscribed to. + */ +export async function getFollowingIds(followerId: string): Promise { + const items = await listAll( + 'user_subscriptions', + `follower_id="${followerId}"`, + '-created' + ).catch(() => [] as UserSubscription[]); + return items.map((s) => s.followee_id); +} + +/** + * Returns the count of subscribers (followers) for a given user. + */ +export async function getFollowerCount(followeeId: string): Promise { + return countCollection('user_subscriptions', `followee_id="${followeeId}"`).catch(() => 0); +} + +/** + * Returns the count of accounts a user is following. + */ +export async function getFollowingCount(followerId: string): Promise { + return countCollection('user_subscriptions', `follower_id="${followerId}"`).catch(() => 0); +} + +/** + * Public profile data for a user. + */ +export interface PublicProfile { + id: string; + username: string; + avatar_url?: string; + created: string; + followerCount: number; + followingCount: number; +} + +/** + * Returns a user's public profile (no sensitive fields) by username. + */ +export async function getPublicProfile(username: string): Promise { + const user = await getUserByUsername(username); + if (!user) return null; + const [followerCount, followingCount] = await Promise.all([ + getFollowerCount(user.id), + getFollowingCount(user.id) + ]); + return { + id: user.id, + username: user.username, + avatar_url: user.avatar_url, + created: user.created, + followerCount, + followingCount + }; +} + +/** + * Returns a user's public library: books they have saved or are reading. + * Only includes books with progress or explicit saves (user_library). + */ +export async function getUserPublicLibrary( + userId: string +): Promise> { + const [allBooks, progressList, savedEntries] = await Promise.all([ + listBooks(), + listAll('progress', `user_id="${userId}"`, '-updated').catch(() => [] as Progress[]), + listAll<{ id: string; slug: string; saved_at: string }>( + 'user_library', + `user_id="${userId}"`, + '-saved_at' + ).catch(() => [] as { id: string; slug: string; saved_at: string }[]) + ]); + + const bookMap = new Map(allBooks.map((b) => [b.slug, b])); + const result: Array<{ book: Book; chapter: number | null; saved: boolean }> = []; + const seen = new Set(); + + // Books with progress first (most recently read) + for (const p of progressList) { + const book = bookMap.get(p.slug); + if (!book || seen.has(p.slug)) continue; + seen.add(p.slug); + result.push({ book, chapter: p.chapter, saved: false }); + } + + // Saved-only books next + for (const e of savedEntries) { + const book = bookMap.get(e.slug); + if (!book || seen.has(e.slug)) continue; + seen.add(e.slug); + result.push({ book, chapter: null, saved: true }); + } + + // Mark saved flag for books that are both in progress AND saved + const savedSlugs = new Set(savedEntries.map((e) => e.slug)); + return result.map((r) => ({ ...r, saved: savedSlugs.has(r.book.slug) })); +} + +/** + * Returns the currently-reading books (books with progress, not completed) + * for a given user ID. + */ +export async function getUserCurrentlyReading( + userId: string +): Promise> { + const [allBooks, progressList] = await Promise.all([ + listBooks(), + listAll('progress', `user_id="${userId}"`, '-updated').catch(() => [] as Progress[]) + ]); + const bookMap = new Map(allBooks.map((b) => [b.slug, b])); + return progressList + .filter((p) => { + const book = bookMap.get(p.slug); + return book && p.chapter > 0 && p.chapter < book.total_chapters; + }) + .slice(0, 10) + .map((p) => ({ book: bookMap.get(p.slug)!, chapter: p.chapter })); +} + +/** + * Returns recently-updated books from ALL users that followerId is subscribed to. + * Deduplicates across followed users; sorts by most recently updated. + */ +export async function getSubscriptionFeed( + followerId: string, + limit = 12 +): Promise> { + const followingIds = await getFollowingIds(followerId); + if (followingIds.length === 0) return []; + + // Fetch all users we follow (for display names) + const token = await getToken(); + const userFetches = followingIds.map((id) => + fetch(`${PB_URL}/api/collections/app_users/records/${id}`, { + headers: { Authorization: `Bearer ${token}` } + }) + .then((r) => (r.ok ? (r.json() as Promise) : null)) + .catch(() => null) + ); + const users = (await Promise.all(userFetches)).filter(Boolean) as User[]; + const userMap = new Map(users.map((u) => [u.id, u])); + + // Fetch progress for each followed user + const progressFetches = followingIds.map((id) => + listAll('progress', `user_id="${id}"`, '-updated').catch(() => [] as Progress[]) + ); + const allProgressArrays = await Promise.all(progressFetches); + + const allBooks = await listBooks(); + const bookMap = new Map(allBooks.map((b) => [b.slug, b])); + + // Merge: per slug take the most-recent progress entry + const seen = new Set(); + const feed: Array<{ book: Book; readerUsername: string; updated: string }> = []; + + for (let i = 0; i < followingIds.length; i++) { + const uid = followingIds[i]; + const username = userMap.get(uid)?.username ?? 'unknown'; + for (const p of allProgressArrays[i]) { + if (seen.has(p.slug)) continue; + const book = bookMap.get(p.slug); + if (!book) continue; + seen.add(p.slug); + feed.push({ book, readerUsername: username, updated: p.updated }); + } + } + + // Sort by most recently read across all followed users + feed.sort((a, b) => b.updated.localeCompare(a.updated)); + return feed.slice(0, limit).map(({ book, readerUsername }) => ({ book, readerUsername })); +} diff --git a/v3/ui/src/lib/server/presignCache.ts b/v3/ui/src/lib/server/presignCache.ts new file mode 100644 index 0000000..23f3f3b --- /dev/null +++ b/v3/ui/src/lib/server/presignCache.ts @@ -0,0 +1,118 @@ +/** + * Valkey-backed presign URL cache (v3). + * + * Replaces the in-process Map from v2. All presign URLs are stored in Valkey + * (Redis-compatible) with native TTL, so: + * - Cache survives UI process restarts. + * - Cache is shared across multiple UI replicas (if scaled horizontally). + * - No manual sweep timer needed — Valkey expires entries automatically. + * + * MinIO presigned audio URLs are valid for 1 hour (set by the backend). + * We cache them for 50 minutes so the browser always gets a URL with at + * least 10 minutes of remaining validity. + * + * Voice-sample URLs use the same cache with key "sample:". + * + * Connection: + * VALKEY_URL env var (default: redis://valkey:6379) + * ioredis handles reconnection automatically. + */ + +import Redis from 'ioredis'; + +const AUDIO_TTL_S = 50 * 60; // 50 minutes in seconds (Valkey TTL is in seconds) + +// Lazily-initialised singleton client. +let _client: Redis | null = null; + +function client(): Redis { + if (!_client) { + const url = process.env.VALKEY_URL ?? 'redis://valkey:6379'; + _client = new Redis(url, { + // Reconnect automatically with exponential backoff (ioredis default). + // lazyConnect: false means the connection is established immediately. + lazyConnect: false, + // Log connection errors to stderr; do not crash the process. + enableOfflineQueue: true, + maxRetriesPerRequest: 2, + }); + _client.on('error', (err: Error) => { + console.error('[presignCache] Valkey error:', err.message); + }); + } + return _client; +} + +// ── Key helpers ─────────────────────────────────────────────────────────────── + +/** Cache key for a chapter audio presigned URL. */ +export function audioKey(slug: string, n: number, voice: string): string { + return `audio:${slug}:${n}:${voice}`; +} + +/** Cache key for a voice-sample presigned URL. */ +export function sampleKey(voice: string): string { + return `sample:${voice}`; +} + +// ── Public API ──────────────────────────────────────────────────────────────── + +/** Return the cached URL for key, or null if absent / expired. */ +export async function get(key: string): Promise { + try { + return await client().get(key); + } catch (err) { + console.error('[presignCache] get error:', err); + return null; + } +} + +/** Store a presigned URL under key for ttlSeconds seconds. */ +export async function set(key: string, url: string, ttlSeconds = AUDIO_TTL_S): Promise { + try { + await client().set(key, url, 'EX', ttlSeconds); + } catch (err) { + console.error('[presignCache] set error:', err); + } +} + +/** Invalidate a specific key (e.g. after audio generation to force refresh). */ +export async function invalidate(key: string): Promise { + try { + await client().del(key); + } catch (err) { + console.error('[presignCache] invalidate error:', err); + } +} + +/** + * Disconnect from Valkey — called on graceful shutdown. + * ioredis queues commands during reconnects; calling quit() drains the queue + * and closes the connection cleanly. + */ +export async function drain(): Promise { + if (_client) { + try { + await _client.quit(); + } catch { + // ignore — process is exiting anyway + } + _client = null; + } +} + +/** + * Returns the approximate number of keys matching the libnovel presign prefix. + * Used for health/debug only — not called in the hot path. + */ +export async function size(): Promise { + try { + // DBSIZE returns the total key count in the current DB. + // For a precise count of just our keys, use SCAN with a pattern. + const keys = await client().keys('audio:*'); + const sampleKeys = await client().keys('sample:*'); + return keys.length + sampleKeys.length; + } catch { + return -1; + } +} diff --git a/v3/ui/src/lib/server/scraper.ts b/v3/ui/src/lib/server/scraper.ts new file mode 100644 index 0000000..cdecfc6 --- /dev/null +++ b/v3/ui/src/lib/server/scraper.ts @@ -0,0 +1,36 @@ +/** + * Backend API helper. + * + * Centralises the BACKEND_URL constant and provides a thin fetch wrapper that: + * - Resolves paths relative to BACKEND_API_URL. + * - Throws 502 on network errors (unreachable backend). + * - Re-throws SvelteKit `error()` objects so callers can still short-circuit. + * - Passes a RequestInit through verbatim so callers keep full control. + * + * Import only from server-side modules (`+server.ts`, `*.server.ts`). + */ + +import { error } from '@sveltejs/kit'; +import { env } from '$env/dynamic/private'; + +export const BACKEND_URL = env.BACKEND_API_URL ?? 'http://localhost:8080'; + +/** + * Fetch a path on the backend, throwing a 502 on network failures. + * + * The `path` must start with `/` (e.g. `/api/voices`). + * + * SvelteKit `error()` exceptions are always re-thrown so callers can + * short-circuit correctly inside their own catch blocks. + */ +export async function backendFetch(path: string, init?: RequestInit): Promise { + try { + return await fetch(`${BACKEND_URL}${path}`, init); + } catch (e) { + // Re-throw SvelteKit HTTP errors so they propagate to the framework. + if (e instanceof Error && 'status' in e) throw e; + throw error(502, 'Could not reach backend'); + } +} + + diff --git a/v3/ui/src/lib/types.ts b/v3/ui/src/lib/types.ts new file mode 100644 index 0000000..5662dff --- /dev/null +++ b/v3/ui/src/lib/types.ts @@ -0,0 +1,63 @@ +/** + * Shared domain types for the LibNovel UI. + * + * Server-only types (full PocketBase record shapes) live in + * src/lib/server/pocketbase.ts. This file holds the types that are + * safe to import in both server and client code. + */ + +// ── Auth / User ────────────────────────────────────────────────────────────── + +export interface AuthUser { + id: string; + username: string; + email: string; + role: 'admin' | 'user'; + avatarUrl?: string; +} + +// ── Books ──────────────────────────────────────────────────────────────────── + +export interface BookSummary { + id: string; + slug: string; + title: string; + author: string; + coverUrl: string; + status: string; + /** Reading progress 0–100, if the user has any. */ + progress?: number; +} + +export interface ChapterRef { + number: number; + title: string; +} + +// ── Comments ───────────────────────────────────────────────────────────────── + +export interface BookComment { + id: string; + slug: string; + user_id: string; + username: string; + body: string; + upvotes: number; + downvotes: number; + created: string; + parent_id?: string; + replies?: BookComment[]; +} + +// ── Audio ──────────────────────────────────────────────────────────────────── + +export type AudioStatus = 'idle' | 'loading' | 'generating' | 'ready' | 'error'; +export type NextStatus = 'none' | 'prefetching' | 'prefetched' | 'failed'; + +// ── User settings ───────────────────────────────────────────────────────────── + +export interface UserSettings { + voice: string; + speed: number; + autoNext: boolean; +} diff --git a/v3/ui/src/lib/utils.ts b/v3/ui/src/lib/utils.ts new file mode 100644 index 0000000..35de52b --- /dev/null +++ b/v3/ui/src/lib/utils.ts @@ -0,0 +1,13 @@ +import { clsx, type ClassValue } from 'clsx'; +import { twMerge } from 'tailwind-merge'; + +/** + * Merge Tailwind classes safely, resolving conflicts via tailwind-merge + * and collapsing falsy values via clsx. + * + * Usage: + * cn('px-4 py-2', isActive && 'bg-brand', className) + */ +export function cn(...inputs: ClassValue[]): string { + return twMerge(clsx(inputs)); +} diff --git a/v3/ui/src/routes/+layout.server.ts b/v3/ui/src/routes/+layout.server.ts new file mode 100644 index 0000000..bafc1b4 --- /dev/null +++ b/v3/ui/src/routes/+layout.server.ts @@ -0,0 +1,32 @@ +import { redirect } from '@sveltejs/kit'; +import type { LayoutServerLoad } from './$types'; +import { getSettings } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +// Routes that are accessible without being logged in +const PUBLIC_ROUTES = new Set(['/login']); + +export const load: LayoutServerLoad = async ({ locals, url }) => { + if (!PUBLIC_ROUTES.has(url.pathname) && !locals.user) { + redirect(302, `/login`); + } + + let settings = { autoNext: false, voice: 'af_bella', speed: 1.0 }; + try { + const row = await getSettings(locals.sessionId, locals.user?.id); + if (row) { + settings = { + autoNext: row.auto_next ?? false, + voice: row.voice ?? 'af_bella', + speed: row.speed ?? 1.0 + }; + } + } catch (e) { + log.warn('layout', 'failed to load settings', { err: String(e) }); + } + + return { + user: locals.user, + settings + }; +}; diff --git a/v3/ui/src/routes/+layout.svelte b/v3/ui/src/routes/+layout.svelte new file mode 100644 index 0000000..c4ce279 --- /dev/null +++ b/v3/ui/src/routes/+layout.svelte @@ -0,0 +1,642 @@ + + + + + + libnovel + + + + + +
    + + {#if navigating} +
    +
    +
    + {/if} +
    + + + + {#if data.user && menuOpen} +
    + (menuOpen = false)} + class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/books') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}" + > + Library + + (menuOpen = false)} + class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/browse') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}" + > + Discover + + (menuOpen = false)} + class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname === '/profile' ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}" + > + Profile ({data.user.username}) + + {#if data.user?.role === 'admin'} +
    +

    Admin

    + (menuOpen = false)} + class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/admin/scrape') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}" + > + Scrape tasks + + (menuOpen = false)} + class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname === '/admin/audio' ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}" + > + Audio cache + + (menuOpen = false)} + class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/admin/audio-jobs') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}" + > + Audio jobs + + {/if} +
    +
    + +
    +
    + {/if} +
    + +
    + {#key page.url.pathname + page.url.search} + {@render children()} + {/key} +
    + + +
    + + +{#if audioStore.active} +
    + + + {#if chapterDrawerOpen && audioStore.chapters.length > 0} + + {/if} + + + {#if audioStore.status === 'generating' || audioStore.status === 'loading'} +
    +
    +
    + {:else if audioStore.status === 'ready'} + +
    + +
    + {/if} + +
    + + + + + {#if audioStore.status === 'ready'} + + + + + + + + + + + + + + + {:else if audioStore.status === 'generating'} + + + + + + {/if} + + + {#if audioStore.slug && audioStore.chapter > 0} + + {#if audioStore.cover} + + {:else} + +
    + + + +
    + {/if} +
    + {/if} + + + +
    +
    +{/if} diff --git a/v3/ui/src/routes/+page.server.ts b/v3/ui/src/routes/+page.server.ts new file mode 100644 index 0000000..abd05b8 --- /dev/null +++ b/v3/ui/src/routes/+page.server.ts @@ -0,0 +1,59 @@ +import type { PageServerLoad } from './$types'; +import { + listBooks, + recentlyAddedBooks, + allProgress, + getHomeStats, + getSubscriptionFeed +} from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; +import type { Book, Progress } from '$lib/server/pocketbase'; + +export const load: PageServerLoad = async ({ locals }) => { + let allBooks: Book[] = []; + let recentBooks: Book[] = []; + let progressList: Progress[] = []; + let stats = { totalBooks: 0, totalChapters: 0 }; + + try { + [allBooks, recentBooks, progressList, stats] = await Promise.all([ + listBooks(), + recentlyAddedBooks(8), + allProgress(locals.sessionId, locals.user?.id), + getHomeStats() + ]); + } catch (e) { + log.error('home', 'failed to load home data', { err: String(e) }); + } + + // Build slug → book lookup + const bookMap = new Map(allBooks.map((b) => [b.slug, b])); + + // Continue reading: progress entries joined with book data, most recent first + const continueReading = progressList + .filter((p) => bookMap.has(p.slug)) + .slice(0, 6) + .map((p) => ({ book: bookMap.get(p.slug)!, chapter: p.chapter })); + + // Recently updated: deduplicate against continueReading slugs + const inProgressSlugs = new Set(continueReading.map((c) => c.book.slug)); + const recentlyUpdated = recentBooks.filter((b) => !inProgressSlugs.has(b.slug)).slice(0, 6); + + // Subscription feed — only when logged in + const subscriptionFeed = locals.user + ? await getSubscriptionFeed(locals.user.id, 12).catch((e) => { + log.error('home', 'failed to load subscription feed', { err: String(e) }); + return [] as Awaited>; + }) + : []; + + return { + continueReading, + recentlyUpdated, + subscriptionFeed, + stats: { + ...stats, + booksInProgress: continueReading.length + } + }; +}; diff --git a/v3/ui/src/routes/+page.svelte b/v3/ui/src/routes/+page.svelte new file mode 100644 index 0000000..4c2cc23 --- /dev/null +++ b/v3/ui/src/routes/+page.svelte @@ -0,0 +1,202 @@ + + + + libnovel + + + +
    +
    +

    {data.stats.totalBooks}

    +

    Books

    +
    +
    +

    {data.stats.totalChapters.toLocaleString()}

    +

    Chapters

    +
    +
    +

    {data.stats.booksInProgress}

    +

    In progress

    +
    +
    + + +{#if data.continueReading.length > 0} +
    +
    +

    Continue Reading

    + View all +
    + +
    +{/if} + + +{#if data.recentlyUpdated.length > 0} +
    +
    +

    Recently Updated

    + View all +
    + +
    +{/if} + + +{#if data.continueReading.length === 0 && data.recentlyUpdated.length === 0} +
    +

    Your library is empty

    +

    Discover novels and scrape them into your library.

    + + Discover Novels + +
    +{/if} + + +{#if data.subscriptionFeed.length > 0} +
    +
    +

    From People You Follow

    +
    + +
    +{/if} diff --git a/v3/ui/src/routes/admin/audio-jobs/+page.server.ts b/v3/ui/src/routes/admin/audio-jobs/+page.server.ts new file mode 100644 index 0000000..83baa67 --- /dev/null +++ b/v3/ui/src/routes/admin/audio-jobs/+page.server.ts @@ -0,0 +1,17 @@ +import { redirect } from '@sveltejs/kit'; +import type { PageServerLoad } from './$types'; +import { listAudioJobs } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +export const load: PageServerLoad = async ({ locals }) => { + if (locals.user?.role !== 'admin') { + redirect(302, '/'); + } + + const jobs = await listAudioJobs().catch((e) => { + log.warn('admin/audio-jobs', 'failed to load audio jobs', { err: String(e) }); + return []; + }); + + return { jobs }; +}; diff --git a/v3/ui/src/routes/admin/audio-jobs/+page.svelte b/v3/ui/src/routes/admin/audio-jobs/+page.svelte new file mode 100644 index 0000000..53591bc --- /dev/null +++ b/v3/ui/src/routes/admin/audio-jobs/+page.svelte @@ -0,0 +1,153 @@ + + + + Audio jobs — libnovel admin + + +
    +
    +
    +

    Audio jobs

    +

    + {stats.total} total · + {stats.done} done · + {#if stats.failed > 0} + {stats.failed} failed · + {/if} + {#if stats.inFlight > 0} + {stats.inFlight} in-flight + {:else} + 0 in-flight + {/if} +

    +
    +
    + + + + + {#if filtered.length === 0} +

    + {q.trim() ? 'No results.' : 'No audio jobs yet.'} +

    + {:else} +
    + + + + + + + + + + + + + {#each filtered as job} + + + + + + + + + {#if job.error_message} + + + + {/if} + {/each} + +
    BookCh.VoiceStatusStartedDuration
    + + {job.slug} + + {job.chapter}{job.voice} + {job.status} + {fmtDate(job.started)}{duration(job.started, job.finished)}
    {job.error_message}
    +
    + {/if} +
    diff --git a/v3/ui/src/routes/admin/audio/+page.server.ts b/v3/ui/src/routes/admin/audio/+page.server.ts new file mode 100644 index 0000000..9d18725 --- /dev/null +++ b/v3/ui/src/routes/admin/audio/+page.server.ts @@ -0,0 +1,17 @@ +import { redirect } from '@sveltejs/kit'; +import type { PageServerLoad } from './$types'; +import { listAudioCache } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +export const load: PageServerLoad = async ({ locals }) => { + if (locals.user?.role !== 'admin') { + redirect(302, '/'); + } + + const entries = await listAudioCache().catch((e) => { + log.warn('admin/audio', 'failed to load audio cache', { err: String(e) }); + return []; + }); + + return { entries }; +}; diff --git a/v3/ui/src/routes/admin/audio/+page.svelte b/v3/ui/src/routes/admin/audio/+page.svelte new file mode 100644 index 0000000..e3a8a91 --- /dev/null +++ b/v3/ui/src/routes/admin/audio/+page.svelte @@ -0,0 +1,93 @@ + + + + Audio cache — libnovel admin + + +
    +
    +

    Audio cache

    +

    {entries.length} cached audio file{entries.length !== 1 ? 's' : ''}

    +
    + + + + + {#if filtered.length === 0} +

    + {q.trim() ? 'No results.' : 'Audio cache is empty.'} +

    + {:else} +
    + + + + + + + + + + + + {#each filtered as entry} + {@const parts = parseKey(entry.cache_key)} + + + + + + + + {/each} + +
    BookChapterVoiceFilenameUpdated
    + + {parts.slug} + + {parts.chapter}{parts.voice} + {entry.filename} + {fmtDate(entry.updated)}
    +
    + {/if} +
    diff --git a/v3/ui/src/routes/admin/scrape/+page.server.ts b/v3/ui/src/routes/admin/scrape/+page.server.ts new file mode 100644 index 0000000..53651b4 --- /dev/null +++ b/v3/ui/src/routes/admin/scrape/+page.server.ts @@ -0,0 +1,27 @@ +import { redirect } from '@sveltejs/kit'; +import type { PageServerLoad } from './$types'; +import { listScrapingTasks } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +export const load: PageServerLoad = async ({ locals }) => { + if (locals.user?.role !== 'admin') { + redirect(302, '/'); + } + + const [tasks, statusRes] = await Promise.all([ + listScrapingTasks().catch((e) => { + log.warn('admin/scrape', 'failed to load tasks', { err: String(e) }); + return []; + }), + backendFetch('/api/scrape/status').catch(() => null) + ]); + + let running = false; + if (statusRes?.ok) { + const body = await statusRes.json().catch(() => null); + running = body?.running ?? false; + } + + return { tasks, running }; +}; diff --git a/v3/ui/src/routes/admin/scrape/+page.svelte b/v3/ui/src/routes/admin/scrape/+page.svelte new file mode 100644 index 0000000..dd3f185 --- /dev/null +++ b/v3/ui/src/routes/admin/scrape/+page.svelte @@ -0,0 +1,238 @@ + + + + Scrape tasks — libnovel admin + + +
    +
    +
    +

    Scrape tasks

    +

    + Job status: + {#if running} + Running + {:else} + Idle + {/if} +

    +
    + + +
    + +
    +
    + + +
    +

    Scrape a single book

    +
    + + +
    + {#if scrapeError} +

    {scrapeError}

    + {/if} +
    + + + {#if tasks.length === 0} +

    No scrape tasks yet.

    + {:else} +
    + + + + + + + + + + + + + + + + {#each tasks as task} + + + + + + + + + + + + {#if task.error_message} + + + + {/if} + {/each} + +
    KindStatusBooksChaptersSkippedErrorsStartedDurationActions
    + {task.kind} + {#if task.target_url} +
    + + {task.target_url.replace('https://novelfire.net/book/', '')} + + {/if} +
    + {task.status} + {task.books_found ?? 0}{task.chapters_scraped ?? 0}{task.chapters_skipped ?? 0}{task.errors ?? 0}{fmtDate(task.started)}{duration(task.started, task.finished)} + {#if task.status === 'pending'} + + {#if cancelErrors[task.id]} +

    {cancelErrors[task.id]}

    + {/if} + {/if} +
    {task.error_message}
    +
    + {/if} +
    diff --git a/v3/ui/src/routes/api/admin/scrape/+server.ts b/v3/ui/src/routes/api/admin/scrape/+server.ts new file mode 100644 index 0000000..d8855fb --- /dev/null +++ b/v3/ui/src/routes/api/admin/scrape/+server.ts @@ -0,0 +1,21 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { backendFetch } from '$lib/server/scraper'; + +/** + * GET /api/admin/scrape/status + * Admin-only proxy to the Go backend's /api/scrape/status endpoint. + */ +export const GET: RequestHandler = async ({ locals }) => { + if (!locals.user || locals.user.role !== 'admin') { + throw error(403, 'Forbidden'); + } + try { + const res = await backendFetch('/api/scrape/status'); + if (!res.ok) return json({ running: false }); + const data = await res.json(); + return json({ running: data.running ?? false }); + } catch { + return json({ running: false }); + } +}; diff --git a/v3/ui/src/routes/api/audio/[slug]/[n]/+server.ts b/v3/ui/src/routes/api/audio/[slug]/[n]/+server.ts new file mode 100644 index 0000000..8ec2760 --- /dev/null +++ b/v3/ui/src/routes/api/audio/[slug]/[n]/+server.ts @@ -0,0 +1,65 @@ +import { error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +/** + * POST /api/audio/[slug]/[n] + * Proxies the audio generation request to the backend's /api/audio endpoint. + * Keeps the backend URL server-side — the browser never needs to know it. + * + * Body: { voice?: string } + * + * Responses: + * 200 { status: "done" } — audio already cached; client should call + * GET /api/presign/audio to obtain a direct MinIO presigned URL. + * 202 { task_id: string, status: "pending"|"generating" } — generation + * enqueued; poll GET /api/audio/status/[slug]/[n]?voice=... until done. + */ +export const POST: RequestHandler = async ({ params, request }) => { + const { slug, n } = params; + const chapter = parseInt(n, 10); + if (!slug || !chapter || chapter < 1) { + error(400, 'Invalid slug or chapter number'); + } + + let body: { voice?: string } = {}; + try { + body = await request.json(); + } catch { + // empty body is fine — scraper will use defaults + } + + const scraperRes = await backendFetch(`/api/audio/${slug}/${chapter}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); + + if (!scraperRes.ok) { + const text = await scraperRes.text().catch(() => ''); + log.error('audio', 'backend audio generation failed', { slug, chapter, status: scraperRes.status, body: text }); + error(scraperRes.status as Parameters[0], text || 'Audio generation failed'); + } + + const data = (await scraperRes.json()) as + | { url: string; status: 'done' } + | { task_id: string; status: string }; + + // 202 Accepted: generation enqueued — return task_id + status for polling. + if (scraperRes.status === 202 || 'task_id' in data) { + return new Response(JSON.stringify(data), { + status: 202, + headers: { 'Content-Type': 'application/json' } + }); + } + + // 200: audio was already cached. + // Return status only — no url — so the client calls GET /api/presign/audio + // and streams directly from MinIO instead of through the Node.js server. + return new Response( + JSON.stringify({ status: 'done' }), + { headers: { 'Content-Type': 'application/json' } } + ); +}; + diff --git a/v3/ui/src/routes/api/audio/status/[slug]/[n]/+server.ts b/v3/ui/src/routes/api/audio/status/[slug]/[n]/+server.ts new file mode 100644 index 0000000..fce9e33 --- /dev/null +++ b/v3/ui/src/routes/api/audio/status/[slug]/[n]/+server.ts @@ -0,0 +1,64 @@ +import { error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +/** + * GET /api/audio/status/[slug]/[n]?voice=... + * Proxies the audio generation status check to the backend's + * GET /api/audio/status/{slug}/{n} endpoint. + * + * Possible responses passed through to the client: + * {"status":"done"} — audio ready; no url + * {"status":"pending"|"generating","task_id":"..."} — in progress + * {"status":"idle"} — no job yet + * {"status":"failed","error":"..."} — last job failed + * + * When status is "done" the scraper's internal proxy URL is stripped — the + * client must call GET /api/presign/audio to obtain a direct MinIO presigned + * URL. This avoids streaming audio through the Node.js server. + */ +export const GET: RequestHandler = async ({ params, url }) => { + const { slug, n } = params; + const chapter = parseInt(n, 10); + if (!slug || !chapter || chapter < 1) { + error(400, 'Invalid slug or chapter number'); + } + + const voice = url.searchParams.get('voice') ?? ''; + const qs = new URLSearchParams(); + if (voice) qs.set('voice', voice); + + const scraperRes = await backendFetch( + `/api/audio/status/${slug}/${chapter}?${qs.toString()}` + ); + + if (!scraperRes.ok) { + const text = await scraperRes.text().catch(() => ''); + log.error('audio', 'backend audio status check failed', { + slug, + chapter, + status: scraperRes.status, + body: text + }); + error(scraperRes.status as Parameters[0], text || 'Status check failed'); + } + + const data = (await scraperRes.json()) as { + status: string; + task_id?: string; + url?: string; + error?: string; + }; + + // Strip the backend's internal proxy URL from "done" responses. + // The client will call GET /api/presign/audio to get a direct MinIO URL, + // avoiding streaming audio through the Node.js server. + if (data.status === 'done') { + delete data.url; + } + + return new Response(JSON.stringify(data), { + headers: { 'Content-Type': 'application/json' } + }); +}; diff --git a/v3/ui/src/routes/api/auth/change-password/+server.ts b/v3/ui/src/routes/api/auth/change-password/+server.ts new file mode 100644 index 0000000..dba2d4e --- /dev/null +++ b/v3/ui/src/routes/api/auth/change-password/+server.ts @@ -0,0 +1,47 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { changePassword } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * POST /api/auth/change-password + * Body: { currentPassword: string, newPassword: string } + * Requires authentication. + */ +export const POST: RequestHandler = async ({ request, locals }) => { + if (!locals.user) { + error(401, 'Not authenticated'); + } + + let body: { currentPassword?: string; newPassword?: string }; + try { + body = await request.json(); + } catch { + error(400, 'Invalid JSON body'); + } + + const currentPassword = body.currentPassword ?? ''; + const newPassword = body.newPassword ?? ''; + + if (!currentPassword || !newPassword) { + error(400, 'currentPassword and newPassword are required'); + } + + if (newPassword.length < 4) { + error(400, 'New password must be at least 4 characters'); + } + + try { + const ok = await changePassword(locals.user.id, currentPassword, newPassword); + if (!ok) { + error(401, 'Current password is incorrect'); + } + } catch (e: unknown) { + // Re-throw SvelteKit errors as-is + if (e && typeof e === 'object' && 'status' in e) throw e; + log.error('api/auth/change-password', 'unexpected error', { err: String(e) }); + error(500, 'An error occurred. Please try again.'); + } + + return json({ ok: true }); +}; diff --git a/v3/ui/src/routes/api/auth/login/+server.ts b/v3/ui/src/routes/api/auth/login/+server.ts new file mode 100644 index 0000000..5d04a36 --- /dev/null +++ b/v3/ui/src/routes/api/auth/login/+server.ts @@ -0,0 +1,75 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { loginUser, mergeSessionProgress, createUserSession } from '$lib/server/pocketbase'; +import { createAuthToken } from '../../../../hooks.server'; +import { log } from '$lib/server/logger'; +import { randomBytes } from 'node:crypto'; + +const AUTH_COOKIE = 'libnovel_auth'; +const ONE_YEAR = 60 * 60 * 24 * 365; + +/** + * POST /api/auth/login + * Body: { username: string, password: string } + * Returns: { token: string, user: { id, username, role } } + * + * Sets the libnovel_auth cookie and returns the raw token value so the + * iOS app can persist it for subsequent requests. + */ +export const POST: RequestHandler = async ({ request, cookies, locals }) => { + let body: { username?: string; password?: string }; + try { + body = await request.json(); + } catch { + error(400, 'Invalid JSON body'); + } + + const username = (body.username ?? '').trim(); + const password = body.password ?? ''; + + if (!username || !password) { + error(400, 'Username and password are required'); + } + + let user; + try { + user = await loginUser(username, password); + } catch (e) { + log.error('api/auth/login', 'unexpected error', { username, err: String(e) }); + error(500, 'An error occurred. Please try again.'); + } + + if (!user) { + error(401, 'Invalid username or password'); + } + + // Merge anonymous session progress (non-fatal) + mergeSessionProgress(locals.sessionId, user.id).catch((e) => + log.warn('api/auth/login', 'mergeSessionProgress failed (non-fatal)', { err: String(e) }) + ); + + const authSessionId = randomBytes(16).toString('hex'); + + const userAgent = request.headers.get('user-agent') ?? ''; + const ip = + request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ?? + request.headers.get('x-real-ip') ?? + ''; + createUserSession(user.id, authSessionId, userAgent, ip).catch((e) => + log.warn('api/auth/login', 'createUserSession failed (non-fatal)', { err: String(e) }) + ); + + const token = createAuthToken(user.id, user.username, user.role ?? 'user', authSessionId); + + cookies.set(AUTH_COOKIE, token, { + path: '/', + httpOnly: true, + sameSite: 'lax', + maxAge: ONE_YEAR + }); + + return json({ + token, + user: { id: user.id, username: user.username, role: user.role ?? 'user' } + }); +}; diff --git a/v3/ui/src/routes/api/auth/logout/+server.ts b/v3/ui/src/routes/api/auth/logout/+server.ts new file mode 100644 index 0000000..9321e34 --- /dev/null +++ b/v3/ui/src/routes/api/auth/logout/+server.ts @@ -0,0 +1,15 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; + +const AUTH_COOKIE = 'libnovel_auth'; + +/** + * POST /api/auth/logout + * Clears the auth cookie and returns { ok: true }. + * Does not revoke the session record from PocketBase — + * for full revocation use DELETE /api/sessions/[id] first. + */ +export const POST: RequestHandler = async ({ cookies }) => { + cookies.delete(AUTH_COOKIE, { path: '/' }); + return json({ ok: true }); +}; diff --git a/v3/ui/src/routes/api/auth/me/+server.ts b/v3/ui/src/routes/api/auth/me/+server.ts new file mode 100644 index 0000000..7ac173d --- /dev/null +++ b/v3/ui/src/routes/api/auth/me/+server.ts @@ -0,0 +1,22 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { getUserByUsername } from '$lib/server/pocketbase'; + +/** + * GET /api/auth/me + * Returns the currently authenticated user from the request's auth cookie. + * Returns 401 if not authenticated. + */ +export const GET: RequestHandler = async ({ locals }) => { + if (!locals.user) { + error(401, 'Not authenticated'); + } + // Fetch full record from PocketBase to get avatar_url + const record = await getUserByUsername(locals.user.username).catch(() => null); + return json({ + id: locals.user.id, + username: locals.user.username, + role: locals.user.role, + avatar_url: record?.avatar_url ?? null + }); +}; diff --git a/v3/ui/src/routes/api/auth/register/+server.ts b/v3/ui/src/routes/api/auth/register/+server.ts new file mode 100644 index 0000000..58d0be7 --- /dev/null +++ b/v3/ui/src/routes/api/auth/register/+server.ts @@ -0,0 +1,84 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { createUser, mergeSessionProgress, createUserSession } from '$lib/server/pocketbase'; +import { createAuthToken } from '../../../../hooks.server'; +import { log } from '$lib/server/logger'; +import { randomBytes } from 'node:crypto'; + +const AUTH_COOKIE = 'libnovel_auth'; +const ONE_YEAR = 60 * 60 * 24 * 365; + +/** + * POST /api/auth/register + * Body: { username: string, password: string } + * Returns: { token: string, user: { id, username, role } } + * + * Sets the libnovel_auth cookie and returns the raw token value so the + * iOS app can persist it for subsequent requests. + */ +export const POST: RequestHandler = async ({ request, cookies, locals }) => { + let body: { username?: string; password?: string }; + try { + body = await request.json(); + } catch { + error(400, 'Invalid JSON body'); + } + + const username = (body.username ?? '').trim(); + const password = body.password ?? ''; + + if (!username || !password) { + error(400, 'Username and password are required'); + } + if (username.length < 3 || username.length > 32) { + error(400, 'Username must be between 3 and 32 characters'); + } + if (!/^[a-zA-Z0-9_-]+$/.test(username)) { + error(400, 'Username may only contain letters, numbers, underscores and hyphens'); + } + if (password.length < 8) { + error(400, 'Password must be at least 8 characters'); + } + + let user; + try { + user = await createUser(username, password); + } catch (e: unknown) { + const msg = e instanceof Error ? e.message : 'Registration failed.'; + if (msg.includes('Username already taken')) { + error(409, 'That username is already taken'); + } + log.error('api/auth/register', 'unexpected error', { username, err: String(e) }); + error(500, 'An error occurred. Please try again.'); + } + + // Merge anonymous session progress (non-fatal) + mergeSessionProgress(locals.sessionId, user.id).catch((e) => + log.warn('api/auth/register', 'mergeSessionProgress failed (non-fatal)', { err: String(e) }) + ); + + const authSessionId = randomBytes(16).toString('hex'); + + const userAgent = request.headers.get('user-agent') ?? ''; + const ip = + request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ?? + request.headers.get('x-real-ip') ?? + ''; + createUserSession(user.id, authSessionId, userAgent, ip).catch((e) => + log.warn('api/auth/register', 'createUserSession failed (non-fatal)', { err: String(e) }) + ); + + const token = createAuthToken(user.id, user.username, user.role ?? 'user', authSessionId); + + cookies.set(AUTH_COOKIE, token, { + path: '/', + httpOnly: true, + sameSite: 'lax', + maxAge: ONE_YEAR + }); + + return json({ + token, + user: { id: user.id, username: user.username, role: user.role ?? 'user' } + }); +}; diff --git a/v3/ui/src/routes/api/book/[slug]/+server.ts b/v3/ui/src/routes/api/book/[slug]/+server.ts new file mode 100644 index 0000000..8b8684b --- /dev/null +++ b/v3/ui/src/routes/api/book/[slug]/+server.ts @@ -0,0 +1,109 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { getBook, listChapterIdx, getProgress, isBookSaved } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +/** + * GET /api/book/[slug] + * Returns book metadata, chapter list, progress, and library status. + * + * If the book is not yet in PocketBase, asks the backend to enqueue a scrape + * task and returns 202 with { scraping: true, task_id }. + * The client should poll and retry once the task completes. + */ +export const GET: RequestHandler = async ({ params, locals }) => { + const { slug } = params; + + // Try PocketBase first + let book = await getBook(slug).catch((e) => { + log.error('api/book', 'getBook failed', { slug, err: String(e) }); + return null; + }); + + if (book) { + let chapters, progress, saved; + try { + [chapters, progress, saved] = await Promise.all([ + listChapterIdx(slug), + getProgress(locals.sessionId, slug, locals.user?.id), + isBookSaved(locals.sessionId, slug, locals.user?.id) + ]); + } catch (e) { + log.error('api/book', 'failed to load book detail data', { slug, err: String(e) }); + error(500, 'Failed to load book'); + } + + return json({ + book, + chapters, + in_lib: true, + saved, + last_chapter: progress?.chapter ?? null, + scraping: false, + task_id: null + }); + } + + // Fall back to backend: enqueue scrape task if not in library. + try { + const res = await backendFetch(`/api/book-preview/${encodeURIComponent(slug)}`); + + if (res.status === 202) { + const body: { task_id: string; message: string } = await res.json(); + log.info('api/book', 'scrape task enqueued', { slug, task_id: body.task_id }); + return json({ scraping: true, task_id: body.task_id, in_lib: false }, { status: 202 }); + } + + if (!res.ok) { + log.warn('api/book', 'book-preview returned error', { slug, status: res.status }); + error(404, `Book "${slug}" not found`); + } + + // 200 — book was already in library + const preview: { + in_lib: boolean; + meta: { + slug: string; + title: string; + author: string; + cover: string; + status: string; + genres: string[]; + summary: string; + total_chapters: number; + source_url: string; + }; + chapters: { number: number; title: string; date?: string }[]; + } = await res.json(); + + const previewBook = { + id: '', + slug: preview.meta.slug || slug, + title: preview.meta.title, + author: preview.meta.author, + cover: preview.meta.cover, + status: preview.meta.status, + genres: preview.meta.genres ?? [], + summary: preview.meta.summary, + total_chapters: preview.meta.total_chapters, + source_url: preview.meta.source_url, + ranking: 0, + meta_updated: '' + }; + + return json({ + book: previewBook, + chapters: preview.chapters, + in_lib: true, + saved: false, + last_chapter: null, + scraping: false, + task_id: null + }); + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; + log.error('api/book', 'book-preview fetch failed', { slug, err: String(e) }); + error(404, `Book "${slug}" not found`); + } +}; diff --git a/v3/ui/src/routes/api/browse-page/+server.ts b/v3/ui/src/routes/api/browse-page/+server.ts new file mode 100644 index 0000000..02b0bcb --- /dev/null +++ b/v3/ui/src/routes/api/browse-page/+server.ts @@ -0,0 +1,76 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +/** + * GET /api/browse-page?page=2&genre=all&sort=popular&status=all&q= + * + * Thin proxy to the Go backend's /api/catalogue endpoint. + * Used by the infinite-scroll browse page to append subsequent pages + * without a full SSR navigation. + * + * Normalises the catalogue response to the { novels, page, hasNext } shape + * expected by the client-side infinite scroll in +page.svelte. + */ +export const GET: RequestHandler = async ({ url }) => { + const page = url.searchParams.get('page') ?? '1'; + const genre = url.searchParams.get('genre') ?? 'all'; + const sort = url.searchParams.get('sort') ?? 'popular'; + const status = url.searchParams.get('status') ?? 'all'; + const q = url.searchParams.get('q') ?? ''; + + const params = new URLSearchParams({ page, genre, sort, status }); + if (q.trim().length >= 2) { + params.set('q', q.trim()); + } + + try { + const res = await backendFetch(`/api/catalogue?${params.toString()}`); + if (!res.ok) { + log.error('browse-page', 'backend returned error', { status: res.status }); + throw error(502, `Browse fetch failed: ${res.status}`); + } + const data: { + books: Array<{ + slug: string; + title: string; + author: string; + cover: string; + status: string; + genres: string[]; + total_chapters: number; + source_url: string; + ranking: number; + rating: number; + }>; + page: number; + has_next: boolean; + } = await res.json(); + + // Normalise to the shape the +page.svelte client-side fetch expects. + const novels = (data.books ?? []).map((book) => ({ + slug: book.slug, + title: book.title, + cover: book.cover, + rank: book.ranking > 0 ? `#${book.ranking}` : '', + rating: book.rating > 0 ? String(book.rating) : '', + chapters: book.total_chapters > 0 ? `${book.total_chapters} chapters` : '', + url: book.source_url ?? '', + author: book.author, + status: book.status, + genres: book.genres ?? [], + source_url: book.source_url + })); + + return json({ + novels, + page: data.page ?? (parseInt(page, 10) || 1), + hasNext: data.has_next ?? false + }); + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; + log.error('browse-page', 'network error', { err: String(e) }); + throw error(502, 'Could not reach browse service'); + } +}; diff --git a/v3/ui/src/routes/api/chapter-text-preview/[slug]/[n]/+server.ts b/v3/ui/src/routes/api/chapter-text-preview/[slug]/[n]/+server.ts new file mode 100644 index 0000000..016e48f --- /dev/null +++ b/v3/ui/src/routes/api/chapter-text-preview/[slug]/[n]/+server.ts @@ -0,0 +1,44 @@ +import { error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +/** + * GET /api/chapter-text-preview/[slug]/[n] + * Proxies to the backend's /api/chapter-text-preview endpoint. + * Used client-side when the normal chapter path returns no content + * (chapter indexed but not yet scraped to MinIO). + */ +export const GET: RequestHandler = async ({ params, url }) => { + const { slug, n } = params; + const chapter = parseInt(n, 10); + if (!slug || !chapter || chapter < 1) { + error(400, 'Invalid slug or chapter number'); + } + + // Forward optional query params (chapter_url, title) if present + const qs = new URLSearchParams(); + const chapterUrl = url.searchParams.get('chapter_url'); + const title = url.searchParams.get('title'); + if (chapterUrl) qs.set('chapter_url', chapterUrl); + if (title) qs.set('title', title); + + const scraperRes = await backendFetch( + `/api/chapter-text-preview/${encodeURIComponent(slug)}/${chapter}?${qs.toString()}` + ).catch((e) => { + log.error('chapter-preview', 'scraper fetch failed', { slug, chapter, err: String(e) }); + return null; + }); + + if (!scraperRes || !scraperRes.ok) { + const status = scraperRes?.status ?? 502; + log.error('chapter-preview', 'scraper returned error', { slug, chapter, status }); + error(status as Parameters[0], 'Chapter preview not available'); + } + + const data = await scraperRes.json(); + + return new Response(JSON.stringify(data), { + headers: { 'Content-Type': 'application/json' } + }); +}; diff --git a/v3/ui/src/routes/api/chapter/[slug]/[n]/+server.ts b/v3/ui/src/routes/api/chapter/[slug]/[n]/+server.ts new file mode 100644 index 0000000..f79492b --- /dev/null +++ b/v3/ui/src/routes/api/chapter/[slug]/[n]/+server.ts @@ -0,0 +1,126 @@ +import { json, error } from '@sveltejs/kit'; +import { marked } from 'marked'; +import type { RequestHandler } from './$types'; +import { getBook, listChapterIdx } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +/** + * GET /api/chapter/[slug]/[n] + * Returns rendered chapter HTML, navigation info, and voice list. + * Supports ?preview=1&chapter_url=...&title=... for un-scraped books. + * + * Response shape mirrors ChapterResponse in the iOS APIClient. + */ +export const GET: RequestHandler = async ({ params, url, locals }) => { + const { slug } = params; + const n = parseInt(params.n, 10); + + if (!n || n < 1) error(400, 'Invalid chapter number'); + + const isPreview = url.searchParams.get('preview') === '1'; + const chapterUrl = url.searchParams.get('chapter_url') ?? ''; + const chapterTitle = url.searchParams.get('title') ?? ''; + + if (isPreview) { + // Preview path: scrape live, nothing from PocketBase/MinIO + const previewParams = new URLSearchParams(); + if (chapterUrl) previewParams.set('chapter_url', chapterUrl); + if (chapterTitle) previewParams.set('title', chapterTitle); + + let chapterData: { slug: string; number: number; title: string; text: string; url: string }; + try { + const res = await backendFetch( + `/api/chapter-text-preview/${encodeURIComponent(slug)}/${n}?${previewParams.toString()}` + ); + if (!res.ok) { + log.error('api/chapter', 'chapter-text-preview returned error', { slug, n, status: res.status }); + error(404, `Chapter ${n} not found`); + } + chapterData = await res.json(); + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; + log.error('api/chapter', 'chapter-text-preview fetch failed', { slug, n, err: String(e) }); + error(502, 'Could not fetch chapter preview'); + } + + const html = chapterData.text + ? '

    ' + chapterData.text.replace(/\n{2,}/g, '

    ').replace(/\n/g, '
    ') + '

    ' + : ''; + + let voices: string[] = []; + try { + const vRes = await backendFetch('/api/voices'); + if (vRes.ok) { + const d = (await vRes.json()) as { voices: string[] }; + voices = d.voices ?? []; + } + } catch { + // Non-critical + } + + const pb = await getBook(slug).catch(() => null); + + return json({ + book: { slug, title: pb?.title ?? slug, cover: pb?.cover ?? '' }, + chapter: { id: '', slug, number: n, title: chapterData.title || `Chapter ${n}`, date_label: '' }, + html, + voices, + prev: null, + next: null, + chapters: [], + is_preview: true + }); + } + + // Normal path: PocketBase + MinIO + const [book, chapters, voicesRes] = await Promise.all([ + getBook(slug), + listChapterIdx(slug), + backendFetch('/api/voices').catch(() => null) + ]); + + if (!book) error(404, `Book "${slug}" not found`); + + const chapterIdx = chapters.find((c) => c.number === n); + if (!chapterIdx) error(404, `Chapter ${n} not found`); + + let voices: string[] = []; + try { + if (voicesRes?.ok) { + const data = (await voicesRes.json()) as { voices: string[] }; + voices = data.voices ?? []; + } + } catch { + // Non-critical + } + + let html = ''; + try { + const res = await backendFetch(`/api/chapter-markdown/${encodeURIComponent(slug)}/${n}`); + if (!res.ok) { + log.error('api/chapter', 'chapter-markdown returned error', { slug, n, status: res.status }); + error(res.status === 404 ? 404 : 502, res.status === 404 ? `Chapter ${n} not found` : 'Could not fetch chapter content'); + } + const markdown = await res.text(); + html = marked(markdown) as string; + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; + log.error('api/chapter', 'failed to fetch chapter content', { slug, n, err: String(e) }); + error(502, 'Could not fetch chapter content'); + } + + const prevChapter = chapters.find((c) => c.number === n - 1) ?? null; + const nextChapter = chapters.find((c) => c.number === n + 1) ?? null; + + return json({ + book: { slug: book.slug, title: book.title, cover: book.cover ?? '' }, + chapter: chapterIdx, + html, + voices, + prev: prevChapter ? prevChapter.number : null, + next: nextChapter ? nextChapter.number : null, + chapters: chapters.map((c) => ({ number: c.number, title: c.title })), + is_preview: false + }); +}; diff --git a/v3/ui/src/routes/api/comment/[id]/+server.ts b/v3/ui/src/routes/api/comment/[id]/+server.ts new file mode 100644 index 0000000..8bb998b --- /dev/null +++ b/v3/ui/src/routes/api/comment/[id]/+server.ts @@ -0,0 +1,26 @@ +import { error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { deleteComment } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * DELETE /api/comment/[id] + * Deletes a comment and its replies. Only the comment owner may delete. + * Requires authentication. + */ +export const DELETE: RequestHandler = async ({ params, locals }) => { + if (!locals.user) error(401, 'Login required'); + + const { id } = params; + + try { + await deleteComment(id, locals.user.id); + return new Response(null, { status: 204 }); + } catch (e) { + const msg = String(e); + if (msg.includes('Not authorized')) error(403, 'Not authorized to delete this comment'); + if (msg.includes('not found')) error(404, 'Comment not found'); + log.error('api/comment/[id]', 'deleteComment failed', { id, err: msg }); + error(500, 'Failed to delete comment'); + } +}; diff --git a/v3/ui/src/routes/api/comment/[id]/vote/+server.ts b/v3/ui/src/routes/api/comment/[id]/vote/+server.ts new file mode 100644 index 0000000..5a7526e --- /dev/null +++ b/v3/ui/src/routes/api/comment/[id]/vote/+server.ts @@ -0,0 +1,33 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { voteComment } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * POST /api/comment/[id]/vote + * Body: { vote: 'up' | 'down' } + * Casts, changes, or toggles off a vote on a comment. + * Works for both authenticated and anonymous users (session-scoped). + * Returns the updated comment. + */ +export const POST: RequestHandler = async ({ params, request, locals }) => { + const { id } = params; + let body: { vote?: string }; + try { + body = await request.json(); + } catch { + error(400, 'Invalid JSON body'); + } + + if (body.vote !== 'up' && body.vote !== 'down') { + error(400, 'vote must be "up" or "down"'); + } + + try { + const updated = await voteComment(id, body.vote, locals.sessionId, locals.user?.id); + return json(updated); + } catch (e) { + log.error('api/comment/[id]/vote', 'voteComment failed', { id, err: String(e) }); + error(500, 'Failed to record vote'); + } +}; diff --git a/v3/ui/src/routes/api/comments/[slug]/+server.ts b/v3/ui/src/routes/api/comments/[slug]/+server.ts new file mode 100644 index 0000000..b90e559 --- /dev/null +++ b/v3/ui/src/routes/api/comments/[slug]/+server.ts @@ -0,0 +1,102 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { + listComments, + listReplies, + createComment, + getMyVotes, + type CommentSort +} from '$lib/server/pocketbase'; +import { presignAvatarUrl } from '$lib/server/minio'; +import { log } from '$lib/server/logger'; + +/** + * GET /api/comments/[slug]?sort=new|top + * Returns top-level comments + their replies + current visitor's votes + avatar URLs. + * Response: { comments: BookComment[], myVotes: Record, avatarUrls: Record } + * Each top-level comment has a `replies` array attached. + */ +export const GET: RequestHandler = async ({ params, url, locals }) => { + const { slug } = params; + const sortParam = url.searchParams.get('sort') ?? 'new'; + const sort: CommentSort = sortParam === 'top' ? 'top' : 'new'; + + try { + const topLevel = await listComments(slug, sort); + + // Fetch replies for all top-level comments in parallel + const repliesPerComment = await Promise.all(topLevel.map((c) => listReplies(c.id))); + const allReplies = repliesPerComment.flat(); + + // Build comment+reply list for vote lookup + const allIds = [...topLevel.map((c) => c.id), ...allReplies.map((r) => r.id)]; + const myVotes = await getMyVotes(allIds, locals.sessionId, locals.user?.id); + + // Attach replies to each top-level comment + const comments = topLevel.map((c, i) => ({ + ...c, + replies: repliesPerComment[i] + })); + + // Batch-resolve avatar presign URLs for all unique user_ids + const allComments = [...topLevel, ...allReplies]; + const uniqueUserIds = [...new Set(allComments.map((c) => c.user_id).filter(Boolean))]; + const avatarEntries = await Promise.all( + uniqueUserIds.map(async (userId) => { + try { + const url = await presignAvatarUrl(userId); + return [userId, url] as [string, string | null]; + } catch { + return [userId, null] as [string, null]; + } + }) + ); + const avatarUrls: Record = {}; + for (const [userId, url] of avatarEntries) { + if (url) avatarUrls[userId] = url; + } + + return json({ comments, myVotes, avatarUrls }); + } catch (e) { + log.error('api/comments/[slug]', 'listComments failed', { slug, err: String(e) }); + error(500, 'Failed to load comments'); + } +}; + +/** + * POST /api/comments/[slug] + * Body: { body: string, parent_id?: string } + * Creates a new comment or reply. Requires authentication. + */ +export const POST: RequestHandler = async ({ params, request, locals }) => { + if (!locals.user) error(401, 'Login required to comment'); + + const { slug } = params; + let body: { body?: string; parent_id?: string }; + try { + body = await request.json(); + } catch { + error(400, 'Invalid JSON body'); + } + + const text = (body.body ?? '').trim(); + if (!text) error(400, 'Comment body is required'); + if (text.length > 2000) error(400, 'Comment is too long (max 2000 characters)'); + + // Enforce 1-level depth: parent_id must be a top-level comment + const parentId = body.parent_id?.trim() || undefined; + + try { + const comment = await createComment( + slug, + text, + locals.user.id, + locals.user.username, + parentId + ); + return json(comment, { status: 201 }); + } catch (e) { + log.error('api/comments/[slug]', 'createComment failed', { slug, err: String(e) }); + error(500, 'Failed to post comment'); + } +}; diff --git a/v3/ui/src/routes/api/home/+server.ts b/v3/ui/src/routes/api/home/+server.ts new file mode 100644 index 0000000..62584cd --- /dev/null +++ b/v3/ui/src/routes/api/home/+server.ts @@ -0,0 +1,65 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { + listBooks, + recentlyAddedBooks, + allProgress, + getHomeStats, + getSubscriptionFeed +} from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; +import type { Book, Progress } from '$lib/server/pocketbase'; + +/** + * GET /api/home + * Returns home screen data: continue-reading list, recently updated books, stats, + * and subscription feed (books recently read by followed users). + * Requires authentication (enforced by layout guard). + */ +export const GET: RequestHandler = async ({ locals }) => { + let allBooks: Book[] = []; + let recentBooks: Book[] = []; + let progressList: Progress[] = []; + let stats = { totalBooks: 0, totalChapters: 0 }; + + try { + [allBooks, recentBooks, progressList, stats] = await Promise.all([ + listBooks(), + recentlyAddedBooks(8), + allProgress(locals.sessionId, locals.user?.id), + getHomeStats() + ]); + } catch (e) { + log.error('api/home', 'failed to load home data', { err: String(e) }); + } + + const bookMap = new Map(allBooks.map((b) => [b.slug, b])); + + const continueReading = progressList + .filter((p) => bookMap.has(p.slug)) + .slice(0, 6) + .map((p) => ({ book: bookMap.get(p.slug)!, chapter: p.chapter })); + + const inProgressSlugs = new Set(continueReading.map((c) => c.book.slug)); + const recentlyUpdated = recentBooks.filter((b) => !inProgressSlugs.has(b.slug)).slice(0, 6); + + // Subscription feed — only available for logged-in users with following + let subscriptionFeed: Array<{ book: Book; readerUsername: string }> = []; + if (locals.user?.id) { + subscriptionFeed = await getSubscriptionFeed(locals.user.id).catch(() => []); + } + + return json({ + continue_reading: continueReading, + recently_updated: recentlyUpdated, + stats: { + totalBooks: stats.totalBooks, + totalChapters: stats.totalChapters, + booksInProgress: continueReading.length + }, + subscription_feed: subscriptionFeed.map((item) => ({ + book: item.book, + readerUsername: item.readerUsername + })) + }); +}; diff --git a/v3/ui/src/routes/api/library/+server.ts b/v3/ui/src/routes/api/library/+server.ts new file mode 100644 index 0000000..6f1c1eb --- /dev/null +++ b/v3/ui/src/routes/api/library/+server.ts @@ -0,0 +1,61 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { listBooks, allProgress, getSavedSlugs } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * GET /api/library + * Returns the user's library: books they have started reading or explicitly saved. + * Each item includes the book record, the last chapter read, and saved_at timestamp. + * + * Response shape mirrors LibraryItem in the iOS APIClient. + */ +export const GET: RequestHandler = async ({ locals }) => { + let allBooks: Awaited>; + let progressList: Awaited>; + let savedSlugs: Set; + + try { + [allBooks, progressList, savedSlugs] = await Promise.all([ + listBooks(), + allProgress(locals.sessionId, locals.user?.id), + getSavedSlugs(locals.sessionId, locals.user?.id) + ]); + } catch (e) { + log.error('api/library', 'failed to load library data', { err: String(e) }); + allBooks = []; + progressList = []; + savedSlugs = new Set(); + } + + const progressMap: Record = {}; + const progressUpdatedMap: Record = {}; + for (const p of progressList) { + progressMap[p.slug] = p.chapter; + progressUpdatedMap[p.slug] = p.updated; + } + + const progressSlugs = new Set(progressList.map((p) => p.slug)); + const books = allBooks.filter((b) => progressSlugs.has(b.slug) || savedSlugs.has(b.slug)); + + const withProgress = books.filter((b) => progressSlugs.has(b.slug)); + const savedOnly = books + .filter((b) => !progressSlugs.has(b.slug)) + .sort((a, b) => (a.title ?? '').localeCompare(b.title ?? '')); + + withProgress.sort((a, b) => { + const ta = progressUpdatedMap[a.slug] ?? ''; + const tb = progressUpdatedMap[b.slug] ?? ''; + return tb.localeCompare(ta); + }); + + const ordered = [...withProgress, ...savedOnly]; + + const items = ordered.map((book) => ({ + book, + last_chapter: progressMap[book.slug] ?? null, + saved_at: progressUpdatedMap[book.slug] ?? new Date().toISOString() + })); + + return json(items); +}; diff --git a/v3/ui/src/routes/api/library/[slug]/+server.ts b/v3/ui/src/routes/api/library/[slug]/+server.ts new file mode 100644 index 0000000..b0f9243 --- /dev/null +++ b/v3/ui/src/routes/api/library/[slug]/+server.ts @@ -0,0 +1,34 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { saveBook, unsaveBook } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * POST /api/library/[slug] + * Save a book to the user's personal library. + */ +export const POST: RequestHandler = async ({ params, locals }) => { + const { slug } = params; + try { + await saveBook(locals.sessionId, slug, locals.user?.id); + } catch (e) { + log.error('library', 'saveBook failed', { slug, err: String(e) }); + error(500, 'Failed to save book'); + } + return json({ ok: true }); +}; + +/** + * DELETE /api/library/[slug] + * Remove a book from the user's personal library. + */ +export const DELETE: RequestHandler = async ({ params, locals }) => { + const { slug } = params; + try { + await unsaveBook(locals.sessionId, slug, locals.user?.id); + } catch (e) { + log.error('library', 'unsaveBook failed', { slug, err: String(e) }); + error(500, 'Failed to remove book'); + } + return json({ ok: true }); +}; diff --git a/v3/ui/src/routes/api/presign/audio/+server.ts b/v3/ui/src/routes/api/presign/audio/+server.ts new file mode 100644 index 0000000..0b5725c --- /dev/null +++ b/v3/ui/src/routes/api/presign/audio/+server.ts @@ -0,0 +1,47 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { presignAudio } from '$lib/server/minio'; +import { log } from '$lib/server/logger'; +import * as cache from '$lib/server/presignCache'; + +/** + * GET /api/presign/audio?slug=...&n=...&voice=... + * Returns a presigned MinIO URL for the audio file so the browser + * can stream it directly without going through the server. + * Returns 404 when the audio has not been generated yet. + * + * Results are cached in-process for 50 minutes (MinIO URLs are valid 1 hour) + * to avoid a backend + MinIO round-trip on every "Play" click. + */ +export const GET: RequestHandler = async ({ url }) => { + const slug = url.searchParams.get('slug'); + // Accept both 'n' (web) and 'chapter' (iOS) as the chapter number param + const n = parseInt(url.searchParams.get('n') ?? url.searchParams.get('chapter') ?? '', 10); + const voice = url.searchParams.get('voice') ?? ''; + + if (!slug || !n || n < 1) { + error(400, 'Missing slug or n'); + } + + const cacheKey = cache.audioKey(slug, n, voice); + + // Fast path: return cached URL if still valid. + const cached = await cache.get(cacheKey); + if (cached) { + return json({ url: cached }); + } + + // Slow path: call backend → MinIO presign. + try { + const presignedUrl = await presignAudio(slug, n, voice || undefined); + await cache.set(cacheKey, presignedUrl); + return json({ url: presignedUrl }); + } catch (e) { + const status = (e as { status?: number }).status; + if (status === 404) { + error(404, 'Audio not found'); + } + log.error('presign', 'presign audio failed', { slug, n, err: String(e) }); + error(500, `Could not get presigned URL: ${e}`); + } +}; diff --git a/v3/ui/src/routes/api/presign/voice-sample/+server.ts b/v3/ui/src/routes/api/presign/voice-sample/+server.ts new file mode 100644 index 0000000..773572c --- /dev/null +++ b/v3/ui/src/routes/api/presign/voice-sample/+server.ts @@ -0,0 +1,40 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { presignVoiceSample } from '$lib/server/minio'; +import * as cache from '$lib/server/presignCache'; + +/** + * GET /api/presign/voice-sample?voice=af_bella + * Returns a presigned URL for the voice sample audio file. + * Returns 404 if the sample has not been generated yet. + * + * Results are cached in-process for 50 minutes to avoid a backend + MinIO + * round-trip on every voice-selection preview play. + */ +export const GET: RequestHandler = async ({ url }) => { + const voice = url.searchParams.get('voice'); + if (!voice) { + error(400, 'Missing voice parameter'); + } + + const cacheKey = cache.sampleKey(voice); + + // Fast path: return cached URL if still valid. + const cached = await cache.get(cacheKey); + if (cached) { + return json({ url: cached }); + } + + // Slow path: call backend → MinIO presign. + try { + const presignedUrl = await presignVoiceSample(voice); + await cache.set(cacheKey, presignedUrl); + return json({ url: presignedUrl }); + } catch (e) { + const status = (e as { status?: number }).status; + if (status === 404) { + error(404, 'Voice sample not found'); + } + error(502, `Failed to presign voice sample: ${e}`); + } +}; diff --git a/v3/ui/src/routes/api/profile/avatar/+server.ts b/v3/ui/src/routes/api/profile/avatar/+server.ts new file mode 100644 index 0000000..4ca1afb --- /dev/null +++ b/v3/ui/src/routes/api/profile/avatar/+server.ts @@ -0,0 +1,81 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { presignAvatarUploadUrl, presignAvatarUrl } from '$lib/server/minio'; +import { updateUserAvatarUrl, getUserByUsername } from '$lib/server/pocketbase'; + +const ALLOWED_TYPES = ['image/jpeg', 'image/png', 'image/webp']; + +/** + * POST /api/profile/avatar + * Body: JSON { mime_type: "image/jpeg" | "image/png" | "image/webp" } + * + * Returns a short-lived presigned PUT URL pointing at MinIO (public endpoint) + * so the client can upload the image bytes directly, bypassing the server. + * After the PUT completes, the client must call PATCH /api/profile/avatar + * with the returned key to record it in PocketBase. + * + * Returns: { upload_url: string, key: string } + */ +export const POST: RequestHandler = async ({ request, locals }) => { + if (!locals.user) error(401, 'Not authenticated'); + + let mimeType = 'image/jpeg'; + try { + const body = await request.json(); + if (body?.mime_type) mimeType = body.mime_type; + } catch { + // default to jpeg if body is missing/invalid + } + + if (!ALLOWED_TYPES.includes(mimeType)) { + error(400, `Unsupported image type: ${mimeType}. Allowed: jpeg, png, webp`); + } + + const { uploadUrl, key } = await presignAvatarUploadUrl(locals.user.id, mimeType); + return json({ upload_url: uploadUrl, key }); +}; + +/** + * PATCH /api/profile/avatar + * Body: JSON { key: string } + * + * Called after the client has successfully PUT the image to MinIO via the + * presigned URL. Records the object key in PocketBase and returns a fresh + * presigned GET URL for immediate display. + * + * Returns: { avatar_url: string | null } + */ +export const PATCH: RequestHandler = async ({ request, locals }) => { + if (!locals.user) error(401, 'Not authenticated'); + + let key: string | undefined; + try { + const body = await request.json(); + if (typeof body?.key === 'string') key = body.key; + } catch { + error(400, 'Invalid JSON body'); + } + + if (!key) error(400, 'Missing "key" field'); + + await updateUserAvatarUrl(locals.user.id, key); + + const avatarUrl = await presignAvatarUrl(locals.user.id); + return json({ avatar_url: avatarUrl }); +}; + +/** + * GET /api/profile/avatar + * Returns a presigned GET URL for the current user's avatar, or null if none set. + */ +export const GET: RequestHandler = async ({ locals }) => { + if (!locals.user) error(401, 'Not authenticated'); + + const record = await getUserByUsername(locals.user.username).catch(() => null); + if (!record?.avatar_url) { + return json({ avatar_url: null }); + } + + const avatarUrl = await presignAvatarUrl(locals.user.id); + return json({ avatar_url: avatarUrl }); +}; diff --git a/v3/ui/src/routes/api/progress/+server.ts b/v3/ui/src/routes/api/progress/+server.ts new file mode 100644 index 0000000..98e3c59 --- /dev/null +++ b/v3/ui/src/routes/api/progress/+server.ts @@ -0,0 +1,27 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { setProgress } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * POST /api/progress + * Body: { slug: string, chapter: number } + * Records the user's reading position. + * When the user is logged in, progress is keyed by user_id so it syncs across devices. + * When anonymous, progress is keyed by the session cookie. + */ +export const POST: RequestHandler = async ({ request, locals }) => { + const body = await request.json().catch(() => null); + + if (!body || typeof body.slug !== 'string' || typeof body.chapter !== 'number') { + error(400, 'Invalid body — expected { slug, chapter }'); + } + + try { + await setProgress(locals.sessionId, body.slug, body.chapter, locals.user?.id); + } catch (e) { + log.error('progress', 'setProgress failed', { slug: body.slug, chapter: body.chapter, err: String(e) }); + error(500, 'Failed to save progress'); + } + return json({ ok: true }); +}; diff --git a/v3/ui/src/routes/api/progress/[slug]/+server.ts b/v3/ui/src/routes/api/progress/[slug]/+server.ts new file mode 100644 index 0000000..498703e --- /dev/null +++ b/v3/ui/src/routes/api/progress/[slug]/+server.ts @@ -0,0 +1,54 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { setProgress, deleteProgress } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * POST /api/progress/[slug] + * Body: { chapter: number } + * Records the user's reading position for a specific book. + * + * This is a slug-in-path variant of POST /api/progress (which takes slug in body). + * Used by the iOS app where slug is part of the URL path. + */ +export const POST: RequestHandler = async ({ params, request, locals }) => { + const { slug } = params; + const body = await request.json().catch(() => null); + + if (!body || typeof body.chapter !== 'number') { + error(400, 'Invalid body — expected { chapter: number }'); + } + + try { + await setProgress(locals.sessionId, slug, body.chapter, locals.user?.id); + } catch (e) { + log.error('api/progress/[slug]', 'setProgress failed', { + slug, + chapter: body.chapter, + err: String(e) + }); + error(500, 'Failed to save progress'); + } + + return json({ ok: true }); +}; + +/** + * DELETE /api/progress/[slug] + * Removes reading progress for a specific book (removes from library/continue reading). + */ +export const DELETE: RequestHandler = async ({ params, locals }) => { + const { slug } = params; + + try { + await deleteProgress(locals.sessionId, slug, locals.user?.id); + } catch (e) { + log.error('api/progress/[slug]', 'deleteProgress failed', { + slug, + err: String(e) + }); + error(500, 'Failed to delete progress'); + } + + return json({ ok: true }); +}; diff --git a/v3/ui/src/routes/api/progress/audio-time/+server.ts b/v3/ui/src/routes/api/progress/audio-time/+server.ts new file mode 100644 index 0000000..1659ba2 --- /dev/null +++ b/v3/ui/src/routes/api/progress/audio-time/+server.ts @@ -0,0 +1,56 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { setAudioTime, getAudioTime } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * GET /api/progress/audio-time?slug=&chapter= + * Returns the last saved audio position for a chapter, or null. + */ +export const GET: RequestHandler = async ({ url, locals }) => { + const slug = url.searchParams.get('slug'); + const chapterParam = url.searchParams.get('chapter'); + + if (!slug || !chapterParam) { + error(400, 'Missing slug or chapter query params'); + } + + const chapter = parseInt(chapterParam, 10); + if (isNaN(chapter)) { + error(400, 'chapter must be a number'); + } + + try { + const audioTime = await getAudioTime(locals.sessionId, slug, chapter, locals.user?.id); + return json({ audioTime }); + } catch (e) { + log.error('audio-time', 'GET failed', { slug, chapter, err: String(e) }); + error(500, 'Failed to load audio time'); + } +}; + +/** + * PATCH /api/progress/audio-time + * Body: { slug: string, chapter: number, audioTime: number } + * Saves the current audio playback position. + */ +export const PATCH: RequestHandler = async ({ request, locals }) => { + const body = await request.json().catch(() => null); + + if ( + !body || + typeof body.slug !== 'string' || + typeof body.chapter !== 'number' || + typeof body.audioTime !== 'number' + ) { + error(400, 'Invalid body — expected { slug, chapter, audioTime }'); + } + + try { + await setAudioTime(locals.sessionId, body.slug, body.chapter, body.audioTime, locals.user?.id); + } catch (e) { + log.error('audio-time', 'PATCH failed', { slug: body.slug, chapter: body.chapter, err: String(e) }); + error(500, 'Failed to save audio time'); + } + return json({ ok: true }); +}; diff --git a/v3/ui/src/routes/api/ranking/+server.ts b/v3/ui/src/routes/api/ranking/+server.ts new file mode 100644 index 0000000..b98b312 --- /dev/null +++ b/v3/ui/src/routes/api/ranking/+server.ts @@ -0,0 +1,25 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +/** + * GET /api/ranking + * Proxies to the Go backend's /api/ranking endpoint. + * Returns the top-ranked novels list as JSON. + */ +export const GET: RequestHandler = async () => { + try { + const res = await backendFetch('/api/ranking'); + if (!res.ok) { + log.error('api/ranking', 'backend returned error', { status: res.status }); + error(502, `Ranking fetch failed: ${res.status}`); + } + const data = await res.json(); + return json(data); + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; + log.error('api/ranking', 'network error', { err: String(e) }); + error(502, 'Could not load ranking'); + } +}; diff --git a/v3/ui/src/routes/api/scrape/+server.ts b/v3/ui/src/routes/api/scrape/+server.ts new file mode 100644 index 0000000..f5d02a0 --- /dev/null +++ b/v3/ui/src/routes/api/scrape/+server.ts @@ -0,0 +1,61 @@ +/** + * POST /api/scrape + * + * Proxies scrape requests to the Go backend. + * Admin-only — returns 403 if the authenticated user is not an admin. + * + * Request body (JSON): + * { "url": "https://novelfire.net/book/..." } — scrape a single book + * {} — scrape the full catalogue + * + * Responses mirror the Go scraper: + * 202 Accepted — job enqueued + * 409 Conflict — a scrape job is already running + * 400 Bad Request + * 403 Forbidden — not an admin + */ + +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +export const POST: RequestHandler = async ({ request, locals }) => { + // Admin guard + if (!locals.user || locals.user.role !== 'admin') { + throw error(403, 'Forbidden'); + } + + let body: { url?: string } = {}; + try { + body = await request.json(); + } catch { + // empty body is fine — means "scrape all" + } + + // Decide which scraper endpoint to call + const isBookScrape = typeof body.url === 'string' && body.url.length > 0; + const endpoint = isBookScrape ? '/scrape/book' : '/scrape'; + + let res: Response; + try { + res = await backendFetch(endpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: isBookScrape ? JSON.stringify({ url: body.url }) : undefined + }); + } catch (e) { + log.error('scrape', 'backend proxy network error', { endpoint, err: String(e) }); + throw error(502, 'Could not reach backend'); + } + + if (!res.ok && res.status >= 500) { + const text = await res.text().catch(() => ''); + log.error('scrape', 'backend returned error', { endpoint, status: res.status, body: text }); + } + + const data = await res.json().catch(() => ({})); + + // Pass through the status code from the Go backend (202, 409, 400, …) + return json(data, { status: res.status }); +}; diff --git a/v3/ui/src/routes/api/scrape/cancel/[id]/+server.ts b/v3/ui/src/routes/api/scrape/cancel/[id]/+server.ts new file mode 100644 index 0000000..b0843c8 --- /dev/null +++ b/v3/ui/src/routes/api/scrape/cancel/[id]/+server.ts @@ -0,0 +1,40 @@ +/** + * POST /api/scrape/cancel/[id] + * + * Admin-only proxy that cancels a pending scrape (or audio) task by ID. + * Forwards the request to the Go backend POST /api/cancel-task/{id}. + * + * Responses: + * 200 OK — task cancelled + * 403 Forbidden — not an admin + * 409 Conflict — task cannot be cancelled (already running/done/not found) + */ + +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +export const POST: RequestHandler = async ({ params, locals }) => { + if (!locals.user || locals.user.role !== 'admin') { + throw error(403, 'Forbidden'); + } + + const { id } = params; + if (!id) { + throw error(400, 'Missing task id'); + } + + let res: Response; + try { + res = await backendFetch(`/api/cancel-task/${encodeURIComponent(id)}`, { + method: 'POST' + }); + } catch (e) { + log.error('scrape/cancel', 'network error cancelling task', { id, err: String(e) }); + throw error(502, 'Could not reach backend'); + } + + const data = await res.json().catch(() => ({})); + return json(data, { status: res.status }); +}; diff --git a/v3/ui/src/routes/api/scrape/range/+server.ts b/v3/ui/src/routes/api/scrape/range/+server.ts new file mode 100644 index 0000000..baaa19d --- /dev/null +++ b/v3/ui/src/routes/api/scrape/range/+server.ts @@ -0,0 +1,59 @@ +/** + * POST /api/scrape/range + * + * Proxies range-scrape requests to the Go backend at POST /scrape/book/range. + * Admin-only. + * + * Request body (JSON): + * { "url": "https://novelfire.net/book/...", "from": 50, "to": 100 } + * "to" is optional — omit to scrape from "from" to the end. + * + * Responses mirror the Go scraper: + * 202 Accepted — job enqueued + * 409 Conflict — a scrape job is already running + * 400 Bad Request + * 403 Forbidden — not an admin + */ + +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +export const POST: RequestHandler = async ({ request, locals }) => { + // Admin guard + if (!locals.user || locals.user.role !== 'admin') { + throw error(403, 'Forbidden'); + } + + let body: { url?: string; from?: number; to?: number } = {}; + try { + body = await request.json(); + } catch { + throw error(400, 'Invalid JSON body'); + } + + if (!body.url || typeof body.from !== 'number') { + throw error(400, 'url and from are required'); + } + + let res: Response; + try { + res = await backendFetch('/scrape/book/range', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url: body.url, from: body.from, to: body.to }) + }); + } catch (e) { + log.error('scrape/range', 'backend proxy network error', { err: String(e) }); + throw error(502, 'Could not reach backend'); + } + + if (!res.ok && res.status >= 500) { + const text = await res.text().catch(() => ''); + log.error('scrape/range', 'backend returned error', { status: res.status, body: text }); + } + + const data = await res.json().catch(() => ({})); + return json(data, { status: res.status }); +}; diff --git a/v3/ui/src/routes/api/search/+server.ts b/v3/ui/src/routes/api/search/+server.ts new file mode 100644 index 0000000..54dfd59 --- /dev/null +++ b/v3/ui/src/routes/api/search/+server.ts @@ -0,0 +1,34 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +/** + * GET /api/search?q= + * Proxies to the Go backend's /api/search endpoint. + * Returns: { results, local_count, remote_count } + * + * Response shape mirrors SearchResponse in the iOS APIClient. + */ +export const GET: RequestHandler = async ({ url }) => { + const q = url.searchParams.get('q') ?? ''; + + if (q.trim().length < 2) { + return json({ results: [], local_count: 0, remote_count: 0 }); + } + + const apiURL = `/api/search?q=${encodeURIComponent(q.trim())}`; + try { + const res = await backendFetch(apiURL); + if (!res.ok) { + log.error('api/search', 'backend returned error', { status: res.status, q }); + error(502, `Search failed: ${res.status}`); + } + const data = await res.json(); + return json(data); + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; + log.error('api/search', 'network error', { q, err: String(e) }); + error(502, 'Could not reach backend'); + } +}; diff --git a/v3/ui/src/routes/api/sessions/+server.ts b/v3/ui/src/routes/api/sessions/+server.ts new file mode 100644 index 0000000..5feb689 --- /dev/null +++ b/v3/ui/src/routes/api/sessions/+server.ts @@ -0,0 +1,32 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { listUserSessions } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * GET /api/sessions + * Returns all active sessions for the logged-in user. + */ +export const GET: RequestHandler = async ({ locals }) => { + if (!locals.user) { + error(401, 'Not logged in'); + } + + try { + const sessions = await listUserSessions(locals.user.id); + // Don't expose raw session_id to the client — only the record ID for revocation + const safe = sessions.map((s) => ({ + id: s.id, + user_agent: s.user_agent, + ip: s.ip, + created_at: s.created_at, + last_seen: s.last_seen, + // Tell the client whether this is the currently active session + is_current: s.session_id === locals.user!.authSessionId + })); + return json({ sessions: safe }); + } catch (e) { + log.error('sessions', 'GET failed', { err: String(e) }); + error(500, 'Failed to load sessions'); + } +}; diff --git a/v3/ui/src/routes/api/sessions/[id]/+server.ts b/v3/ui/src/routes/api/sessions/[id]/+server.ts new file mode 100644 index 0000000..f40774a --- /dev/null +++ b/v3/ui/src/routes/api/sessions/[id]/+server.ts @@ -0,0 +1,41 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { revokeUserSession } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * DELETE /api/sessions/[id] + * Revokes a specific session by its PocketBase record ID. + * Only the owner can revoke their own sessions. + */ +export const DELETE: RequestHandler = async ({ params, locals, cookies }) => { + if (!locals.user) { + error(401, 'Not logged in'); + } + + const recordId = params.id; + if (!recordId) { + error(400, 'Session ID required'); + } + + try { + const ok = await revokeUserSession(recordId, locals.user.id); + if (!ok) { + error(404, 'Session not found or not yours'); + } + + // If the user is terminating their own current session, clear their auth cookie + // so they get logged out immediately (the hook would do this on the next request anyway, + // but clearing it here gives instant feedback for the "end this session" flow). + // For other sessions, we leave the cookie intact. + // We detect "current session" via authSessionId — but since the client sends the + // record ID (not the session_id), we rely on the UI to redirect after ending its own session. + + log.info('sessions', 'session revoked', { recordId, userId: locals.user.id }); + return json({ ok: true }); + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; // re-throw SvelteKit errors + log.error('sessions', 'DELETE failed', { recordId, err: String(e) }); + error(500, 'Failed to revoke session'); + } +}; diff --git a/v3/ui/src/routes/api/settings/+server.ts b/v3/ui/src/routes/api/settings/+server.ts new file mode 100644 index 0000000..0e27ad4 --- /dev/null +++ b/v3/ui/src/routes/api/settings/+server.ts @@ -0,0 +1,49 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { getSettings, saveSettings } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * GET /api/settings + * Returns the current user's settings (auto_next, voice, speed). + * Returns defaults if no settings record exists yet. + */ +export const GET: RequestHandler = async ({ locals }) => { + try { + const settings = await getSettings(locals.sessionId, locals.user?.id); + return json({ + autoNext: settings?.auto_next ?? false, + voice: settings?.voice ?? 'af_bella', + speed: settings?.speed ?? 1.0 + }); + } catch (e) { + log.error('settings', 'GET failed', { err: String(e) }); + error(500, 'Failed to load settings'); + } +}; + +/** + * PUT /api/settings + * Body: { autoNext: boolean, voice: string, speed: number } + * Saves user preferences. + */ +export const PUT: RequestHandler = async ({ request, locals }) => { + const body = await request.json().catch(() => null); + + if ( + !body || + typeof body.autoNext !== 'boolean' || + typeof body.voice !== 'string' || + typeof body.speed !== 'number' + ) { + error(400, 'Invalid body — expected { autoNext, voice, speed }'); + } + + try { + await saveSettings(locals.sessionId, body, locals.user?.id); + } catch (e) { + log.error('settings', 'PUT failed', { err: String(e) }); + error(500, 'Failed to save settings'); + } + return json({ ok: true }); +}; diff --git a/v3/ui/src/routes/api/users/[username]/+server.ts b/v3/ui/src/routes/api/users/[username]/+server.ts new file mode 100644 index 0000000..7c8f5b5 --- /dev/null +++ b/v3/ui/src/routes/api/users/[username]/+server.ts @@ -0,0 +1,46 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { getPublicProfile, getSubscription } from '$lib/server/pocketbase'; +import { presignAvatarUrl } from '$lib/server/minio'; +import { log } from '$lib/server/logger'; + +/** + * GET /api/users/[username] + * Returns public profile info + whether the current user is subscribed. + */ +export const GET: RequestHandler = async ({ params, locals }) => { + const { username } = params; + + try { + const profile = await getPublicProfile(username); + if (!profile) error(404, `User "${username}" not found`); + + // Resolve avatar presigned URL if set + let avatarUrl: string | null = null; + if (profile.avatar_url) { + avatarUrl = await presignAvatarUrl(profile.id).catch(() => null); + } + + // Is the current logged-in user subscribed? + let isSubscribed = false; + if (locals.user && locals.user.id !== profile.id) { + const sub = await getSubscription(locals.user.id, profile.id).catch(() => null); + isSubscribed = !!sub; + } + + return json({ + id: profile.id, + username: profile.username, + avatarUrl, + created: profile.created, + followerCount: profile.followerCount, + followingCount: profile.followingCount, + isSubscribed, + isSelf: locals.user?.id === profile.id + }); + } catch (e) { + if ((e as { status?: number }).status === 404) throw e; + log.error('api/users', 'failed to load profile', { username, err: String(e) }); + error(500, 'Failed to load profile'); + } +}; diff --git a/v3/ui/src/routes/api/users/[username]/library/+server.ts b/v3/ui/src/routes/api/users/[username]/library/+server.ts new file mode 100644 index 0000000..8fcb739 --- /dev/null +++ b/v3/ui/src/routes/api/users/[username]/library/+server.ts @@ -0,0 +1,43 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { + getUserByUsername, + getUserPublicLibrary, + getUserCurrentlyReading +} from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * GET /api/users/[username]/library + * Returns the public library + currently-reading list for a user. + * Does not require authentication — all data is public. + */ +export const GET: RequestHandler = async ({ params }) => { + const { username } = params; + + const user = await getUserByUsername(username).catch(() => null); + if (!user) error(404, `User "${username}" not found`); + + try { + const [currentlyReading, library] = await Promise.all([ + getUserCurrentlyReading(user.id), + getUserPublicLibrary(user.id) + ]); + + return json({ + currently_reading: currentlyReading.map((item) => ({ + book: item.book, + last_chapter: item.chapter, + saved: false + })), + library: library.map((item) => ({ + book: item.book, + last_chapter: item.chapter, + saved: item.saved + })) + }); + } catch (e) { + log.error('api/users/library', 'failed to load library', { username, err: String(e) }); + error(500, 'Failed to load library'); + } +}; diff --git a/v3/ui/src/routes/api/users/[username]/subscribe/+server.ts b/v3/ui/src/routes/api/users/[username]/subscribe/+server.ts new file mode 100644 index 0000000..c381dfc --- /dev/null +++ b/v3/ui/src/routes/api/users/[username]/subscribe/+server.ts @@ -0,0 +1,48 @@ +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { + getUserByUsername, + subscribe, + unsubscribe, + getSubscription +} from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +/** + * POST /api/users/[username]/subscribe — subscribe to a user + * DELETE /api/users/[username]/subscribe — unsubscribe + * Requires authentication. + */ +export const POST: RequestHandler = async ({ params, locals }) => { + if (!locals.user) error(401, 'Login required'); + + const { username } = params; + const target = await getUserByUsername(username).catch(() => null); + if (!target) error(404, `User "${username}" not found`); + if (locals.user.id === target.id) error(400, 'Cannot subscribe to yourself'); + + try { + await subscribe(locals.user.id, target.id); + const sub = await getSubscription(locals.user.id, target.id); + return json({ subscribed: true, subId: sub?.id ?? null }); + } catch (e) { + log.error('api/users/subscribe', 'subscribe failed', { username, err: String(e) }); + error(500, 'Failed to subscribe'); + } +}; + +export const DELETE: RequestHandler = async ({ params, locals }) => { + if (!locals.user) error(401, 'Login required'); + + const { username } = params; + const target = await getUserByUsername(username).catch(() => null); + if (!target) error(404, `User "${username}" not found`); + + try { + await unsubscribe(locals.user.id, target.id); + return json({ subscribed: false }); + } catch (e) { + log.error('api/users/subscribe', 'unsubscribe failed', { username, err: String(e) }); + error(500, 'Failed to unsubscribe'); + } +}; diff --git a/v3/ui/src/routes/api/voices/+server.ts b/v3/ui/src/routes/api/voices/+server.ts new file mode 100644 index 0000000..8438ed4 --- /dev/null +++ b/v3/ui/src/routes/api/voices/+server.ts @@ -0,0 +1,21 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { backendFetch } from '$lib/server/scraper'; + +/** + * GET /api/voices + * Proxies the voice list from the backend → Kokoro. + * Returns { voices: string[] } + */ +export const GET: RequestHandler = async () => { + try { + const res = await backendFetch('/api/voices'); + if (!res.ok) { + return json({ voices: [] }); + } + const data = (await res.json()) as { voices: string[] }; + return json({ voices: data.voices ?? [] }); + } catch { + return json({ voices: [] }); + } +}; diff --git a/v3/ui/src/routes/books/+page.server.ts b/v3/ui/src/routes/books/+page.server.ts new file mode 100644 index 0000000..6d0ec72 --- /dev/null +++ b/v3/ui/src/routes/books/+page.server.ts @@ -0,0 +1,57 @@ +import { error } from '@sveltejs/kit'; +import type { PageServerLoad } from './$types'; +import { listBooks, allProgress, getSavedSlugs } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +export const load: PageServerLoad = async ({ locals }) => { + let allBooks: Awaited>; + let progressList: Awaited>; + let savedSlugs: Set; + + try { + [allBooks, progressList, savedSlugs] = await Promise.all([ + listBooks(), + allProgress(locals.sessionId, locals.user?.id), + getSavedSlugs(locals.sessionId, locals.user?.id) + ]); + } catch (e) { + log.error('books', 'failed to load library data', { err: String(e) }); + allBooks = []; + progressList = []; + savedSlugs = new Set(); + } + + // Build a quick lookup: slug → last chapter read + const progressMap: Record = {}; + for (const p of progressList) { + progressMap[p.slug] = p.chapter; + } + + // Library = books the user has started reading OR explicitly saved + const progressSlugs = new Set(progressList.map((p) => p.slug)); + const books = allBooks.filter((b) => progressSlugs.has(b.slug) || savedSlugs.has(b.slug)); + + // Sort: books with progress first (most-recently-read order is implicit via progressList), + // then saved-only books alphabetically. + const withProgress = books.filter((b) => progressSlugs.has(b.slug)); + const savedOnly = books + .filter((b) => !progressSlugs.has(b.slug)) + .sort((a, b) => (a.title ?? '').localeCompare(b.title ?? '')); + + // Re-sort withProgress by most recent progress update + const progressUpdatedMap: Record = {}; + for (const p of progressList) { + progressUpdatedMap[p.slug] = p.updated; + } + withProgress.sort((a, b) => { + const ta = progressUpdatedMap[a.slug] ?? ''; + const tb = progressUpdatedMap[b.slug] ?? ''; + return tb.localeCompare(ta); // descending — most recently read first + }); + + return { + books: [...withProgress, ...savedOnly], + progressMap, + savedSlugs: [...savedSlugs] + }; +}; diff --git a/v3/ui/src/routes/books/+page.svelte b/v3/ui/src/routes/books/+page.svelte new file mode 100644 index 0000000..c54affa --- /dev/null +++ b/v3/ui/src/routes/books/+page.svelte @@ -0,0 +1,99 @@ + + + + Library — libnovel + + +
    +

    Library

    +

    + {data.books?.length ?? 0} book{(data.books?.length ?? 0) !== 1 ? 's' : ''} +

    +
    + +{#if !data.books?.length} +
    +

    Your library is empty.

    +

    + Books you start reading or save from + Discover + will appear here. +

    +
    +{:else} + +{/if} diff --git a/v3/ui/src/routes/books/[slug]/+page.server.ts b/v3/ui/src/routes/books/[slug]/+page.server.ts new file mode 100644 index 0000000..8005d92 --- /dev/null +++ b/v3/ui/src/routes/books/[slug]/+page.server.ts @@ -0,0 +1,121 @@ +import { error } from '@sveltejs/kit'; +import type { PageServerLoad } from './$types'; +import { getBook, listChapterIdx, getProgress, isBookSaved } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +export const load: PageServerLoad = async ({ params, locals }) => { + const { slug } = params; + + // Try fetching from PocketBase first + let book = await getBook(slug).catch((e) => { + log.error('books', 'getBook failed', { slug, err: String(e) }); + return null; + }); + + if (book) { + // Book is in the library — normal path + let chapters, progress, saved; + try { + [chapters, progress, saved] = await Promise.all([ + listChapterIdx(slug), + getProgress(locals.sessionId, slug, locals.user?.id), + isBookSaved(locals.sessionId, slug, locals.user?.id) + ]); + } catch (e) { + log.error('books', 'failed to load book page data', { slug, err: String(e) }); + throw error(500, 'Failed to load book'); + } + + return { + book, + chapters, + inLib: true, + saved, + lastChapter: progress?.chapter ?? null, + isAdmin: locals.user?.role === 'admin', + isLoggedIn: !!locals.user, + currentUserId: locals.user?.id ?? '', + // Not scraping + scraping: false, + taskId: null as string | null + }; + } + + // Book not in PocketBase — ask backend to enqueue a scrape task. + try { + const res = await backendFetch(`/api/book-preview/${encodeURIComponent(slug)}`); + + if (res.status === 202) { + // Scrape task enqueued — show "scraping" placeholder page. + const body: { task_id: string; message: string } = await res.json(); + log.info('books', 'scrape task enqueued for book', { slug, task_id: body.task_id }); + return { + book: null, + chapters: [], + inLib: false, + saved: false, + lastChapter: null, + isAdmin: locals.user?.role === 'admin', + isLoggedIn: !!locals.user, + currentUserId: locals.user?.id ?? '', + scraping: true, + taskId: body.task_id + }; + } + + if (!res.ok) { + log.warn('books', 'book-preview returned error', { slug, status: res.status }); + error(404, `Book "${slug}" not found`); + } + + // 200 — book was already in library when backend checked + const preview: { + in_lib: boolean; + meta: { + slug: string; + title: string; + author: string; + cover: string; + status: string; + genres: string[]; + summary: string; + total_chapters: number; + source_url: string; + }; + chapters: { number: number; title: string; date?: string }[]; + } = await res.json(); + + const previewBook = { + id: '', + slug: preview.meta.slug || slug, + title: preview.meta.title, + author: preview.meta.author, + cover: preview.meta.cover, + status: preview.meta.status, + genres: preview.meta.genres ?? [], + summary: preview.meta.summary, + total_chapters: preview.meta.total_chapters, + source_url: preview.meta.source_url, + ranking: 0, + meta_updated: '' + }; + + return { + book: previewBook, + chapters: preview.chapters, + inLib: true, + saved: false, + lastChapter: null, + isAdmin: locals.user?.role === 'admin', + isLoggedIn: !!locals.user, + currentUserId: locals.user?.id ?? '', + scraping: false, + taskId: null as string | null + }; + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; + log.error('books', 'book-preview fetch failed', { slug, err: String(e) }); + error(404, `Book "${slug}" not found`); + } +}; diff --git a/v3/ui/src/routes/books/[slug]/+page.svelte b/v3/ui/src/routes/books/[slug]/+page.svelte new file mode 100644 index 0000000..d95fd61 --- /dev/null +++ b/v3/ui/src/routes/books/[slug]/+page.svelte @@ -0,0 +1,420 @@ + + + + {data.scraping ? 'Scraping…' : data.book?.title ?? 'Book'} — libnovel + + +{#if data.scraping} + +
    + + + + +
    +

    Scraping in progress…

    +

    + Fetching the first 20 chapters. Refresh the page in a minute. +

    + {#if data.taskId} +

    task: {data.taskId}

    + {/if} +
    + ← Home +
    + +{:else} +{@const book = data.book!} + + +
    + + {#if book.cover} + + {/if} + + +
    + +
    + + {#if book.cover} + {book.title} + {/if} + + +
    + +
    +

    {book.title}

    + {#if !data.inLib} + + not in library + + {/if} +
    + + + {#if book.author} +

    {book.author}

    + {/if} + + +
    + {#if book.status} + {book.status} + {/if} + {#each genres as genre} + {genre} + {/each} +
    + + + {#if book.summary} +
    +

    + {book.summary} +

    + {#if book.summary.length > 220} + + {/if} +
    + {/if} + + + +
    +
    + + +
    + {#if data.lastChapter} + + Continue ch.{data.lastChapter} + + {/if} + {#if chapterList.length > 0} + + {data.inLib ? 'Start from ch.1' : 'Preview ch.1'} + + {/if} + {#if data.inLib} + + {/if} +
    +
    +
    + + +
    + + + + + +
    + Chapters + {#if chapterList.length > 0} + + {#if data.lastChapter && data.lastChapter > 0} + Reading ch.{data.lastChapter} of {chapterList.length} + {:else} + {chapterList.length} chapter{chapterList.length === 1 ? '' : 's'} + {/if} + + {/if} +
    + + + +
    + + + {#if data.isAdmin && book.source_url} +
    + + + {#if adminOpen} +
    + +
    + + {#if scrapeResult} + + {scrapeResult === 'queued' ? 'Queued.' : scrapeResult === 'busy' ? 'Scraper busy.' : 'Error.'} + + {/if} +
    + + +
    +
    + + +
    +
    + + +
    + + {#if rangeResult} + + {rangeResult === 'queued' ? 'Range scrape queued.' : rangeResult === 'busy' ? 'Scraper busy.' : 'Error queuing.'} + + {/if} +
    +
    + {/if} +
    + {/if} +
    + + + + +{/if} diff --git a/v3/ui/src/routes/books/[slug]/chapters/+page.server.ts b/v3/ui/src/routes/books/[slug]/chapters/+page.server.ts new file mode 100644 index 0000000..9352791 --- /dev/null +++ b/v3/ui/src/routes/books/[slug]/chapters/+page.server.ts @@ -0,0 +1,32 @@ +import { error } from '@sveltejs/kit'; +import type { PageServerLoad } from './$types'; +import { getBook, listChapterIdx, getProgress } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; + +export const load: PageServerLoad = async ({ params, locals }) => { + const { slug } = params; + + const book = await getBook(slug).catch((e) => { + log.error('chapters', 'getBook failed', { slug, err: String(e) }); + return null; + }); + + if (!book) error(404, `Book "${slug}" not found`); + + let chapters, progress; + try { + [chapters, progress] = await Promise.all([ + listChapterIdx(slug), + getProgress(locals.sessionId, slug, locals.user?.id) + ]); + } catch (e) { + log.error('chapters', 'failed to load chapters', { slug, err: String(e) }); + throw error(500, 'Failed to load chapters'); + } + + return { + book: { slug: book.slug, title: book.title, cover: book.cover ?? '', totalChapters: book.total_chapters }, + chapters, + lastChapter: progress?.chapter ?? null + }; +}; diff --git a/v3/ui/src/routes/books/[slug]/chapters/+page.svelte b/v3/ui/src/routes/books/[slug]/chapters/+page.svelte new file mode 100644 index 0000000..89a8ed9 --- /dev/null +++ b/v3/ui/src/routes/books/[slug]/chapters/+page.svelte @@ -0,0 +1,203 @@ + + + + {data.book.title} — Chapters — libnovel + + + +
    + + + + + Back + + / +

    {data.book.title}

    +
    + + +
    + + + + + {#if searchQuery} + + {/if} +
    + + +{#if !searchQuery && totalGroups > 1} +
    + {#each Array(totalGroups) as _, i} + + {/each} +
    +{/if} + + +{#if data.lastChapter && data.lastChapter > 0 && !searchQuery && activeGroup !== currentGroup} + +{/if} + + +{#if visibleChapters.length === 0} + {#if searchQuery} +

    No chapters match "{searchQuery}"

    + {:else} +

    No chapters available yet.

    + {/if} +{:else} + + {#if searchQuery} +

    {visibleChapters.length} result{visibleChapters.length === 1 ? '' : 's'}

    + {/if} + + + + + {#if !searchQuery && totalGroups > 1} +
    + {#each Array(totalGroups) as _, i} + + {/each} +
    + {/if} +{/if} diff --git a/v3/ui/src/routes/books/[slug]/chapters/[n]/+page.server.ts b/v3/ui/src/routes/books/[slug]/chapters/[n]/+page.server.ts new file mode 100644 index 0000000..911e531 --- /dev/null +++ b/v3/ui/src/routes/books/[slug]/chapters/[n]/+page.server.ts @@ -0,0 +1,138 @@ +import { error } from '@sveltejs/kit'; +import { marked } from 'marked'; +import type { PageServerLoad } from './$types'; +import { getBook, listChapterIdx } from '$lib/server/pocketbase'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +export const load: PageServerLoad = async ({ params, url, locals }) => { + const { slug } = params; + const n = parseInt(params.n, 10); + + if (!n || n < 1) error(400, 'Invalid chapter number'); + + const isPreview = url.searchParams.get('preview') === '1'; + const chapterUrl = url.searchParams.get('chapter_url') ?? ''; + const chapterTitle = url.searchParams.get('title') ?? ''; + + if (isPreview) { + // ── Preview path: scrape chapter live, nothing from PocketBase/MinIO ── + const previewParams = new URLSearchParams(); + if (chapterUrl) previewParams.set('chapter_url', chapterUrl); + if (chapterTitle) previewParams.set('title', chapterTitle); + + let chapterData: { slug: string; number: number; title: string; text: string; url: string }; + try { + const res = await backendFetch( + `/api/chapter-text-preview/${encodeURIComponent(slug)}/${n}?${previewParams.toString()}` + ); + if (!res.ok) { + log.error('chapter', 'chapter-text-preview returned error', { slug, n, status: res.status }); + error(404, `Chapter ${n} not found`); + } + chapterData = await res.json(); + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; + log.error('chapter', 'chapter-text-preview fetch failed', { slug, n, err: String(e) }); + error(502, 'Could not fetch chapter preview'); + } + + // Wrap plain text in minimal HTML paragraphs for display + const html = chapterData.text + ? '

    ' + chapterData.text.replace(/\n{2,}/g, '

    ').replace(/\n/g, '
    ') + '

    ' + : ''; + + // Fetch voices (non-critical for preview) + let voices: string[] = []; + try { + const vRes = await backendFetch('/api/voices'); + if (vRes.ok) { + const d = (await vRes.json()) as { voices: string[] }; + voices = d.voices ?? []; + } + } catch { + // Non-critical + } + + // Try to get book title/cover from PocketBase for breadcrumbs; fall back to slug + const pb = await getBook(slug).catch(() => null); + + return { + book: { + slug, + title: pb?.title ?? slug, + cover: pb?.cover ?? '' + }, + chapter: { + id: '', + slug, + number: n, + title: chapterData.title || `Chapter ${n}`, + date_label: '' + }, + html, + voices, + prev: null as number | null, + next: null as number | null, + chapters: [] as { number: number; title: string }[], + sessionId: locals.sessionId, + isPreview: true + }; + } + + // ── Normal path: fetch from PocketBase + MinIO ───────────────────────── + // Fetch book metadata, chapter index, and voice list in parallel + const [book, chapters, voicesRes] = await Promise.all([ + getBook(slug), + listChapterIdx(slug), + backendFetch('/api/voices').catch(() => null) + ]); + + if (!book) error(404, `Book "${slug}" not found`); + + const chapterIdx = chapters.find((c) => c.number === n); + if (!chapterIdx) error(404, `Chapter ${n} not found`); + + // Parse voices — fall back to a minimal default list on error + let voices: string[] = []; + try { + if (voicesRes?.ok) { + const data = (await voicesRes.json()) as { voices: string[] }; + voices = data.voices ?? []; + } + } catch { + // Non-critical — UI will use store default + } + + // Fetch chapter markdown directly from the backend (server-side MinIO read) + let html = ''; + try { + const res = await backendFetch(`/api/chapter-markdown/${encodeURIComponent(slug)}/${n}`); + if (!res.ok) { + log.error('chapter', 'chapter-markdown returned error', { slug, n, status: res.status }); + error(res.status === 404 ? 404 : 502, res.status === 404 ? `Chapter ${n} not found` : 'Could not fetch chapter content'); + } + const markdown = await res.text(); + html = marked(markdown) as string; + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; + // Don't hard-fail — show empty content with error message + log.error('chapter', 'failed to fetch chapter content', { slug, n, err: String(e) }); + error(502, 'Could not fetch chapter content'); + } + + const prevChapter = chapters.find((c) => c.number === n - 1) ?? null; + const nextChapter = chapters.find((c) => c.number === n + 1) ?? null; + + return { + book: { slug: book.slug, title: book.title, cover: book.cover ?? '' }, + chapter: chapterIdx, + html, + voices, + prev: prevChapter ? prevChapter.number : null, + next: nextChapter ? nextChapter.number : null, + chapters: chapters.map((c) => ({ number: c.number, title: c.title })), + sessionId: locals.sessionId, + isPreview: false + }; +}; diff --git a/v3/ui/src/routes/books/[slug]/chapters/[n]/+page.svelte b/v3/ui/src/routes/books/[slug]/chapters/[n]/+page.svelte new file mode 100644 index 0000000..de7212c --- /dev/null +++ b/v3/ui/src/routes/books/[slug]/chapters/[n]/+page.svelte @@ -0,0 +1,161 @@ + + + + {data.chapter.title || `Chapter ${data.chapter.number}`} — {data.book.title} — libnovel + + + +
    + + + + + Chapters + + +
    + {#if data.prev} + + ← Ch.{data.prev} + + {/if} + {#if data.next} + + Ch.{data.next} → + + {/if} +
    +
    + + +
    +

    + {data.chapter.title || `Chapter ${data.chapter.number}`} +

    + {#if wordCount > 0} +

    {wordCount.toLocaleString()} words

    + {/if} +
    + + +{#if !data.isPreview} + +{:else} +
    + Preview chapter — audio not available for books outside the library. +
    +{/if} + + +{#if fetchingContent} +
    + + + + + Fetching chapter… +
    +{:else if !html} +
    +

    {fetchError || 'Chapter content not available.'}

    +
    +{:else} +
    + {@html html} +
    +{/if} + + +
    + {#if data.prev} + + ← Previous chapter + + {:else} +
    + {/if} + {#if data.next} + + Next chapter → + + {/if} +
    diff --git a/v3/ui/src/routes/browse/+page.server.ts b/v3/ui/src/routes/browse/+page.server.ts new file mode 100644 index 0000000..14655f8 --- /dev/null +++ b/v3/ui/src/routes/browse/+page.server.ts @@ -0,0 +1,125 @@ +import { error } from '@sveltejs/kit'; +import type { PageServerLoad, Actions } from './$types'; +import { log } from '$lib/server/logger'; +import { backendFetch } from '$lib/server/scraper'; + +export interface NovelListing { + slug: string; + title: string; + cover: string; + rank: string; + rating: string; + chapters: string; + url: string; + // enriched fields + author?: string; + status?: string; + genres?: string[]; + source_url?: string; +} + +// Shape returned by GET /api/catalogue on the Go scraper. +interface CatalogueBook { + slug: string; + title: string; + author: string; + cover: string; + status: string; + genres: string[]; + summary: string; + total_chapters: number; + source_url: string; + ranking: number; + rating: number; +} + +interface CatalogueResponse { + books: CatalogueBook[]; + page: number; + limit: number; + total: number; + has_next: boolean; +} + +function bookToListing(book: CatalogueBook): NovelListing { + return { + slug: book.slug, + title: book.title, + cover: book.cover, + rank: book.ranking > 0 ? `#${book.ranking}` : '', + rating: book.rating > 0 ? String(book.rating) : '', + chapters: book.total_chapters > 0 ? `${book.total_chapters} chapters` : '', + url: book.source_url ?? '', + author: book.author, + status: book.status, + genres: book.genres ?? [], + source_url: book.source_url + }; +} + +export const load: PageServerLoad = async ({ url, locals }) => { + const page = url.searchParams.get('page') ?? '1'; + const genre = url.searchParams.get('genre') ?? 'all'; + const sort = url.searchParams.get('sort') ?? 'popular'; + const status = url.searchParams.get('status') ?? 'all'; + const q = url.searchParams.get('q') ?? ''; + + const params = new URLSearchParams({ page, genre, sort, status }); + if (q.trim().length >= 2) { + params.set('q', q.trim()); + } + + let novels: NovelListing[] = []; + let pageNum = parseInt(page, 10) || 1; + let hasNext = false; + + try { + const res = await backendFetch(`/api/catalogue?${params.toString()}`); + if (!res.ok) { + log.error('browse', 'catalogue returned error', { status: res.status }); + throw error(502, `Catalogue fetch failed: ${res.status}`); + } + const data: CatalogueResponse = await res.json(); + novels = (data.books ?? []).map(bookToListing); + pageNum = data.page ?? 1; + hasNext = data.has_next ?? false; + } catch (e) { + if (e instanceof Error && 'status' in e) throw e; + log.error('browse', 'catalogue network error', { err: String(e) }); + throw error(502, 'Could not reach catalogue service'); + } + + return { + novels, + page: pageNum, + hasNext, + genre, + sort, + status, + isAdmin: locals.user?.role === 'admin', + searchQuery: q.trim().length >= 2 ? q.trim() : '', + searchLocalCount: 0, + searchRemoteCount: 0 + }; +}; + +// Admin action: trigger a full catalogue scrape (refreshes ranking + library). +export const actions: Actions = { + refresh: async ({ locals, fetch }) => { + if (!locals.user || locals.user.role !== 'admin') { + throw error(403, 'Forbidden'); + } + try { + const res = await fetch('/api/scrape', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({}) + }); + if (res.status === 409) return { status: 'busy' }; + if (!res.ok) return { status: 'error' }; + return { status: 'queued' }; + } catch { + return { status: 'error' }; + } + } +}; diff --git a/v3/ui/src/routes/browse/+page.svelte b/v3/ui/src/routes/browse/+page.svelte new file mode 100644 index 0000000..a87c67d --- /dev/null +++ b/v3/ui/src/routes/browse/+page.svelte @@ -0,0 +1,700 @@ + + + + Discover — libnovel + + + +
    +

    Discover

    +

    + {#if isSearchView} + {novels.length} result{novels.length !== 1 ? 's' : ''} for "{data.searchQuery}" + {#if data.searchLocalCount > 0 || data.searchRemoteCount > 0} + ({data.searchLocalCount} local, {data.searchRemoteCount} from novelfire) + {/if} + {:else if isRankView} + {#if novels.length > 0} + {novels.length} novels ranked from last catalogue scrape + {:else} + No ranking data — run a full catalogue scrape to populate + {/if} + {:else} + Browse novels from novelfire.net + {/if} +

    +
    + + +{#if form} + {#if form.status === 'queued'} +
    + Full catalogue scrape queued. Library and ranking will update as books are processed. +
    + {:else if form.status === 'busy'} +
    + A scrape job is already running. Check back once it finishes. +
    + {:else if form.status === 'error'} +
    + Failed to queue scrape. Check that the scraper service is reachable. +
    + {/if} +{/if} + + +
    + +
    + + + {#if data.searchQuery} + + Clear + + {/if} +
    + + + + + +
    + + +
    + + + {#if data.isAdmin} +
    { + refreshing = true; + return async ({ update }) => { + await update(); + refreshing = false; + }; + }} + > + +
    + {/if} +
    + + +{#if !filtersOpen && hasActiveFilters} +

    + {filterSummary()} + clear +

    +{/if} + + +{#if filtersOpen} + + {#if data.isAdmin} +
    { + refreshing = true; + return async ({ update }) => { + await update(); + refreshing = false; + }; + }} + class="sm:hidden mb-2" + > + +
    + {/if} + +
    + + +
    +
    + + +
    + +
    + + +
    + +
    + + +
    +
    + + {#if isRankView} +

    Genre & status filters apply to Browse only

    + {/if} + +
    + + Reset + + +
    +
    +{/if} + + +{#if novels.length === 0} +
    +

    {isSearchView ? 'No results found.' : isRankView ? 'No ranking data.' : 'No novels found.'}

    +

    + {#if isSearchView} + Try a different search term. + {:else if isRankView} + {#if data.isAdmin} + Click Refresh catalogue above to trigger a full catalogue scrape. + {:else} + Ask an admin to run a catalogue scrape. + {/if} + {:else} + Try different filters or check back later. + {/if} +

    +
    + +{:else if view === 'grid'} + + + +{:else} + +
    + {#each novels as novel} + {@const isLoading = loadingSlug === novel.slug} +
    + + {#if novel.rank} + {novel.rank} + {/if} + + +
    + {#if novel.cover} + {novel.title} + {:else} +
    + + + +
    + {/if} + {#if isLoading} +
    + + + + +
    + {/if} +
    + + +
    + {#if novel.slug} + handleNovelClick(novel.slug)} + class="text-sm font-semibold transition-colors line-clamp-1 + {isLoading ? 'text-amber-400' : 'text-zinc-100 hover:text-amber-400'}" + > + {novel.title} + + {:else} + {novel.title} + {/if} +
    + {#if novel.author} + {novel.author} + {/if} + {#if novel.status} + {novel.status} + {:else if novel.chapters} + {novel.chapters} + {/if} + {#if novel.rating} + ★ {novel.rating} + {/if} + {#if novel.genres?.length} + {#each novel.genres.slice(0, 3) as genre} + {genre} + {/each} + {/if} +
    +
    + + + {#if data.isAdmin && novel.url} +
    + {#if scrapeResult[novel.slug] === 'queued'} + Queued + {:else if scrapeResult[novel.slug] === 'busy'} + Busy + {:else if scrapeResult[novel.slug] === 'error'} + Error + {:else} + + {/if} +
    + {/if} + + + {#if novel.source_url || novel.url} + + + + + + {/if} +
    + {/each} +
    +{/if} + + +{#if !isRankView && !isSearchView} + {#if hasNext} + +
    + {/if} + + + {#if loadingMore} +
    + + + + +
    + {:else if !hasNext && novels.length > 0} +

    All novels loaded

    + {/if} +{/if} + + +{#if showScrollTop} + +{/if} diff --git a/v3/ui/src/routes/disclaimer/+page.svelte b/v3/ui/src/routes/disclaimer/+page.svelte new file mode 100644 index 0000000..db34bbe --- /dev/null +++ b/v3/ui/src/routes/disclaimer/+page.svelte @@ -0,0 +1,34 @@ + + Disclaimer — libnovel + + +
    +

    Disclaimer

    + +
    +

    + libnovel is a personal reading tool that indexes and caches publicly accessible novel content + from third-party sources, primarily novelfire.net. + It is not affiliated with, endorsed by, or in any way officially connected to those sources. +

    + +

    + All novel titles, cover images, chapter text, and related materials are the property of their + respective authors and publishers. libnovel does not claim ownership of any of this content. + The content is reproduced solely for personal, non-commercial reading convenience. +

    + +

    + If you are a rights holder and believe your work is being used without authorisation, please + refer to our DMCA policy + for instructions on how to request removal. +

    + +

    + libnovel makes no warranties regarding the accuracy, completeness, or timeliness of any + content displayed. Use of this site is at your own risk. +

    + +

    Last updated: {new Date().getFullYear()}

    +
    +
    diff --git a/v3/ui/src/routes/dmca/+page.svelte b/v3/ui/src/routes/dmca/+page.svelte new file mode 100644 index 0000000..8a73ae6 --- /dev/null +++ b/v3/ui/src/routes/dmca/+page.svelte @@ -0,0 +1,45 @@ + + DMCA — libnovel + + +
    +

    DMCA Takedown Policy

    + +
    +

    + libnovel respects the intellectual property rights of authors, publishers, and other content + creators. If you believe that content available through this site infringes your copyright, + please send a written takedown notice to the contact address below. +

    + +

    Your notice must include

    +
      +
    1. Your full legal name and contact information (email address).
    2. +
    3. A description of the copyrighted work you claim has been infringed.
    4. +
    5. The specific URL(s) on this site where the allegedly infringing content appears.
    6. +
    7. + A statement that you have a good-faith belief that the use is not authorised by the copyright + owner, its agent, or the law. +
    8. +
    9. + A statement, made under penalty of perjury, that the information in your notice is accurate + and that you are the copyright owner or authorised to act on their behalf. +
    10. +
    11. Your electronic or physical signature.
    12. +
    + +

    How to submit

    +

    + Send your notice by email to dmca@libnovel.local. + We will review valid notices and remove or disable access to the identified content promptly. +

    + +

    Counter-notices

    +

    + If you believe content was removed in error, you may submit a counter-notice to the same + address with the information required under 17 U.S.C. § 512(g)(3). +

    + +

    Last updated: {new Date().getFullYear()}

    +
    +
    diff --git a/v3/ui/src/routes/health/+server.ts b/v3/ui/src/routes/health/+server.ts new file mode 100644 index 0000000..a7b2832 --- /dev/null +++ b/v3/ui/src/routes/health/+server.ts @@ -0,0 +1,6 @@ +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; + +export const GET: RequestHandler = () => { + return json({ status: 'ok' }); +}; diff --git a/v3/ui/src/routes/login/+page.server.ts b/v3/ui/src/routes/login/+page.server.ts new file mode 100644 index 0000000..69e6211 --- /dev/null +++ b/v3/ui/src/routes/login/+page.server.ts @@ -0,0 +1,142 @@ +import { fail, redirect } from '@sveltejs/kit'; +import type { Actions, PageServerLoad } from './$types'; +import { loginUser, createUser, mergeSessionProgress, createUserSession } from '$lib/server/pocketbase'; +import { createAuthToken } from '../../hooks.server'; +import { log } from '$lib/server/logger'; +import { randomBytes } from 'node:crypto'; + +const AUTH_COOKIE = 'libnovel_auth'; +const ONE_YEAR = 60 * 60 * 24 * 365; + +export const load: PageServerLoad = async ({ locals }) => { + // Already logged in — send to home + if (locals.user) { + redirect(302, '/'); + } + return {}; +}; + +export const actions: Actions = { + login: async ({ request, cookies, locals }) => { + const data = await request.formData(); + const username = (data.get('username') as string | null)?.trim() ?? ''; + const password = (data.get('password') as string | null) ?? ''; + + if (!username || !password) { + return fail(400, { action: 'login', error: 'Username and password are required.' }); + } + + let user; + try { + user = await loginUser(username, password); + } catch (err) { + log.error('auth', 'login unexpected error', { username, err: String(err) }); + return fail(500, { action: 'login', error: 'An error occurred. Please try again.' }); + } + + if (!user) { + return fail(401, { action: 'login', error: 'Invalid username or password.' }); + } + + // Merge any anonymous session progress into the user's account so that + // chapters read before logging in are preserved and portable across devices. + mergeSessionProgress(locals.sessionId, user.id).catch((err) => + log.warn('auth', 'login: mergeSessionProgress failed (non-fatal)', { err: String(err) }) + ); + + // Create a unique auth session ID for this login + const authSessionId = randomBytes(16).toString('hex'); + + // Record the session in PocketBase (best-effort, non-fatal) + const userAgent = request.headers.get('user-agent') ?? ''; + const ip = + request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ?? + request.headers.get('x-real-ip') ?? + ''; + createUserSession(user.id, authSessionId, userAgent, ip).catch((err) => + log.warn('auth', 'login: createUserSession failed (non-fatal)', { err: String(err) }) + ); + + const token = createAuthToken(user.id, user.username, user.role ?? 'user', authSessionId); + cookies.set(AUTH_COOKIE, token, { + path: '/', + httpOnly: true, + sameSite: 'lax', + maxAge: ONE_YEAR + }); + + redirect(302, '/'); + }, + + register: async ({ request, cookies, locals }) => { + const data = await request.formData(); + const username = (data.get('username') as string | null)?.trim() ?? ''; + const password = (data.get('password') as string | null) ?? ''; + const confirm = (data.get('confirm') as string | null) ?? ''; + + if (!username || !password) { + return fail(400, { action: 'register', error: 'Username and password are required.' }); + } + if (username.length < 3 || username.length > 32) { + return fail(400, { + action: 'register', + error: 'Username must be between 3 and 32 characters.' + }); + } + if (!/^[a-zA-Z0-9_-]+$/.test(username)) { + return fail(400, { + action: 'register', + error: 'Username may only contain letters, numbers, underscores and hyphens.' + }); + } + if (password.length < 8) { + return fail(400, { + action: 'register', + error: 'Password must be at least 8 characters.' + }); + } + if (password !== confirm) { + return fail(400, { action: 'register', error: 'Passwords do not match.' }); + } + + let user; + try { + user = await createUser(username, password); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : 'Registration failed.'; + if (msg.includes('Username already taken')) { + return fail(409, { action: 'register', error: 'That username is already taken.' }); + } + log.error('auth', 'register unexpected error', { username, err: String(err) }); + return fail(500, { action: 'register', error: 'An error occurred. Please try again.' }); + } + + // Merge any anonymous session progress into the newly created account. + mergeSessionProgress(locals.sessionId, user.id).catch((err) => + log.warn('auth', 'register: mergeSessionProgress failed (non-fatal)', { err: String(err) }) + ); + + // Create a unique auth session ID for this registration + const authSessionId = randomBytes(16).toString('hex'); + + // Record the session in PocketBase (best-effort, non-fatal) + const userAgent = request.headers.get('user-agent') ?? ''; + const ip = + request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ?? + request.headers.get('x-real-ip') ?? + ''; + createUserSession(user.id, authSessionId, userAgent, ip).catch((err) => + log.warn('auth', 'register: createUserSession failed (non-fatal)', { err: String(err) }) + ); + + const token = createAuthToken(user.id, user.username, user.role ?? 'user', authSessionId); + cookies.set(AUTH_COOKIE, token, { + path: '/', + httpOnly: true, + sameSite: 'lax', + maxAge: ONE_YEAR + }); + + redirect(302, '/'); + } +}; diff --git a/v3/ui/src/routes/login/+page.svelte b/v3/ui/src/routes/login/+page.svelte new file mode 100644 index 0000000..77a5a13 --- /dev/null +++ b/v3/ui/src/routes/login/+page.svelte @@ -0,0 +1,136 @@ + + + + Sign in — libnovel + + +
    +
    + +
    + + +
    + + {#if form?.error && (form?.action === mode || !form?.action)} +
    + {form.error} +
    + {/if} + + {#if mode === 'login'} +
    +
    + + +
    +
    + + +
    + +
    + {:else} +
    +
    + + +

    3–32 characters: letters, numbers, _ or -

    +
    +
    + + +

    At least 8 characters

    +
    +
    + + +
    + +
    + {/if} +
    +
    diff --git a/v3/ui/src/routes/logout/+page.server.ts b/v3/ui/src/routes/logout/+page.server.ts new file mode 100644 index 0000000..9af7c4b --- /dev/null +++ b/v3/ui/src/routes/logout/+page.server.ts @@ -0,0 +1,11 @@ +import { redirect } from '@sveltejs/kit'; +import type { Actions } from './$types'; + +const AUTH_COOKIE = 'libnovel_auth'; + +export const actions: Actions = { + default: async ({ cookies }) => { + cookies.delete(AUTH_COOKIE, { path: '/' }); + redirect(302, '/login'); + } +}; diff --git a/v3/ui/src/routes/privacy/+page.svelte b/v3/ui/src/routes/privacy/+page.svelte new file mode 100644 index 0000000..e90e99b --- /dev/null +++ b/v3/ui/src/routes/privacy/+page.svelte @@ -0,0 +1,55 @@ + + Privacy Policy — libnovel + + +
    +

    Privacy Policy

    + +
    +

    + This policy describes what limited data libnovel collects and how it is used. +

    + +

    Data we collect

    +
      +
    • + Session cookies — a short-lived cookie is set when you + visit the site to track reading progress across pages. No account is required. +
    • +
    • + Account data (optional) — if you create an account, + we store your username and a hashed password. No email address is required. +
    • +
    • + Reading progress — the last chapter you read for each + book is stored server-side, tied to your session or account, so you can resume reading. +
    • +
    • + Saved books — books you explicitly bookmark are stored + server-side tied to your session or account. +
    • +
    + +

    What we do not collect

    +
      +
    • No email addresses (unless you choose to provide one).
    • +
    • No tracking pixels, analytics scripts, or third-party ad networks.
    • +
    • No selling or sharing of data with third parties.
    • +
    + +

    Third-party content

    +

    + Cover images and chapter content are fetched from third-party sources (e.g. + novelfire.net). + Your browser may make requests directly to those domains when loading images. +

    + +

    Data deletion

    +

    + You can delete your reading progress and saved books from your profile page at any time. + To request full account deletion, contact us via the contact address listed in our DMCA policy. +

    + +

    Last updated: {new Date().getFullYear()}

    +
    +
    diff --git a/v3/ui/src/routes/profile/+page.server.ts b/v3/ui/src/routes/profile/+page.server.ts new file mode 100644 index 0000000..587337b --- /dev/null +++ b/v3/ui/src/routes/profile/+page.server.ts @@ -0,0 +1,79 @@ +import { fail, redirect } from '@sveltejs/kit'; +import type { Actions, PageServerLoad } from './$types'; +import { changePassword, listUserSessions, getUserByUsername } from '$lib/server/pocketbase'; +import { presignAvatarUrl } from '$lib/server/minio'; +import { log } from '$lib/server/logger'; + +export const load: PageServerLoad = async ({ locals }) => { + if (!locals.user) { + redirect(302, '/login'); + } + + let sessions: Awaited> = []; + try { + sessions = await listUserSessions(locals.user.id); + } catch (e) { + log.warn('profile', 'listUserSessions failed (non-fatal)', { err: String(e) }); + } + + // Fetch avatar presigned URL if user has one + let avatarUrl: string | null = null; + try { + const record = await getUserByUsername(locals.user.username); + if (record?.avatar_url) { + avatarUrl = await presignAvatarUrl(locals.user.id); + } + } catch (e) { + log.warn('profile', 'avatar fetch failed (non-fatal)', { err: String(e) }); + } + + return { + user: locals.user, + avatarUrl, + sessions: sessions.map((s) => ({ + id: s.id, + user_agent: s.user_agent, + ip: s.ip, + created_at: s.created_at, + last_seen: s.last_seen, + is_current: s.session_id === locals.user!.authSessionId + })) + }; +}; + +export const actions: Actions = { + changePassword: async ({ request, locals }) => { + if (!locals.user) { + return fail(401, { error: 'Not logged in.' }); + } + + const data = await request.formData(); + const current = (data.get('current') as string | null) ?? ''; + const next = (data.get('next') as string | null) ?? ''; + const confirm = (data.get('confirm') as string | null) ?? ''; + + if (!current || !next || !confirm) { + return fail(400, { error: 'All fields are required.' }); + } + if (next.length < 8) { + return fail(400, { error: 'New password must be at least 8 characters.' }); + } + if (next !== confirm) { + return fail(400, { error: 'New passwords do not match.' }); + } + + let ok: boolean; + try { + ok = await changePassword(locals.user.id, current, next); + } catch (e) { + log.error('profile', 'changePassword failed', { err: String(e) }); + return fail(500, { error: 'An error occurred. Please try again.' }); + } + + if (!ok) { + return fail(401, { error: 'Current password is incorrect.' }); + } + + return { success: true }; + } +}; diff --git a/v3/ui/src/routes/profile/+page.svelte b/v3/ui/src/routes/profile/+page.svelte new file mode 100644 index 0000000..883d91f --- /dev/null +++ b/v3/ui/src/routes/profile/+page.svelte @@ -0,0 +1,474 @@ + + + + Profile — libnovel + + +{#if cropFile && browser} + {#await import('$lib/components/AvatarCropModal.svelte') then { default: AvatarCropModal }} + + {/await} +{/if} + + + + +
    +
    + +
    + + +
    + +
    +

    {data.user.username}

    +

    {data.user.role}

    + {#if avatarError} +

    {avatarError}

    + {:else} +

    Click avatar to change photo

    + {/if} +
    +
    + + +
    +

    Reading settings

    + + +
    + + {#if !voicesLoaded} +
    + {:else if voices.length === 0} + + {:else} + + {/if} +
    + + +
    + + +
    + 0.5x + 3.0x +
    +
    + + + + +
    + + {#if settingsSaved} + Saved! + {/if} +
    +
    + + +
    +

    Active sessions

    +

    These are all devices currently signed into your account. End any session you don't recognise.

    + + {#if revokeError} +
    + {revokeError} +
    + {/if} + + {#if sessions.length === 0} +

    No session records found. Sessions are tracked from the next login.

    + {:else} +
      + {#each sessions as session (session.id)} +
    • +
      +
      + {parseUA(session.user_agent)} + {#if session.is_current} + This session + {/if} +
      + {#if session.ip} +

      {session.ip}

      + {/if} +

      + Signed in {formatDate(session.created_at)} + {#if session.last_seen && session.last_seen !== session.created_at} + · Last seen {formatDate(session.last_seen)} + {/if} +

      +
      + +
    • + {/each} +
    + {/if} +
    + + +
    +

    Change password

    + + {#if form?.error} +
    + {form.error} +
    + {/if} + + {#if pwSuccess} +
    + Password changed successfully. +
    + {/if} + +
    { + pwSubmitting = true; + return async ({ update }) => { + pwSubmitting = false; + await update(); + }; + }} + class="space-y-4" + > +
    + + +
    +
    + + +
    +
    + + +
    + +
    +
    +
    diff --git a/v3/ui/src/routes/users/[username]/+page.server.ts b/v3/ui/src/routes/users/[username]/+page.server.ts new file mode 100644 index 0000000..fa9cfe6 --- /dev/null +++ b/v3/ui/src/routes/users/[username]/+page.server.ts @@ -0,0 +1,59 @@ +import { error } from '@sveltejs/kit'; +import type { PageServerLoad } from './$types'; +import { + getPublicProfile, + getSubscription, + getUserPublicLibrary, + getUserCurrentlyReading +} from '$lib/server/pocketbase'; +import { presignAvatarUrl } from '$lib/server/minio'; +import { log } from '$lib/server/logger'; + +export const load: PageServerLoad = async ({ params, locals }) => { + const { username } = params; + + const profile = await getPublicProfile(username).catch(() => null); + if (!profile) error(404, `User "${username}" not found`); + + // Resolve avatar + let avatarUrl: string | null = null; + if (profile.avatar_url) { + avatarUrl = await presignAvatarUrl(profile.id).catch(() => null); + } + + // Subscription state for the logged-in visitor + let isSubscribed = false; + const isSelf = locals.user?.id === profile.id; + if (locals.user && !isSelf) { + const sub = await getSubscription(locals.user.id, profile.id).catch(() => null); + isSubscribed = !!sub; + } + + // Load public library + currently reading in parallel + const [library, currentlyReading] = await Promise.all([ + getUserPublicLibrary(profile.id).catch((e) => { + log.error('users/profile', 'getUserPublicLibrary failed', { username, err: String(e) }); + return [] as Awaited>; + }), + getUserCurrentlyReading(profile.id).catch((e) => { + log.error('users/profile', 'getUserCurrentlyReading failed', { username, err: String(e) }); + return [] as Awaited>; + }) + ]); + + return { + profile: { + id: profile.id, + username: profile.username, + created: profile.created, + followerCount: profile.followerCount, + followingCount: profile.followingCount + }, + avatarUrl, + isSubscribed, + isSelf, + isLoggedIn: !!locals.user, + library, + currentlyReading + }; +}; diff --git a/v3/ui/src/routes/users/[username]/+page.svelte b/v3/ui/src/routes/users/[username]/+page.svelte new file mode 100644 index 0000000..cde8afc --- /dev/null +++ b/v3/ui/src/routes/users/[username]/+page.svelte @@ -0,0 +1,225 @@ + + + + {data.profile.username} — libnovel + + + +
    + +
    + {#if data.avatarUrl} + {data.profile.username} + {:else} +
    + {initials(data.profile.username)} +
    + {/if} +
    + + +
    +

    {data.profile.username}

    +

    Joined {joinDate(data.profile.created)}

    + + +
    + + {followerCount} + followers + + + {data.profile.followingCount} + following + +
    + + + {#if data.isLoggedIn && !data.isSelf} + + {:else if !data.isLoggedIn} + + Follow + + {/if} +
    +
    + + +{#if data.currentlyReading.length > 0} +
    +

    Currently Reading

    + +
    +{/if} + + +{#if data.library.length > 0} +
    +

    + Library + ({data.library.length}) +

    + +
    +{/if} + + +{#if data.library.length === 0 && data.currentlyReading.length === 0} +
    + + + +

    No books in library yet.

    +
    +{/if} diff --git a/v3/ui/static/apple-touch-icon.png b/v3/ui/static/apple-touch-icon.png new file mode 100644 index 0000000..07ba559 Binary files /dev/null and b/v3/ui/static/apple-touch-icon.png differ diff --git a/v3/ui/static/favicon-16.png b/v3/ui/static/favicon-16.png new file mode 100644 index 0000000..90771d0 Binary files /dev/null and b/v3/ui/static/favicon-16.png differ diff --git a/v3/ui/static/favicon-32.png b/v3/ui/static/favicon-32.png new file mode 100644 index 0000000..c9cca0b Binary files /dev/null and b/v3/ui/static/favicon-32.png differ diff --git a/v3/ui/static/favicon.ico b/v3/ui/static/favicon.ico new file mode 100644 index 0000000..9aea163 Binary files /dev/null and b/v3/ui/static/favicon.ico differ diff --git a/v3/ui/static/icon-192.png b/v3/ui/static/icon-192.png new file mode 100644 index 0000000..17b9214 Binary files /dev/null and b/v3/ui/static/icon-192.png differ diff --git a/v3/ui/static/icon-512.png b/v3/ui/static/icon-512.png new file mode 100644 index 0000000..3135efa Binary files /dev/null and b/v3/ui/static/icon-512.png differ diff --git a/v3/ui/static/robots.txt b/v3/ui/static/robots.txt new file mode 100644 index 0000000..b6dd667 --- /dev/null +++ b/v3/ui/static/robots.txt @@ -0,0 +1,3 @@ +# allow crawling everything by default +User-agent: * +Disallow: diff --git a/v3/ui/svelte.config.js b/v3/ui/svelte.config.js new file mode 100644 index 0000000..6bfb3c4 --- /dev/null +++ b/v3/ui/svelte.config.js @@ -0,0 +1,10 @@ +import adapter from '@sveltejs/adapter-node'; + +/** @type {import('@sveltejs/kit').Config} */ +const config = { + kit: { + adapter: adapter() + } +}; + +export default config; diff --git a/v3/ui/tsconfig.json b/v3/ui/tsconfig.json new file mode 100644 index 0000000..2c2ed3c --- /dev/null +++ b/v3/ui/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "./.svelte-kit/tsconfig.json", + "compilerOptions": { + "rewriteRelativeImportExtensions": true, + "allowJs": true, + "checkJs": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "sourceMap": true, + "strict": true, + "moduleResolution": "bundler" + } + // Path aliases are handled by https://svelte.dev/docs/kit/configuration#alias + // except $lib which is handled by https://svelte.dev/docs/kit/configuration#files + // + // To make changes to top-level options such as include and exclude, we recommend extending + // the generated config; see https://svelte.dev/docs/kit/configuration#typescript +} diff --git a/v3/ui/vite.config.ts b/v3/ui/vite.config.ts new file mode 100644 index 0000000..bb50a3d --- /dev/null +++ b/v3/ui/vite.config.ts @@ -0,0 +1,18 @@ +import { sveltekit } from '@sveltejs/kit/vite'; +import tailwindcss from '@tailwindcss/vite'; +import { defineConfig } from 'vite'; + +export default defineConfig({ + plugins: [tailwindcss(), sveltekit()], + ssr: { + // Force these packages to be bundled into the server output rather than + // treated as external requires. The production Docker image has no + // node_modules, so anything used in server-side code must be inlined. + noExternal: ['marked'], + // cropperjs is DOM-only (used inside $effect); exclude from SSR bundle. + external: ['cropperjs'] + }, + optimizeDeps: { + include: ['cropperjs'] + } +});