diff --git a/.env.example b/.env.example deleted file mode 100644 index d1e4792..0000000 --- a/.env.example +++ /dev/null @@ -1,41 +0,0 @@ -# libnovel scraper — environment overrides -# Copy to .env and adjust values; do NOT commit this file with real secrets. - -# Browserless API token (leave empty to disable auth) -BROWSERLESS_TOKEN= - -# Number of concurrent browser sessions in Browserless -BROWSERLESS_CONCURRENT=10 - -# Queue depth before Browserless returns 429 -BROWSERLESS_QUEUED=100 - -# Per-session timeout in ms -BROWSERLESS_TIMEOUT=60000 - -# Optional webhook URL for Browserless error alerts (leave empty to disable) -ERROR_ALERT_URL= - -# Which Browserless strategy the scraper uses: content | scrape | cdp | direct -BROWSERLESS_STRATEGY=direct - -# Strategy for URL retrieval (chapter list). Uses browserless content strategy by default. -# Set to direct to use plain HTTP, or content/scrape/cdp for browserless. -BROWSERLESS_URL_STRATEGY=content - -# Chapter worker goroutines (0 = NumCPU inside the container) -SCRAPER_WORKERS=0 - -# Host path to mount as the static output directory -STATIC_ROOT=./static/books - -# ── Kokoro-FastAPI TTS ──────────────────────────────────────────────────────── -# Base URL for the Kokoro-FastAPI service. When running via docker-compose the -# default (http://kokoro:8880) is wired in automatically; override here only if -# you are pointing at an external or GPU instance. -KOKORO_URL=http://kokoro:8880 - -# Default voice used for chapter narration. -# Single voices: af_bella, af_sky, af_heart, am_adam, … -# Mixed voices: af_bella+af_sky or af_bella(2)+af_sky(1) (weighted blend) -KOKORO_VOICE=af_bella diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 621fdc5..3231de1 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -2,105 +2,75 @@ name: CI on: push: - branches: ["main", "master"] + tags-ignore: + - "v*" paths: - - "scraper/**" - - ".gitea/workflows/**" - pull_request: - branches: ["main", "master"] - paths: - - "scraper/**" - - ".gitea/workflows/**" + - "backend/**" + - "ui/**" + - ".gitea/workflows/ci.yaml" -defaults: - run: - working-directory: scraper +concurrency: + group: ${{ gitea.workflow }}-${{ gitea.ref }} + cancel-in-progress: true jobs: - # ── lint & vet ─────────────────────────────────────────────────────────────── - lint: - name: Lint + # ── Go: vet + build + test ──────────────────────────────────────────────── + backend: + name: Backend runs-on: ubuntu-latest + defaults: + run: + working-directory: backend steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version-file: scraper/go.mod - cache-dependency-path: scraper/go.sum + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum - name: go vet run: go vet ./... - - name: staticcheck - run: | - go install honnef.co/go/tools/cmd/staticcheck@latest - staticcheck ./... + - name: Build backend + run: go build -o /dev/null ./cmd/backend - # ── tests ──────────────────────────────────────────────────────────────────── - test: - name: Test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 + - name: Build runner + run: go build -o /dev/null ./cmd/runner - - uses: actions/setup-go@v5 - with: - go-version-file: scraper/go.mod - cache-dependency-path: scraper/go.sum + - name: Build healthcheck + run: go build -o /dev/null ./cmd/healthcheck + + - name: Build pocketbase + run: go build -o /dev/null ./cmd/pocketbase - name: Run tests - run: go test -race -count=1 -timeout=60s ./... + run: go test -short -race -count=1 -timeout=60s ./... - # ── build binary ───────────────────────────────────────────────────────────── - build: - name: Build + # ── UI: type-check + build ──────────────────────────────────────────────── + ui: + name: UI runs-on: ubuntu-latest - needs: [lint, test] + defaults: + run: + working-directory: ui steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/setup-node@v4 with: - go-version-file: scraper/go.mod - cache-dependency-path: scraper/go.sum + node-version: "22" + cache: npm + cache-dependency-path: ui/package-lock.json - - name: Build binary - run: | - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ - go build -ldflags="-s -w" -o bin/scraper ./cmd/scraper + - name: Install dependencies + run: npm ci - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: scraper-linux-amd64 - path: scraper/bin/scraper - retention-days: 7 + - name: Check Paraglide codegen is up to date + run: npm run paraglide && git diff --exit-code src/lib/paraglide/ - # ── docker build (& push) ──────────────────────────────────────────────────── - # Uncomment once the runner has Docker available and a registry is configured. - # - # docker: - # name: Docker - # runs-on: ubuntu-latest - # needs: [lint, test] - # # Only push images on commits to the default branch, not on PRs. - # # if: github.event_name == 'push' - # steps: - # - uses: actions/checkout@v4 - # - # - name: Log in to Gitea registry - # uses: docker/login-action@v3 - # with: - # registry: gitea.kalekber.cc - # username: ${{ secrets.REGISTRY_USER }} - # password: ${{ secrets.REGISTRY_TOKEN }} - # - # - name: Build and push - # uses: docker/build-push-action@v5 - # with: - # context: ./scraper - # push: true - # tags: | - # gitea.kalekber.cc/kamil/libnovel:latest - # gitea.kalekber.cc/kamil/libnovel:${{ gitea.sha }} + - name: Type check + run: npm run check + + - name: Build + run: npm run build diff --git a/.gitea/workflows/release.yaml b/.gitea/workflows/release.yaml new file mode 100644 index 0000000..5c1b1b4 --- /dev/null +++ b/.gitea/workflows/release.yaml @@ -0,0 +1,207 @@ +name: Release + +on: + push: + tags: + - "v*" # e.g. v1.0.0, v1.2.3 + +concurrency: + group: ${{ gitea.workflow }}-${{ gitea.ref }} + cancel-in-progress: true + +jobs: + # ── backend: vet & test ─────────────────────────────────────────────────────── + test-backend: + name: Test backend + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: backend/go.mod + cache-dependency-path: backend/go.sum + + - name: go vet + working-directory: backend + run: go vet ./... + + - name: Run tests + working-directory: backend + run: go test -short -race -count=1 -timeout=60s ./... + + # ── ui: type-check & build ──────────────────────────────────────────────────── + check-ui: + name: Test UI + runs-on: ubuntu-latest + defaults: + run: + working-directory: ui + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: ui/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Type check + run: npm run check + + - name: Build + run: npm run build + + # ── docker: build + push all images via docker bake ────────────────────────── + docker: + name: Build and push images + runs-on: ubuntu-latest + needs: [test-backend, check-ui] + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Compute version tags + id: ver + run: | + V="${{ gitea.ref_name }}" + VER="${V#v}" + echo "version=$VER" >> "$GITHUB_OUTPUT" + echo "major_minor=$(echo "$VER" | cut -d. -f1-2)" >> "$GITHUB_OUTPUT" + + - name: Build and push all images + uses: docker/bake-action@v6 + with: + files: docker-bake.hcl + set: | + *.output=type=image,push=true + env: + VERSION: ${{ steps.ver.outputs.version }} + MAJOR_MINOR: ${{ steps.ver.outputs.major_minor }} + COMMIT: ${{ gitea.sha }} + BUILD_TIME: ${{ gitea.event.head_commit.timestamp }} + + # ── deploy: sync docker-compose.yml + restart prod ─────────────────────────── + # Runs after all images are pushed to Docker Hub. + # Copies the compose file from the tagged commit to the server, pulls the new + # images, and restarts only the services whose image or config changed. + # --remove-orphans cleans up containers no longer defined in the compose file + # (e.g. the now-removed pb-init container). + # + # Required Gitea secrets: + # PROD_HOST — prod server IP or hostname + # PROD_USER — SSH login user (typically root) + # PROD_SSH_KEY — private key whose public half is in authorized_keys + # PROD_SSH_KNOWN_HOSTS — output of: ssh-keyscan -H + deploy-prod: + name: Deploy to prod + runs-on: ubuntu-latest + needs: [docker] + steps: + - uses: actions/checkout@v4 + + - name: Install SSH key + run: | + mkdir -p ~/.ssh + printf '%s\n' "${{ secrets.PROD_SSH_KEY }}" > ~/.ssh/deploy_key + chmod 600 ~/.ssh/deploy_key + printf '%s\n' "${{ secrets.PROD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + + - name: Copy docker-compose.yml to prod + run: | + scp -i ~/.ssh/deploy_key \ + docker-compose.yml \ + "${{ secrets.PROD_USER }}@${{ secrets.PROD_HOST }}:/opt/libnovel/docker-compose.yml" + + - name: Pull new images and restart changed services + run: | + ssh -i ~/.ssh/deploy_key \ + "${{ secrets.PROD_USER }}@${{ secrets.PROD_HOST }}" \ + 'set -euo pipefail + cd /opt/libnovel + doppler run -- docker compose pull backend runner ui caddy pocketbase + doppler run -- docker compose stop backend runner ui caddy pocketbase + doppler run -- docker compose up -d --remove-orphans backend runner ui caddy pocketbase' + + # ── deploy homelab runner ───────────────────────────────────────────────────── + # Syncs the homelab runner compose file and restarts the runner service. + # + # Required Gitea secrets: + # HOMELAB_HOST — homelab server IP (192.168.0.109) + # HOMELAB_USER — SSH login user (typically root) + # HOMELAB_SSH_KEY — private key whose public half is in authorized_keys + # HOMELAB_SSH_KNOWN_HOSTS — output of: ssh-keyscan -H + deploy-homelab: + name: Deploy to homelab + runs-on: ubuntu-latest + needs: [docker] + steps: + - uses: actions/checkout@v4 + + - name: Install SSH key + run: | + mkdir -p ~/.ssh + printf '%s\n' "${{ secrets.HOMELAB_SSH_KEY }}" > ~/.ssh/homelab_key + chmod 600 ~/.ssh/homelab_key + + - name: Copy docker-compose.yml to homelab + run: | + scp -i ~/.ssh/homelab_key \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + homelab/docker-compose.yml \ + "${{ secrets.HOMELAB_USER }}@${{ secrets.HOMELAB_HOST }}:/opt/libnovel-runner/docker-compose.yml" + + - name: Pull new runner image and restart + run: | + ssh -i ~/.ssh/homelab_key \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "${{ secrets.HOMELAB_USER }}@${{ secrets.HOMELAB_HOST }}" \ + 'set -euo pipefail + cd /opt/libnovel-runner + doppler run --project libnovel --config prd_homelab -- docker compose pull runner + doppler run --project libnovel --config prd_homelab -- docker compose up -d runner' + + # ── Gitea release ───────────────────────────────────────────────────────────── + release: + name: Gitea Release + runs-on: ubuntu-latest + needs: [docker] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Extract release notes from tag commit + id: notes + run: | + set -euo pipefail + # Subject line (first line of commit message) → release title + SUBJECT=$(git log -1 --format="%s" "${{ gitea.sha }}") + # Body (everything after the blank line) → release body + BODY=$(git log -1 --format="%b" "${{ gitea.sha }}" | sed '/^Co-Authored-By:/d' | sed '/^[[:space:]]*$/{ N; /^\n$/d }' | sed 's/^[[:space:]]*$//' | awk 'NF || !p; {p = !NF}') + echo "title=${SUBJECT}" >> "$GITHUB_OUTPUT" + # Use a heredoc delimiter to safely handle multi-line body + { + echo "body<> "$GITHUB_OUTPUT" + + - name: Create release + uses: https://gitea.com/actions/gitea-release-action@v1 + with: + token: ${{ secrets.GITEA_TOKEN }} + title: ${{ steps.notes.outputs.title }} + body: ${{ steps.notes.outputs.body }} diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..2432eda --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Auto-recompile paraglide messages when ui/messages/*.json files are staged. +# Prevents svelte-check / CI failures caused by stale generated JS files. + +set -euo pipefail + +STAGED=$(git diff --cached --name-only) + +if echo "$STAGED" | grep -q '^ui/messages/'; then + echo "[pre-commit] ui/messages/*.json changed — recompiling paraglide..." + (cd ui && npm run paraglide --silent) + git add -f ui/src/lib/paraglide/messages/ + echo "[pre-commit] paraglide output re-staged." +fi diff --git a/.gitignore b/.gitignore index 0e45f0f..e3eda84 100644 --- a/.gitignore +++ b/.gitignore @@ -5,15 +5,18 @@ /dist/ # ── Compiled binaries ────────────────────────────────────────────────────────── -scraper/bin/ +backend/bin/ +backend/backend +backend/runner -# ── Scraped output (large, machine-generated) ────────────────────────────────── - -/static/books # ── Environment & secrets ────────────────────────────────────────────────────── +# Secrets are managed by Doppler — never commit .env files. .env .env.* -!.env.example +.env.local + +# ── CrowdSec — generated bouncer API key ────────────────────────────────────── +crowdsec/.crowdsec.env # ── OS artefacts ─────────────────────────────────────────────────────────────── .DS_Store @@ -25,3 +28,7 @@ Thumbs.db *.swp *.swo *~ +.opencode/ + +# ── Playwright MCP browser session data ──────────────────────────────────────── +.playwright-mcp/ diff --git a/AGENTS.md b/AGENTS.md index 23e4bae..671ce06 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,89 +1,193 @@ -# libnovel Project +# LibNovel v2 — Agent Context -Go web scraper for novelfire.net with TTS support via Kokoro-FastAPI. +This file is the root-level knowledge base for LLM coding agents (OpenCode, Claude, Cursor, Copilot, etc.). +Sub-directories have their own `AGENTS.md` with deeper context (e.g. `ios/AGENTS.md`). -## Architecture +--- + +## Stack + +| Layer | Technology | +|---|---| +| UI | SvelteKit 2 + Svelte 5, TypeScript, TailwindCSS | +| Backend / Runner | Go (single repo, two binaries: `backend`, `runner`) | +| iOS app | SwiftUI, iOS 17+, Swift 5.9+ | +| Database | PocketBase (SQLite) + MinIO (object storage) | +| Search | Meilisearch | +| Queue | Asynq over Redis (local) / Valkey (prod) | +| Scraping | Novelfire scraper in `backend/novelfire/` | + +--- + +## Repository Layout ``` -scraper/ -├── cmd/scraper/main.go # Entry point: 'run' (one-shot) and 'serve' (HTTP server) -├── internal/ -│ ├── orchestrator/orchestrator.go # Coordinates catalogue walk, metadata extraction, chapter scraping -│ ├── browser/ # Browser client (content/scrape/cdp strategies) via Browserless -│ ├── novelfire/scraper.go # novelfire.net specific scraping logic -│ ├── server/server.go # HTTP API (POST /scrape, POST /scrape/book) -│ ├── writer/writer.go # File writer (metadata.yaml, chapter .md files) -│ └── scraper/interfaces.go # NovelScraper interface definition -└── static/books/ # Output directory for scraped content +. +├── .gitea/workflows/ # CI/CD — Gitea Actions (NOT .github/) +├── .opencode/ # OpenCode agent config (memory, skills) +├── backend/ # Go backend + runner (single module) +├── caddy/ # Caddy reverse proxy Dockerfile +├── homelab/ # Homelab docker-compose + observability stack +├── ios/ # SwiftUI iOS app (see ios/AGENTS.md) +├── scripts/ # Utility scripts +├── ui/ # SvelteKit UI +├── docker-compose.yml # Prod compose (all services) +├── AGENTS.md # This file +└── opencode.json # OpenCode config ``` -## Key Concepts +--- -- **Orchestrator**: Manages concurrency - catalogue streaming → per-book metadata goroutines → chapter worker pool -- **Browser Client**: 3 strategies (content/scrape/cdp) via Browserless Chrome container -- **Writer**: Writes metadata.yaml and chapter markdown files to `static/books/{slug}/vol-0/1-50/` -- **Server**: HTTP API with async scrape jobs, UI for browsing books/chapters, chapter-text endpoint for TTS +## CI/CD — Gitea Actions -## Commands +- Workflows live in `.gitea/workflows/` — **not** `.github/workflows/` +- Self-hosted Gitea instance; use `gitea.ref_name` / `gitea.sha` (not `github.*`) +- Two workflows: + - `ci.yaml` — runs on every push to `main` (test + type-check) + - `release.yaml` — runs on `v*` tags (build Docker images, upload source maps, create Gitea release) +- Secrets: `DOCKER_USER`, `DOCKER_TOKEN`, `GITEA_TOKEN`, `GLITCHTIP_AUTH_TOKEN` + +### Git credentials + +Credentials are embedded in the remote URL — no `HOME=/root` or credential helper needed for push: + +``` +https://kamil:95782641Apple%24@gitea.kalekber.cc/kamil/libnovel.git +``` + +All git commands still use `HOME=/root` prefix for consistency (picks up `/root/.gitconfig` for user name/email), but push auth works without it. + +### Releasing a new version ```bash -# Build -cd scraper && go build -o bin/scraper ./cmd/scraper - -# One-shot scrape (full catalogue) -./bin/scraper run - -# Single book -./bin/scraper run --url https://novelfire.net/book/xxx - -# HTTP server -./bin/scraper serve - -# Tests -cd scraper && go test ./... +HOME=/root git tag v2.6.X -m "Short title" +HOME=/root git push origin v3-cleanup --tags ``` -## Environment Variables +CI will build all Docker images, upload source maps to GlitchTip, and create a Gitea release automatically. -| Variable | Description | Default | -|----------|-------------|---------| -| BROWSERLESS_URL | Browserless Chrome endpoint | http://localhost:3030 | -| BROWSERLESS_STRATEGY | content \| scrape \| cdp | content | -| SCRAPER_WORKERS | Chapter goroutines | NumCPU | -| SCRAPER_STATIC_ROOT | Output directory | ./static/books | -| SCRAPER_HTTP_ADDR | HTTP listen address | :8080 | -| KOKORO_URL | Kokoro TTS endpoint | http://localhost:8880 | -| KOKORO_VOICE | Default TTS voice | af_bella | -| LOG_LEVEL | debug \| info \| warn \| error | info | +--- -## Docker +## GlitchTip Error Tracking + +- Instance: `https://errors.libnovel.cc/` +- Org: `libnovel` +- Projects: `ui` (id/1), `backend` (id/2), `runner` (id/3) +- Tool: `glitchtip-cli` v0.1.0 + +### Per-service DSNs (stored in Doppler) + +| Service | Doppler key | GlitchTip project | +|---|---|---| +| UI (SvelteKit) | `PUBLIC_GLITCHTIP_DSN` | ui (1) | +| Backend (Go) | `GLITCHTIP_DSN_BACKEND` | backend (2) | +| Runner (Go) | `GLITCHTIP_DSN_RUNNER` | runner (3) | + +### Source map upload flow (release.yaml) + +The correct order is **critical** — uploading before `releases new` results in 0 files shown in GlitchTip UI: + +``` +glitchtip-cli sourcemaps inject ./build # inject debug IDs +glitchtip-cli releases new # MUST come before upload +glitchtip-cli sourcemaps upload ./build \ + --release # associate files with release +glitchtip-cli releases finalize # mark release complete +``` + +--- + +## Infrastructure + +| Environment | Host | Path | Doppler config | +|---|---|---|---| +| Prod | `165.22.70.138` | `/opt/libnovel/` | `prd` | +| Homelab runner | `192.168.0.109` | `/opt/libnovel-runner/` | `prd_homelab` | + +### Docker Compose — always use Doppler ```bash -docker-compose up -d # Starts browserless, kokoro, scraper +# Prod +doppler run --project libnovel --config prd -- docker compose + +# Homelab full-stack (runs from .bak file on server) +doppler run --project libnovel --config prd_homelab -- docker compose -f homelab/docker-compose.yml.bak + +# Homelab runner only +doppler run --project libnovel --config prd_homelab -- docker compose -f homelab/runner/docker-compose.yml ``` -## Code Patterns +- Prod runner has `profiles: [runner]` — `docker compose up -d` will NOT accidentally start it +- When deploying, always sync `docker-compose.yml` to the server before running `up -d` +- **Caddyfile is NOT in git** — lives at `/opt/libnovel/Caddyfile` on prod server only. Edit directly on the server and restart the `caddy` container. -- Uses `log/slog` for structured logging -- Context-based cancellation throughout -- Worker pool pattern in orchestrator (channel + goroutines) -- Mutex for single async job (409 on concurrent scrape requests) +--- -## AI Context Tips +## Observability -- Primary files to modify: `orchestrator.go`, `server.go`, `scraper.go`, `browser/*.go` -- To add new source: implement `NovelScraper` interface from `internal/scraper/interfaces.go` -- Skip `static/` directory - generated content, not source +| Tool | Purpose | +|---|---| +| GlitchTip | Error tracking (UI + backend + runner) | +| Grafana Faro | RUM / Web Vitals (collector at `faro.libnovel.cc/collect`) → Alloy (port 12347) | +| OpenTelemetry | Distributed tracing (OTLP → cloudflared → OTel collector → Tempo) | +| Grafana | Dashboards at `https://grafana.libnovel.cc` | -## Speed Up AI Sessions (Optional) +### Grafana dashboards: `homelab/otel/grafana/provisioning/dashboards/` -For faster AI context loading, use **Context7** (free, local indexing): +Key dashboards: +- `backend.json` — Backend logs (Loki: `{service_name="backend"}`, plain text) +- `runner.json` — Runner logs (Loki: `{service_name="runner"}`) + Asynq Prometheus metrics +- `web-vitals.json` — Web Vitals (Loki: `{service_name="unknown_service"} kind=measurement` + pattern parser) +- `catalogue.json` — Scrape progress (Loki: `{service_name="runner"} | json | body="..."`) -```bash -# Install and index once -npx @context7/cli@latest index --path . --ignore .aiignore +### Data pipeline (2026-04-07 working state) -# After first run, AI tools will query the index instead of re-scanning files -``` +**Browser → Grafana Faro:** +Browser sends RUM data → `https://faro.libnovel.cc/collect` → **Alloy** `faro.receiver` (port 12347) → Loki (logs/exceptions) + OTel collector → **Tempo** (traces) -VSCode extension: https://marketplace.visualstudio.com/items?itemName=context7.context7 +**Backend/Runner → OTel:** +Backend/Runner Go SDK → `https://otel.libnovel.cc` (cloudflared tunnel) → **OTel collector** (port 4318) → Tempo (traces) + Loki (logs via `otlphttp/loki` exporter) +Runner also sends to **Alloy** `otelcol.receiver.otlp` (port 4318) → `otelcol.exporter.loki` → Loki + +### Loki log format per service + +- `service_name="backend"`: Plain text (e.g. `backend: asynq task dispatch enabled`) +- `service_name="runner"`: JSON with `body`, `attributes{slug,chapters,page}`, `severity` +- `service_name="unknown_service"`: Faro RUM text format (e.g. `kind=measurement lcp=5428.0 ...`) + +### OTel Collector ports (homelab) + +- gRPC: `4317` — receives from cloudflared (`otel.libnovel.cc`) +- HTTP: `4318` — receives from cloudflared + Alloy +- Metrics: `8888` + +### Known issues / pending fixes + +- Web Vitals use `service_name="unknown_service"` (Faro SDK doesn't set service.name in browser) — works with `unknown_service` label +- Runner logs go to both Alloy→Loki AND OTel collector→Loki (dual pipeline — intentional for resilience) + +--- + +## Go Backend + +- Primary files: `orchestrator.go`, `server/handlers_*.go`, `novelfire/scraper.go`, `storage/hybrid.go`, `storage/pocketbase.go` +- Store interface: `store.go` — never touch MinIO/PocketBase clients directly outside `storage/` +- Two binaries built from the same module: `backend` (HTTP API) and `runner` (Asynq worker) + +--- + +## SvelteKit UI + +- Source: `ui/src/` +- i18n: Paraglide — translation files in `ui/messages/*.json` (5 locales) +- Auth debug bypass: `GET /api/auth/debug-login?token=&username=&next=` + +--- + +## iOS App + +Full context in `ios/AGENTS.md`. Quick notes: +- SwiftUI, iOS 17+, `@Observable` for new types +- Download key separator: `::` (not `-`) +- Voice fallback: book override → global default → `"af_bella"` +- Offline pattern: `NetworkMonitor` env object + `OfflineBanner` + `ErrorAlertModifier` diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ccf68e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,97 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Environment / Secrets + +All project environment variables are stored in **Doppler**. When you need to access any secret or env var (e.g. API tokens, database URLs, credentials), fetch them via: + +```bash +doppler run -- # inject all secrets into a command +doppler secrets get SECRET_NAME # inspect a specific secret +``` + +Never use `.env` files. Do not ask the user to provide secrets manually — they are available via Doppler. + +## Commands + +### Docker (via `just` — the primary way to run services) +All services use Doppler for secrets injection. The `just` commands handle this automatically. + +```bash +just up # Start all services in background +just up-fg # Start all services, stream logs +just down # Stop all services +just down-volumes # Full reset (destructive — removes all volumes) +just build # Rebuild all Docker images +just build-svc backend # Rebuild a specific service +just restart # Stop + rebuild + start +just logs # Tail all logs +just log backend # Tail a specific service +just shell backend # Open shell in running container +just init # One-shot init: MinIO buckets, PocketBase collections, Postgres +``` + +### Backend (Go) +```bash +cd backend +go vet ./... +go test -short -race -count=1 -timeout=60s ./... +go test -short -race -count=1 -run TestFoo ./internal/somepackage/ +go build ./cmd/backend +go build ./cmd/runner +``` + +### Frontend (SvelteKit) +```bash +cd ui +npm run dev # Dev server at localhost:5173 +npm run build # Production build +npm run check # svelte-check (type-check) +npm run paraglide # Regenerate i18n messages (run after editing messages/*.json) +``` + +## Architecture + +Three services communicate via PocketBase records and a Redis/Valkey task queue: + +**Backend** (`backend/cmd/backend`) — HTTP REST API. Handles reads, enqueues tasks to Redis via Asynq, returns presigned MinIO URLs. Minimal processing; delegates heavy work to the runner. + +**Runner** (`backend/cmd/runner`) — Asynq task worker. Processes scraping, TTS audio generation, AI text/image generation. Reads/writes PocketBase and MinIO directly. + +**UI** (`ui/`) — SvelteKit 2 + Svelte 5 SSR app. Consumes the backend API. Uses Paraglide JS for i18n (5 locales). + +### Data layer +| Service | Role | +|---------|------| +| **PocketBase** (SQLite) | Auth, structured records (books, chapters, tasks, subscriptions) | +| **MinIO** (S3-compatible) | Object storage — chapter text, audio files, images | +| **Meilisearch** | Full-text search (runner indexes, backend reads) | +| **Redis/Valkey** | Asynq task queue + presigned URL cache | + +### Key backend packages +- `internal/backend/` — HTTP handlers and server setup +- `internal/runner/` — Task processor implementations +- `internal/storage/` — Unified MinIO + PocketBase interface (all data access goes through here) +- `internal/orchestrator/` — Task orchestration across services +- `internal/taskqueue/` — Enqueue helpers (backend side) +- `internal/asynqqueue/` — Asynq queue setup (runner side) +- `internal/config/` — Environment variable loading (Doppler-injected at runtime, no .env files) +- `internal/presigncache/` — Redis cache for MinIO presigned URLs + +### UI routing conventions (SvelteKit) +- `+page.svelte` / `+page.server.ts` — Page + server-side load +- `+layout.svelte` / `+layout.server.ts` — Layouts +- `routes/api/` — API routes (`+server.ts`) +- `lib/audio.svelte.ts` — Client-side audio playback store (Svelte 5 runes) + +## Key Conventions + +- **Svelte 5 runes only** — use `$state`, `$derived`, `$effect`; do not use Svelte 4 stores or reactive statements. +- **Modern Go idioms** — structured logging via `log/slog`, OpenTelemetry tracing throughout. +- **No direct MinIO/PocketBase client calls** outside the `internal/storage/` package. +- **Secrets via Doppler** — never use `.env` files. All secrets are injected by Doppler CLI. +- **CI/CD is Gitea Actions** (`.gitea/workflows/`), not GitHub Actions. Use `gitea.ref_name`/`gitea.sha` variables. +- **Git hooks** in `.githooks/` — enable with `just setup`. +- **i18n**: translation files live in `ui/messages/{en,es,fr,de,pt}.json`; run `npm run paraglide` after editing them. +- **Error tracking**: GlitchTip with per-service DSNs (backend id/2, runner id/3, UI id/1) stored in Doppler. diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000..8fdc7b5 --- /dev/null +++ b/Caddyfile @@ -0,0 +1,291 @@ +# v3/Caddyfile +# +# Caddy reverse proxy for LibNovel v3. +# Custom build includes github.com/mholt/caddy-ratelimit. +# +# Environment variables consumed (set in docker-compose.yml): +# DOMAIN — public hostname, e.g. libnovel.example.com +# Use "localhost" for local dev (no TLS cert attempted). +# CADDY_ACME_EMAIL — Let's Encrypt notification email (empty = no email) +# +# Routing rules (main domain): +# /health → backend:8080 (liveness probe) +# /scrape* → backend:8080 (Go admin scrape endpoints) +# /api/book-preview/* → backend:8080 (live scrape, no store write) +# /api/chapter-text/* → backend:8080 (chapter markdown from MinIO) +# /api/chapter-markdown/* → backend:8080 (chapter markdown from MinIO) +# /api/reindex/* → backend:8080 (rebuild chapter index) +# /api/cover/* → backend:8080 (proxy cover image) +# /api/audio-proxy/* → backend:8080 (proxy generated audio) +# /avatars/* → minio:9000 (presigned avatar GETs) +# /audio/* → minio:9000 (presigned audio GETs) +# /chapters/* → minio:9000 (presigned chapter GETs) +# /* (everything else) → ui:3000 (SvelteKit — handles all +# remaining /api/* routes) +# +# Subdomain routing: +# feedback.libnovel.cc → fider:3000 (user feedback / feature requests) +# errors.libnovel.cc → glitchtip-web:8000 (error tracking) +# analytics.libnovel.cc → umami:3000 (page analytics) +# logs.libnovel.cc → dozzle:8080 (Docker log viewer) +# uptime.libnovel.cc → uptime-kuma:3001 (uptime monitoring) +# push.libnovel.cc → gotify:80 (push notifications) +# search.libnovel.cc → meilisearch:7700 (search index — homelab runner) +# +# Routes intentionally removed from direct-to-backend: +# /api/scrape/* — SvelteKit has /api/scrape/ counterparts +# that enforce auth; routing directly would +# bypass SK middleware. +# /api/chapter-text-preview/* — Same: SvelteKit owns +# /api/chapter-text-preview/[slug]/[n]. +# /api/browse — Endpoint removed; browse snapshot system +# was deleted. + +{ + # Email for Let's Encrypt ACME account registration. + # When CADDY_ACME_EMAIL is set this expands to e.g. "email you@example.com". + # When unset the variable expands to an empty string and Caddy ignores it. + email {$CADDY_ACME_EMAIL:} + + # CrowdSec bouncer — streams decisions from the CrowdSec LAPI every 15s. + # CROWDSEC_API_KEY is injected at runtime via crowdsec/.crowdsec.env. + # The default "disabled" placeholder makes the bouncer fail-open (warn, + # pass traffic) when no key is configured — Caddy still starts cleanly. + crowdsec { + api_url http://crowdsec:8080 + api_key {$CROWDSEC_API_KEY:disabled} + ticker_interval 15s + } + + # ── Redis TCP proxy via layer4 ──────────────────────────────────────────── + # Exposes prod Redis over TLS for Asynq job enqueueing from the homelab runner. + # Listens on :6380 (all interfaces). TLS is terminated here using the cert + # for redis.libnovel.cc; traffic is proxied to the local Redis sidecar. + # Requires the caddy-l4 module in the custom Caddy build. + layer4 { + :6380 { + route { + tls { + connection_policy { + match { + sni redis.libnovel.cc + } + } + } + proxy { + upstream redis:6379 + } + } + } + } +} + +(security_headers) { + header { + # Prevent clickjacking + X-Frame-Options "SAMEORIGIN" + # Prevent MIME-type sniffing + X-Content-Type-Options "nosniff" + # Minimal referrer info for cross-origin requests + Referrer-Policy "strict-origin-when-cross-origin" + # Restrict powerful browser features + Permissions-Policy "camera=(), microphone=(), geolocation=(), payment=()" + # Enforce HTTPS for 1 year (includeSubDomains) + Strict-Transport-Security "max-age=31536000; includeSubDomains" + # Enable XSS filter in older browsers + X-XSS-Protection "1; mode=block" + # Remove server identity header + -Server + } +} + +{$DOMAIN:localhost} { + import security_headers + + # ── CrowdSec bouncer ────────────────────────────────────────────────────── + # Checks every incoming request against CrowdSec decisions. + # Banned IPs receive a 403; all others pass through unchanged. + route { + crowdsec + } + + # ── Rate limiting ───────────────────────────────────────────────────────── + # Auth endpoints: strict — 10 req/min per IP + rate_limit { + zone auth_zone { + match { + path /api/auth/login /api/auth/register /api/auth/change-password + } + key {remote_host} + window 1m + events 10 + } + } + + # Admin scrape endpoints: moderate — 20 req/min per IP + rate_limit { + zone scrape_zone { + match { + path /scrape* + } + key {remote_host} + window 1m + events 20 + } + } + + # Global: 300 req/min per IP (covers everything) + rate_limit { + zone global_zone { + key {remote_host} + window 1m + events 300 + } + } + + # ── Liveness probe ──────────────────────────────────────────────────────── + handle /health { + reverse_proxy backend:8080 + } + + # ── Scrape task creation (Go backend only) ──────────────────────────────── + handle /scrape* { + reverse_proxy backend:8080 + } + + # ── Backend-only API paths ──────────────────────────────────────────────── + # These paths are served exclusively by the Go backend and have no + # SvelteKit counterpart. Routing them here skips SK intentionally. + handle /api/book-preview/* { + reverse_proxy backend:8080 + } + handle /api/chapter-text/* { + reverse_proxy backend:8080 + } + handle /api/chapter-markdown/* { + reverse_proxy backend:8080 + } + handle /api/reindex/* { + reverse_proxy backend:8080 + } + handle /api/cover/* { + reverse_proxy backend:8080 + } + handle /api/audio-proxy/* { + reverse_proxy backend:8080 + } + + # ── MinIO bucket paths (presigned URLs) ────────────────────────────────── + # MinIO path-style presigned URLs include the bucket name as the first + # path segment. MINIO_PUBLIC_ENDPOINT points here, so Caddy must proxy + # these paths directly to MinIO — no auth layer needed (the presigned + # signature itself enforces access and expiry). + handle /avatars/* { + reverse_proxy minio:9000 + } + handle /audio/* { + reverse_proxy minio:9000 + } + handle /chapters/* { + reverse_proxy minio:9000 + } + + # ── SvelteKit UI (catch-all — includes all remaining /api/* routes) ─────── + handle { + reverse_proxy ui:3000 { + # Active health check: Caddy polls /health every 5 s and marks the + # upstream down immediately when it fails. Combined with + # lb_try_duration this means Watchtower container replacements + # show the maintenance page within a few seconds instead of + # hanging or returning a raw connection error to the browser. + health_uri /health + health_interval 5s + health_timeout 2s + health_status 200 + + # If the upstream is down, fail fast (don't retry for longer than + # 3 s) and let Caddy's handle_errors 502/503 take over. + lb_try_duration 3s + } + } + + # ── Caddy-level error pages ─────────────────────────────────────────────── + # These fire when the upstream (backend or ui) is completely unreachable. + # SvelteKit's own +error.svelte handles application-level errors (404, 500). + handle_errors 404 { + root * /srv/errors + rewrite * /404.html + file_server + } + handle_errors 500 { + root * /srv/errors + rewrite * /500.html + file_server + } + handle_errors 502 { + root * /srv/errors + rewrite * /502.html + file_server + } + handle_errors 503 { + root * /srv/errors + rewrite * /503.html + file_server + } + handle_errors 504 { + root * /srv/errors + rewrite * /504.html + file_server + } + + # ── Logging ─────────────────────────────────────────────────────────────── + # JSON log file read by CrowdSec for threat detection. + log { + output file /var/log/caddy/access.log { + roll_size 100MiB + roll_keep 5 + roll_keep_for 720h + } + format json + } +} + +# ── Tooling subdomains ──────────────────────────────────────────────────────── +# feedback.libnovel.cc, errors.libnovel.cc, analytics.libnovel.cc, +# logs.libnovel.cc, uptime.libnovel.cc, push.libnovel.cc, grafana.libnovel.cc +# are now routed via Cloudflare Tunnel directly to the homelab (192.168.0.109). +# No Caddy rules needed here — Cloudflare handles TLS termination and routing. + +# ── PocketBase: exposed for homelab runner task polling ─────────────────────── +# Allows the homelab runner to claim tasks and write results via the PB API. +# Admin UI is also accessible here for convenience. +pb.libnovel.cc { + import security_headers + reverse_proxy pocketbase:8090 +} + +# ── MinIO S3 API: exposed for homelab runner object writes ──────────────────── +# The homelab runner connects here as MINIO_ENDPOINT to PutObject audio/chapters. +# Also used as MINIO_PUBLIC_ENDPOINT for presigned URL generation. +storage.libnovel.cc { + import security_headers + reverse_proxy minio:9000 +} + +# ── Meilisearch: exposed for homelab runner search indexing ────────────────── +# The homelab runner connects here as MEILI_URL to index books after scraping. +# Protected by MEILI_MASTER_KEY bearer token — Meilisearch enforces auth on +# every request; Caddy just terminates TLS. +search.libnovel.cc { + import security_headers + reverse_proxy meilisearch:7700 +} + +# ── Redis TLS cert anchor ───────────────────────────────────────────────────── +# This virtual host exists solely so Caddy obtains and caches a TLS certificate +# for redis.libnovel.cc. The layer4 block above uses that cert to terminate TLS +# on :6380 (Asynq job-queue channel from prod → homelab Redis). +# The HTTP route itself just returns 404 — no real traffic expected here. +redis.libnovel.cc { + respond 404 +} diff --git a/DOCKERFILE_ANALYSIS.md b/DOCKERFILE_ANALYSIS.md new file mode 100644 index 0000000..8f0f299 --- /dev/null +++ b/DOCKERFILE_ANALYSIS.md @@ -0,0 +1,225 @@ +# Dockerfile Dependency Analysis + +## Current Image Sizes + +| Image | Size | Status | +|-------|------|--------| +| backend | 179MB | ✅ Good | +| runner | 178MB | ✅ Good | +| pocketbase | 37MB | ✅ Excellent | +| caddy | 114MB | ✅ Good | +| ui | **413MB** | ⚠️ **LARGE** | + +--- + +## UI Dependencies Analysis (413MB image) + +### Production Dependencies (package.json) + +| Package | Used? | Size Impact | Notes | +|---------|-------|-------------|-------| +| `@aws-sdk/client-s3` | ❌ **UNUSED** | ~100MB | **REMOVE** - Not imported anywhere | +| `@aws-sdk/s3-request-presigner` | ❌ **UNUSED** | ~50MB | **REMOVE** - Not imported anywhere | +| `@grafana/faro-web-sdk` | ✅ Used | ~2MB | Keep - RUM tracking | +| `@inlang/paraglide-js` | ✅ Used | ~1MB | Keep - i18n | +| `@opentelemetry/*` (5 packages) | ✅ Used | ~15MB | Keep - Server-side tracing | +| `@sentry/sveltekit` | ✅ Used | ~10MB | Keep - Error tracking | +| `cropperjs` | ✅ Used | ~500KB | Keep - Avatar cropping | +| `ioredis` | ✅ Used | ~5MB | Keep - Redis client (server-side) | +| `marked` | ✅ Used | ~500KB | Keep - Markdown parsing | +| `pocketbase` | ✅ Used | ~200KB | Keep - PocketBase client | +| **EXTRANEOUS** | | | | +| `@playwright/test` | ❌ **EXTRANEOUS** | ~50MB | **REMOVE** - Should be devDependency | +| `playwright-core` | ❌ **EXTRANEOUS** | ~50MB | **REMOVE** - Should be devDependency | +| `playwright` | ❌ **EXTRANEOUS** | ~50MB | **REMOVE** - Should be devDependency | + +**Total waste: ~300MB (AWS SDK + Playwright)** + +### Why AWS SDK is in dependencies? + +Checking git history... it was likely added for direct S3 uploads but never actually used. The backend handles all S3 operations. + +--- + +## Backend Dependencies Analysis (179MB image) + +### Docker Image Breakdown + +```dockerfile +FROM alpine:3.21 # ~7MB base +RUN apk add ffmpeg # ~40MB (needed for audio transcoding) +RUN apk add ca-certificates # ~200KB (needed for HTTPS) +COPY /out/backend # ~130MB (Go binary + stdlib) +``` + +**All dependencies justified:** +- ✅ `ffmpeg` - Required for pocket-tts WAV→MP3 transcoding +- ✅ `ca-certificates` - Required for HTTPS connections to external services +- ✅ Go binary includes all dependencies (static linking) + +### Go Module Analysis + +```bash +go list -m all | wc -l # 169 modules +``` + +Go binaries are statically linked, so unused imports don't increase image size. The build process with `-ldflags="-s -w"` strips symbols and debug info. + +**Optimization already applied:** ✅ +- CGO_ENABLED=0 (static linking, no libc dependency) +- -ldflags="-s -w" (strip symbols, ~20% size reduction) +- BuildKit cache mounts (faster rebuilds) + +--- + +## Caddy Dependencies Analysis (114MB image) + +```dockerfile +FROM caddy:2-alpine # ~50MB base +COPY /usr/bin/caddy # ~60MB (with 3 plugins) +COPY errors/ # ~4MB (error page assets) +``` + +**Plugins in use:** +- ✅ `caddy-ratelimit` - Used for API rate limiting +- ✅ `caddy-crowdsec-bouncer` - Used for CrowdSec integration +- ✅ `caddy-l4` - Used for TCP/UDP proxying (Redis TLS proxy) + +**All plugins justified** - actively used in production Caddyfile. + +--- + +## Recommendations + +### 1. Remove AWS SDK from UI (PRIORITY: HIGH) + +**Impact:** ~150MB reduction (413MB → 263MB, 36% smaller) + +```bash +cd ui +npm uninstall @aws-sdk/client-s3 @aws-sdk/s3-request-presigner +``` + +**Risk:** None - confirmed unused via grep + +### 2. Remove Playwright from production (PRIORITY: HIGH) + +**Impact:** ~150MB reduction (263MB → 113MB, 57% smaller) + +**Issue:** Playwright packages are marked as "extraneous" - they're installed but not in package.json. This happens when: +- Someone ran `npm install playwright` without `--save-dev` +- package-lock.json got corrupted + +**Fix:** +```bash +cd ui +rm -rf node_modules package-lock.json +npm install +``` + +This will regenerate package-lock.json without the extraneous packages. + +### 3. Consider distroless for backend/runner (OPTIONAL) + +**Impact:** ~10-15MB reduction per image + +**Current:** Alpine + ffmpeg (required) +**Alternative:** Use distroless + statically compiled ffmpeg + +**Tradeoff:** +- Pros: Smaller attack surface, smaller image +- Cons: Harder to debug, need to bundle ffmpeg binary +- Verdict: **NOT WORTH IT** - ffmpeg from apk is well-maintained + +### 4. Use .dockerignore (ALREADY GOOD ✅) + +Both UI and backend have proper .dockerignore files: +- ✅ node_modules excluded (ui) +- ✅ build artifacts excluded +- ✅ .git excluded + +--- + +## Expected Results After Cleanup + +| Image | Before | After | Savings | +|-------|--------|-------|---------| +| backend | 179MB | 179MB | 0MB (already optimal) | +| runner | 178MB | 178MB | 0MB (already optimal) | +| pocketbase | 37MB | 37MB | 0MB (already optimal) | +| caddy | 114MB | 114MB | 0MB (already optimal) | +| ui | **413MB** | **~110MB** | **~300MB (73% smaller)** | + +**Total deployment size reduction:** ~300MB +**Deployment time improvement:** ~20-30s faster (less to pull from Docker Hub) + +--- + +## Action Plan + +```bash +# 1. Clean up UI dependencies +cd ui +npm uninstall @aws-sdk/client-s3 @aws-sdk/s3-request-presigner +rm -rf node_modules package-lock.json +npm install + +# 2. Verify no imports remain +grep -r "@aws-sdk" src/ # Should return nothing +grep -r "playwright" src/ # Should return nothing + +# 3. Test build locally +npm run build + +# 4. Commit changes +git add package.json package-lock.json +git commit -m "chore: remove unused AWS SDK and Playwright dependencies from UI + +- Remove @aws-sdk/client-s3 and @aws-sdk/s3-request-presigner (~150MB) +- Remove extraneous Playwright packages (~150MB) +- UI image size: 413MB → ~110MB (73% smaller) + +All S3 operations are handled by the backend, not the UI." + +# 5. Tag and deploy +git tag v4.3.7 -m "chore: remove unused dependencies, reduce UI image by 73%" +git push origin --tags +``` + +--- + +## Backend/Runner Go Dependencies (For Reference) + +The Go images are already well-optimized. Here are the main dependencies: + +**Backend (179MB):** +- PocketBase SDK +- MinIO SDK (S3) +- Meilisearch SDK +- Redis SDK (ioredis equivalent) +- HTTP router (chi) +- OpenTelemetry SDK + +**Runner (178MB):** +- Same as backend +- + Chromedp (headless Chrome for scraping) +- + Audio processing libs + +All are actively used - no dead code found. + +--- + +## Conclusion + +**Current state:** +- Backend, runner, pocketbase, caddy: ✅ Already well-optimized +- UI: ⚠️ Carrying 300MB of unused dependencies + +**Impact of cleanup:** +- 73% smaller UI image +- Faster deployments +- Lower bandwidth costs +- Cleaner dependency tree + +**Effort:** ~5 minutes (remove 2 packages + regenerate lockfile) +**Risk:** Very low (confirmed unused via code search) diff --git a/HOMELAB_SECRETS_SETUP.md b/HOMELAB_SECRETS_SETUP.md new file mode 100644 index 0000000..fe6d59a --- /dev/null +++ b/HOMELAB_SECRETS_SETUP.md @@ -0,0 +1,60 @@ +# Homelab Deployment Secrets Setup + +The release workflow now includes automatic deployment to the homelab runner server. You need to add these secrets to Gitea. + +## Required Secrets + +Go to: `https://gitea.kalekber.cc/kamil/libnovel/settings/secrets/actions` + +### 1. HOMELAB_HOST +``` +192.168.0.109 +``` + +### 2. HOMELAB_USER +``` +root +``` + +### 3. HOMELAB_SSH_KEY + +If you want to use the same SSH key as prod: +- Copy the value from `PROD_SSH_KEY` secret + +If you want a separate key: +```bash +# On your local machine or CI runner +cat ~/.ssh/id_rsa # or your preferred key +``` + +### 4. HOMELAB_SSH_KNOWN_HOSTS + +Run this when the homelab server is reachable: +```bash +ssh-keyscan -H 192.168.0.109 2>/dev/null +``` + +Expected output format: +``` +|1|base64hash...|192.168.0.109 ssh-rsa AAAAB3NzaC... +|1|base64hash...|192.168.0.109 ecdsa-sha2-nistp256 AAAAE2... +|1|base64hash...|192.168.0.109 ssh-ed25519 AAAAC3... +``` + +## Testing + +After adding the secrets, the next release (e.g., v4.1.10) will automatically: +1. Build all Docker images +2. Deploy to prod (165.22.70.138) ✅ +3. Deploy to homelab (192.168.0.109) ✅ NEW +4. Create a Gitea release + +Both deployments run in parallel for faster releases. + +## Troubleshooting + +If the homelab deployment fails: +- Check that the secrets are set correctly +- Verify SSH access: `ssh root@192.168.0.109` +- Check Doppler config exists: `doppler configs --project libnovel` +- Manually test: `cd /opt/libnovel-runner && doppler run --project libnovel --config prd_homelab -- docker compose pull runner` diff --git a/README.md b/README.md new file mode 100644 index 0000000..47563c7 --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# LibNovel + +Self-hosted audiobook platform. Go backend + SvelteKit UI + MinIO/PocketBase/Meilisearch. + +## Requirements + +- Docker + Docker Compose +- [just](https://github.com/casey/just) +- [Doppler CLI](https://docs.doppler.com/docs/install-cli) + +## Setup + +```sh +doppler login +doppler setup # project=libnovel, config=prd +``` + +## Usage + +```sh +just up # start everything +just down # stop +just logs # tail all logs +just log backend # tail one service +just build # rebuild images +just restart # down + up +just secrets # view/edit secrets +``` + +## Secrets + +Managed via Doppler (`project=libnovel`, `config=prd`). No `.env` files. + +To add or update a secret: + +```sh +doppler secrets set MY_SECRET=value +``` diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..d32a9c3 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,13 @@ +# Exclude compiled binaries +bin/ + +# Exclude test binaries produced by `go test -c` +*.test + +# Git history is not needed inside the image +.git/ + +# Editor/OS noise +.DS_Store +*.swp +*.swo diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..5293c86 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,67 @@ +# syntax=docker/dockerfile:1 +FROM golang:1.26.1-alpine AS builder +WORKDIR /app + +# Download modules into the BuildKit cache so they survive across builds. +# This layer is only invalidated when go.mod or go.sum changes. +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/root/go/pkg/mod \ + go mod download + +COPY . . + +ARG VERSION=dev +ARG COMMIT=unknown + +# Build all three binaries in a single layer so the Go compiler can reuse +# intermediate object files. Both cache mounts are preserved between builds: +# /root/go/pkg/mod — downloaded module source +# /root/.cache/go-build — compiled package objects (incremental recompile) +RUN --mount=type=cache,target=/root/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=linux go build \ + -ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT}" \ + -o /out/backend ./cmd/backend && \ + CGO_ENABLED=0 GOOS=linux go build \ + -ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT}" \ + -o /out/runner ./cmd/runner && \ + CGO_ENABLED=0 GOOS=linux go build \ + -ldflags="-s -w" \ + -o /out/healthcheck ./cmd/healthcheck && \ + CGO_ENABLED=0 GOOS=linux go build \ + -ldflags="-s -w" \ + -o /out/pocketbase ./cmd/pocketbase + +# ── backend service ────────────────────────────────────────────────────────── +# Uses Alpine (not distroless) so ffmpeg is available for on-demand voice +# sample generation via pocket-tts (WAV→MP3 transcoding). +FROM alpine:3.21 AS backend +RUN apk add --no-cache ffmpeg ca-certificates && \ + addgroup -S appgroup && adduser -S appuser -G appgroup +COPY --from=builder /out/healthcheck /healthcheck +COPY --from=builder /out/backend /backend +USER appuser +ENTRYPOINT ["/backend"] + +# ── pocketbase service ─────────────────────────────────────────────────────── +# Runs the custom PocketBase binary with Go migrations baked in. +# On every `serve` startup it applies any pending migrations automatically. +# Data is stored in /pb_data (mounted as a Docker volume in production). +FROM alpine:3.21 AS pocketbase +RUN apk add --no-cache ca-certificates wget +COPY --from=builder /out/pocketbase /pocketbase +RUN mkdir -p /pb_data +VOLUME /pb_data +EXPOSE 8090 +CMD ["/pocketbase", "serve", "--dir", "/pb_data", "--http", "0.0.0.0:8090"] + +# ── runner service ─────────────────────────────────────────────────────────── +# Uses Alpine (not distroless) so ffmpeg is available for WAV→MP3 transcoding +# when pocket-tts voices are used. +FROM alpine:3.21 AS runner +RUN apk add --no-cache ffmpeg ca-certificates && \ + addgroup -S appgroup && adduser -S appuser -G appgroup +COPY --from=builder /out/healthcheck /healthcheck +COPY --from=builder /out/runner /runner +USER appuser +ENTRYPOINT ["/runner"] diff --git a/backend/cmd/backend/main.go b/backend/cmd/backend/main.go new file mode 100644 index 0000000..caafa9a --- /dev/null +++ b/backend/cmd/backend/main.go @@ -0,0 +1,260 @@ +// Command backend is the LibNovel HTTP API server. +// +// It exposes all endpoints consumed by the SvelteKit UI: book/chapter reads, +// scrape-task creation, presigned MinIO URLs, audio-task creation, reading +// progress, live novelfire.net search, and Kokoro voice list. +// +// All heavy lifting (scraping, TTS generation) is delegated to the runner +// binary via PocketBase task records. The backend never scrapes directly. +// +// Usage: +// +// backend # start HTTP server (blocks until SIGINT/SIGTERM) +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + "github.com/getsentry/sentry-go" + "github.com/hibiken/asynq" + "github.com/libnovel/backend/internal/asynqqueue" + "github.com/libnovel/backend/internal/backend" + "github.com/libnovel/backend/internal/cfai" + "github.com/libnovel/backend/internal/config" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/meili" + "github.com/libnovel/backend/internal/otelsetup" + "github.com/libnovel/backend/internal/pockettts" + "github.com/libnovel/backend/internal/storage" + "github.com/libnovel/backend/internal/taskqueue" +) + +// version and commit are set at build time via -ldflags. +var ( + version = "dev" + commit = "unknown" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "backend: fatal: %v\n", err) + os.Exit(1) + } +} + +func run() error { + cfg := config.Load() + + // ── Sentry / GlitchTip error tracking ──────────────────────────────────── + if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" { + if err := sentry.Init(sentry.ClientOptions{ + Dsn: dsn, + Release: version + "@" + commit, + TracesSampleRate: 0.1, + }); err != nil { + fmt.Fprintf(os.Stderr, "backend: sentry init warning: %v\n", err) + } else { + defer sentry.Flush(2 * time.Second) + } + } + + // ── Logger ─────────────────────────────────────────────────────────────── + log := buildLogger(cfg.LogLevel) + log.Info("backend starting", + "version", version, + "commit", commit, + "addr", cfg.HTTP.Addr, + ) + + // ── Context: cancel on SIGINT / SIGTERM ────────────────────────────────── + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + // ── OpenTelemetry tracing + logs ────────────────────────────────────────── + otelShutdown, otelLog, err := otelsetup.Init(ctx, version) + if err != nil { + return fmt.Errorf("init otel: %w", err) + } + if otelShutdown != nil { + defer otelShutdown() + // Replace the plain slog logger with the OTel-bridged one so all + // structured log lines are forwarded to Loki with trace IDs attached. + log = otelLog + log.Info("otel tracing + logs enabled", "endpoint", os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")) + } + + // ── Storage ────────────────────────────────────────────────────────────── + store, err := storage.NewStore(ctx, cfg, log) + if err != nil { + return fmt.Errorf("init storage: %w", err) + } + + // ── Kokoro (voice list only; audio generation is done by the runner) ───── + var kokoroClient kokoro.Client + if cfg.Kokoro.URL != "" { + kokoroClient = kokoro.New(cfg.Kokoro.URL) + log.Info("kokoro voices enabled", "url", cfg.Kokoro.URL) + } else { + log.Info("KOKORO_URL not set — voice list will use built-in fallback") + kokoroClient = &noopKokoro{} + } + + // ── Pocket-TTS (voice list + sample generation; audio generation is the runner's job) ── + var pocketTTSClient pockettts.Client + if cfg.PocketTTS.URL != "" { + pocketTTSClient = pockettts.New(cfg.PocketTTS.URL) + log.Info("pocket-tts voices enabled", "url", cfg.PocketTTS.URL) + } else { + log.Info("POCKET_TTS_URL not set — pocket-tts voices unavailable in backend") + } + + // ── Cloudflare Workers AI (voice sample generation + audio-stream live TTS) ── + var cfaiClient cfai.Client + if cfg.CFAI.AccountID != "" && cfg.CFAI.APIToken != "" { + cfaiClient = cfai.New(cfg.CFAI.AccountID, cfg.CFAI.APIToken, cfg.CFAI.Model) + log.Info("cloudflare AI TTS enabled", "model", cfg.CFAI.Model) + } else { + log.Info("CFAI_ACCOUNT_ID/CFAI_API_TOKEN not set — CF AI voices unavailable in backend") + } + + // ── Cloudflare Workers AI Image Generation ──────────────────────────────── + var imageGenClient cfai.ImageGenClient + if cfg.CFAI.AccountID != "" && cfg.CFAI.APIToken != "" { + imageGenClient = cfai.NewImageGen(cfg.CFAI.AccountID, cfg.CFAI.APIToken) + log.Info("cloudflare AI image generation enabled") + } else { + log.Info("CFAI_ACCOUNT_ID/CFAI_API_TOKEN not set — image generation unavailable") + } + + // ── Cloudflare Workers AI Text Generation ───────────────────────────────── + var textGenClient cfai.TextGenClient + if cfg.CFAI.AccountID != "" && cfg.CFAI.APIToken != "" { + textGenClient = cfai.NewTextGen(cfg.CFAI.AccountID, cfg.CFAI.APIToken) + log.Info("cloudflare AI text generation enabled") + } else { + log.Info("CFAI_ACCOUNT_ID/CFAI_API_TOKEN not set — text generation unavailable") + } + + // ── Meilisearch (search reads only; indexing is the runner's job) ──────── + var searchIndex meili.Client + if cfg.Meilisearch.URL != "" { + searchIndex = meili.New(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey) + log.Info("meilisearch search enabled", "url", cfg.Meilisearch.URL) + } else { + log.Info("MEILI_URL not set — search will use PocketBase substring fallback") + searchIndex = meili.NoopClient{} + } + + // ── Task Producer ──────────────────────────────────────────────────────── + // When REDIS_ADDR is set the backend dual-writes: PocketBase record (audit) + // + Asynq job (immediate delivery). Otherwise it writes to PocketBase only + // and the runner picks up on the next poll tick. + var producer taskqueue.Producer = store + if cfg.Redis.Addr != "" { + redisOpt, parseErr := parseRedisOpt(cfg.Redis) + if parseErr != nil { + return fmt.Errorf("parse REDIS_ADDR: %w", parseErr) + } + asynqProducer := asynqqueue.NewProducer(store, redisOpt, log) + defer asynqProducer.Close() //nolint:errcheck + producer = asynqProducer + log.Info("backend: asynq task dispatch enabled", "addr", cfg.Redis.Addr) + } else { + log.Info("backend: poll-mode task dispatch (REDIS_ADDR not set)") + } + + // ── Backend server ─────────────────────────────────────────────────────── + srv := backend.New( + backend.Config{ + Addr: cfg.HTTP.Addr, + DefaultVoice: cfg.Kokoro.DefaultVoice, + Version: version, + Commit: commit, + AdminToken: cfg.HTTP.AdminToken, + }, + backend.Dependencies{ + BookReader: store, + RankingStore: store, + AudioStore: store, + TranslationStore: store, + PresignStore: store, + ProgressStore: store, + CoverStore: store, + ChapterImageStore: store, + Producer: producer, + TaskReader: store, + ImportFileStore: store, + SearchIndex: searchIndex, + Kokoro: kokoroClient, + PocketTTS: pocketTTSClient, + CFAI: cfaiClient, + ImageGen: imageGenClient, + TextGen: textGenClient, + BookWriter: store, + AIJobStore: store, + BookAdminStore: store, + NotificationStore: store, + Log: log, + }, + ) + + return srv.ListenAndServe(ctx) +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +func buildLogger(level string) *slog.Logger { + var lvl slog.Level + switch level { + case "debug": + lvl = slog.LevelDebug + case "warn": + lvl = slog.LevelWarn + case "error": + lvl = slog.LevelError + default: + lvl = slog.LevelInfo + } + return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl})) +} + +// noopKokoro is a no-op implementation used when KOKORO_URL is not set. +// The backend only uses Kokoro for the voice list; audio generation is the +// runner's responsibility. With no URL the built-in fallback list is served. +type noopKokoro struct{} + +func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) { + return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)") +} + +func (n *noopKokoro) StreamAudioMP3(_ context.Context, _, _ string) (io.ReadCloser, error) { + return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)") +} + +func (n *noopKokoro) StreamAudioWAV(_ context.Context, _, _ string) (io.ReadCloser, error) { + return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)") +} + +func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) { + return nil, nil +} + +// parseRedisOpt converts a config.Redis into an asynq.RedisConnOpt. +// Handles full "redis://" / "rediss://" URLs and plain "host:port". +func parseRedisOpt(cfg config.Redis) (asynq.RedisConnOpt, error) { + addr := cfg.Addr + if len(addr) > 7 && (addr[:8] == "redis://" || (len(addr) > 8 && addr[:9] == "rediss://")) { + return asynq.ParseRedisURI(addr) + } + return asynq.RedisClientOpt{ + Addr: addr, + Password: cfg.Password, + }, nil +} diff --git a/backend/cmd/backend/main_test.go b/backend/cmd/backend/main_test.go new file mode 100644 index 0000000..edd47dc --- /dev/null +++ b/backend/cmd/backend/main_test.go @@ -0,0 +1,57 @@ +package main + +import ( + "os" + "testing" +) + +// TestBuildLogger verifies that buildLogger returns a non-nil logger for each +// supported log level string and for unknown values. +func TestBuildLogger(t *testing.T) { + for _, level := range []string{"debug", "info", "warn", "error", "unknown", ""} { + l := buildLogger(level) + if l == nil { + t.Errorf("buildLogger(%q) returned nil", level) + } + } +} + +// TestNoopKokoro verifies that the no-op Kokoro stub returns the expected +// sentinel error from GenerateAudio and nil, nil from ListVoices. +func TestNoopKokoro(t *testing.T) { + noop := &noopKokoro{} + + _, err := noop.GenerateAudio(t.Context(), "text", "af_bella") + if err == nil { + t.Fatal("noopKokoro.GenerateAudio: expected error, got nil") + } + + voices, err := noop.ListVoices(t.Context()) + if err != nil { + t.Fatalf("noopKokoro.ListVoices: unexpected error: %v", err) + } + if voices != nil { + t.Fatalf("noopKokoro.ListVoices: expected nil slice, got %v", voices) + } +} + +// TestRunStorageUnreachable verifies that run() fails fast and returns a +// descriptive error when PocketBase is unreachable. +func TestRunStorageUnreachable(t *testing.T) { + // Point at an address nothing is listening on. + t.Setenv("POCKETBASE_URL", "http://127.0.0.1:19999") + // Use a fast listen address so we don't accidentally start a real server. + t.Setenv("BACKEND_HTTP_ADDR", "127.0.0.1:0") + + err := run() + if err == nil { + t.Fatal("run() should have returned an error when storage is unreachable") + } + + t.Logf("got expected error: %v", err) +} + +// TestMain runs the test suite. No special setup required. +func TestMain(m *testing.M) { + os.Exit(m.Run()) +} diff --git a/backend/cmd/healthcheck/main.go b/backend/cmd/healthcheck/main.go new file mode 100644 index 0000000..4b7aa20 --- /dev/null +++ b/backend/cmd/healthcheck/main.go @@ -0,0 +1,89 @@ +// healthcheck is a static binary used by Docker HEALTHCHECK CMD in distroless +// images (which have no shell, wget, or curl). +// +// Two modes: +// +// 1. HTTP mode (default): +// /healthcheck +// Performs GET ; exits 0 if HTTP 2xx/3xx, 1 otherwise. +// Example: /healthcheck http://localhost:8080/health +// +// 2. File-liveness mode: +// /healthcheck file +// Reads , parses its content as RFC3339 timestamp, and exits 1 if the +// timestamp is older than . Used by the runner service which +// writes /tmp/runner.alive on every successful poll. +// Example: /healthcheck file /tmp/runner.alive 120 +package main + +import ( + "fmt" + "net/http" + "os" + "strconv" + "time" +) + +func main() { + if len(os.Args) > 1 && os.Args[1] == "file" { + checkFile() + return + } + checkHTTP() +} + +// checkHTTP performs a GET request and exits 0 on success, 1 on failure. +func checkHTTP() { + url := "http://localhost:8080/health" + if len(os.Args) > 1 { + url = os.Args[1] + } + resp, err := http.Get(url) //nolint:gosec,noctx + if err != nil { + fmt.Fprintf(os.Stderr, "healthcheck: %v\n", err) + os.Exit(1) + } + resp.Body.Close() + if resp.StatusCode >= 400 { + fmt.Fprintf(os.Stderr, "healthcheck: status %d\n", resp.StatusCode) + os.Exit(1) + } +} + +// checkFile reads a timestamp from a file and exits 1 if it is older than the +// given max age. Usage: /healthcheck file +func checkFile() { + if len(os.Args) < 4 { + fmt.Fprintln(os.Stderr, "healthcheck file: usage: /healthcheck file ") + os.Exit(1) + } + path := os.Args[2] + maxAgeSec, err := strconv.ParseInt(os.Args[3], 10, 64) + if err != nil { + fmt.Fprintf(os.Stderr, "healthcheck file: invalid max_age_seconds %q: %v\n", os.Args[3], err) + os.Exit(1) + } + + data, err := os.ReadFile(path) + if err != nil { + fmt.Fprintf(os.Stderr, "healthcheck file: cannot read %s: %v\n", path, err) + os.Exit(1) + } + + ts, err := time.Parse(time.RFC3339, string(data)) + if err != nil { + // Fallback: use file mtime if content is not a valid timestamp. + info, statErr := os.Stat(path) + if statErr != nil { + fmt.Fprintf(os.Stderr, "healthcheck file: cannot stat %s: %v\n", path, statErr) + os.Exit(1) + } + ts = info.ModTime() + } + + age := time.Since(ts) + if age > time.Duration(maxAgeSec)*time.Second { + fmt.Fprintf(os.Stderr, "healthcheck file: %s is %.0fs old (max %ds)\n", path, age.Seconds(), maxAgeSec) + os.Exit(1) + } +} diff --git a/backend/cmd/pocketbase/main.go b/backend/cmd/pocketbase/main.go new file mode 100644 index 0000000..aa57f26 --- /dev/null +++ b/backend/cmd/pocketbase/main.go @@ -0,0 +1,47 @@ +// Command pocketbase is a thin wrapper that runs PocketBase as a Go framework +// with version-controlled Go migrations. +// +// On every `serve`, PocketBase automatically applies any pending migrations from +// the migrations/ package before accepting traffic. +// +// Usage (Docker): +// +// ./pocketbase serve --dir /pb_data --http 0.0.0.0:8090 +// +// Migration workflow: +// +// # Generate a timestamped stub: +// go run ./cmd/pocketbase migrate create "description" +// # Apply manually (also runs automatically on serve): +// go run ./cmd/pocketbase migrate up +// # Revert last migration: +// go run ./cmd/pocketbase migrate down 1 +// # After migrating an existing install, mark existing schema as done: +// go run ./cmd/pocketbase migrate history-sync +package main + +import ( + "log" + + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/plugins/migratecmd" + + // Register all migrations via init(). + _ "github.com/libnovel/backend/migrations" +) + +func main() { + app := pocketbase.New() + + // Register the migrate sub-command. + // Automigrate: false — migrations are written by hand, never auto-generated + // from Admin UI changes. Pending migrations still apply automatically on + // every `serve` regardless of this flag. + migratecmd.MustRegister(app, app.RootCmd, migratecmd.Config{ + Automigrate: false, + }) + + if err := app.Start(); err != nil { + log.Fatal(err) + } +} diff --git a/backend/cmd/runner/main.go b/backend/cmd/runner/main.go new file mode 100644 index 0000000..5ed7bb9 --- /dev/null +++ b/backend/cmd/runner/main.go @@ -0,0 +1,263 @@ +// Command runner is the homelab worker binary. +// +// It polls PocketBase for pending scrape and audio tasks, executes them, and +// writes results back. It connects directly to PocketBase and MinIO using +// admin credentials loaded from environment variables. +// +// Usage: +// +// runner # start polling loop (blocks until SIGINT/SIGTERM) +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "os/signal" + "runtime" + "syscall" + "time" + + "github.com/getsentry/sentry-go" + "github.com/libnovel/backend/internal/asynqqueue" + "github.com/libnovel/backend/internal/browser" + "github.com/libnovel/backend/internal/cfai" + "github.com/libnovel/backend/internal/config" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/libretranslate" + "github.com/libnovel/backend/internal/meili" + "github.com/libnovel/backend/internal/novelfire" + "github.com/libnovel/backend/internal/otelsetup" + "github.com/libnovel/backend/internal/pockettts" + "github.com/libnovel/backend/internal/runner" + "github.com/libnovel/backend/internal/storage" + "github.com/libnovel/backend/internal/taskqueue" + "github.com/libnovel/backend/internal/webpush" +) + +// version and commit are set at build time via -ldflags. +var ( + version = "dev" + commit = "unknown" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "runner: fatal: %v\n", err) + os.Exit(1) + } +} + +func run() error { + cfg := config.Load() + + // ── Sentry / GlitchTip error tracking ──────────────────────────────────── + if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" { + if err := sentry.Init(sentry.ClientOptions{ + Dsn: dsn, + Release: version + "@" + commit, + TracesSampleRate: 0.1, + }); err != nil { + fmt.Fprintf(os.Stderr, "runner: sentry init warning: %v\n", err) + } else { + defer sentry.Flush(2 * time.Second) + } + } + + // ── Logger ────────────────────────────────────────────────────────────── + log := buildLogger(cfg.LogLevel) + log.Info("runner starting", + "version", version, + "commit", commit, + "worker_id", cfg.Runner.WorkerID, + ) + + // ── Context: cancel on SIGINT / SIGTERM ───────────────────────────────── + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + // ── OpenTelemetry tracing + logs ───────────────────────────────────────── + otelShutdown, otelLog, err := otelsetup.Init(ctx, version) + if err != nil { + return fmt.Errorf("init otel: %w", err) + } + if otelShutdown != nil { + defer otelShutdown() + // Switch to the OTel-bridged logger so all structured log lines are + // forwarded to Loki with trace IDs attached. + log = otelLog + log.Info("otel tracing + logs enabled", "endpoint", os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")) + } + + // ── Storage ───────────────────────────────────────────────────────────── + store, err := storage.NewStore(ctx, cfg, log) + if err != nil { + return fmt.Errorf("init storage: %w", err) + } + + // ── Browser / Scraper ─────────────────────────────────────────────────── + workers := cfg.Runner.Workers + if workers <= 0 { + workers = runtime.NumCPU() + } + timeout := cfg.Runner.Timeout + if timeout <= 0 { + timeout = 90 * time.Second + } + + browserClient := browser.NewDirectClient(browser.Config{ + MaxConcurrent: workers, + Timeout: timeout, + }) + novel := novelfire.New(browserClient, log) + + // ── Kokoro ────────────────────────────────────────────────────────────── + var kokoroClient kokoro.Client + if cfg.Kokoro.URL != "" { + kokoroClient = kokoro.New(cfg.Kokoro.URL) + log.Info("kokoro TTS enabled", "url", cfg.Kokoro.URL) + } else { + log.Warn("KOKORO_URL not set — kokoro voice tasks will fail") + kokoroClient = &noopKokoro{} + } + + // ── pocket-tts ────────────────────────────────────────────────────────── + var pocketTTSClient pockettts.Client + if cfg.PocketTTS.URL != "" { + pocketTTSClient = pockettts.New(cfg.PocketTTS.URL) + log.Info("pocket-tts enabled", "url", cfg.PocketTTS.URL) + } else { + log.Warn("POCKET_TTS_URL not set — pocket-tts voice tasks will fail") + } + + // ── Cloudflare Workers AI ──────────────────────────────────────────────── + var cfaiClient cfai.Client + if cfg.CFAI.AccountID != "" && cfg.CFAI.APIToken != "" { + cfaiClient = cfai.New(cfg.CFAI.AccountID, cfg.CFAI.APIToken, cfg.CFAI.Model) + log.Info("cloudflare AI TTS enabled", "model", cfg.CFAI.Model) + } else { + log.Info("CFAI_ACCOUNT_ID/CFAI_API_TOKEN not set — CF AI voice tasks will fail") + } + + // ── LibreTranslate ────────────────────────────────────────────────────── + ltClient := libretranslate.New(cfg.LibreTranslate.URL, cfg.LibreTranslate.APIKey) + if ltClient != nil { + log.Info("libretranslate enabled", "url", cfg.LibreTranslate.URL) + } else { + log.Info("LIBRETRANSLATE_URL not set — machine translation disabled") + } + + // ── Meilisearch ───────────────────────────────────────────────────────── + var searchIndex meili.Client + if cfg.Meilisearch.URL != "" { + if err := meili.Configure(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey); err != nil { + log.Warn("meilisearch configure failed — search indexing disabled", "err", err) + searchIndex = meili.NoopClient{} + } else { + searchIndex = meili.New(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey) + log.Info("meilisearch enabled", "url", cfg.Meilisearch.URL) + } + } else { + log.Info("MEILI_URL not set — search indexing disabled") + searchIndex = meili.NoopClient{} + } + + // ── Runner ────────────────────────────────────────────────────────────── + rCfg := runner.Config{ + WorkerID: cfg.Runner.WorkerID, + PollInterval: cfg.Runner.PollInterval, + MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape, + MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio, + MaxConcurrentTranslation: cfg.Runner.MaxConcurrentTranslation, + OrchestratorWorkers: workers, + MetricsAddr: cfg.Runner.MetricsAddr, + CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval, + CatalogueRequestDelay: cfg.Runner.CatalogueRequestDelay, + SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh, + RedisAddr: cfg.Redis.Addr, + RedisPassword: cfg.Redis.Password, + } + + // In Asynq mode the Consumer is a thin wrapper: claim/heartbeat/reap are + // no-ops, but FinishAudioTask / FinishScrapeTask / FailTask write back to + // PocketBase as before. + var consumer taskqueue.Consumer = store + if cfg.Redis.Addr != "" { + log.Info("runner: asynq mode — using Redis for task dispatch", "addr", cfg.Redis.Addr) + consumer = asynqqueue.NewConsumer(store) + } else { + log.Info("runner: poll mode — using PocketBase for task dispatch") + } + + // ── Web Push ───────────────────────────────────────────────────────────── + var pushSender *webpush.Sender + if cfg.VAPID.PublicKey != "" && cfg.VAPID.PrivateKey != "" { + pushSender = webpush.New(cfg.VAPID.PublicKey, cfg.VAPID.PrivateKey, cfg.VAPID.Subject, log) + log.Info("runner: web push notifications enabled") + } else { + log.Info("runner: VAPID_PUBLIC_KEY/VAPID_PRIVATE_KEY not set — push notifications disabled") + } + + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: store, + BookReader: store, + AudioStore: store, + CoverStore: store, + TranslationStore: store, + BookImport: storage.NewBookImporter(store), + ImportChapterStore: store, + ChapterIngester: store, + SearchIndex: searchIndex, + Novel: novel, + Kokoro: kokoroClient, + PocketTTS: pocketTTSClient, + CFAI: cfaiClient, + LibreTranslate: ltClient, + Notifier: store, + WebPush: pushSender, + Store: store, + Log: log, + } + r := runner.New(rCfg, deps) + + return r.Run(ctx) +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +func buildLogger(level string) *slog.Logger { + var lvl slog.Level + switch level { + case "debug": + lvl = slog.LevelDebug + case "warn": + lvl = slog.LevelWarn + case "error": + lvl = slog.LevelError + default: + lvl = slog.LevelInfo + } + return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl})) +} + +// noopKokoro is a no-op implementation used when KOKORO_URL is not set. +type noopKokoro struct{} + +func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) { + return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)") +} + +func (n *noopKokoro) StreamAudioMP3(_ context.Context, _, _ string) (io.ReadCloser, error) { + return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)") +} + +func (n *noopKokoro) StreamAudioWAV(_ context.Context, _, _ string) (io.ReadCloser, error) { + return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)") +} + +func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) { + return nil, nil +} diff --git a/backend/go.mod b/backend/go.mod new file mode 100644 index 0000000..e03ca8c --- /dev/null +++ b/backend/go.mod @@ -0,0 +1,101 @@ +module github.com/libnovel/backend + +go 1.26.1 + +require ( + github.com/SherClockHolmes/webpush-go v1.4.0 + github.com/getsentry/sentry-go v0.43.0 + github.com/hibiken/asynq v0.26.0 + github.com/hibiken/asynq/x v0.0.0-20260203063626-d704b68a426d + github.com/meilisearch/meilisearch-go v0.36.1 + github.com/minio/minio-go/v7 v7.0.98 + github.com/pdfcpu/pdfcpu v0.11.1 + github.com/pocketbase/pocketbase v0.36.9 + github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.18.0 + github.com/yuin/goldmark v1.8.2 + go.opentelemetry.io/contrib/bridges/otelslog v0.17.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 + go.opentelemetry.io/otel v1.42.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.18.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0 + go.opentelemetry.io/otel/log v0.18.0 + go.opentelemetry.io/otel/sdk v1.42.0 + go.opentelemetry.io/otel/sdk/log v0.18.0 + golang.org/x/net v0.52.0 +) + +require ( + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/disintegration/imaging v1.6.2 // indirect + github.com/domodwyer/mailyak/v3 v3.6.2 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect + github.com/ganigeorgiev/fexpr v0.5.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect + github.com/hhrutter/lzw v1.0.0 // indirect + github.com/hhrutter/pkcs7 v0.2.0 // indirect + github.com/hhrutter/tiff v1.0.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.21 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pocketbase/dbx v1.12.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect + go.opentelemetry.io/otel/metric v1.42.0 // indirect + go.opentelemetry.io/otel/trace v1.42.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/image v0.38.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect + google.golang.org/grpc v1.79.2 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + modernc.org/libc v1.70.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.48.2 // indirect +) diff --git a/backend/go.sum b/backend/go.sum new file mode 100644 index 0000000..d06d1e7 --- /dev/null +++ b/backend/go.sum @@ -0,0 +1,340 @@ +github.com/SherClockHolmes/webpush-go v1.4.0 h1:ocnzNKWN23T9nvHi6IfyrQjkIc0oJWv1B1pULsf9i3s= +github.com/SherClockHolmes/webpush-go v1.4.0/go.mod h1:XSq8pKX11vNV8MJEMwjrlTkxhAj1zKfxmyhdV7Pd6UA= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY= +github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8= +github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/ganigeorgiev/fexpr v0.5.0 h1:XA9JxtTE/Xm+g/JFI6RfZEHSiQlk+1glLvRK1Lpv/Tk= +github.com/ganigeorgiev/fexpr v0.5.0/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= +github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4= +github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= +github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hhrutter/lzw v1.0.0 h1:laL89Llp86W3rRs83LvKbwYRx6INE8gDn0XNb1oXtm0= +github.com/hhrutter/lzw v1.0.0/go.mod h1:2HC6DJSn/n6iAZfgM3Pg+cP1KxeWc3ezG8bBqW5+WEo= +github.com/hhrutter/pkcs7 v0.2.0 h1:i4HN2XMbGQpZRnKBLsUwO3dSckzgX142TNqY/KfXg+I= +github.com/hhrutter/pkcs7 v0.2.0/go.mod h1:aEzKz0+ZAlz7YaEMY47jDHL14hVWD6iXt0AgqgAvWgE= +github.com/hhrutter/tiff v1.0.2 h1:7H3FQQpKu/i5WaSChoD1nnJbGx4MxU5TlNqqpxw55z8= +github.com/hhrutter/tiff v1.0.2/go.mod h1:pcOeuK5loFUE7Y/WnzGw20YxUdnqjY1P0Jlcieb/cCw= +github.com/hibiken/asynq v0.26.0 h1:1Zxr92MlDnb1Zt/QR5g2vSCqUS03i95lUfqx5X7/wrw= +github.com/hibiken/asynq v0.26.0/go.mod h1:Qk4e57bTnWDoyJ67VkchuV6VzSM9IQW2nPvAGuDyw58= +github.com/hibiken/asynq/x v0.0.0-20260203063626-d704b68a426d h1:Ld5m8EIK5QVOq/owOexKIbETij3skACg4eU1pArHsrw= +github.com/hibiken/asynq/x v0.0.0-20260203063626-d704b68a426d/go.mod h1:hhpStehaxSGg3ib9wJXzw5AXY1YS6lQ9BNavAgPbIhE= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= +github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/meilisearch/meilisearch-go v0.36.1 h1:mJTCJE5g7tRvaqKco6DfqOuJEjX+rRltDEnkEC02Y0M= +github.com/meilisearch/meilisearch-go v0.36.1/go.mod h1:hWcR0MuWLSzHfbz9GGzIr3s9rnXLm1jqkmHkJPbUSvM= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pdfcpu/pdfcpu v0.11.1 h1:htHBSkGH5jMKWC6e0sihBFbcKZ8vG1M67c8/dJxhjas= +github.com/pdfcpu/pdfcpu v0.11.1/go.mod h1:pP3aGga7pRvwFWAm9WwFvo+V68DfANi9kxSQYioNYcw= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA= +github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= +github.com/pocketbase/pocketbase v0.36.9 h1:x3mXMB4AwhTzJ34JZpZR7IQyUih7Fx1l86r0V/k4oW8= +github.com/pocketbase/pocketbase v0.36.9/go.mod h1:t3sMcAxGHrDAXNcZ+65cZxBMpFP1vBdI9DrghB4n5Gw= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= +github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/bridges/otelslog v0.17.0 h1:NFIS6x7wyObQ7cR84x7bt1sr8nYBx89s3x3GwRjw40k= +go.opentelemetry.io/contrib/bridges/otelslog v0.17.0/go.mod h1:39SaByOyDMRMe872AE7uelMuQZidIw7LLFAnQi0FWTE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= +go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.18.0 h1:icqq3Z34UrEFk2u+HMhTtRsvo7Ues+eiJVjaJt62njs= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.18.0/go.mod h1:W2m8P+d5Wn5kipj4/xmbt9uMqezEKfBjzVJadfABSBE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0 h1:uLXP+3mghfMf7XmV4PkGfFhFKuNWoCvvx5wP/wOXo0o= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0/go.mod h1:v0Tj04armyT59mnURNUJf7RCKcKzq+lgJs6QSjHjaTc= +go.opentelemetry.io/otel/log v0.18.0 h1:XgeQIIBjZZrliksMEbcwMZefoOSMI1hdjiLEiiB0bAg= +go.opentelemetry.io/otel/log v0.18.0/go.mod h1:KEV1kad0NofR3ycsiDH4Yjcoj0+8206I6Ox2QYFSNgI= +go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= +go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= +go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= +go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= +go.opentelemetry.io/otel/sdk/log v0.18.0 h1:n8OyZr7t7otkeTnPTbDNom6rW16TBYGtvyy2Gk6buQw= +go.opentelemetry.io/otel/sdk/log v0.18.0/go.mod h1:C0+wxkTwKpOCZLrlJ3pewPiiQwpzycPI/u6W0Z9fuYk= +go.opentelemetry.io/otel/sdk/log/logtest v0.18.0 h1:l3mYuPsuBx6UKE47BVcPrZoZ0q/KER57vbj2qkgDLXA= +go.opentelemetry.io/otel/sdk/log/logtest v0.18.0/go.mod h1:7cHtiVJpZebB3wybTa4NG+FUo5NPe3PROz1FqB0+qdw= +go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= +go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= +go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= +go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= +golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= +golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= +google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= +google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= +modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.32.0 h1:hjG66bI/kqIPX1b2yT6fr/jt+QedtP2fqojG2VrFuVw= +modernc.org/ccgo/v4 v4.32.0/go.mod h1:6F08EBCx5uQc38kMGl+0Nm0oWczoo1c7cgpzEry7Uc0= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw= +modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.48.2 h1:5CnW4uP8joZtA0LedVqLbZV5GD7F/0x91AXeSyjoh5c= +modernc.org/sqlite v1.48.2/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/backend/internal/asynqqueue/consumer.go b/backend/internal/asynqqueue/consumer.go new file mode 100644 index 0000000..a7d02f2 --- /dev/null +++ b/backend/internal/asynqqueue/consumer.go @@ -0,0 +1,81 @@ +package asynqqueue + +import ( + "context" + "time" + + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/taskqueue" +) + +// Consumer wraps the PocketBase-backed Consumer for result write-back only. +// +// When using Asynq, the runner no longer polls for scrape/audio work — Asynq +// delivers those tasks via the ServeMux handlers. However translation tasks +// live in PocketBase (not Redis), so ClaimNextTranslationTask and HeartbeatTask +// still delegate to the underlying PocketBase consumer. +// +// ClaimNextAudioTask, ClaimNextScrapeTask are no-ops here because Asynq owns +// those responsibilities. +type Consumer struct { + pb taskqueue.Consumer // underlying PocketBase consumer (for write-back) +} + +// NewConsumer wraps an existing PocketBase Consumer. +func NewConsumer(pb taskqueue.Consumer) *Consumer { + return &Consumer{pb: pb} +} + +// ── Write-back (delegated to PocketBase) ────────────────────────────────────── + +func (c *Consumer) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error { + return c.pb.FinishScrapeTask(ctx, id, result) +} + +func (c *Consumer) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error { + return c.pb.FinishAudioTask(ctx, id, result) +} + +func (c *Consumer) FinishTranslationTask(ctx context.Context, id string, result domain.TranslationResult) error { + return c.pb.FinishTranslationTask(ctx, id, result) +} + +func (c *Consumer) FinishImportTask(ctx context.Context, id string, result domain.ImportResult) error { + return c.pb.FinishImportTask(ctx, id, result) +} + +func (c *Consumer) FailTask(ctx context.Context, id, errMsg string) error { + return c.pb.FailTask(ctx, id, errMsg) +} + +// ── No-ops (Asynq owns claiming / heartbeating / reaping) ─────────────────── + +func (c *Consumer) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) { + return domain.ScrapeTask{}, false, nil +} + +func (c *Consumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) { + return domain.AudioTask{}, false, nil +} + +// ClaimNextTranslationTask delegates to PocketBase because translation tasks +// are stored in PocketBase (not Redis/Asynq) and must still be polled directly. +func (c *Consumer) ClaimNextTranslationTask(ctx context.Context, workerID string) (domain.TranslationTask, bool, error) { + return c.pb.ClaimNextTranslationTask(ctx, workerID) +} + +// ClaimNextImportTask delegates to PocketBase because import tasks +// are stored in PocketBase (not Redis/Asynq) and must still be polled directly. +func (c *Consumer) ClaimNextImportTask(ctx context.Context, workerID string) (domain.ImportTask, bool, error) { + return c.pb.ClaimNextImportTask(ctx, workerID) +} + +func (c *Consumer) HeartbeatTask(ctx context.Context, id string) error { + return c.pb.HeartbeatTask(ctx, id) +} + +// ReapStaleTasks delegates to PocketBase so stale translation tasks are reset +// to pending and can be reclaimed. +func (c *Consumer) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) { + return c.pb.ReapStaleTasks(ctx, staleAfter) +} diff --git a/backend/internal/asynqqueue/producer.go b/backend/internal/asynqqueue/producer.go new file mode 100644 index 0000000..ad099a2 --- /dev/null +++ b/backend/internal/asynqqueue/producer.go @@ -0,0 +1,135 @@ +package asynqqueue + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + + "github.com/hibiken/asynq" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/taskqueue" +) + +// Producer dual-writes every task: first to PocketBase (via pb, for audit / +// UI status), then to Redis via Asynq so the runner picks it up immediately. +type Producer struct { + pb taskqueue.Producer // underlying PocketBase producer + client *asynq.Client + log *slog.Logger +} + +// NewProducer wraps an existing PocketBase Producer with Asynq dispatch. +func NewProducer(pb taskqueue.Producer, redisOpt asynq.RedisConnOpt, log *slog.Logger) *Producer { + return &Producer{ + pb: pb, + client: asynq.NewClient(redisOpt), + log: log, + } +} + +// Close shuts down the underlying Asynq client connection. +func (p *Producer) Close() error { + return p.client.Close() +} + +// CreateScrapeTask creates a PocketBase record then enqueues an Asynq job. +func (p *Producer) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) { + id, err := p.pb.CreateScrapeTask(ctx, kind, targetURL, fromChapter, toChapter) + if err != nil { + return "", err + } + + payload := ScrapePayload{ + PBTaskID: id, + Kind: kind, + TargetURL: targetURL, + FromChapter: fromChapter, + ToChapter: toChapter, + } + taskType := TypeScrapeBook + if kind == "catalogue" { + taskType = TypeScrapeCatalogue + } + if err := p.enqueue(ctx, taskType, payload); err != nil { + // Non-fatal: PB record exists; runner will pick it up on next poll. + p.log.Warn("asynq enqueue scrape failed (task still in PB, runner will poll)", + "task_id", id, "err", err) + return id, nil + } + return id, nil +} + +// CreateAudioTask creates a PocketBase record then enqueues an Asynq job. +func (p *Producer) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) { + id, err := p.pb.CreateAudioTask(ctx, slug, chapter, voice) + if err != nil { + return "", err + } + + payload := AudioPayload{ + PBTaskID: id, + Slug: slug, + Chapter: chapter, + Voice: voice, + } + if err := p.enqueue(ctx, TypeAudioGenerate, payload); err != nil { + // Non-fatal: PB record exists; runner will pick it up on next poll. + p.log.Warn("asynq enqueue audio failed (task still in PB, runner will poll)", + "task_id", id, "err", err) + return id, nil + } + return id, nil +} + +// CreateTranslationTask creates a PocketBase record. Translation tasks are +// not currently dispatched via Asynq — the runner picks them up via polling. +func (p *Producer) CreateTranslationTask(ctx context.Context, slug string, chapter int, lang string) (string, error) { + return p.pb.CreateTranslationTask(ctx, slug, chapter, lang) +} + +// CreateImportTask creates a PocketBase record then enqueues an Asynq job for PDF/EPUB import. +func (p *Producer) CreateImportTask(ctx context.Context, task domain.ImportTask) (string, error) { + id, err := p.pb.CreateImportTask(ctx, task) + if err != nil { + return "", err + } + + payload := ImportPayload{ + PBTaskID: id, + Slug: task.Slug, + Title: task.Title, + FileType: task.FileType, + ObjectKey: task.ObjectKey, + ChaptersKey: task.ChaptersKey, + } + if err := p.enqueue(ctx, TypeImportBook, payload); err != nil { + // Non-fatal: PB record exists; runner will pick it up on next poll. + p.log.Warn("asynq enqueue import failed (task still in PB, runner will poll)", + "task_id", id, "err", err) + return id, nil + } + return id, nil +} + +// CancelTask delegates to PocketBase; Asynq jobs may already be running and +// cannot be reliably cancelled, so we only update the audit record. +func (p *Producer) CancelTask(ctx context.Context, id string) error { + return p.pb.CancelTask(ctx, id) +} + +// CancelAudioTasksBySlug delegates to PocketBase to cancel all pending/running +// audio tasks for slug. +func (p *Producer) CancelAudioTasksBySlug(ctx context.Context, slug string) (int, error) { + return p.pb.CancelAudioTasksBySlug(ctx, slug) +} + +// enqueue serialises payload and dispatches it to Asynq. +func (p *Producer) enqueue(_ context.Context, taskType string, payload any) error { + b, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("marshal payload: %w", err) + } + _, err = p.client.Enqueue(asynq.NewTask(taskType, b)) + return err +} diff --git a/backend/internal/asynqqueue/tasks.go b/backend/internal/asynqqueue/tasks.go new file mode 100644 index 0000000..5c2c709 --- /dev/null +++ b/backend/internal/asynqqueue/tasks.go @@ -0,0 +1,57 @@ +// Package asynqqueue provides Asynq-backed implementations of the +// taskqueue.Producer and taskqueue.Consumer interfaces. +// +// Architecture: +// - Producer: dual-writes — creates a PocketBase record for audit/UI, then +// enqueues an Asynq job so the runner picks it up immediately (sub-ms). +// - Consumer: thin wrapper used only for result write-back (FinishAudioTask, +// FinishScrapeTask, FailTask). ClaimNext*/Heartbeat/Reap are no-ops because +// Asynq owns those responsibilities. +// - Handlers: asynq.HandlerFunc wrappers that decode job payloads and invoke +// the existing runner logic (runScrapeTask / runAudioTask). +// +// Fallback: when REDIS_ADDR is empty the caller should use the plain +// storage.Store (PocketBase-polling) implementation unchanged. +package asynqqueue + +// Queue names — keep all jobs on the default queue for now. +// Add separate queues (e.g. "audio", "scrape") later if you need priority. +const QueueDefault = "default" + +// Task type constants used for Asynq routing. +const ( + TypeAudioGenerate = "audio:generate" + TypeScrapeBook = "scrape:book" + TypeScrapeCatalogue = "scrape:catalogue" + TypeImportBook = "import:book" +) + +// AudioPayload is the Asynq job payload for audio generation tasks. +type AudioPayload struct { + // PBTaskID is the PocketBase record ID created before enqueueing. + // The handler uses it to write results back via Consumer.FinishAudioTask. + PBTaskID string `json:"pb_task_id"` + Slug string `json:"slug"` + Chapter int `json:"chapter"` + Voice string `json:"voice"` +} + +// ScrapePayload is the Asynq job payload for scrape tasks. +type ScrapePayload struct { + // PBTaskID is the PocketBase record ID created before enqueueing. + PBTaskID string `json:"pb_task_id"` + Kind string `json:"kind"` // "catalogue", "book", or "book_range" + TargetURL string `json:"target_url"` // empty for catalogue tasks + FromChapter int `json:"from_chapter"` // 0 unless Kind=="book_range" + ToChapter int `json:"to_chapter"` // 0 unless Kind=="book_range" +} + +// ImportPayload is the Asynq job payload for PDF/EPUB import tasks. +type ImportPayload struct { + PBTaskID string `json:"pb_task_id"` + Slug string `json:"slug"` + Title string `json:"title"` + FileType string `json:"file_type"` // "pdf" or "epub" + ObjectKey string `json:"object_key"` // MinIO path to uploaded file + ChaptersKey string `json:"chapters_key"` // MinIO path to pre-parsed chapters JSON +} diff --git a/backend/internal/backend/epub.go b/backend/internal/backend/epub.go new file mode 100644 index 0000000..cca5cf4 --- /dev/null +++ b/backend/internal/backend/epub.go @@ -0,0 +1,143 @@ +package backend + +import ( + "archive/zip" + "bytes" + "fmt" + "strings" +) + +type epubChapter struct { + Number int + Title string + HTML string +} + +func generateEPUB(slug, title, author string, chapters []epubChapter) ([]byte, error) { + var buf bytes.Buffer + w := zip.NewWriter(&buf) + + // 1. mimetype — MUST be first, MUST be uncompressed (Store method) + mw, err := w.CreateHeader(&zip.FileHeader{ + Name: "mimetype", + Method: zip.Store, + }) + if err != nil { + return nil, err + } + mw.Write([]byte("application/epub+zip")) + + // 2. META-INF/container.xml + addFile(w, "META-INF/container.xml", containerXML()) + + // 3. OEBPS/style.css + addFile(w, "OEBPS/style.css", epubCSS()) + + // 4. OEBPS/content.opf + addFile(w, "OEBPS/content.opf", contentOPF(slug, title, author, chapters)) + + // 5. OEBPS/toc.ncx + addFile(w, "OEBPS/toc.ncx", tocNCX(slug, title, chapters)) + + // 6. Chapter files + for _, ch := range chapters { + name := fmt.Sprintf("OEBPS/chapter-%04d.xhtml", ch.Number) + addFile(w, name, chapterXHTML(ch)) + } + + w.Close() + return buf.Bytes(), nil +} + +func addFile(w *zip.Writer, name, content string) { + f, _ := w.Create(name) + f.Write([]byte(content)) +} + +func containerXML() string { + return ` + + + + +` +} + +func contentOPF(slug, title, author string, chapters []epubChapter) string { + var items, spine strings.Builder + for _, ch := range chapters { + id := fmt.Sprintf("ch%04d", ch.Number) + href := fmt.Sprintf("chapter-%04d.xhtml", ch.Number) + items.WriteString(fmt.Sprintf(` `+"\n", id, href)) + spine.WriteString(fmt.Sprintf(` `+"\n", id)) + } + return fmt.Sprintf(` + + + %s + %s + %s + en + + + + +%s + +%s +`, escapeXML(title), escapeXML(author), slug, items.String(), spine.String()) +} + +func tocNCX(slug, title string, chapters []epubChapter) string { + var points strings.Builder + for i, ch := range chapters { + chTitle := ch.Title + if chTitle == "" { + chTitle = fmt.Sprintf("Chapter %d", ch.Number) + } + points.WriteString(fmt.Sprintf(` + %s + + `+"\n", i+1, i+1, escapeXML(chTitle), ch.Number)) + } + return fmt.Sprintf(` + + + + %s + +%s +`, slug, escapeXML(title), points.String()) +} + +func chapterXHTML(ch epubChapter) string { + title := ch.Title + if title == "" { + title = fmt.Sprintf("Chapter %d", ch.Number) + } + return fmt.Sprintf(` + + +%s + +

%s

+%s + +`, escapeXML(title), escapeXML(title), ch.HTML) +} + +func epubCSS() string { + return `body { font-family: Georgia, serif; font-size: 1em; line-height: 1.6; margin: 1em 2em; } +h1.chapter-title { font-size: 1.4em; margin-bottom: 1em; } +p { margin: 0 0 0.8em 0; text-indent: 1.5em; } +p:first-of-type { text-indent: 0; } +` +} + +func escapeXML(s string) string { + s = strings.ReplaceAll(s, "&", "&") + s = strings.ReplaceAll(s, "<", "<") + s = strings.ReplaceAll(s, ">", ">") + s = strings.ReplaceAll(s, `"`, """) + return s +} diff --git a/backend/internal/backend/handlers.go b/backend/internal/backend/handlers.go new file mode 100644 index 0000000..8bfcd52 --- /dev/null +++ b/backend/internal/backend/handlers.go @@ -0,0 +1,2163 @@ +package backend + +// handlers.go — all HTTP request handlers for the backend server. +// +// Handler naming mirrors the route table in server.go: +// handleScrapeCatalogue, handleScrapeBook, handleScrapeBookRange +// handleScrapeStatus, handleScrapeTasks +// handleBrowse, handleSearch +// handleGetRanking, handleGetCover +// handleBookPreview, handleChapterText, handleChapterTextPreview, handleChapterMarkdown, handleReindex +// handleAudioGenerate, handleAudioStatus, handleAudioProxy, handleAudioStream +// handleTTSAnnounce +// handleVoices +// handlePresignChapter, handlePresignAudio, handlePresignVoiceSample +// handlePresignAvatarUpload, handlePresignAvatar +// handleGetProgress, handleSetProgress, handleDeleteProgress +// +// Key design choices vs. old scraper: +// - POST /scrape* creates a PocketBase task record and returns 202 with the +// task_id — it does NOT run the orchestrator inline. +// - POST /api/audio creates a PocketBase audio task and returns 202 — the +// runner binary executes TTS generation asynchronously. +// - GET /api/audio/status polls PocketBase for the task record status. +// - GET /api/audio-proxy reads the completed audio object from MinIO via a +// presigned URL redirect (the runner has already uploaded the bytes). +// - GET /api/browse and /api/search fetch novelfire.net live (no MinIO cache). +// - GET /api/cover redirects to the source cover URL live. +// - GET /api/ranking reads from the PocketBase ranking collection (populated +// by the runner after each catalogue scrape). +// - GET /api/book-preview returns stored data when in library, or enqueues a +// scrape task and returns 202 when not. The backend never scrapes directly. +// - GET /api/chapter-text-preview scrapes a chapter live from novelfire.net +// directly (no runner task, no store writes). Used for unscraped books. + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/libnovel/backend/internal/cfai" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/meili" + "github.com/libnovel/backend/internal/novelfire/htmlutil" + "github.com/libnovel/backend/internal/pockettts" + "github.com/libnovel/backend/internal/scraper" + "github.com/yuin/goldmark" +) + +const ( + novelFireBase = "https://novelfire.net" + novelFireDomain = "novelfire.net" +) + +// ── Scrape task creation ─────────────────────────────────────────────────────── + +// handleScrapeCatalogue handles POST /scrape. +// Creates a "catalogue" scrape task in PocketBase and returns 202 with the task ID. +func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) { + taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "catalogue", "", 0, 0) + if err != nil { + s.deps.Log.Error("handleScrapeCatalogue: CreateScrapeTask failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to create task") + return + } + writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"}) +} + +// handleScrapeBook handles POST /scrape/book. +// Body: {"url": "https://novelfire.net/book/..."} +func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) { + var body struct { + URL string `json:"url"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { + jsonError(w, http.StatusBadRequest, `request body must be JSON with "url" field`) + return + } + taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "book", body.URL, 0, 0) + if err != nil { + s.deps.Log.Error("handleScrapeBook: CreateScrapeTask failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to create task") + return + } + writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"}) +} + +// handleScrapeBookRange handles POST /scrape/book/range. +// Body: {"url": "...", "from": N, "to": M} +func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) { + var body struct { + URL string `json:"url"` + From int `json:"from"` + To int `json:"to"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { + jsonError(w, http.StatusBadRequest, `request body must be JSON with "url" field`) + return + } + taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "book_range", body.URL, body.From, body.To) + if err != nil { + s.deps.Log.Error("handleScrapeBookRange: CreateScrapeTask failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to create task") + return + } + writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"}) +} + +// handleCancelTask handles POST /api/cancel-task/{id}. +// Transitions a pending task (scrape or audio) to status=cancelled. +// Returns 404 if the task does not exist, 409 if it cannot be cancelled +// (e.g. already running/done). +func (s *Server) handleCancelTask(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + if id == "" { + jsonError(w, http.StatusBadRequest, "missing task id") + return + } + if err := s.deps.Producer.CancelTask(r.Context(), id); err != nil { + s.deps.Log.Warn("handleCancelTask: CancelTask failed", "id", id, "err", err) + jsonError(w, http.StatusConflict, "could not cancel task: "+err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "cancelled", "id": id}) +} + +// ── Scrape task status / history ─────────────────────────────────────────────── + +// handleScrapeStatus handles GET /api/scrape/status. +// Returns the most recent scrape task status (or {"running":false} if none). +func (s *Server) handleScrapeStatus(w http.ResponseWriter, r *http.Request) { + tasks, err := s.deps.TaskReader.ListScrapeTasks(r.Context()) + if err != nil { + s.deps.Log.Error("handleScrapeStatus: ListScrapeTasks failed", "err", err) + writeJSON(w, 0, map[string]bool{"running": false}) + return + } + running := false + for _, t := range tasks { + if t.Status == domain.TaskStatusRunning || t.Status == domain.TaskStatusPending { + running = true + break + } + } + writeJSON(w, 0, map[string]bool{"running": running}) +} + +// handleScrapeTasks handles GET /api/scrape/tasks. +// Returns all scrape task records from PocketBase, newest first. +func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) { + tasks, err := s.deps.TaskReader.ListScrapeTasks(r.Context()) + if err != nil { + s.deps.Log.Error("handleScrapeTasks: ListScrapeTasks failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to list tasks") + return + } + if tasks == nil { + tasks = []domain.ScrapeTask{} + } + writeJSON(w, 0, tasks) +} + +// ── Browse & search ──────────────────────────────────────────────────────────── + +// NovelListing represents a single novel entry from the novelfire browse/search page. +type NovelListing struct { + Slug string `json:"slug"` + Title string `json:"title"` + Cover string `json:"cover"` + Rank string `json:"rank"` + Rating string `json:"rating"` + Chapters string `json:"chapters"` + URL string `json:"url"` +} + +// handleSearch handles GET /api/search. +// Query params: q (min 2 chars), source ("local"|"remote"|"all", default "all") +// +// Local search is powered by Meilisearch when configured; falls back to a +// substring match against PocketBase book records otherwise. +func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("q") + if len([]rune(q)) < 2 { + jsonError(w, http.StatusBadRequest, "query must be at least 2 characters") + return + } + + source := r.URL.Query().Get("source") + if source == "" { + source = "all" + } + + ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second) + defer cancel() + + var localResults, remoteResults []NovelListing + + // Local search: Meilisearch → PocketBase substring fallback + if source == "local" || source == "all" { + meiliBooks, meiliErr := s.deps.SearchIndex.Search(ctx, q, 50) + if meiliErr == nil && len(meiliBooks) > 0 { + for _, b := range meiliBooks { + localResults = append(localResults, NovelListing{ + Slug: b.Slug, + Title: b.Title, + Cover: b.Cover, + URL: b.SourceURL, + }) + } + } else { + // Fallback: substring match against PocketBase + books, err := s.deps.BookReader.ListBooks(ctx) + if err != nil { + s.deps.Log.Warn("search: ListBooks failed", "err", err) + } else { + qLower := strings.ToLower(q) + for _, b := range books { + if strings.Contains(strings.ToLower(b.Title), qLower) || + strings.Contains(strings.ToLower(b.Author), qLower) { + localResults = append(localResults, NovelListing{ + Slug: b.Slug, + Title: b.Title, + Cover: b.Cover, + URL: b.SourceURL, + }) + } + } + } + } + } + + // Remote search (novelfire.net) + if source == "remote" || source == "all" { + searchURL := novelFireBase + "/search?keyword=" + url.QueryEscape(q) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil) + if err == nil { + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + if resp, fetchErr := http.DefaultClient.Do(req); fetchErr == nil { + defer resp.Body.Close() + if resp.StatusCode == http.StatusOK { + parsed, _ := parseBrowsePage(resp.Body) + remoteResults = parsed + } + } + } + } + + // Merge: local first, de-duplicate remote + localSlugs := make(map[string]bool, len(localResults)) + for _, item := range localResults { + localSlugs[item.Slug] = true + } + combined := make([]NovelListing, 0, len(localResults)+len(remoteResults)) + combined = append(combined, localResults...) + for _, item := range remoteResults { + if !localSlugs[item.Slug] { + combined = append(combined, item) + } + } + + writeJSON(w, 0, map[string]any{ + "results": combined, + "local_count": len(localResults), + "remote_count": len(remoteResults), + }) +} + +// ── Ranking ──────────────────────────────────────────────────────────────────── + +// handleGetRanking handles GET /api/ranking. +// Returns all ranking items sorted by rank ascending. +func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) { + items, err := s.deps.RankingStore.ReadRankingItems(r.Context()) + if err != nil { + s.deps.Log.Error("handleGetRanking: ReadRankingItems failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to read ranking") + return + } + if items == nil { + items = []domain.RankingItem{} + } + writeJSON(w, 0, items) +} + +// handleGetCover handles GET /api/cover/{domain}/{slug}. +// Serves the cover image directly from MinIO when available; falls back to a +// redirect to the stored cover URL from PocketBase when the cover has not yet +// been downloaded to MinIO. +func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + http.Error(w, "missing slug", http.StatusBadRequest) + return + } + + // Fast path: serve from MinIO if the cover has been downloaded. + if s.deps.CoverStore != nil { + data, ct, ok, err := s.deps.CoverStore.GetCover(r.Context(), slug) + if err != nil { + s.deps.Log.Warn("handleGetCover: GetCover error", "slug", slug, "err", err) + } + if ok && len(data) > 0 { + if ct == "" { + ct = "image/jpeg" + } + w.Header().Set("Content-Type", ct) + w.Header().Set("Cache-Control", "public, max-age=86400") + _, _ = w.Write(data) + return + } + } + + // Fallback: read the stored cover URL from PocketBase and redirect to it. + // This avoids the broken cdn.novelfire.net domain and uses the actual URL + // scraped from the source. If the book is not found, return 404. + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), slug) + if err != nil { + s.deps.Log.Warn("handleGetCover: ReadMetadata error", "slug", slug, "err", err) + http.Error(w, "cover not found", http.StatusNotFound) + return + } + if !ok || meta.Cover == "" || strings.HasPrefix(meta.Cover, "/api/cover/") { + http.Error(w, "cover not found", http.StatusNotFound) + return + } + http.Redirect(w, r, meta.Cover, http.StatusFound) +} + +// ── Preview (live scrape, no store writes) ───────────────────────────────────── + +// handleBookPreview handles GET /api/book-preview/{slug}. +// +// If the book is already in the library (PocketBase), returns its metadata and +// chapter index immediately (200). +// +// If the book is not yet in the library, enqueues a "book" scrape task and +// returns 202 Accepted with the task_id. The runner will scrape the book +// asynchronously; the client should poll GET /api/scrape/status or +// GET /api/scrape/tasks to detect completion, then re-request this endpoint. +// +// The backend never scrapes directly — all scraping is the runner's job. +func (s *Server) handleBookPreview(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + + ctx := r.Context() + + meta, inLib, err := s.deps.BookReader.ReadMetadata(ctx, slug) + if err != nil { + s.deps.Log.Warn("book-preview: ReadMetadata failed", "slug", slug, "err", err) + inLib = false + } + + if inLib { + // Fast path: book is already scraped — return stored data. + chapters, cerr := s.deps.BookReader.ListChapters(ctx, slug) + if cerr != nil { + s.deps.Log.Warn("book-preview: ListChapters failed", "slug", slug, "err", cerr) + } + writeJSON(w, 0, map[string]any{ + "in_lib": true, + "meta": meta, + "chapters": chapters, + }) + return + } + + // Book not in library — enqueue a range scrape task for the first 20 chapters + // so the user can start reading quickly. Remaining chapters can be scraped + // later via the book detail page or the admin scrape panel. + bookURL := r.URL.Query().Get("source_url") + if bookURL == "" { + bookURL = fmt.Sprintf("%s/book/%s", novelFireBase, slug) + } + + const previewFrom, previewTo = 1, 20 + taskID, err := s.deps.Producer.CreateScrapeTask(ctx, "book_range", bookURL, previewFrom, previewTo) + if err != nil { + s.deps.Log.Error("book-preview: CreateScrapeTask failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, "failed to enqueue scrape task") + return + } + + s.deps.Log.Info("book-preview: enqueued range scrape task", "slug", slug, "task_id", taskID, + "from", previewFrom, "to", previewTo) + writeJSON(w, http.StatusAccepted, map[string]any{ + "in_lib": false, + "task_id": taskID, + "message": fmt.Sprintf("scraping first %d chapters; poll /api/scrape/tasks for completion", previewTo), + }) +} + +// ── Chapter text ─────────────────────────────────────────────────────────────── + +// handleChapterText handles GET /api/chapter-text/{slug}/{n}. +// Returns plain text (markdown stripped) of a stored chapter. +func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + http.NotFound(w, r) + return + } + raw, err := s.deps.BookReader.ReadChapter(r.Context(), slug, n) + if err != nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + fmt.Fprint(w, stripMarkdown(raw)) +} + +// handleChapterMarkdown handles GET /api/chapter-markdown/{slug}/{n}. +// +// Returns the raw markdown content of a stored chapter directly from MinIO. +// This is used by the SvelteKit UI as a simpler alternative to presign+fetch: +// it avoids the need for the SvelteKit server to reach MinIO directly, and +// gives a clean 404 when the chapter has not been scraped yet. +func (s *Server) handleChapterMarkdown(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest) + return + } + raw, err := s.deps.BookReader.ReadChapter(r.Context(), slug, n) + if err != nil { + s.deps.Log.Warn("chapter-markdown: not found in MinIO", "slug", slug, "n", n, "err", err) + http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "text/markdown; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + fmt.Fprint(w, raw) +} + +// handleChapterTextPreview handles GET /api/chapter-text-preview/{slug}/{n}. +// +// Fetches a chapter live from novelfire.net and returns its plain text without +// writing anything to PocketBase or MinIO. This is the preview path used when +// a chapter has not yet been scraped into the library. +// +// Optional query params: +// +// chapter_url — the canonical chapter URL (preferred over constructing one) +// title — hint for the chapter title (used when the page title is empty) +// +// Response: {"slug":string,"number":int,"title":string,"text":string,"url":string} +func (s *Server) handleChapterTextPreview(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + jsonError(w, http.StatusBadRequest, "invalid slug or chapter number") + return + } + + // Determine the chapter URL to fetch. + chapterURL := r.URL.Query().Get("chapter_url") + if chapterURL == "" { + // Best-effort: novelfire chapter URLs follow /book/{slug}/chapter-{n} + chapterURL = fmt.Sprintf("%s/book/%s/chapter-%d", novelFireBase, slug, n) + } + + titleHint := r.URL.Query().Get("title") + + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + + // Fetch the chapter page. + req, err := http.NewRequestWithContext(ctx, http.MethodGet, chapterURL, nil) + if err != nil { + s.deps.Log.Error("chapter-text-preview: build request failed", "url", chapterURL, "err", err) + jsonError(w, http.StatusInternalServerError, "failed to build request") + return + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + s.deps.Log.Warn("chapter-text-preview: fetch failed", "url", chapterURL, "err", err) + jsonError(w, http.StatusBadGateway, "failed to fetch chapter") + return + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + jsonError(w, http.StatusNotFound, "chapter not found") + return + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + s.deps.Log.Warn("chapter-text-preview: upstream error", + "url", chapterURL, "status", resp.StatusCode, "body_snippet", string(body)) + jsonError(w, http.StatusBadGateway, fmt.Sprintf("upstream returned %d", resp.StatusCode)) + return + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + s.deps.Log.Error("chapter-text-preview: read body failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to read response") + return + } + + // Parse HTML and extract the #content node. + root, err := htmlutil.ParseHTML(string(bodyBytes)) + if err != nil { + s.deps.Log.Error("chapter-text-preview: html parse failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to parse chapter HTML") + return + } + + container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"}) + if container == nil { + s.deps.Log.Warn("chapter-text-preview: #content not found", "url", chapterURL) + jsonError(w, http.StatusNotFound, "chapter content not found on page") + return + } + + markdownText := htmlutil.NodeToMarkdown(container) + plainText := stripMarkdown(markdownText) + + // Extract the chapter title from the page or <h1> if not hinted. + chapterTitle := titleHint + if chapterTitle == "" { + // Try <h1 class="chapter-title"> first, then <h2 class="chapter-title"> + for _, tag := range []string{"h1", "h2", "h3"} { + if node := htmlutil.FindFirst(root, scraper.Selector{Tag: tag, Class: "chapter-title"}); node != nil { + chapterTitle = strings.TrimSpace(htmlutil.TextContent(node)) + break + } + } + } + if chapterTitle == "" { + chapterTitle = fmt.Sprintf("Chapter %d", n) + } + + writeJSON(w, 0, map[string]any{ + "slug": slug, + "number": n, + "title": chapterTitle, + "text": plainText, + "url": chapterURL, + }) +} + +// handleReindex handles POST /api/reindex/{slug}. +// Rebuilds the chapters_idx PocketBase collection for a book from MinIO objects. +func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + + count, err := s.deps.BookReader.ReindexChapters(r.Context(), slug) + if err != nil { + s.deps.Log.Error("reindex failed", "slug", slug, "indexed", count, "err", err) + writeJSON(w, http.StatusInternalServerError, map[string]any{ + "error": err.Error(), + "indexed": count, + }) + return + } + + s.deps.Log.Info("reindex complete", "slug", slug, "indexed", count) + writeJSON(w, 0, map[string]any{"slug": slug, "indexed": count}) +} + +// handleDedupChapters handles POST /api/admin/dedup-chapters/{slug}. +// Removes duplicate chapters_idx records for a book, keeping the latest record +// per chapter number. Returns the number of duplicate records deleted. +func (s *Server) handleDedupChapters(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + + deleted, err := s.deps.BookWriter.DeduplicateChapters(r.Context(), slug) + if err != nil { + s.deps.Log.Error("dedup-chapters failed", "slug", slug, "err", err) + writeJSON(w, http.StatusInternalServerError, map[string]any{ + "error": err.Error(), + "deleted": deleted, + }) + return + } + + s.deps.Log.Info("dedup-chapters complete", "slug", slug, "deleted", deleted) + writeJSON(w, 0, map[string]any{"slug": slug, "deleted": deleted}) +} + +// ── Audio ────────────────────────────────────────────────────────────────────── + +// handleAudioGenerate handles POST /api/audio/{slug}/{n}. +// Creates an audio_jobs task in PocketBase (runner executes asynchronously). +// Returns 200 immediately if audio already exists in MinIO. +// Returns 202 with the task_id if a new task was created. +func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + jsonError(w, http.StatusBadRequest, "invalid chapter") + return + } + + voice := s.cfg.DefaultVoice + var body struct { + Voice string `json:"voice"` + } + if r.Body != nil { + _ = json.NewDecoder(r.Body).Decode(&body) + } + if body.Voice != "" { + voice = body.Voice + } + + cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice) + + // Fast path: audio already in MinIO + audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice) + if s.deps.AudioStore.AudioExists(r.Context(), audioKey) { + proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice) + writeJSON(w, 0, map[string]string{"url": proxyURL, "status": "done"}) + return + } + + // Check if a task is already pending/running + task, found, _ := s.deps.TaskReader.GetAudioTask(r.Context(), cacheKey) + if found && (task.Status == domain.TaskStatusPending || task.Status == domain.TaskStatusRunning) { + writeJSON(w, http.StatusAccepted, map[string]string{ + "task_id": task.ID, + "status": string(task.Status), + }) + return + } + + // Create a new audio task + taskID, err := s.deps.Producer.CreateAudioTask(r.Context(), slug, n, voice) + if err != nil { + s.deps.Log.Error("handleAudioGenerate: CreateAudioTask failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to create audio task") + return + } + + writeJSON(w, http.StatusAccepted, map[string]string{ + "task_id": taskID, + "status": "pending", + }) +} + +// handleAudioStatus handles GET /api/audio/status/{slug}/{n}. +// Polls PocketBase for the audio task status. +// Query params: voice (optional, defaults to DefaultVoice) +func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + jsonError(w, http.StatusBadRequest, "invalid params") + return + } + + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.cfg.DefaultVoice + } + + // Fast path: audio exists in MinIO + audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice) + if s.deps.AudioStore.AudioExists(r.Context(), audioKey) { + proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice) + writeJSON(w, 0, map[string]string{ + "status": "done", + "url": proxyURL, + }) + return + } + + cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice) + task, found, _ := s.deps.TaskReader.GetAudioTask(r.Context(), cacheKey) + if !found { + writeJSON(w, 0, map[string]string{"status": "idle"}) + return + } + + resp := map[string]string{ + "status": string(task.Status), + "task_id": task.ID, + } + if task.Status == domain.TaskStatusFailed && task.ErrorMessage != "" { + resp["error"] = task.ErrorMessage + } + writeJSON(w, 0, resp) +} + +// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}. +// Redirects to a presigned MinIO URL for the generated audio object. +// Query params: voice (optional, defaults to DefaultVoice) +func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + http.NotFound(w, r) + return + } + + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.cfg.DefaultVoice + } + + audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice) + if !s.deps.AudioStore.AudioExists(r.Context(), audioKey) { + http.Error(w, "audio not generated yet", http.StatusNotFound) + return + } + + presignURL, err := s.deps.PresignStore.PresignAudio(r.Context(), audioKey, 1*time.Hour) + if err != nil { + s.deps.Log.Error("handleAudioProxy: PresignAudio failed", "slug", slug, "n", n, "err", err) + http.Error(w, "presign failed", http.StatusInternalServerError) + return + } + + http.Redirect(w, r, presignURL, http.StatusFound) +} + +// handleAudioStream handles GET /api/audio-stream/{slug}/{n}. +// +// Fast path: if audio already exists in MinIO, redirects to the presigned URL +// (same as handleAudioProxy) — the client plays from storage immediately. +// +// Slow path (first request): streams audio directly to the client while +// simultaneously uploading it to MinIO. After the stream completes, subsequent +// requests hit the fast path and skip TTS generation entirely. +// +// Query params: +// +// voice (optional, defaults to DefaultVoice) +// format (optional, "mp3" or "wav"; defaults to "mp3") +// +// Using format=wav skips the ffmpeg transcode for pocket-tts voices, delivering +// raw WAV frames to the client with lower latency at the cost of larger files. +func (s *Server) handleAudioStream(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + jsonError(w, http.StatusBadRequest, "invalid chapter") + return + } + + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.cfg.DefaultVoice + } + + format := r.URL.Query().Get("format") + if format != "wav" { + format = "mp3" + } + + contentType := "audio/mpeg" + if format == "wav" { + contentType = "audio/wav" + } + + audioKey := s.deps.AudioStore.AudioObjectKeyExt(slug, n, voice, format) + + // ── Fast path: already in MinIO ─────────────────────────────────────────── + if s.deps.AudioStore.AudioExists(r.Context(), audioKey) { + presignURL, err := s.deps.PresignStore.PresignAudio(r.Context(), audioKey, 1*time.Hour) + if err != nil { + s.deps.Log.Error("handleAudioStream: PresignAudio failed", "slug", slug, "n", n, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + http.Redirect(w, r, presignURL, http.StatusFound) + return + } + + // ── Slow path: generate + stream + save ─────────────────────────────────── + + // Read the chapter text. + raw, err := s.deps.BookReader.ReadChapter(r.Context(), slug, n) + if err != nil { + s.deps.Log.Error("handleAudioStream: ReadChapter failed", "slug", slug, "n", n, "err", err) + jsonError(w, http.StatusNotFound, "chapter not found") + return + } + text := stripMarkdown(raw) + if text == "" { + jsonError(w, http.StatusUnprocessableEntity, "chapter text is empty") + return + } + + // Open the TTS stream (WAV or MP3 depending on format param). + var audioStream io.ReadCloser + if format == "wav" { + if cfai.IsCFAIVoice(voice) { + if s.deps.CFAI == nil { + jsonError(w, http.StatusServiceUnavailable, "cloudflare AI TTS not configured") + return + } + audioStream, err = s.deps.CFAI.StreamAudioWAV(r.Context(), text, voice) + } else if pockettts.IsPocketTTSVoice(voice) { + if s.deps.PocketTTS == nil { + jsonError(w, http.StatusServiceUnavailable, "pocket-tts not configured") + return + } + audioStream, err = s.deps.PocketTTS.StreamAudioWAV(r.Context(), text, voice) + } else { + if s.deps.Kokoro == nil { + jsonError(w, http.StatusServiceUnavailable, "kokoro not configured") + return + } + audioStream, err = s.deps.Kokoro.StreamAudioWAV(r.Context(), text, voice) + } + } else { + if cfai.IsCFAIVoice(voice) { + if s.deps.CFAI == nil { + jsonError(w, http.StatusServiceUnavailable, "cloudflare AI TTS not configured") + return + } + audioStream, err = s.deps.CFAI.StreamAudioMP3(r.Context(), text, voice) + } else if pockettts.IsPocketTTSVoice(voice) { + if s.deps.PocketTTS == nil { + jsonError(w, http.StatusServiceUnavailable, "pocket-tts not configured") + return + } + audioStream, err = s.deps.PocketTTS.StreamAudioMP3(r.Context(), text, voice) + } else { + if s.deps.Kokoro == nil { + jsonError(w, http.StatusServiceUnavailable, "kokoro not configured") + return + } + audioStream, err = s.deps.Kokoro.StreamAudioMP3(r.Context(), text, voice) + } + } + if err != nil { + s.deps.Log.Error("handleAudioStream: TTS stream failed", "slug", slug, "n", n, "voice", voice, "format", format, "err", err) + jsonError(w, http.StatusInternalServerError, "tts stream failed") + return + } + defer audioStream.Close() + + // Tee: every byte read from audioStream is written to both the HTTP + // response and a pipe that feeds the MinIO upload goroutine. + pr, pw := io.Pipe() + + // MinIO upload runs concurrently. Size -1 triggers multipart upload. + uploadDone := make(chan error, 1) + go func() { + uploadDone <- s.deps.AudioStore.PutAudioStream( + context.Background(), // use background — request ctx may cancel after client disconnects + audioKey, pr, -1, contentType, + ) + }() + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Accel-Buffering", "no") // disable nginx/caddy buffering + w.WriteHeader(http.StatusOK) + + flusher, canFlush := w.(http.Flusher) + + tee := io.TeeReader(audioStream, pw) + buf := make([]byte, 32*1024) + for { + nr, readErr := tee.Read(buf) + if nr > 0 { + if _, writeErr := w.Write(buf[:nr]); writeErr != nil { + // Client disconnected — abort upload pipe so goroutine exits. + pw.CloseWithError(writeErr) + <-uploadDone + return + } + if canFlush { + flusher.Flush() + } + } + if readErr != nil { + if readErr == io.EOF { + break + } + s.deps.Log.Warn("handleAudioStream: read error mid-stream", "err", readErr) + pw.CloseWithError(readErr) + <-uploadDone + return + } + } + + // Signal end of stream to the MinIO upload goroutine. + pw.Close() + if uploadErr := <-uploadDone; uploadErr != nil { + s.deps.Log.Error("handleAudioStream: MinIO upload failed", "key", audioKey, "err", uploadErr) + // Audio was already streamed to the client — just log; don't error. + // The next request will re-stream since the object is absent. + } + // Note: we do not call FinishAudioTask here — the backend has no Consumer. + // handleAudioStatus fast-paths on AudioExists, so the UI will see "done" + // on its next poll as soon as the MinIO object is present. +} + +// handleTTSAnnounce handles GET /api/tts-announce. +// +// Streams a short TTS clip for arbitrary text — used by the UI to announce +// the upcoming chapter number/title through the real <audio> element instead +// of the Web Speech API (which is silently muted on mobile after the audio +// session ends). +// +// Query params: +// - text — the text to synthesize (required, max 300 chars) +// - voice — voice ID (defaults to server default) +// - format — "mp3" or "wav" (default "mp3") +// +// No MinIO caching — announcement clips are tiny and ephemeral. +func (s *Server) handleTTSAnnounce(w http.ResponseWriter, r *http.Request) { + text := r.URL.Query().Get("text") + if text == "" { + jsonError(w, http.StatusBadRequest, "text is required") + return + } + if len(text) > 300 { + text = text[:300] + } + + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.cfg.DefaultVoice + } + + format := r.URL.Query().Get("format") + if format != "wav" { + format = "mp3" + } + + contentType := "audio/mpeg" + if format == "wav" { + contentType = "audio/wav" + } + + var ( + audioStream io.ReadCloser + err error + ) + + if format == "wav" { + if cfai.IsCFAIVoice(voice) { + if s.deps.CFAI == nil { + jsonError(w, http.StatusServiceUnavailable, "cloudflare AI TTS not configured") + return + } + audioStream, err = s.deps.CFAI.StreamAudioWAV(r.Context(), text, voice) + } else if pockettts.IsPocketTTSVoice(voice) { + if s.deps.PocketTTS == nil { + jsonError(w, http.StatusServiceUnavailable, "pocket-tts not configured") + return + } + audioStream, err = s.deps.PocketTTS.StreamAudioWAV(r.Context(), text, voice) + } else { + if s.deps.Kokoro == nil { + jsonError(w, http.StatusServiceUnavailable, "kokoro not configured") + return + } + audioStream, err = s.deps.Kokoro.StreamAudioWAV(r.Context(), text, voice) + } + } else { + if cfai.IsCFAIVoice(voice) { + if s.deps.CFAI == nil { + jsonError(w, http.StatusServiceUnavailable, "cloudflare AI TTS not configured") + return + } + audioStream, err = s.deps.CFAI.StreamAudioMP3(r.Context(), text, voice) + } else if pockettts.IsPocketTTSVoice(voice) { + if s.deps.PocketTTS == nil { + jsonError(w, http.StatusServiceUnavailable, "pocket-tts not configured") + return + } + audioStream, err = s.deps.PocketTTS.StreamAudioMP3(r.Context(), text, voice) + } else { + if s.deps.Kokoro == nil { + jsonError(w, http.StatusServiceUnavailable, "kokoro not configured") + return + } + audioStream, err = s.deps.Kokoro.StreamAudioMP3(r.Context(), text, voice) + } + } + if err != nil { + s.deps.Log.Error("handleTTSAnnounce: TTS stream failed", "voice", voice, "err", err) + jsonError(w, http.StatusInternalServerError, "tts stream failed") + return + } + defer audioStream.Close() + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Accel-Buffering", "no") + w.WriteHeader(http.StatusOK) + + flusher, canFlush := w.(http.Flusher) + buf := make([]byte, 32*1024) + for { + nr, readErr := audioStream.Read(buf) + if nr > 0 { + if _, writeErr := w.Write(buf[:nr]); writeErr != nil { + return + } + if canFlush { + flusher.Flush() + } + } + if readErr != nil { + break + } + } +} +// +// CF AI voices are batch-only and can take 1-2+ minutes to generate a full +// chapter. This endpoint generates only the FIRST chunk of text (~1 800 chars, +// roughly 1-2 minutes of audio) so the client can start playing immediately +// while the full audio is generated in the background by the runner. +// +// Fast path: if a preview object already exists in MinIO, redirects to its +// presigned URL (no regeneration). +// +// Slow path: generates the first chunk via CF AI, streams the MP3 bytes to the +// client, and simultaneously uploads to MinIO under a "_preview" key so future +// requests hit the fast path. +// +// Only CF AI voices are expected here. Calling this with a Kokoro/PocketTTS +// voice falls back to the normal audio-stream endpoint behaviour. +// +// Query params: +// +// voice (required — must be a cfai: voice) +func (s *Server) handleAudioPreview(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + jsonError(w, http.StatusBadRequest, "invalid chapter") + return + } + + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.cfg.DefaultVoice + } + + if s.deps.CFAI == nil { + jsonError(w, http.StatusServiceUnavailable, "cloudflare AI TTS not configured") + return + } + + // Preview key: same as normal key with a "_preview" suffix before the extension. + // e.g. slug/1/cfai:luna_preview.mp3 + previewKey := s.deps.AudioStore.AudioObjectKeyExt(slug, n, voice+"_preview", "mp3") + + // ── Fast path: preview already in MinIO ────────────────────────────────── + if s.deps.AudioStore.AudioExists(r.Context(), previewKey) { + presignURL, err := s.deps.PresignStore.PresignAudio(r.Context(), previewKey, 1*time.Hour) + if err != nil { + s.deps.Log.Error("handleAudioPreview: PresignAudio failed", "slug", slug, "n", n, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + http.Redirect(w, r, presignURL, http.StatusFound) + return + } + + // ── Slow path: generate first chunk + stream + save ────────────────────── + + // Read the chapter text. + raw, err := s.deps.BookReader.ReadChapter(r.Context(), slug, n) + if err != nil { + s.deps.Log.Error("handleAudioPreview: ReadChapter failed", "slug", slug, "n", n, "err", err) + jsonError(w, http.StatusNotFound, "chapter not found") + return + } + text := stripMarkdown(raw) + if text == "" { + jsonError(w, http.StatusUnprocessableEntity, "chapter text is empty") + return + } + + // Take only the first ~1 800 characters — one CF AI chunk, roughly 1-2 min. + const previewChars = 1800 + firstChunk := text + if len([]rune(text)) > previewChars { + runes := []rune(text) + firstChunk = string(runes[:previewChars]) + // Walk back to last sentence boundary (. ! ?) to avoid a mid-word cut. + for i := previewChars - 1; i > previewChars/2; i-- { + r := runes[i] + if r == '.' || r == '!' || r == '?' || r == '\n' { + firstChunk = string(runes[:i+1]) + break + } + } + } + + // Generate the preview chunk via CF AI. + mp3, err := s.deps.CFAI.GenerateAudio(r.Context(), firstChunk, voice) + if err != nil { + s.deps.Log.Error("handleAudioPreview: GenerateAudio failed", "slug", slug, "n", n, "voice", voice, "err", err) + jsonError(w, http.StatusInternalServerError, "tts generation failed") + return + } + + // Upload to MinIO in the background so the next request hits the fast path. + go func() { + if uploadErr := s.deps.AudioStore.PutAudio( + context.Background(), previewKey, mp3, + ); uploadErr != nil { + s.deps.Log.Error("handleAudioPreview: MinIO upload failed", "key", previewKey, "err", uploadErr) + } + }() + + w.Header().Set("Content-Type", "audio/mpeg") + w.Header().Set("Content-Length", strconv.Itoa(len(mp3))) + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(mp3) +} + +// ── Translation ──────────────────────────────────────────────────────────────── + +// supportedTranslationLangs is the set of target locales the backend accepts. +// Source is always "en". +var supportedTranslationLangs = map[string]bool{ + "ru": true, "id": true, "pt": true, "fr": true, +} + +// handleTranslationGenerate handles POST /api/translation/{slug}/{n}. +// Query params: lang (required, one of ru|id|pt|fr) +// +// Returns 200 immediately if translation already exists in MinIO. +// Returns 202 with task_id if a new task was created. +// Returns 503 if TranslationStore is nil (feature disabled). +func (s *Server) handleTranslationGenerate(w http.ResponseWriter, r *http.Request) { + if s.deps.TranslationStore == nil { + jsonError(w, http.StatusServiceUnavailable, "machine translation not configured") + return + } + + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + jsonError(w, http.StatusBadRequest, "invalid chapter") + return + } + + lang := r.URL.Query().Get("lang") + if !supportedTranslationLangs[lang] { + jsonError(w, http.StatusBadRequest, "unsupported lang; use ru, id, pt, or fr") + return + } + + cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, lang) + + // Fast path: translation already in MinIO + key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n) + if s.deps.TranslationStore.TranslationExists(r.Context(), key) { + writeJSON(w, 0, map[string]string{"status": "done", "lang": lang}) + return + } + + // Check if a task is already pending/running + task, found, _ := s.deps.TaskReader.GetTranslationTask(r.Context(), cacheKey) + if found && (task.Status == domain.TaskStatusPending || task.Status == domain.TaskStatusRunning) { + writeJSON(w, http.StatusAccepted, map[string]string{ + "task_id": task.ID, + "status": string(task.Status), + "lang": lang, + }) + return + } + + // Create a new translation task + taskID, err := s.deps.Producer.CreateTranslationTask(r.Context(), slug, n, lang) + if err != nil { + s.deps.Log.Error("handleTranslationGenerate: CreateTranslationTask failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to create translation task") + return + } + + writeJSON(w, http.StatusAccepted, map[string]string{ + "task_id": taskID, + "status": "pending", + "lang": lang, + }) +} + +// handleTranslationStatus handles GET /api/translation/status/{slug}/{n}. +// Query params: lang (required) +func (s *Server) handleTranslationStatus(w http.ResponseWriter, r *http.Request) { + if s.deps.TranslationStore == nil { + writeJSON(w, 0, map[string]string{"status": "unavailable"}) + return + } + + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + jsonError(w, http.StatusBadRequest, "invalid params") + return + } + + lang := r.URL.Query().Get("lang") + if !supportedTranslationLangs[lang] { + jsonError(w, http.StatusBadRequest, "unsupported lang") + return + } + + // Fast path: translation exists in MinIO + key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n) + if s.deps.TranslationStore.TranslationExists(r.Context(), key) { + writeJSON(w, 0, map[string]string{"status": "done", "lang": lang}) + return + } + + cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, lang) + task, found, _ := s.deps.TaskReader.GetTranslationTask(r.Context(), cacheKey) + if !found { + writeJSON(w, 0, map[string]string{"status": "idle", "lang": lang}) + return + } + + resp := map[string]string{ + "status": string(task.Status), + "task_id": task.ID, + "lang": lang, + } + if task.Status == domain.TaskStatusFailed && task.ErrorMessage != "" { + resp["error"] = task.ErrorMessage + } + writeJSON(w, 0, resp) +} + +// handleTranslationRead handles GET /api/translation/{slug}/{n}. +// Query params: lang (required) +// +// Returns {"html": "<p>...</p>", "lang": "ru"} from the MinIO-cached translation. +// Returns 404 when the translation has not been generated yet. +func (s *Server) handleTranslationRead(w http.ResponseWriter, r *http.Request) { + if s.deps.TranslationStore == nil { + http.Error(w, `{"error":"machine translation not configured"}`, http.StatusServiceUnavailable) + return + } + + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + jsonError(w, http.StatusBadRequest, "invalid params") + return + } + + lang := r.URL.Query().Get("lang") + if !supportedTranslationLangs[lang] { + jsonError(w, http.StatusBadRequest, "unsupported lang") + return + } + + key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n) + md, err := s.deps.TranslationStore.GetTranslation(r.Context(), key) + if err != nil { + s.deps.Log.Warn("handleTranslationRead: translation not found", "slug", slug, "n", n, "lang", lang, "err", err) + jsonError(w, http.StatusNotFound, "translation not available") + return + } + + var buf bytes.Buffer + if err := goldmark.Convert([]byte(md), &buf); err != nil { + s.deps.Log.Error("handleTranslationRead: markdown conversion failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to render translation") + return + } + + // Translated chapter content is immutable once generated — cache aggressively. + // The browser and any intermediary (CDN, SvelteKit fetch cache) can reuse this + // response for 1 hour without hitting MinIO again. + w.Header().Set("Cache-Control", "public, max-age=3600, stale-while-revalidate=86400") + writeJSON(w, 0, map[string]string{"html": buf.String(), "lang": lang}) +} + +// handleAdminTranslationJobs handles GET /api/admin/translation/jobs. +// Returns the full list of translation jobs sorted by started descending. +func (s *Server) handleAdminTranslationJobs(w http.ResponseWriter, r *http.Request) { + tasks, err := s.deps.TaskReader.ListTranslationTasks(r.Context()) + if err != nil { + s.deps.Log.Error("handleAdminTranslationJobs: ListTranslationTasks failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to list translation jobs") + return + } + type jobRow struct { + ID string `json:"id"` + CacheKey string `json:"cache_key"` + Slug string `json:"slug"` + Chapter int `json:"chapter"` + Lang string `json:"lang"` + Status string `json:"status"` + WorkerID string `json:"worker_id"` + ErrorMessage string `json:"error_message"` + Started string `json:"started"` + Finished string `json:"finished"` + } + rows := make([]jobRow, 0, len(tasks)) + for _, t := range tasks { + rows = append(rows, jobRow{ + ID: t.ID, + CacheKey: t.CacheKey, + Slug: t.Slug, + Chapter: t.Chapter, + Lang: t.Lang, + Status: string(t.Status), + WorkerID: t.WorkerID, + ErrorMessage: t.ErrorMessage, + Started: t.Started.Format(time.RFC3339), + Finished: t.Finished.Format(time.RFC3339), + }) + } + writeJSON(w, 0, map[string]any{"jobs": rows}) +} + +// handleAdminTranslationBulk handles POST /api/admin/translation/bulk. +// Body: {"slug": "...", "lang": "ru", "from": 1, "to": 50} +// Enqueues one translation task per chapter in the range [from, to] inclusive. +func (s *Server) handleAdminTranslationBulk(w http.ResponseWriter, r *http.Request) { + var body struct { + Slug string `json:"slug"` + Lang string `json:"lang"` + From int `json:"from"` + To int `json:"to"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + jsonError(w, http.StatusBadRequest, "invalid JSON body") + return + } + if body.Slug == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if !supportedTranslationLangs[body.Lang] { + jsonError(w, http.StatusBadRequest, "unsupported lang; use ru, id, pt, or fr") + return + } + if body.From < 1 || body.To < body.From { + jsonError(w, http.StatusBadRequest, "from must be >= 1 and to must be >= from") + return + } + if body.To-body.From > 999 { + jsonError(w, http.StatusBadRequest, "range too large; max 1000 chapters per request") + return + } + + var taskIDs []string + for n := body.From; n <= body.To; n++ { + id, err := s.deps.Producer.CreateTranslationTask(r.Context(), body.Slug, n, body.Lang) + if err != nil { + s.deps.Log.Error("handleAdminTranslationBulk: CreateTranslationTask failed", + "slug", body.Slug, "chapter", n, "lang", body.Lang, "err", err) + jsonError(w, http.StatusInternalServerError, + fmt.Sprintf("failed to create task for chapter %d: %s", n, err)) + return + } + taskIDs = append(taskIDs, id) + } + + writeJSON(w, http.StatusAccepted, map[string]any{ + "enqueued": len(taskIDs), + "task_ids": taskIDs, + }) +} + +// ── Admin Audio ──────────────────────────────────────────────────────────────── + +// handleAdminAudioJobs handles GET /api/admin/audio/jobs. +// Returns all audio jobs, optionally filtered by slug (?slug=...). +// Sorted by started descending. +func (s *Server) handleAdminAudioJobs(w http.ResponseWriter, r *http.Request) { + tasks, err := s.deps.TaskReader.ListAudioTasks(r.Context()) + if err != nil { + s.deps.Log.Error("handleAdminAudioJobs: ListAudioTasks failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to list audio jobs") + return + } + + // Optional slug filter. + slugFilter := r.URL.Query().Get("slug") + + type jobRow struct { + ID string `json:"id"` + CacheKey string `json:"cache_key"` + Slug string `json:"slug"` + Chapter int `json:"chapter"` + Voice string `json:"voice"` + Status string `json:"status"` + WorkerID string `json:"worker_id"` + ErrorMessage string `json:"error_message"` + Started string `json:"started"` + Finished string `json:"finished"` + } + rows := make([]jobRow, 0, len(tasks)) + for _, t := range tasks { + if slugFilter != "" && t.Slug != slugFilter { + continue + } + rows = append(rows, jobRow{ + ID: t.ID, + CacheKey: t.CacheKey, + Slug: t.Slug, + Chapter: t.Chapter, + Voice: t.Voice, + Status: string(t.Status), + WorkerID: t.WorkerID, + ErrorMessage: t.ErrorMessage, + Started: t.Started.Format(time.RFC3339), + Finished: t.Finished.Format(time.RFC3339), + }) + } + writeJSON(w, 0, map[string]any{"jobs": rows, "total": len(rows)}) +} + +// handleAdminAudioBulk handles POST /api/admin/audio/bulk. +// Body: {"slug": "...", "voice": "af_bella", "from": 1, "to": 100, "skip_existing": true} +// +// Enqueues one audio task per chapter in [from, to]. +// skip_existing (default true): skip chapters already cached in MinIO — use this +// to resume a previously interrupted bulk job. +// force: if true, enqueue even when a pending/running task already exists. +// Max 1000 chapters per request. +func (s *Server) handleAdminAudioBulk(w http.ResponseWriter, r *http.Request) { + var body struct { + Slug string `json:"slug"` + Voice string `json:"voice"` + From int `json:"from"` + To int `json:"to"` + SkipExisting *bool `json:"skip_existing"` // pointer so we can detect omission + Force bool `json:"force"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + jsonError(w, http.StatusBadRequest, "invalid JSON body") + return + } + if body.Slug == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if body.Voice == "" { + body.Voice = s.cfg.DefaultVoice + } + if body.From < 1 || body.To < body.From { + jsonError(w, http.StatusBadRequest, "from must be >= 1 and to must be >= from") + return + } + if body.To-body.From > 999 { + jsonError(w, http.StatusBadRequest, "range too large; max 1000 chapters per request") + return + } + + // skip_existing defaults to true (resume-friendly). + skipExisting := true + if body.SkipExisting != nil { + skipExisting = *body.SkipExisting + } + + var taskIDs []string + skipped := 0 + + for n := body.From; n <= body.To; n++ { + // Skip chapters already cached in MinIO. + if skipExisting { + audioKey := s.deps.AudioStore.AudioObjectKey(body.Slug, n, body.Voice) + if s.deps.AudioStore.AudioExists(r.Context(), audioKey) { + skipped++ + continue + } + } + + // Skip chapters with an active (pending/running) task unless force=true. + if !body.Force { + cacheKey := fmt.Sprintf("%s/%d/%s", body.Slug, n, body.Voice) + existing, found, _ := s.deps.TaskReader.GetAudioTask(r.Context(), cacheKey) + if found && (existing.Status == domain.TaskStatusPending || existing.Status == domain.TaskStatusRunning) { + skipped++ + continue + } + } + + id, err := s.deps.Producer.CreateAudioTask(r.Context(), body.Slug, n, body.Voice) + if err != nil { + s.deps.Log.Error("handleAdminAudioBulk: CreateAudioTask failed", + "slug", body.Slug, "chapter", n, "voice", body.Voice, "err", err) + jsonError(w, http.StatusInternalServerError, + fmt.Sprintf("failed to create task for chapter %d: %s", n, err)) + return + } + taskIDs = append(taskIDs, id) + } + + writeJSON(w, http.StatusAccepted, map[string]any{ + "enqueued": len(taskIDs), + "skipped": skipped, + "task_ids": taskIDs, + }) +} + +// handleAdminAudioCancelBulk handles POST /api/admin/audio/cancel-bulk. +// Body: {"slug": "..."} +// Cancels all pending and running audio tasks for the given slug. +func (s *Server) handleAdminAudioCancelBulk(w http.ResponseWriter, r *http.Request) { + var body struct { + Slug string `json:"slug"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + jsonError(w, http.StatusBadRequest, "invalid JSON body") + return + } + if body.Slug == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + + cancelled, err := s.deps.Producer.CancelAudioTasksBySlug(r.Context(), body.Slug) + if err != nil { + s.deps.Log.Error("handleAdminAudioCancelBulk: CancelAudioTasksBySlug failed", + "slug", body.Slug, "err", err) + jsonError(w, http.StatusInternalServerError, "failed to cancel tasks") + return + } + + writeJSON(w, 0, map[string]any{"cancelled": cancelled}) +} + +// ── Voices ───────────────────────────────────────────────────────────────────── +// Returns {"voices": [...]} — merged list from Kokoro and pocket-tts. +func (s *Server) handleVoices(w http.ResponseWriter, r *http.Request) { + writeJSON(w, 0, map[string]any{"voices": s.voices(r.Context())}) +} + +// ── Presigned URLs ───────────────────────────────────────────────────────────── + +// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}. +func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + jsonError(w, http.StatusBadRequest, "invalid params") + return + } + + u, err := s.deps.PresignStore.PresignChapter(r.Context(), slug, n, 15*time.Minute) + if err != nil { + s.deps.Log.Error("presign chapter failed", "slug", slug, "n", n, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + writeJSON(w, 0, map[string]string{"url": u}) +} + +// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}. +// Query params: voice (optional) +func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + jsonError(w, http.StatusBadRequest, "invalid params") + return + } + + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.cfg.DefaultVoice + } + + key := s.deps.AudioStore.AudioObjectKey(slug, n, voice) + if !s.deps.AudioStore.AudioExists(r.Context(), key) { + http.NotFound(w, r) + return + } + + u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour) + if err != nil { + s.deps.Log.Error("presign audio failed", "slug", slug, "n", n, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + writeJSON(w, 0, map[string]string{"url": u}) +} + +// voiceSampleText is the phrase synthesised for every voice sample. +const voiceSampleText = "Hello! This is a preview of what I sound like. I hope you enjoy listening to your stories with my voice." + +// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}. +// If the sample has not been generated yet it synthesises it on the fly via +// the appropriate TTS engine (Kokoro for kokoro voices, pocket-tts for +// pocket-tts voices), stores the result in MinIO, and returns the presigned URL. +func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) { + voice := r.PathValue("voice") + if voice == "" { + jsonError(w, http.StatusBadRequest, "missing voice") + return + } + + key := kokoro.VoiceSampleKey(voice) + if cfai.IsCFAIVoice(voice) { + key = cfai.VoiceSampleKey(voice) + } + + // Generate sample on demand when it is not in MinIO yet. + if !s.deps.AudioStore.AudioExists(r.Context(), key) { + s.deps.Log.Info("generating voice sample on demand", "voice", voice) + + var ( + mp3 []byte + err error + ) + if cfai.IsCFAIVoice(voice) { + if s.deps.CFAI == nil { + jsonError(w, http.StatusServiceUnavailable, "cloudflare AI TTS not configured") + return + } + mp3, err = s.deps.CFAI.GenerateAudio(r.Context(), voiceSampleText, voice) + } else if pockettts.IsPocketTTSVoice(voice) { + if s.deps.PocketTTS == nil { + jsonError(w, http.StatusServiceUnavailable, "pocket-tts not configured") + return + } + mp3, err = s.deps.PocketTTS.GenerateAudio(r.Context(), voiceSampleText, voice) + } else { + mp3, err = s.deps.Kokoro.GenerateAudio(r.Context(), voiceSampleText, voice) + } + if err != nil { + s.deps.Log.Error("voice sample generation failed", "voice", voice, "err", err) + jsonError(w, http.StatusInternalServerError, "voice sample generation failed") + return + } + if err := s.deps.AudioStore.PutAudio(r.Context(), key, mp3); err != nil { + s.deps.Log.Error("voice sample upload failed", "voice", voice, "err", err) + jsonError(w, http.StatusInternalServerError, "voice sample upload failed") + return + } + } + + u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour) + if err != nil { + s.deps.Log.Error("presign voice sample failed", "voice", voice, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + writeJSON(w, 0, map[string]string{"url": u}) +} + +// handleAvatarUpload handles PUT /api/avatar-upload/{userId}. +// The request body must be the raw image bytes; Content-Type must be +// image/jpeg, image/png, or image/webp. +// +// This endpoint is called by the SvelteKit server (not the browser directly), +// so MinIO credentials and internal networking are not a concern. +// +// Returns: { "key": "<objectKey>" } +func (s *Server) handleAvatarUpload(w http.ResponseWriter, r *http.Request) { + userID := r.PathValue("userId") + if userID == "" { + jsonError(w, http.StatusBadRequest, "missing userId") + return + } + + ct := r.Header.Get("Content-Type") + var ext string + switch { + case strings.HasPrefix(ct, "image/jpeg"): + ext = "jpg" + case strings.HasPrefix(ct, "image/png"): + ext = "png" + case strings.HasPrefix(ct, "image/webp"): + ext = "webp" + default: + jsonError(w, http.StatusBadRequest, "unsupported content-type; use image/jpeg, image/png, or image/webp") + return + } + + const maxSize = 5 << 20 // 5 MiB + data, err := io.ReadAll(io.LimitReader(r.Body, maxSize+1)) + if err != nil { + jsonError(w, http.StatusBadRequest, "failed to read body") + return + } + if len(data) > maxSize { + jsonError(w, http.StatusRequestEntityTooLarge, "image too large (max 5 MiB)") + return + } + if len(data) == 0 { + jsonError(w, http.StatusBadRequest, "empty body") + return + } + + key, err := s.deps.PresignStore.PutAvatar(r.Context(), userID, ext, ct, data) + if err != nil { + s.deps.Log.Error("avatar upload failed", "userId", userID, "err", err) + jsonError(w, http.StatusInternalServerError, "upload failed") + return + } + writeJSON(w, 0, map[string]string{"key": key}) +} + +// handlePresignAvatarUpload handles GET /api/presign/avatar-upload/{userId}. +// Query params: ext (jpg|png|webp, defaults to jpg) +func (s *Server) handlePresignAvatarUpload(w http.ResponseWriter, r *http.Request) { + userID := r.PathValue("userId") + if userID == "" { + jsonError(w, http.StatusBadRequest, "missing userId") + return + } + + ext := r.URL.Query().Get("ext") + switch ext { + case "jpg", "jpeg": + ext = "jpg" + case "png": + ext = "png" + case "webp": + ext = "webp" + default: + ext = "jpg" + } + + uploadURL, key, err := s.deps.PresignStore.PresignAvatarUpload(r.Context(), userID, ext) + if err != nil { + s.deps.Log.Error("presign avatar upload failed", "userId", userID, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + writeJSON(w, 0, map[string]string{"upload_url": uploadURL, "key": key}) +} + +// handlePresignAvatar handles GET /api/presign/avatar/{userId}. +func (s *Server) handlePresignAvatar(w http.ResponseWriter, r *http.Request) { + userID := r.PathValue("userId") + if userID == "" { + jsonError(w, http.StatusBadRequest, "missing userId") + return + } + + u, found, err := s.deps.PresignStore.PresignAvatarURL(r.Context(), userID) + if err != nil { + s.deps.Log.Error("presign avatar failed", "userId", userID, "err", err) + jsonError(w, http.StatusInternalServerError, "presign failed") + return + } + if !found { + http.NotFound(w, r) + return + } + writeJSON(w, 0, map[string]string{"url": u}) +} + +// ── Progress ─────────────────────────────────────────────────────────────────── + +// handleGetProgress handles GET /api/progress. +// Returns {"slug": chapterNum, "slug_ts": timestampMs, ...} +func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) { + sid := ensureSession(w, r) + entries, err := s.deps.ProgressStore.AllProgress(r.Context(), sid) + if err != nil { + s.deps.Log.Error("AllProgress failed", "err", err) + entries = nil + } + + progress := make(map[string]any, len(entries)*2) + for _, p := range entries { + progress[p.Slug] = p.Chapter + progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli() + } + writeJSON(w, 0, progress) +} + +// handleSetProgress handles POST /api/progress/{slug}. +// Body: {"chapter": N} +func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) { + sid := ensureSession(w, r) + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + + var body struct { + Chapter int `json:"chapter"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 { + jsonError(w, http.StatusBadRequest, "invalid body") + return + } + + p := domain.ReadingProgress{ + Slug: slug, + Chapter: body.Chapter, + UpdatedAt: time.Now(), + } + if err := s.deps.ProgressStore.SetProgress(r.Context(), sid, p); err != nil { + s.deps.Log.Error("SetProgress failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, "store error") + return + } + writeJSON(w, 0, map[string]string{}) +} + +// handleDeleteProgress handles DELETE /api/progress/{slug}. +func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) { + sid := ensureSession(w, r) + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + + if err := s.deps.ProgressStore.DeleteProgress(r.Context(), sid, slug); err != nil { + s.deps.Log.Error("DeleteProgress failed", "slug", slug, "err", err) + // non-fatal + } + writeJSON(w, 0, map[string]string{}) +} + +// ── Catalogue (Meilisearch-backed browse + search) ──────────────────────────── + +// handleCatalogue handles GET /api/catalogue. +// +// Provides unified browse + search over the locally-indexed book catalogue +// via Meilisearch. Unlike /api/browse this never fetches novelfire.net live — +// it is entirely served from the Meilisearch index populated by the runner. +// +// Query params: +// +// q — full-text search query (optional) +// genre — genre filter, e.g. "fantasy" or "all" (default "all") +// status — status filter: "ongoing", "completed", or "all" (default "all") +// sort — "popular" (default) | "new" | "top-rated" | "rank" +// page — 1-indexed page number (default 1) +// limit — items per page (default 20, max 100) +func (s *Server) handleCatalogue(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + + genre := q.Get("genre") + if genre == "" { + genre = "all" + } + status := q.Get("status") + if status == "" { + status = "all" + } + sort := q.Get("sort") + if sort == "" { + sort = "popular" + } + + page, _ := strconv.Atoi(q.Get("page")) + if page <= 0 { + page = 1 + } + limit, _ := strconv.Atoi(q.Get("limit")) + if limit <= 0 { + limit = 20 + } + if limit > 100 { + limit = 100 + } + + // Admin users (identified by bearer token) see all non-archived books + // including those marked admin_only. + isAdmin := s.cfg.AdminToken != "" && + r.Header.Get("Authorization") == "Bearer "+s.cfg.AdminToken + + cq := meili.CatalogueQuery{ + Q: q.Get("q"), + Genre: genre, + Status: status, + Sort: sort, + Page: page, + Limit: limit, + AdminAll: isAdmin, + } + + books, total, facets, err := s.deps.SearchIndex.Catalogue(r.Context(), cq) + if err != nil { + s.deps.Log.Error("handleCatalogue: Catalogue query failed", "err", err) + jsonError(w, http.StatusInternalServerError, "search failed") + return + } + + // Rewrite raw scraped cover URLs to go through the backend cover proxy. + // /api/cover/{domain}/{slug} serves from MinIO when available, otherwise + // redirects to the CDN. This avoids ERR_BLOCKED_BY_ORB when the source + // site returns HTML error pages instead of images. + for i := range books { + if !strings.HasPrefix(books[i].Cover, "/api/cover/") { + books[i].Cover = fmt.Sprintf("/api/cover/novelfire.net/%s", books[i].Slug) + } + } + + hasNext := int64(page*limit) < total + + w.Header().Set("Cache-Control", "public, max-age=60") + writeJSON(w, 0, map[string]any{ + "books": books, + "page": page, + "limit": limit, + "total": total, + "has_next": hasNext, + "facets": map[string]any{ + "genres": facets.Genres, + "statuses": facets.Statuses, + }, + }) +} + +// ── Browse page parsing helpers ──────────────────────────────────────────────── + +// fetchBrowsePage fetches pageURL and parses NovelListings from the HTML. +func (s *Server) fetchBrowsePage(ctx context.Context, pageURL string) ([]NovelListing, bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) + if err != nil { + return nil, false, fmt.Errorf("build request: %w", err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, false, fmt.Errorf("upstream returned %d", resp.StatusCode) + } + + novels, hasNext := parseBrowsePage(resp.Body) + return novels, hasNext, nil +} + +// parseBrowsePage parses a novelfire HTML body and returns novel listings. +// It uses a simple string-scanning approach to avoid importing golang.org/x/net/html +// in this package (that dependency is only in internal/novelfire). +func parseBrowsePage(r io.Reader) ([]NovelListing, bool) { + data, err := io.ReadAll(r) + if err != nil { + return nil, false + } + body := string(data) + + var novels []NovelListing + hasNext := false + + // Detect "next page" link + if strings.Contains(body, `rel="next"`) || + strings.Contains(body, `aria-label="Next"`) || + strings.Contains(body, `class="next"`) { + hasNext = true + } + + // Extract novel slugs and titles using simple regex patterns. + // novelfire.net novel items: <li class="novel-item">...</li> + // Each contains an anchor like <a href="/book/{slug}"> + slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`) + titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`) + coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`) + + slugMatches := slugRe.FindAllStringSubmatch(body, -1) + titleMatches := titleRe.FindAllStringSubmatch(body, -1) + coverMatches := coverRe.FindAllStringSubmatch(body, -1) + + seen := make(map[string]bool) + for i, sm := range slugMatches { + slug := sm[1] + if seen[slug] { + continue + } + seen[slug] = true + + novel := NovelListing{ + Slug: slug, + URL: novelFireBase + "/book/" + slug, + } + if i < len(titleMatches) { + novel.Title = strings.TrimSpace(titleMatches[i][1]) + } + if i < len(coverMatches) { + novel.Cover = coverMatches[i][1] + } + if novel.Title != "" { + novels = append(novels, novel) + } + } + + return novels, hasNext +} + +// ── Markdown stripping ───────────────────────────────────────────────────────── + +// stripMarkdown removes common markdown syntax from src, returning plain text. +func stripMarkdown(src string) string { + src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "") + src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "") + src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "") + src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "") + src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1") + src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "") + src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "") + src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "") + src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n") + return strings.TrimSpace(src) +} + +// ── EPUB export ─────────────────────────────────────────────────────────────── + +// handleExportEPUB handles GET /api/export/{slug}. +// Generates and streams an EPUB file for the book identified by slug. +// Optional query params: from=N&to=N to limit the chapter range (default: all). +func (s *Server) handleExportEPUB(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + + ctx := r.Context() + + // Parse optional from/to range. + fromStr := r.URL.Query().Get("from") + toStr := r.URL.Query().Get("to") + fromN, toN := 0, 0 + if fromStr != "" { + v, err := strconv.Atoi(fromStr) + if err != nil || v < 1 { + jsonError(w, http.StatusBadRequest, "invalid 'from' param") + return + } + fromN = v + } + if toStr != "" { + v, err := strconv.Atoi(toStr) + if err != nil || v < 1 { + jsonError(w, http.StatusBadRequest, "invalid 'to' param") + return + } + toN = v + } + + // Fetch book metadata for title and author. + meta, inLib, err := s.deps.BookReader.ReadMetadata(ctx, slug) + if err != nil || !inLib { + s.deps.Log.Warn("handleExportEPUB: book not found", "slug", slug, "err", err) + jsonError(w, http.StatusNotFound, "book not found") + return + } + + // List all chapters. + chapters, err := s.deps.BookReader.ListChapters(ctx, slug) + if err != nil { + s.deps.Log.Error("handleExportEPUB: ListChapters failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, "failed to list chapters") + return + } + + // Filter chapters by from/to range. + var filtered []epubChapter + for _, ch := range chapters { + if fromN > 0 && ch.Number < fromN { + continue + } + if toN > 0 && ch.Number > toN { + continue + } + + // Fetch markdown from MinIO. + mdText, readErr := s.deps.BookReader.ReadChapter(ctx, slug, ch.Number) + if readErr != nil { + s.deps.Log.Warn("handleExportEPUB: ReadChapter failed", "slug", slug, "n", ch.Number, "err", readErr) + // Skip chapters that cannot be fetched. + continue + } + + // Convert markdown to HTML using goldmark. + md := goldmark.New() + var htmlBuf bytes.Buffer + if convErr := md.Convert([]byte(mdText), &htmlBuf); convErr != nil { + htmlBuf.Reset() + htmlBuf.WriteString("<p>" + mdText + "</p>") + } + + filtered = append(filtered, epubChapter{ + Number: ch.Number, + Title: ch.Title, + HTML: htmlBuf.String(), + }) + } + + if len(filtered) == 0 { + jsonError(w, http.StatusNotFound, "no chapters found in the requested range") + return + } + + epubBytes, err := generateEPUB(slug, meta.Title, meta.Author, filtered) + if err != nil { + s.deps.Log.Error("handleExportEPUB: generateEPUB failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, "failed to generate EPUB") + return + } + + w.Header().Set("Content-Type", "application/epub+zip") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s.epub"`, slug)) + w.Header().Set("Content-Length", strconv.Itoa(len(epubBytes))) + w.WriteHeader(http.StatusOK) + w.Write(epubBytes) +} + +// ── Hardcoded Kokoro voice fallback ─────────────────────────────────────────── + +// kokoroVoiceIDs is the built-in fallback list of Kokoro voice IDs used when +// the Kokoro service is unavailable. +var kokoroVoiceIDs = []string{ + // American English + "af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia", + "af_jessica", "af_kore", "af_nicole", "af_nova", "af_river", + "af_sarah", "af_sky", + "am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam", + "am_michael", "am_onyx", "am_puck", + // British English + "bf_alice", "bf_emma", "bf_lily", + "bm_daniel", "bm_fable", "bm_george", "bm_lewis", + // Spanish + "ef_dora", "em_alex", + // French + "ff_siwis", + // Hindi + "hf_alpha", "hf_beta", "hm_omega", "hm_psi", + // Italian + "if_sara", "im_nicola", + // Japanese + "jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo", + // Portuguese + "pf_dora", "pm_alex", + // Chinese + "zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi", + "zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang", +} diff --git a/backend/internal/backend/handlers_aijobs.go b/backend/internal/backend/handlers_aijobs.go new file mode 100644 index 0000000..719ff59 --- /dev/null +++ b/backend/internal/backend/handlers_aijobs.go @@ -0,0 +1,233 @@ +package backend + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/libnovel/backend/internal/cfai" + "github.com/libnovel/backend/internal/domain" +) + +// ── Cancel registry ──────────────────────────────────────────────────────────── +// cancelJobsMu guards cancelJobs. +var cancelJobsMu sync.Mutex + +// cancelJobs maps a job ID to its CancelFunc. Entries are added when a batch +// job starts and removed when it finishes or is cancelled. +var cancelJobs = map[string]context.CancelFunc{} + +func registerCancelJob(id string, cancel context.CancelFunc) { + cancelJobsMu.Lock() + cancelJobs[id] = cancel + cancelJobsMu.Unlock() +} + +func deregisterCancelJob(id string) { + cancelJobsMu.Lock() + delete(cancelJobs, id) + cancelJobsMu.Unlock() +} + +// ── AI Job list / get / cancel ───────────────────────────────────────────────── + +// handleAdminListAIJobs handles GET /api/admin/ai-jobs. +// Returns all ai_job records sorted by started descending. +func (s *Server) handleAdminListAIJobs(w http.ResponseWriter, r *http.Request) { + if s.deps.AIJobStore == nil { + jsonError(w, http.StatusServiceUnavailable, "ai job store not configured") + return + } + jobs, err := s.deps.AIJobStore.ListAIJobs(r.Context()) + if err != nil { + s.deps.Log.Error("admin: list ai jobs failed", "err", err) + jsonError(w, http.StatusInternalServerError, "list ai jobs: "+err.Error()) + return + } + writeJSON(w, 0, map[string]any{"jobs": jobs}) +} + +// handleAdminGetAIJob handles GET /api/admin/ai-jobs/{id}. +func (s *Server) handleAdminGetAIJob(w http.ResponseWriter, r *http.Request) { + if s.deps.AIJobStore == nil { + jsonError(w, http.StatusServiceUnavailable, "ai job store not configured") + return + } + id := r.PathValue("id") + job, ok, err := s.deps.AIJobStore.GetAIJob(r.Context(), id) + if err != nil { + jsonError(w, http.StatusInternalServerError, err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("job %q not found", id)) + return + } + writeJSON(w, 0, job) +} + +// handleAdminCancelAIJob handles POST /api/admin/ai-jobs/{id}/cancel. +// Marks the job as cancelled in PB and cancels the in-memory context if present. +func (s *Server) handleAdminCancelAIJob(w http.ResponseWriter, r *http.Request) { + if s.deps.AIJobStore == nil { + jsonError(w, http.StatusServiceUnavailable, "ai job store not configured") + return + } + id := r.PathValue("id") + + // Cancel in-memory context if the job is still running in this process. + cancelJobsMu.Lock() + if cancel, ok := cancelJobs[id]; ok { + cancel() + } + cancelJobsMu.Unlock() + + // Mark as cancelled in PB. + if err := s.deps.AIJobStore.UpdateAIJob(r.Context(), id, map[string]any{ + "status": string(domain.TaskStatusCancelled), + "finished": time.Now().Format(time.RFC3339), + }); err != nil { + s.deps.Log.Error("admin: cancel ai job failed", "id", id, "err", err) + jsonError(w, http.StatusInternalServerError, "cancel ai job: "+err.Error()) + return + } + + s.deps.Log.Info("admin: ai job cancelled", "id", id) + writeJSON(w, 0, map[string]any{"cancelled": true}) +} + +// ── Auto-prompt ──────────────────────────────────────────────────────────────── + +// autoPromptRequest is the JSON body for POST /api/admin/image-gen/auto-prompt. +type autoPromptRequest struct { + // Slug is the book slug. + Slug string `json:"slug"` + // Type is "cover" or "chapter". + Type string `json:"type"` + // Chapter number (required when type == "chapter"). + Chapter int `json:"chapter"` + // Model is the text-gen model to use. Defaults to DefaultTextModel. + Model string `json:"model"` +} + +// autoPromptResponse is returned by POST /api/admin/image-gen/auto-prompt. +type autoPromptResponse struct { + Prompt string `json:"prompt"` + Model string `json:"model"` +} + +// handleAdminImageGenAutoPrompt handles POST /api/admin/image-gen/auto-prompt. +// +// Uses the text generation model to create a vivid image generation prompt +// based on the book's description (for covers) or chapter title/content (for chapters). +func (s *Server) handleAdminImageGenAutoPrompt(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil { + jsonError(w, http.StatusServiceUnavailable, "text generation not configured") + return + } + + var req autoPromptRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if req.Type != "cover" && req.Type != "chapter" { + jsonError(w, http.StatusBadRequest, `type must be "cover" or "chapter"`) + return + } + + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", req.Slug)) + return + } + + model := req.Model + if model == "" { + model = string(cfai.DefaultTextModel) + } + + var userPrompt string + if req.Type == "cover" { + userPrompt = fmt.Sprintf( + "Book: \"%s\"\nAuthor: %s\nGenres: %s\n\nDescription:\n%s", + meta.Title, + meta.Author, + strings.Join(meta.Genres, ", "), + meta.Summary, + ) + } else { + // For chapter images, use chapter title if available. + chapterTitle := fmt.Sprintf("Chapter %d", req.Chapter) + if req.Chapter > 0 { + chapters, listErr := s.deps.BookReader.ListChapters(r.Context(), req.Slug) + if listErr == nil { + for _, ch := range chapters { + if ch.Number == req.Chapter { + chapterTitle = ch.Title + break + } + } + } + } + userPrompt = fmt.Sprintf( + "Book: \"%s\"\nGenres: %s\nChapter: %s\n\nBook description:\n%s", + meta.Title, + strings.Join(meta.Genres, ", "), + chapterTitle, + meta.Summary, + ) + } + + systemPrompt := buildAutoPromptSystem(req.Type) + + s.deps.Log.Info("admin: image auto-prompt requested", + "slug", req.Slug, "type", req.Type, "chapter", req.Chapter, "model", model) + + result, genErr := s.deps.TextGen.Generate(r.Context(), cfai.TextRequest{ + Model: cfai.TextModel(model), + Messages: []cfai.TextMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + }, + MaxTokens: 256, + }) + if genErr != nil { + s.deps.Log.Error("admin: auto-prompt failed", "err", genErr) + jsonError(w, http.StatusBadGateway, "text generation failed: "+genErr.Error()) + return + } + + writeJSON(w, 0, autoPromptResponse{ + Prompt: strings.TrimSpace(result), + Model: model, + }) +} + +func buildAutoPromptSystem(imageType string) string { + if imageType == "cover" { + return `You are a professional prompt engineer for AI image generation (Stable Diffusion / FLUX models). ` + + `Given a book's title, genres, and description, write a single vivid image generation prompt ` + + `for a book cover. The prompt should describe the visual composition, art style, lighting, ` + + `and mood without mentioning text or typography. ` + + `Format: comma-separated visual descriptors, 30–60 words. ` + + `Output ONLY the prompt — no explanation, no quotes, no labels.` + } + return `You are a professional prompt engineer for AI image generation (Stable Diffusion / FLUX models). ` + + `Given a book's title, genres, and a specific chapter title, write a single vivid scene illustration prompt. ` + + `Describe the scene, characters, setting, lighting, and art style. ` + + `Format: comma-separated visual descriptors, 30–60 words. ` + + `Output ONLY the prompt — no explanation, no quotes, no labels.` +} diff --git a/backend/internal/backend/handlers_books_admin.go b/backend/internal/backend/handlers_books_admin.go new file mode 100644 index 0000000..b0db8c8 --- /dev/null +++ b/backend/internal/backend/handlers_books_admin.go @@ -0,0 +1,117 @@ +package backend + +import ( + "errors" + "net/http" + + "github.com/libnovel/backend/internal/storage" +) + +// handleAdminArchiveBook handles PATCH /api/admin/books/{slug}/archive. +// Soft-deletes a book by setting archived=true in PocketBase and updating the +// Meilisearch document so it is excluded from all public search results. +// The book data is preserved and can be restored with the unarchive endpoint. +func (s *Server) handleAdminArchiveBook(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + if s.deps.BookAdminStore == nil { + jsonError(w, http.StatusServiceUnavailable, "book admin store not configured") + return + } + + if err := s.deps.BookAdminStore.ArchiveBook(r.Context(), slug); err != nil { + if errors.Is(err, storage.ErrNotFound) { + jsonError(w, http.StatusNotFound, "book not found") + return + } + s.deps.Log.Error("archive book failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, err.Error()) + return + } + + // Update the Meilisearch document so the archived flag takes effect + // immediately in search/catalogue results. + if meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), slug); err == nil && ok { + if upsertErr := s.deps.SearchIndex.UpsertBook(r.Context(), meta); upsertErr != nil { + s.deps.Log.Warn("archive book: meili upsert failed", "slug", slug, "err", upsertErr) + } + } + + s.deps.Log.Info("book archived", "slug", slug) + writeJSON(w, http.StatusOK, map[string]string{"slug": slug, "status": "archived"}) +} + +// handleAdminUnarchiveBook handles PATCH /api/admin/books/{slug}/unarchive. +// Restores a previously archived book by clearing the archived flag, making it +// publicly visible in search and catalogue results again. +func (s *Server) handleAdminUnarchiveBook(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + if s.deps.BookAdminStore == nil { + jsonError(w, http.StatusServiceUnavailable, "book admin store not configured") + return + } + + if err := s.deps.BookAdminStore.UnarchiveBook(r.Context(), slug); err != nil { + if errors.Is(err, storage.ErrNotFound) { + jsonError(w, http.StatusNotFound, "book not found") + return + } + s.deps.Log.Error("unarchive book failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, err.Error()) + return + } + + // Sync the updated archived=false state back to Meilisearch. + if meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), slug); err == nil && ok { + if upsertErr := s.deps.SearchIndex.UpsertBook(r.Context(), meta); upsertErr != nil { + s.deps.Log.Warn("unarchive book: meili upsert failed", "slug", slug, "err", upsertErr) + } + } + + s.deps.Log.Info("book unarchived", "slug", slug) + writeJSON(w, http.StatusOK, map[string]string{"slug": slug, "status": "active"}) +} + +// handleAdminDeleteBook handles DELETE /api/admin/books/{slug}. +// Permanently removes all data for a book: +// - PocketBase books record and all chapters_idx records +// - All MinIO chapter markdown objects and the cover image +// - Meilisearch document +// +// This operation is irreversible. Use the archive endpoint for soft-deletion. +func (s *Server) handleAdminDeleteBook(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + if s.deps.BookAdminStore == nil { + jsonError(w, http.StatusServiceUnavailable, "book admin store not configured") + return + } + + if err := s.deps.BookAdminStore.DeleteBook(r.Context(), slug); err != nil { + if errors.Is(err, storage.ErrNotFound) { + jsonError(w, http.StatusNotFound, "book not found") + return + } + s.deps.Log.Error("delete book failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, err.Error()) + return + } + + // Remove from Meilisearch — best-effort (log on failure, don't fail request). + if err := s.deps.SearchIndex.DeleteBook(r.Context(), slug); err != nil { + s.deps.Log.Warn("delete book: meili delete failed", "slug", slug, "err", err) + } + + s.deps.Log.Info("book deleted", "slug", slug) + writeJSON(w, http.StatusOK, map[string]string{"slug": slug, "status": "deleted"}) +} diff --git a/backend/internal/backend/handlers_catalogue.go b/backend/internal/backend/handlers_catalogue.go new file mode 100644 index 0000000..662276a --- /dev/null +++ b/backend/internal/backend/handlers_catalogue.go @@ -0,0 +1,792 @@ +package backend + +// Catalogue enrichment handlers: tagline, genre tagging, content warnings, +// quality scoring, batch cover regeneration, and per-book metadata refresh. +// +// All generation endpoints are admin-only (enforced by the SvelteKit proxy layer). +// All long-running operations support cancellation via r.Context().Done(). +// Batch operations use an in-memory cancel registry (cancelJobs map) so the +// frontend can send a cancel request by job ID. + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/libnovel/backend/internal/cfai" + "github.com/libnovel/backend/internal/domain" +) + + +// ── Tagline ─────────────────────────────────────────────────────────────── + +// textGenTaglineRequest is the JSON body for POST /api/admin/text-gen/tagline. +type textGenTaglineRequest struct { + Slug string `json:"slug"` + Model string `json:"model"` + MaxTokens int `json:"max_tokens"` +} + +// textGenTaglineResponse is returned by POST /api/admin/text-gen/tagline. +type textGenTaglineResponse struct { + OldTagline string `json:"old_tagline"` + NewTagline string `json:"new_tagline"` + Model string `json:"model"` +} + +// handleAdminTextGenTagline handles POST /api/admin/text-gen/tagline. +// Generates a 1-sentence marketing hook for a book. +func (s *Server) handleAdminTextGenTagline(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil { + jsonError(w, http.StatusServiceUnavailable, "text generation not configured") + return + } + + var req textGenTaglineRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", req.Slug)) + return + } + + model := cfai.TextModel(req.Model) + if model == "" { + model = cfai.DefaultTextModel + } + + system := `You are a copywriter for a web novel platform. ` + + `Given a book's title, genres, and description, write a single punchy tagline ` + + `(one sentence, under 20 words) that hooks a reader. ` + + `Output ONLY the tagline — no quotes, no labels, no explanation.` + + user := fmt.Sprintf("Title: %s\nGenres: %s\n\nDescription:\n%s", + meta.Title, + strings.Join(meta.Genres, ", "), + meta.Summary, + ) + + s.deps.Log.Info("admin: text-gen tagline requested", "slug", req.Slug, "model", model) + + result, genErr := s.deps.TextGen.Generate(r.Context(), cfai.TextRequest{ + Model: model, + Messages: []cfai.TextMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, + MaxTokens: 64, + }) + if genErr != nil { + s.deps.Log.Error("admin: text-gen tagline failed", "err", genErr) + jsonError(w, http.StatusBadGateway, "text generation failed: "+genErr.Error()) + return + } + + writeJSON(w, 0, textGenTaglineResponse{ + OldTagline: "", // BookMeta has no tagline field yet — always empty + NewTagline: strings.TrimSpace(result), + Model: string(model), + }) +} + +// ── Genres ──────────────────────────────────────────────────────────────── + +// textGenGenresRequest is the JSON body for POST /api/admin/text-gen/genres. +type textGenGenresRequest struct { + Slug string `json:"slug"` + Model string `json:"model"` + MaxTokens int `json:"max_tokens"` +} + +// textGenGenresResponse is returned by POST /api/admin/text-gen/genres. +type textGenGenresResponse struct { + CurrentGenres []string `json:"current_genres"` + ProposedGenres []string `json:"proposed_genres"` + Model string `json:"model"` +} + +// handleAdminTextGenGenres handles POST /api/admin/text-gen/genres. +// Suggests a refined genre list based on the book's description. +func (s *Server) handleAdminTextGenGenres(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil { + jsonError(w, http.StatusServiceUnavailable, "text generation not configured") + return + } + + var req textGenGenresRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", req.Slug)) + return + } + + model := cfai.TextModel(req.Model) + if model == "" { + model = cfai.DefaultTextModel + } + + system := `You are a genre classification expert for a web novel platform. ` + + `Given a book's title and description, return a JSON array of 2–6 genre tags. ` + + `Use only well-known web novel genres such as: ` + + `Action, Adventure, Comedy, Drama, Fantasy, Historical, Horror, Isekai, Josei, ` + + `Martial Arts, Mature, Mecha, Mystery, Psychological, Romance, School Life, ` + + `Sci-fi, Seinen, Shoujo, Shounen, Slice of Life, Supernatural, System, Tragedy, Wuxia, Xianxia. ` + + `Output ONLY a raw JSON array of strings — no prose, no markdown, no explanation. ` + + `Example: ["Fantasy","Adventure","Action"]` + + user := fmt.Sprintf("Title: %s\nCurrent genres: %s\n\nDescription:\n%s", + meta.Title, + strings.Join(meta.Genres, ", "), + meta.Summary, + ) + + s.deps.Log.Info("admin: text-gen genres requested", "slug", req.Slug, "model", model) + + raw, genErr := s.deps.TextGen.Generate(r.Context(), cfai.TextRequest{ + Model: model, + Messages: []cfai.TextMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, + MaxTokens: 128, + }) + if genErr != nil { + s.deps.Log.Error("admin: text-gen genres failed", "err", genErr) + jsonError(w, http.StatusBadGateway, "text generation failed: "+genErr.Error()) + return + } + + proposed := parseStringArrayJSON(raw) + + writeJSON(w, 0, textGenGenresResponse{ + CurrentGenres: meta.Genres, + ProposedGenres: proposed, + Model: string(model), + }) +} + +// handleAdminTextGenApplyGenres handles POST /api/admin/text-gen/genres/apply. +// Persists the confirmed genre list to PocketBase. +func (s *Server) handleAdminTextGenApplyGenres(w http.ResponseWriter, r *http.Request) { + if s.deps.BookWriter == nil { + jsonError(w, http.StatusServiceUnavailable, "book writer not configured") + return + } + + var req struct { + Slug string `json:"slug"` + Genres []string `json:"genres"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", req.Slug)) + return + } + + meta.Genres = req.Genres + if err := s.deps.BookWriter.WriteMetadata(r.Context(), meta); err != nil { + s.deps.Log.Error("admin: apply genres failed", "slug", req.Slug, "err", err) + jsonError(w, http.StatusInternalServerError, "write metadata: "+err.Error()) + return + } + + s.deps.Log.Info("admin: genres applied", "slug", req.Slug, "genres", req.Genres) + writeJSON(w, 0, map[string]any{"updated": true}) +} + +// ── Content warnings ────────────────────────────────────────────────────── + +// textGenContentWarningsRequest is the JSON body for POST /api/admin/text-gen/content-warnings. +type textGenContentWarningsRequest struct { + Slug string `json:"slug"` + Model string `json:"model"` + MaxTokens int `json:"max_tokens"` +} + +// textGenContentWarningsResponse is returned by POST /api/admin/text-gen/content-warnings. +type textGenContentWarningsResponse struct { + Warnings []string `json:"warnings"` + Model string `json:"model"` +} + +// handleAdminTextGenContentWarnings handles POST /api/admin/text-gen/content-warnings. +// Detects mature or sensitive themes in a book's description. +func (s *Server) handleAdminTextGenContentWarnings(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil { + jsonError(w, http.StatusServiceUnavailable, "text generation not configured") + return + } + + var req textGenContentWarningsRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", req.Slug)) + return + } + + model := cfai.TextModel(req.Model) + if model == "" { + model = cfai.DefaultTextModel + } + + system := `You are a content moderation assistant for a web novel platform. ` + + `Given a book's title, genres, and description, detect any content warnings that should be shown to readers. ` + + `Choose only relevant warnings from: Violence, Strong Language, Sexual Content, Mature Themes, ` + + `Dark Themes, Gore, Torture, Abuse, Drug Use, Suicide/Self-Harm. ` + + `If the book is clean, return an empty array. ` + + `Output ONLY a raw JSON array of strings — no prose, no markdown. ` + + `Example: ["Violence","Dark Themes"]` + + user := fmt.Sprintf("Title: %s\nGenres: %s\n\nDescription:\n%s", + meta.Title, + strings.Join(meta.Genres, ", "), + meta.Summary, + ) + + s.deps.Log.Info("admin: text-gen content-warnings requested", "slug", req.Slug, "model", model) + + raw, genErr := s.deps.TextGen.Generate(r.Context(), cfai.TextRequest{ + Model: model, + Messages: []cfai.TextMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, + MaxTokens: 128, + }) + if genErr != nil { + s.deps.Log.Error("admin: text-gen content-warnings failed", "err", genErr) + jsonError(w, http.StatusBadGateway, "text generation failed: "+genErr.Error()) + return + } + + warnings := parseStringArrayJSON(raw) + + writeJSON(w, 0, textGenContentWarningsResponse{ + Warnings: warnings, + Model: string(model), + }) +} + +// ── Quality score ───────────────────────────────────────────────────────── + +// textGenQualityScoreRequest is the JSON body for POST /api/admin/text-gen/quality-score. +type textGenQualityScoreRequest struct { + Slug string `json:"slug"` + Model string `json:"model"` + MaxTokens int `json:"max_tokens"` +} + +// textGenQualityScoreResponse is returned by POST /api/admin/text-gen/quality-score. +type textGenQualityScoreResponse struct { + Score int `json:"score"` // 1–5 + Feedback string `json:"feedback"` // brief reasoning + Model string `json:"model"` +} + +// handleAdminTextGenQualityScore handles POST /api/admin/text-gen/quality-score. +// Rates the book description quality on a 1–5 scale with brief feedback. +func (s *Server) handleAdminTextGenQualityScore(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil { + jsonError(w, http.StatusServiceUnavailable, "text generation not configured") + return + } + + var req textGenQualityScoreRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", req.Slug)) + return + } + + model := cfai.TextModel(req.Model) + if model == "" { + model = cfai.DefaultTextModel + } + + system := `You are a book description quality reviewer for a web novel platform. ` + + `Rate the provided description on a scale of 1–5 where: ` + + `1=poor (vague/too short), 2=below average, 3=average, 4=good, 5=excellent (engaging/detailed). ` + + `Respond with ONLY a JSON object: {"score": <1-5>, "feedback": "<one sentence explanation>"}. ` + + `No markdown, no extra text.` + + user := fmt.Sprintf("Title: %s\nGenres: %s\n\nDescription:\n%s", + meta.Title, + strings.Join(meta.Genres, ", "), + meta.Summary, + ) + + s.deps.Log.Info("admin: text-gen quality-score requested", "slug", req.Slug, "model", model) + + raw, genErr := s.deps.TextGen.Generate(r.Context(), cfai.TextRequest{ + Model: model, + Messages: []cfai.TextMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, + MaxTokens: 128, + }) + if genErr != nil { + s.deps.Log.Error("admin: text-gen quality-score failed", "err", genErr) + jsonError(w, http.StatusBadGateway, "text generation failed: "+genErr.Error()) + return + } + + var parsed struct { + Score int `json:"score"` + Feedback string `json:"feedback"` + } + // Strip markdown fences if any. + clean := extractJSONObject(raw) + if err := json.Unmarshal([]byte(clean), &parsed); err != nil { + // Fallback: try to extract a digit. + parsed.Score = 0 + for _, ch := range raw { + if ch >= '1' && ch <= '5' { + parsed.Score = int(ch - '0') + break + } + } + parsed.Feedback = strings.TrimSpace(raw) + } + + writeJSON(w, 0, textGenQualityScoreResponse{ + Score: parsed.Score, + Feedback: parsed.Feedback, + Model: string(model), + }) +} + +// ── Batch cover regeneration ────────────────────────────────────────────── + +// batchCoverEvent is one SSE event emitted during batch cover regeneration. +type batchCoverEvent struct { + // JobID is the opaque identifier clients use to cancel this job. + JobID string `json:"job_id,omitempty"` + Done int `json:"done"` + Total int `json:"total"` + Slug string `json:"slug,omitempty"` + Error string `json:"error,omitempty"` + Skipped bool `json:"skipped,omitempty"` + Finish bool `json:"finish,omitempty"` +} + +// handleAdminBatchCovers handles POST /api/admin/catalogue/batch-covers. +// +// Streams SSE events as it generates covers for every book that has no cover +// stored in MinIO. Each event carries progress info. The final event has Finish=true. +// +// Supports from_item/to_item to process a sub-range of the catalogue (0-based indices). +// Supports job_id to resume a previously interrupted job. +// The job can be cancelled by calling POST /api/admin/ai-jobs/{id}/cancel. +func (s *Server) handleAdminBatchCovers(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil || s.deps.ImageGen == nil { + jsonError(w, http.StatusServiceUnavailable, "image/text generation not configured") + return + } + if s.deps.CoverStore == nil { + jsonError(w, http.StatusServiceUnavailable, "cover store not configured") + return + } + + var reqBody struct { + Model string `json:"model"` + NumSteps int `json:"num_steps"` + Width int `json:"width"` + Height int `json:"height"` + FromItem int `json:"from_item"` + ToItem int `json:"to_item"` + JobID string `json:"job_id"` + } + // Body is optional — defaults used if absent. + json.NewDecoder(r.Body).Decode(&reqBody) //nolint:errcheck + + allBooks, err := s.deps.BookReader.ListBooks(r.Context()) + if err != nil { + jsonError(w, http.StatusInternalServerError, "list books: "+err.Error()) + return + } + + // Apply range filter. + books := allBooks + if reqBody.FromItem > 0 || reqBody.ToItem > 0 { + from := reqBody.FromItem + to := reqBody.ToItem + if to == 0 || to >= len(allBooks) { + to = len(allBooks) - 1 + } + if from < 0 { + from = 0 + } + if from <= to && from < len(allBooks) { + books = allBooks[from : to+1] + } + } + + // SSE headers. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("X-Accel-Buffering", "no") + flusher, canFlush := w.(http.Flusher) + + sseWrite := func(evt batchCoverEvent) { + b, _ := json.Marshal(evt) + fmt.Fprintf(w, "data: %s\n\n", b) + if canFlush { + flusher.Flush() + } + } + + total := len(books) + done := 0 + + // Create or resume PB ai_job and register cancel context. + var pbJobID string + resumeFrom := 0 + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + + if s.deps.AIJobStore != nil { + if reqBody.JobID != "" { + if existing, ok, _ := s.deps.AIJobStore.GetAIJob(r.Context(), reqBody.JobID); ok { + pbJobID = reqBody.JobID + resumeFrom = existing.ItemsDone + done = resumeFrom + _ = s.deps.AIJobStore.UpdateAIJob(r.Context(), pbJobID, map[string]any{ + "status": string(domain.TaskStatusRunning), + "items_total": total, + }) + } + } + if pbJobID == "" { + id, createErr := s.deps.AIJobStore.CreateAIJob(r.Context(), domain.AIJob{ + Kind: "batch-covers", + Status: domain.TaskStatusRunning, + FromItem: reqBody.FromItem, + ToItem: reqBody.ToItem, + ItemsTotal: total, + Started: time.Now(), + }) + if createErr == nil { + pbJobID = id + } + } + if pbJobID != "" { + registerCancelJob(pbJobID, cancel) + defer deregisterCancelJob(pbJobID) + } + } + + // Use pbJobID as the SSE job_id when available, else a random hex fallback. + sseJobID := pbJobID + if sseJobID == "" { + sseJobID = randomHex(8) + ctx2, cancel2 := context.WithCancel(r.Context()) + registerCancelJob(sseJobID, cancel2) + defer deregisterCancelJob(sseJobID) + defer cancel2() + cancel() // replace ctx with ctx2 + ctx = ctx2 + } + + // Send initial event with jobID so frontend can store it for cancellation. + sseWrite(batchCoverEvent{JobID: sseJobID, Done: done, Total: total}) + + for i, book := range books { + if ctx.Err() != nil { + break + } + // Skip already-processed items when resuming. + if i < resumeFrom { + continue + } + + // Check if cover already exists. + hasCover := s.deps.CoverStore.CoverExists(ctx, book.Slug) + if hasCover { + done++ + sseWrite(batchCoverEvent{Done: done, Total: total, Slug: book.Slug, Skipped: true}) + if pbJobID != "" && s.deps.AIJobStore != nil { + _ = s.deps.AIJobStore.UpdateAIJob(r.Context(), pbJobID, map[string]any{"items_done": done}) + } + continue + } + + // Build a prompt from the book metadata. + prompt := buildCoverPrompt(book) + + // Generate the image via CF AI. + imgBytes, genErr := s.deps.ImageGen.GenerateImage(ctx, cfai.ImageRequest{ + Prompt: prompt, + NumSteps: reqBody.NumSteps, + Width: reqBody.Width, + Height: reqBody.Height, + }) + if genErr != nil { + done++ + s.deps.Log.Error("batch-covers: image gen failed", "slug", book.Slug, "err", genErr) + sseWrite(batchCoverEvent{Done: done, Total: total, Slug: book.Slug, Error: genErr.Error()}) + continue + } + + // Save to CoverStore. + if saveErr := s.deps.CoverStore.PutCover(ctx, book.Slug, imgBytes, "image/png"); saveErr != nil { + done++ + s.deps.Log.Error("batch-covers: save failed", "slug", book.Slug, "err", saveErr) + sseWrite(batchCoverEvent{Done: done, Total: total, Slug: book.Slug, Error: saveErr.Error()}) + continue + } + + done++ + s.deps.Log.Info("batch-covers: cover generated", "slug", book.Slug) + sseWrite(batchCoverEvent{Done: done, Total: total, Slug: book.Slug}) + if pbJobID != "" && s.deps.AIJobStore != nil { + _ = s.deps.AIJobStore.UpdateAIJob(r.Context(), pbJobID, map[string]any{"items_done": done}) + } + } + + if pbJobID != "" && s.deps.AIJobStore != nil { + status := domain.TaskStatusDone + if ctx.Err() != nil { + status = domain.TaskStatusCancelled + } + _ = s.deps.AIJobStore.UpdateAIJob(r.Context(), pbJobID, map[string]any{ + "status": string(status), + "items_done": done, + "finished": time.Now().Format(time.RFC3339), + }) + } + + sseWrite(batchCoverEvent{Done: done, Total: total, Finish: true}) +} + +// handleAdminBatchCoversCancel handles POST /api/admin/catalogue/batch-covers/cancel. +// Cancels an in-progress batch cover job by its job ID. +func (s *Server) handleAdminBatchCoversCancel(w http.ResponseWriter, r *http.Request) { + var req struct { + JobID string `json:"job_id"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.JobID == "" { + jsonError(w, http.StatusBadRequest, "job_id is required") + return + } + + cancelJobsMu.Lock() + cancel, ok := cancelJobs[req.JobID] + cancelJobsMu.Unlock() + + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("job %q not found", req.JobID)) + return + } + cancel() + s.deps.Log.Info("batch-covers: job cancelled", "job_id", req.JobID) + writeJSON(w, 0, map[string]any{"cancelled": true}) +} + +// ── Refresh metadata (per-book) ──────────────────────────────────────────── + +// refreshMetadataEvent is one SSE event during per-book metadata refresh. +type refreshMetadataEvent struct { + Step string `json:"step"` // "description" | "tagline" | "cover" + Done bool `json:"done"` + Error string `json:"error,omitempty"` +} + +// handleAdminRefreshMetadata handles POST /api/admin/catalogue/refresh-metadata/{slug}. +// +// Runs description → tagline → cover generation in sequence for a single book +// and streams SSE progress. Interruptable via client disconnect (r.Context()). +func (s *Server) handleAdminRefreshMetadata(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", slug)) + return + } + + // SSE headers. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("X-Accel-Buffering", "no") + flusher, canFlush := w.(http.Flusher) + + sseWrite := func(evt refreshMetadataEvent) { + b, _ := json.Marshal(evt) + fmt.Fprintf(w, "data: %s\n\n", b) + if canFlush { + flusher.Flush() + } + } + + ctx := r.Context() + + // Step 1 — description. + if s.deps.TextGen != nil { + if ctx.Err() == nil { + newDesc, genErr := s.deps.TextGen.Generate(ctx, cfai.TextRequest{ + Model: cfai.DefaultTextModel, + Messages: []cfai.TextMessage{ + {Role: "system", Content: `You are a book description writer for a web novel platform. Write an improved description. Respond with ONLY the new description text — no title, no labels, no markdown.`}, + {Role: "user", Content: fmt.Sprintf("Title: %s\nGenres: %s\n\nCurrent description:\n%s\n\nInstructions: Write a compelling 2–4 sentence description. Keep it spoiler-free and engaging.", meta.Title, strings.Join(meta.Genres, ", "), meta.Summary)}, + }, + MaxTokens: 512, + }) + if genErr == nil && strings.TrimSpace(newDesc) != "" && s.deps.BookWriter != nil { + meta.Summary = strings.TrimSpace(newDesc) + if writeErr := s.deps.BookWriter.WriteMetadata(ctx, meta); writeErr != nil { + sseWrite(refreshMetadataEvent{Step: "description", Error: writeErr.Error()}) + } else { + sseWrite(refreshMetadataEvent{Step: "description"}) + } + } else if genErr != nil { + sseWrite(refreshMetadataEvent{Step: "description", Error: genErr.Error()}) + } + } + } + + // Step 2 — cover. + if s.deps.ImageGen != nil && s.deps.CoverStore != nil { + if ctx.Err() == nil { + prompt := buildCoverPrompt(meta) + imgBytes, genErr := s.deps.ImageGen.GenerateImage(ctx, cfai.ImageRequest{Prompt: prompt}) + if genErr == nil { + if saveErr := s.deps.CoverStore.PutCover(ctx, slug, imgBytes, "image/png"); saveErr != nil { + sseWrite(refreshMetadataEvent{Step: "cover", Error: saveErr.Error()}) + } else { + sseWrite(refreshMetadataEvent{Step: "cover"}) + } + } else { + sseWrite(refreshMetadataEvent{Step: "cover", Error: genErr.Error()}) + } + } + } + + sseWrite(refreshMetadataEvent{Step: "done", Done: true}) +} + +// ── Helpers ─────────────────────────────────────────────────────────────── + +// parseStringArrayJSON extracts a JSON string array from model output, +// tolerating markdown fences and surrounding prose. +func parseStringArrayJSON(raw string) []string { + s := raw + if idx := strings.Index(s, "```json"); idx >= 0 { + s = s[idx+7:] + } else if idx := strings.Index(s, "```"); idx >= 0 { + s = s[idx+3:] + } + if idx := strings.LastIndex(s, "```"); idx >= 0 { + s = s[:idx] + } + start := strings.Index(s, "[") + end := strings.LastIndex(s, "]") + if start < 0 || end <= start { + return nil + } + s = s[start : end+1] + var out []string + json.Unmarshal([]byte(s), &out) //nolint:errcheck + return out +} + +// extractJSONObject finds the first {...} object in a string. +func extractJSONObject(raw string) string { + start := strings.Index(raw, "{") + end := strings.LastIndex(raw, "}") + if start < 0 || end <= start { + return raw + } + return raw[start : end+1] +} + +// buildCoverPrompt constructs a prompt string for cover generation from a book. +func buildCoverPrompt(meta domain.BookMeta) string { + parts := []string{"book cover art"} + if meta.Title != "" { + parts = append(parts, "titled \""+meta.Title+"\"") + } + if len(meta.Genres) > 0 { + parts = append(parts, strings.Join(meta.Genres, ", ")+" genre") + } + if meta.Summary != "" { + summary := meta.Summary + if len(summary) > 200 { + summary = summary[:200] + } + parts = append(parts, summary) + } + return strings.Join(parts, ", ") +} + +// randomHex returns a random hex string of n bytes. +func randomHex(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/backend/internal/backend/handlers_image.go b/backend/internal/backend/handlers_image.go new file mode 100644 index 0000000..c6de202 --- /dev/null +++ b/backend/internal/backend/handlers_image.go @@ -0,0 +1,656 @@ +package backend + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/libnovel/backend/internal/cfai" + "github.com/libnovel/backend/internal/domain" +) + +// handleAdminImageGenModels handles GET /api/admin/image-gen/models. +// Returns the list of supported Cloudflare AI image generation models. +func (s *Server) handleAdminImageGenModels(w http.ResponseWriter, r *http.Request) { + if s.deps.ImageGen == nil { + jsonError(w, http.StatusServiceUnavailable, "image generation not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN missing)") + return + } + models := s.deps.ImageGen.Models() + writeJSON(w, 0, map[string]any{"models": models}) +} + +// imageGenRequest is the JSON body for POST /api/admin/image-gen. +type imageGenRequest struct { + // Prompt is the text description of the desired image. + Prompt string `json:"prompt"` + + // Model is the CF Workers AI model ID (e.g. "@cf/black-forest-labs/flux-2-dev"). + // Defaults to the recommended model for the given type. + Model string `json:"model"` + + // Type is either "cover" or "chapter". + Type string `json:"type"` + + // Slug is the book slug. Required for cover; required for chapter. + Slug string `json:"slug"` + + // Chapter number (1-based). Required when type == "chapter". + Chapter int `json:"chapter"` + + // ReferenceImageB64 is an optional base64-encoded PNG/JPEG reference image. + // When present the img2img path is used. + ReferenceImageB64 string `json:"reference_image_b64"` + + // NumSteps overrides inference steps (default 20). + NumSteps int `json:"num_steps"` + + // Width / Height override output dimensions (0 = model default). + Width int `json:"width"` + Height int `json:"height"` + + // Guidance overrides prompt guidance scale (0 = model default). + Guidance float64 `json:"guidance"` + + // Strength for img2img: 0.0–1.0, default 0.75. + Strength float64 `json:"strength"` + + // SaveToCover when true stores the result as the book cover in MinIO + // (overwriting any existing cover) and sets the book's cover URL. + // Only valid when type == "cover". + SaveToCover bool `json:"save_to_cover"` +} + +// imageGenResponse is the JSON body returned by POST /api/admin/image-gen. +type imageGenResponse struct { + // ImageB64 is the generated image as a base64-encoded PNG string. + ImageB64 string `json:"image_b64"` + // ContentType is "image/png" or "image/jpeg". + ContentType string `json:"content_type"` + // Saved indicates whether the image was persisted to MinIO. + Saved bool `json:"saved"` + // CoverURL is the URL the cover is now served from (only set when Saved==true). + CoverURL string `json:"cover_url,omitempty"` + // Model is the model that was used. + Model string `json:"model"` + // Bytes is the raw image size in bytes. + Bytes int `json:"bytes"` +} + +// handleAdminImageGen handles POST /api/admin/image-gen. +// +// Generates an image using Cloudflare Workers AI and optionally stores it. +// Multipart/form-data is also accepted so the reference image can be uploaded +// directly; otherwise the reference is expected as base64 JSON. +func (s *Server) handleAdminImageGen(w http.ResponseWriter, r *http.Request) { + if s.deps.ImageGen == nil { + jsonError(w, http.StatusServiceUnavailable, "image generation not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN missing)") + return + } + + var req imageGenRequest + var refImageData []byte + + ct := r.Header.Get("Content-Type") + if strings.HasPrefix(ct, "multipart/form-data") { + // Multipart: parse JSON fields from a "json" part + optional "reference" file part. + if err := r.ParseMultipartForm(32 << 20); err != nil { + jsonError(w, http.StatusBadRequest, "parse multipart: "+err.Error()) + return + } + if jsonPart := r.FormValue("json"); jsonPart != "" { + if err := json.Unmarshal([]byte(jsonPart), &req); err != nil { + jsonError(w, http.StatusBadRequest, "parse json field: "+err.Error()) + return + } + } + if f, _, err := r.FormFile("reference"); err == nil { + defer f.Close() + refImageData, _ = io.ReadAll(f) + } + } else { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if req.ReferenceImageB64 != "" { + var decErr error + refImageData, decErr = base64.StdEncoding.DecodeString(req.ReferenceImageB64) + if decErr != nil { + // Try std without padding + refImageData, decErr = base64.RawStdEncoding.DecodeString(req.ReferenceImageB64) + if decErr != nil { + jsonError(w, http.StatusBadRequest, "decode reference_image_b64: "+decErr.Error()) + return + } + } + } + } + + if strings.TrimSpace(req.Prompt) == "" { + jsonError(w, http.StatusBadRequest, "prompt is required") + return + } + if req.Type != "cover" && req.Type != "chapter" { + jsonError(w, http.StatusBadRequest, `type must be "cover" or "chapter"`) + return + } + if req.Slug == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if req.Type == "chapter" && req.Chapter <= 0 { + jsonError(w, http.StatusBadRequest, "chapter must be > 0 when type is chapter") + return + } + + // Resolve model + model := cfai.ImageModel(req.Model) + if model == "" { + if req.Type == "cover" { + model = cfai.DefaultImageModel + } else { + model = cfai.ImageModelFlux2Klein4B + } + } + + imgReq := cfai.ImageRequest{ + Prompt: req.Prompt, + Model: model, + NumSteps: req.NumSteps, + Width: req.Width, + Height: req.Height, + Guidance: req.Guidance, + Strength: req.Strength, + } + + s.deps.Log.Info("admin: image gen requested", + "type", req.Type, "slug", req.Slug, "chapter", req.Chapter, + "model", model, "has_reference", len(refImageData) > 0) + + var imgData []byte + var genErr error + if len(refImageData) > 0 { + imgData, genErr = s.deps.ImageGen.GenerateImageFromReference(r.Context(), imgReq, refImageData) + } else { + imgData, genErr = s.deps.ImageGen.GenerateImage(r.Context(), imgReq) + } + if genErr != nil { + s.deps.Log.Error("admin: image gen failed", "err", genErr) + jsonError(w, http.StatusBadGateway, "image generation failed: "+genErr.Error()) + return + } + + contentType := sniffImageContentType(imgData) + + // ── Optional persistence ────────────────────────────────────────────────── + var saved bool + var coverURL string + + if req.SaveToCover && req.Type == "cover" && s.deps.CoverStore != nil { + if err := s.deps.CoverStore.PutCover(r.Context(), req.Slug, imgData, contentType); err != nil { + s.deps.Log.Error("admin: save generated cover failed", "slug", req.Slug, "err", err) + // Non-fatal: still return the image + } else { + saved = true + coverURL = fmt.Sprintf("/api/cover/novelfire.net/%s", req.Slug) + s.deps.Log.Info("admin: generated cover saved", "slug", req.Slug, "bytes", len(imgData)) + } + } + + // Encode result as base64 + b64 := base64.StdEncoding.EncodeToString(imgData) + + writeJSON(w, 0, imageGenResponse{ + ImageB64: b64, + ContentType: contentType, + Saved: saved, + CoverURL: coverURL, + Model: string(model), + Bytes: len(imgData), + }) +} + +// saveCoverRequest is the JSON body for POST /api/admin/image-gen/save-cover. +type saveCoverRequest struct { + // Slug is the book slug whose cover should be overwritten. + Slug string `json:"slug"` + // ImageB64 is the base64-encoded image bytes (PNG or JPEG). + ImageB64 string `json:"image_b64"` +} + +// handleAdminImageGenSaveCover handles POST /api/admin/image-gen/save-cover. +// +// Accepts a pre-generated image as base64 and stores it as the book cover in +// MinIO, replacing the existing one. Does not call Cloudflare AI at all. +func (s *Server) handleAdminImageGenSaveCover(w http.ResponseWriter, r *http.Request) { + if s.deps.CoverStore == nil { + jsonError(w, http.StatusServiceUnavailable, "cover store not configured") + return + } + + var req saveCoverRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if req.Slug == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if req.ImageB64 == "" { + jsonError(w, http.StatusBadRequest, "image_b64 is required") + return + } + + imgData, err := base64.StdEncoding.DecodeString(req.ImageB64) + if err != nil { + imgData, err = base64.RawStdEncoding.DecodeString(req.ImageB64) + if err != nil { + jsonError(w, http.StatusBadRequest, "decode image_b64: "+err.Error()) + return + } + } + + contentType := sniffImageContentType(imgData) + if err := s.deps.CoverStore.PutCover(r.Context(), req.Slug, imgData, contentType); err != nil { + s.deps.Log.Error("admin: save-cover failed", "slug", req.Slug, "err", err) + jsonError(w, http.StatusInternalServerError, "save cover: "+err.Error()) + return + } + + s.deps.Log.Info("admin: cover saved via image-gen", "slug", req.Slug, "bytes", len(imgData)) + writeJSON(w, 0, map[string]any{ + "saved": true, + "cover_url": fmt.Sprintf("/api/cover/novelfire.net/%s", req.Slug), + "bytes": len(imgData), + }) +} + +// sniffImageContentType returns the MIME type of the image bytes. +func sniffImageContentType(data []byte) string { + if len(data) >= 4 { + // PNG: 0x89 P N G + if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4e && data[3] == 0x47 { + return "image/png" + } + // JPEG: FF D8 FF + if data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF { + return "image/jpeg" + } + // WebP: RIFF....WEBP + if len(data) >= 12 && data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' && + data[8] == 'W' && data[9] == 'E' && data[10] == 'B' && data[11] == 'P' { + return "image/webp" + } + } + return "image/png" +} + +// saveChapterImageRequest is the JSON body for POST /api/admin/image-gen/save-chapter-image. +type saveChapterImageRequest struct { + // Slug is the book slug. + Slug string `json:"slug"` + // Chapter is the 1-based chapter number. + Chapter int `json:"chapter"` + // ImageB64 is the base64-encoded image bytes (PNG or JPEG). + ImageB64 string `json:"image_b64"` +} + +// handleAdminImageGenSaveChapterImage handles POST /api/admin/image-gen/save-chapter-image. +// +// Accepts a pre-generated image as base64 and stores it as the chapter illustration +// in MinIO, replacing the existing one if present. Does not call Cloudflare AI. +func (s *Server) handleAdminImageGenSaveChapterImage(w http.ResponseWriter, r *http.Request) { + if s.deps.ChapterImageStore == nil { + jsonError(w, http.StatusServiceUnavailable, "chapter image store not configured") + return + } + + var req saveChapterImageRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if req.Slug == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if req.Chapter <= 0 { + jsonError(w, http.StatusBadRequest, "chapter must be > 0") + return + } + if req.ImageB64 == "" { + jsonError(w, http.StatusBadRequest, "image_b64 is required") + return + } + + imgData, err := base64.StdEncoding.DecodeString(req.ImageB64) + if err != nil { + imgData, err = base64.RawStdEncoding.DecodeString(req.ImageB64) + if err != nil { + jsonError(w, http.StatusBadRequest, "decode image_b64: "+err.Error()) + return + } + } + + contentType := sniffImageContentType(imgData) + if err := s.deps.ChapterImageStore.PutChapterImage(r.Context(), req.Slug, req.Chapter, imgData, contentType); err != nil { + s.deps.Log.Error("admin: save-chapter-image failed", "slug", req.Slug, "chapter", req.Chapter, "err", err) + jsonError(w, http.StatusInternalServerError, "save chapter image: "+err.Error()) + return + } + + s.deps.Log.Info("admin: chapter image saved", "slug", req.Slug, "chapter", req.Chapter, "bytes", len(imgData)) + writeJSON(w, 0, map[string]any{ + "saved": true, + "image_url": fmt.Sprintf("/api/chapter-image/novelfire.net/%s/%d", req.Slug, req.Chapter), + "bytes": len(imgData), + }) +} + +// handleHeadChapterImage handles HEAD /api/chapter-image/{domain}/{slug}/{n}. +// +// Returns 200 when an image exists for this chapter, 404 otherwise. +// Used by the SSR loader to check existence without downloading the full image. +func (s *Server) handleHeadChapterImage(w http.ResponseWriter, r *http.Request) { + if s.deps.ChapterImageStore == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + slug := r.PathValue("slug") + nStr := r.PathValue("n") + n, err := strconv.Atoi(nStr) + if err != nil || n <= 0 { + w.WriteHeader(http.StatusBadRequest) + return + } + + if s.deps.ChapterImageStore.ChapterImageExists(r.Context(), slug, n) { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusNotFound) + } +} + +// handleGetChapterImage handles GET /api/chapter-image/{domain}/{slug}/{n}. +// +// Serves the stored chapter illustration directly from MinIO. +// Returns 404 when no image has been saved for this chapter. +func (s *Server) handleGetChapterImage(w http.ResponseWriter, r *http.Request) { + if s.deps.ChapterImageStore == nil { + http.NotFound(w, r) + return + } + + slug := r.PathValue("slug") + nStr := r.PathValue("n") + n, err := strconv.Atoi(nStr) + if err != nil || n <= 0 { + jsonError(w, http.StatusBadRequest, "invalid chapter number") + return + } + + data, contentType, ok, err := s.deps.ChapterImageStore.GetChapterImage(r.Context(), slug, n) + if err != nil { + s.deps.Log.Error("chapter-image: get failed", "slug", slug, "n", n, "err", err) + jsonError(w, http.StatusInternalServerError, "could not retrieve chapter image") + return + } + if !ok { + http.NotFound(w, r) + return + } + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(data) +} + +// handleAdminImageGenAsync handles POST /api/admin/image-gen/async. +// +// Fire-and-forget variant: validates the request, creates an ai_job record of +// kind "image-gen", spawns a background goroutine, and returns HTTP 202 with +// {job_id} immediately. The goroutine calls Cloudflare AI, stores the result +// as base64 in the job payload, and marks the job done/failed when finished. +// +// The admin can then review the result via the ai-jobs page and approve +// (save as cover) or reject (discard) the image. +func (s *Server) handleAdminImageGenAsync(w http.ResponseWriter, r *http.Request) { + if s.deps.ImageGen == nil { + jsonError(w, http.StatusServiceUnavailable, "image generation not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN missing)") + return + } + if s.deps.AIJobStore == nil { + jsonError(w, http.StatusServiceUnavailable, "ai job store not configured") + return + } + + var req imageGenRequest + var refImageData []byte + + ct := r.Header.Get("Content-Type") + if strings.HasPrefix(ct, "multipart/form-data") { + if err := r.ParseMultipartForm(32 << 20); err != nil { + jsonError(w, http.StatusBadRequest, "parse multipart: "+err.Error()) + return + } + if jsonPart := r.FormValue("json"); jsonPart != "" { + if err := json.Unmarshal([]byte(jsonPart), &req); err != nil { + jsonError(w, http.StatusBadRequest, "parse json field: "+err.Error()) + return + } + } + if f, _, err := r.FormFile("reference"); err == nil { + defer f.Close() + refImageData, _ = io.ReadAll(f) + } + } else { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if req.ReferenceImageB64 != "" { + var decErr error + refImageData, decErr = base64.StdEncoding.DecodeString(req.ReferenceImageB64) + if decErr != nil { + refImageData, decErr = base64.RawStdEncoding.DecodeString(req.ReferenceImageB64) + if decErr != nil { + jsonError(w, http.StatusBadRequest, "decode reference_image_b64: "+decErr.Error()) + return + } + } + } + } + + if strings.TrimSpace(req.Prompt) == "" { + jsonError(w, http.StatusBadRequest, "prompt is required") + return + } + if req.Type != "cover" && req.Type != "chapter" { + jsonError(w, http.StatusBadRequest, `type must be "cover" or "chapter"`) + return + } + if req.Slug == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if req.Type == "chapter" && req.Chapter <= 0 { + jsonError(w, http.StatusBadRequest, "chapter must be > 0 when type is chapter") + return + } + + // Resolve model. + model := cfai.ImageModel(req.Model) + if model == "" { + if req.Type == "cover" { + model = cfai.DefaultImageModel + } else { + model = cfai.ImageModelFlux2Klein4B + } + } + + // Encode request params as job payload so the UI can reconstruct context. + type jobParams struct { + Prompt string `json:"prompt"` + Type string `json:"type"` + Chapter int `json:"chapter,omitempty"` + NumSteps int `json:"num_steps,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + Guidance float64 `json:"guidance,omitempty"` + Strength float64 `json:"strength,omitempty"` + HasRef bool `json:"has_ref,omitempty"` + } + paramsJSON, _ := json.Marshal(jobParams{ + Prompt: req.Prompt, + Type: req.Type, + Chapter: req.Chapter, + NumSteps: req.NumSteps, + Width: req.Width, + Height: req.Height, + Guidance: req.Guidance, + Strength: req.Strength, + HasRef: len(refImageData) > 0, + }) + + jobID, createErr := s.deps.AIJobStore.CreateAIJob(r.Context(), domain.AIJob{ + Kind: "image-gen", + Slug: req.Slug, + Status: domain.TaskStatusPending, + Model: string(model), + Payload: string(paramsJSON), + Started: time.Now(), + }) + if createErr != nil { + jsonError(w, http.StatusInternalServerError, "create ai job: "+createErr.Error()) + return + } + + jobCtx, jobCancel := context.WithCancel(context.Background()) + registerCancelJob(jobID, jobCancel) + + // Mark running before returning. + _ = s.deps.AIJobStore.UpdateAIJob(r.Context(), jobID, map[string]any{ + "status": string(domain.TaskStatusRunning), + }) + + s.deps.Log.Info("admin: image-gen async started", + "job_id", jobID, "slug", req.Slug, "type", req.Type, "model", model) + + // Capture locals for the goroutine. + store := s.deps.AIJobStore + imageGen := s.deps.ImageGen + coverStore := s.deps.CoverStore + logger := s.deps.Log + capturedReq := req + capturedModel := model + capturedRefImage := refImageData + + go func() { + defer deregisterCancelJob(jobID) + defer jobCancel() + defer func() { + if r := recover(); r != nil { + logger.Error("admin: image-gen goroutine panic", "job_id", jobID, "recover", r) + } + }() + + if jobCtx.Err() != nil { + _ = store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": string(domain.TaskStatusCancelled), + "finished": time.Now().Format(time.RFC3339), + }) + return + } + + imgReq := cfai.ImageRequest{ + Prompt: capturedReq.Prompt, + Model: capturedModel, + NumSteps: capturedReq.NumSteps, + Width: capturedReq.Width, + Height: capturedReq.Height, + Guidance: capturedReq.Guidance, + Strength: capturedReq.Strength, + } + + var imgData []byte + var genErr error + if len(capturedRefImage) > 0 { + imgData, genErr = imageGen.GenerateImageFromReference(jobCtx, imgReq, capturedRefImage) + } else { + imgData, genErr = imageGen.GenerateImage(jobCtx, imgReq) + } + + if genErr != nil { + logger.Error("admin: image-gen async failed", "job_id", jobID, "err", genErr) + _ = store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": string(domain.TaskStatusFailed), + "error_message": genErr.Error(), + "finished": time.Now().Format(time.RFC3339), + }) + return + } + + contentType := sniffImageContentType(imgData) + b64 := base64.StdEncoding.EncodeToString(imgData) + + // Build result payload: include the original params + the generated image. + type resultPayload struct { + Prompt string `json:"prompt"` + Type string `json:"type"` + Chapter int `json:"chapter,omitempty"` + ContentType string `json:"content_type"` + ImageB64 string `json:"image_b64"` + Bytes int `json:"bytes"` + NumSteps int `json:"num_steps,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + Guidance float64 `json:"guidance,omitempty"` + } + resultJSON, _ := json.Marshal(resultPayload{ + Prompt: capturedReq.Prompt, + Type: capturedReq.Type, + Chapter: capturedReq.Chapter, + ContentType: contentType, + ImageB64: b64, + Bytes: len(imgData), + NumSteps: capturedReq.NumSteps, + Width: capturedReq.Width, + Height: capturedReq.Height, + Guidance: capturedReq.Guidance, + }) + + if err := store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": string(domain.TaskStatusDone), + "items_done": 1, + "items_total": 1, + "finished": time.Now().Format(time.RFC3339), + }); err != nil { + logger.Error("admin: image-gen failed to mark job done", "job_id", jobID, "err", err) + } + if err := store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "payload": string(resultJSON), + }); err != nil { + logger.Error("admin: image-gen failed to write job payload", "job_id", jobID, "err", err) + } + + logger.Info("admin: image-gen async done", + "job_id", jobID, "slug", capturedReq.Slug, + "bytes", len(imgData), "content_type", contentType) + + // Suppress unused variable warning for coverStore when SaveToCover is false. + _ = coverStore + }() + + writeJSON(w, http.StatusAccepted, map[string]any{"job_id": jobID}) +} diff --git a/backend/internal/backend/handlers_import.go b/backend/internal/backend/handlers_import.go new file mode 100644 index 0000000..0747525 --- /dev/null +++ b/backend/internal/backend/handlers_import.go @@ -0,0 +1,234 @@ +package backend + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/storage" +) + +type importRequest struct { + Title string `json:"title"` + Author string `json:"author"` + CoverURL string `json:"cover_url"` + Genres []string `json:"genres"` + Summary string `json:"summary"` + BookStatus string `json:"book_status"` // "ongoing" | "completed" | "hiatus" + FileName string `json:"file_name"` + FileType string `json:"file_type"` // "pdf" or "epub" + ObjectKey string `json:"object_key"` // MinIO path to uploaded file +} + +type importResponse struct { + TaskID string `json:"task_id"` + Slug string `json:"slug"` + Preview *importPreview `json:"preview,omitempty"` +} + +type importPreview struct { + Chapters int `json:"chapters"` + FirstLines []string `json:"first_lines"` +} + +func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) { + if s.deps.Producer == nil { + jsonError(w, http.StatusServiceUnavailable, "task queue not configured") + return + } + + ct := r.Header.Get("Content-Type") + var req importRequest + var objectKey string + var chaptersKey string + var chapterCount int + + if strings.HasPrefix(ct, "multipart/form-data") { + if err := r.ParseMultipartForm(32 << 20); err != nil { + jsonError(w, http.StatusBadRequest, "parse multipart: "+err.Error()) + return + } + req.Title = r.FormValue("title") + req.Author = r.FormValue("author") + req.CoverURL = r.FormValue("cover_url") + req.Summary = r.FormValue("summary") + req.BookStatus = r.FormValue("book_status") + if g := r.FormValue("genres"); g != "" { + for _, s := range strings.Split(g, ",") { + if s = strings.TrimSpace(s); s != "" { + req.Genres = append(req.Genres, s) + } + } + } + req.FileName = r.FormValue("file_name") + req.FileType = r.FormValue("file_type") + analyzeOnly := r.FormValue("analyze") == "true" + + file, header, err := r.FormFile("file") + if err != nil { + jsonError(w, http.StatusBadRequest, "parse file: "+err.Error()) + return + } + defer file.Close() + + if req.FileName == "" { + req.FileName = header.Filename + } + if req.FileType == "" { + req.FileType = strings.TrimPrefix(filepath.Ext(header.Filename), ".") + } + + data, err := io.ReadAll(file) + if err != nil { + jsonError(w, http.StatusBadRequest, "read file: "+err.Error()) + return + } + + // Analyze only - just count chapters + if analyzeOnly { + preview := analyzeImportFile(data, req.FileType) + writeJSON(w, 0, importResponse{ + Preview: preview, + }) + return + } + + // Parse PDF/EPUB on the backend (with timeout) and store chapters as JSON. + // The runner only needs to ingest pre-parsed chapters — no PDF parsing on runner. + parseCtx, parseCancel := context.WithTimeout(r.Context(), 3*time.Minute) + defer parseCancel() + chapters, parseErr := storage.ParseImportFile(parseCtx, data, req.FileType) + if parseErr != nil || len(chapters) == 0 { + jsonError(w, http.StatusUnprocessableEntity, "could not parse file: "+func() string { + if parseErr != nil { return parseErr.Error() } + return "no chapters found" + }()) + return + } + + // Store raw file in MinIO (for reference/re-import). + objectKey = fmt.Sprintf("imports/%d_%s", time.Now().Unix(), header.Filename) + if s.deps.ImportFileStore == nil { + jsonError(w, http.StatusInternalServerError, "storage not available") + return + } + if err := s.deps.ImportFileStore.PutImportFile(r.Context(), objectKey, data); err != nil { + jsonError(w, http.StatusInternalServerError, "upload file: "+err.Error()) + return + } + + // Store pre-parsed chapters JSON in MinIO so runner can ingest without re-parsing. + chaptersJSON, _ := json.Marshal(chapters) + chaptersKey = fmt.Sprintf("imports/%d_%s_chapters.json", time.Now().Unix(), strings.TrimSuffix(header.Filename, filepath.Ext(header.Filename))) + if err := s.deps.ImportFileStore.PutImportChapters(r.Context(), chaptersKey, chaptersJSON); err != nil { + jsonError(w, http.StatusInternalServerError, "store chapters: "+err.Error()) + return + } + chapterCount = len(chapters) + } else { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + objectKey = req.ObjectKey + } + + if req.Title == "" { + jsonError(w, http.StatusBadRequest, "title is required") + return + } + if req.FileType != "pdf" && req.FileType != "epub" { + jsonError(w, http.StatusBadRequest, "file_type must be 'pdf' or 'epub'") + return + } + + slug := strings.ToLower(strings.ReplaceAll(req.Title, " ", "-")) + slug = strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' { + return r + } + return -1 + }, slug) + + taskID, err := s.deps.Producer.CreateImportTask(r.Context(), domain.ImportTask{ + Slug: slug, + Title: req.Title, + Author: req.Author, + CoverURL: req.CoverURL, + Genres: req.Genres, + Summary: req.Summary, + BookStatus: req.BookStatus, + FileType: req.FileType, + ObjectKey: objectKey, + ChaptersKey: chaptersKey, + ChaptersTotal: chapterCount, + InitiatorUserID: "", + }) + if err != nil { + jsonError(w, http.StatusInternalServerError, "create import task: "+err.Error()) + return + } + + writeJSON(w, 0, importResponse{ + TaskID: taskID, + Slug: slug, + Preview: &importPreview{Chapters: chapterCount}, + }) +} + +// analyzeImportFile parses the file to count chapters and extract preview lines. +func analyzeImportFile(data []byte, fileType string) *importPreview { + count, firstLines, err := storage.AnalyzeFile(data, fileType) + if err != nil || count == 0 { + // Fall back to rough size estimate so the UI still shows something + count = estimateChapters(data, fileType) + } + return &importPreview{ + Chapters: count, + FirstLines: firstLines, + } +} + +func estimateChapters(data []byte, fileType string) int { + // Rough estimate: ~100KB per chapter for PDF, ~50KB for EPUB + size := len(data) + if fileType == "pdf" { + return size / 100000 + } + return size / 50000 +} + +func (s *Server) handleAdminImportStatus(w http.ResponseWriter, r *http.Request) { + taskID := r.PathValue("id") + if taskID == "" { + jsonError(w, http.StatusBadRequest, "task id required") + return + } + + task, ok, err := s.deps.TaskReader.GetImportTask(r.Context(), taskID) + if err != nil { + jsonError(w, http.StatusInternalServerError, "get task: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, "task not found") + return + } + + writeJSON(w, 0, task) +} + +func (s *Server) handleAdminImportList(w http.ResponseWriter, r *http.Request) { + tasks, err := s.deps.TaskReader.ListImportTasks(r.Context()) + if err != nil { + jsonError(w, http.StatusInternalServerError, "list tasks: "+err.Error()) + return + } + writeJSON(w, 0, map[string]any{"tasks": tasks}) +} \ No newline at end of file diff --git a/backend/internal/backend/handlers_notifications.go b/backend/internal/backend/handlers_notifications.go new file mode 100644 index 0000000..d52b11a --- /dev/null +++ b/backend/internal/backend/handlers_notifications.go @@ -0,0 +1,119 @@ +package backend + +import ( + "encoding/json" + "net/http" +) + +// handleDismissNotification handles DELETE /api/notifications/{id}. +func (s *Server) handleDismissNotification(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + if id == "" { + jsonError(w, http.StatusBadRequest, "notification id required") + return + } + if s.deps.NotificationStore == nil { + jsonError(w, http.StatusServiceUnavailable, "notification store not configured") + return + } + if err := s.deps.NotificationStore.DeleteNotification(r.Context(), id); err != nil { + jsonError(w, http.StatusInternalServerError, "dismiss notification: "+err.Error()) + return + } + writeJSON(w, 0, map[string]any{"success": true}) +} + +// handleClearAllNotifications handles DELETE /api/notifications?user_id=... +func (s *Server) handleClearAllNotifications(w http.ResponseWriter, r *http.Request) { + userID := r.URL.Query().Get("user_id") + if userID == "" { + jsonError(w, http.StatusBadRequest, "user_id required") + return + } + if s.deps.NotificationStore == nil { + jsonError(w, http.StatusServiceUnavailable, "notification store not configured") + return + } + if err := s.deps.NotificationStore.ClearAllNotifications(r.Context(), userID); err != nil { + jsonError(w, http.StatusInternalServerError, "clear notifications: "+err.Error()) + return + } + writeJSON(w, 0, map[string]any{"success": true}) +} + +// handleMarkAllNotificationsRead handles PATCH /api/notifications?user_id=... +func (s *Server) handleMarkAllNotificationsRead(w http.ResponseWriter, r *http.Request) { + userID := r.URL.Query().Get("user_id") + if userID == "" { + jsonError(w, http.StatusBadRequest, "user_id required") + return + } + if s.deps.NotificationStore == nil { + jsonError(w, http.StatusServiceUnavailable, "notification store not configured") + return + } + if err := s.deps.NotificationStore.MarkAllNotificationsRead(r.Context(), userID); err != nil { + jsonError(w, http.StatusInternalServerError, "mark all read: "+err.Error()) + return + } + writeJSON(w, 0, map[string]any{"success": true}) +} + +type notification struct { + ID string `json:"id"` + UserID string `json:"user_id"` + Title string `json:"title"` + Message string `json:"message"` + Link string `json:"link"` + Read bool `json:"read"` +} + +func (s *Server) handleListNotifications(w http.ResponseWriter, r *http.Request) { + userID := r.URL.Query().Get("user_id") + if userID == "" { + jsonError(w, http.StatusBadRequest, "user_id required") + return + } + + if s.deps.NotificationStore == nil { + jsonError(w, http.StatusServiceUnavailable, "notification store not configured") + return + } + + items, err := s.deps.NotificationStore.ListNotifications(r.Context(), userID, 50) + if err != nil { + jsonError(w, http.StatusInternalServerError, "list notifications: "+err.Error()) + return + } + + // Parse each item as notification + notifications := make([]notification, 0, len(items)) + for _, item := range items { + b, _ := json.Marshal(item) + var n notification + json.Unmarshal(b, &n) //nolint:errcheck + notifications = append(notifications, n) + } + + writeJSON(w, 0, map[string]any{"notifications": notifications}) +} + +func (s *Server) handleMarkNotificationRead(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + if id == "" { + jsonError(w, http.StatusBadRequest, "notification id required") + return + } + + if s.deps.NotificationStore == nil { + jsonError(w, http.StatusServiceUnavailable, "notification store not configured") + return + } + + if err := s.deps.NotificationStore.MarkNotificationRead(r.Context(), id); err != nil { + jsonError(w, http.StatusInternalServerError, "mark read: "+err.Error()) + return + } + + writeJSON(w, 0, map[string]any{"success": true}) +} diff --git a/backend/internal/backend/handlers_podcast.go b/backend/internal/backend/handlers_podcast.go new file mode 100644 index 0000000..51a3746 --- /dev/null +++ b/backend/internal/backend/handlers_podcast.go @@ -0,0 +1,524 @@ +package backend + +// handlers_podcast.go — Podcast RSS feed generation and serving. +// +// Public endpoints (no auth required, suitable for podcast clients): +// +// GET /podcast/{slug} — RSS 2.0 feed (slug may end in .xml) +// ?voice=<voice-id> (optional; defaults to DefaultVoice) +// GET /podcast/audio/{slug}/{n}/{voice} — redirects to a presigned MinIO URL +// +// Admin endpoints (Bearer-token protected): +// +// POST /api/admin/podcast — start a podcast generation job +// GET /api/admin/podcast/{slug} — list podcast jobs for slug + +import ( + "context" + "encoding/json" + "encoding/xml" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/libnovel/backend/internal/cfai" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/pockettts" +) + +// ── RSS data structures ──────────────────────────────────────────────────────── + +type podcastFeed struct { + XMLName xml.Name `xml:"rss"` + Version string `xml:"version,attr"` + ItunesNS string `xml:"xmlns:itunes,attr"` + Channel podcastChannel `xml:"channel"` +} + +type podcastChannel struct { + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + Language string `xml:"language"` + Author string `xml:"itunes:author"` + Image *podcastImage `xml:"image,omitempty"` + ItunesImg *itunesCover `xml:"itunes:image,omitempty"` + Items []podcastItem `xml:"item"` +} + +type podcastImage struct { + URL string `xml:"url"` +} + +type itunesCover struct { + Href string `xml:"href,attr"` +} + +type podcastItem struct { + Title string `xml:"title"` + GUID string `xml:"guid"` + PubDate string `xml:"pubDate,omitempty"` + Enclosure *podcastEnclosure `xml:"enclosure,omitempty"` + Episode int `xml:"itunes:episode,omitempty"` +} + +type podcastEnclosure struct { + URL string `xml:"url,attr"` + Type string `xml:"type,attr"` + Length string `xml:"length,attr"` +} + +// ── Public: RSS feed ─────────────────────────────────────────────────────────── + +// handlePodcastFeed handles GET /podcast/{slug}. +// Slug may optionally end with ".xml" (podcast clients expect it). +// Query param: voice (optional; defaults to DefaultVoice). +func (s *Server) handlePodcastFeed(w http.ResponseWriter, r *http.Request) { + slug := strings.TrimSuffix(r.PathValue("slug"), ".xml") + if slug == "" { + http.NotFound(w, r) + return + } + + voice := r.URL.Query().Get("voice") + if voice == "" { + voice = s.cfg.DefaultVoice + } + + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), slug) + if err != nil { + http.Error(w, "read metadata: "+err.Error(), http.StatusInternalServerError) + return + } + if !ok { + http.NotFound(w, r) + return + } + + chapters, err := s.deps.BookReader.ListChapters(r.Context(), slug) + if err != nil { + http.Error(w, "list chapters: "+err.Error(), http.StatusInternalServerError) + return + } + + // Determine base URL from request headers (Caddy sets X-Forwarded-Proto/Host). + scheme := "https" + if r.Header.Get("X-Forwarded-Proto") == "http" || (r.TLS == nil && r.Header.Get("X-Forwarded-Proto") == "") { + scheme = "http" + } + host := r.Host + if fh := r.Header.Get("X-Forwarded-Host"); fh != "" { + host = fh + } + baseURL := scheme + "://" + host + + var items []podcastItem + for _, ch := range chapters { + audioKey := s.deps.AudioStore.AudioObjectKey(slug, ch.Number, voice) + size := s.deps.AudioStore.AudioObjectSize(r.Context(), audioKey) + if size < 0 { + continue // audio not generated yet + } + title := ch.Title + if title == "" { + title = fmt.Sprintf("Chapter %d", ch.Number) + } + audioURL := fmt.Sprintf("%s/podcast/audio/%s/%d/%s", baseURL, slug, ch.Number, voice) + + // pubDate: use chapter creation date if available, else fallback to now. + pubDate := time.Now().Format(time.RFC1123Z) + if ch.Date != "" { + if t, err := time.Parse(time.RFC3339, ch.Date); err == nil { + pubDate = t.Format(time.RFC1123Z) + } + } + + items = append(items, podcastItem{ + Title: title, + GUID: fmt.Sprintf("%s/podcast/%s/%d/%s", baseURL, slug, ch.Number, voice), + PubDate: pubDate, + Enclosure: &podcastEnclosure{ + URL: audioURL, + Type: "audio/mpeg", + Length: fmt.Sprintf("%d", size), + }, + Episode: ch.Number, + }) + } + + desc := meta.Summary + if desc == "" { + desc = meta.Title + } + + feed := podcastFeed{ + Version: "2.0", + ItunesNS: "http://www.itunes.com/dtds/podcast-1.0.dtd", + Channel: podcastChannel{ + Title: meta.Title, + Link: fmt.Sprintf("%s/books/%s", baseURL, slug), + Description: desc, + Language: "en", + Author: meta.Author, + Items: items, + }, + } + if meta.Cover != "" { + feed.Channel.Image = &podcastImage{URL: meta.Cover} + feed.Channel.ItunesImg = &itunesCover{Href: meta.Cover} + } + + w.Header().Set("Content-Type", "application/rss+xml; charset=utf-8") + _, _ = w.Write([]byte(xml.Header)) + enc := xml.NewEncoder(w) + enc.Indent("", " ") + if err := enc.Encode(feed); err != nil { + s.deps.Log.Error("podcast: encode RSS failed", "slug", slug, "err", err) + } +} + +// handlePodcastAudio handles GET /podcast/audio/{slug}/{n}/{voice}. +// Redirects to a 4-hour presigned MinIO URL. No auth required. +func (s *Server) handlePodcastAudio(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + voice := r.PathValue("voice") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 { + http.NotFound(w, r) + return + } + + audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice) + if !s.deps.AudioStore.AudioExists(r.Context(), audioKey) { + http.NotFound(w, r) + return + } + + url, err := s.deps.PresignStore.PresignAudio(r.Context(), audioKey, 4*time.Hour) + if err != nil { + s.deps.Log.Error("podcast: presign audio failed", "slug", slug, "n", n, "err", err) + http.Error(w, "presign failed", http.StatusInternalServerError) + return + } + http.Redirect(w, r, url, http.StatusFound) +} + +// ── Admin: create podcast generation job ────────────────────────────────────── + +type podcastJobRequest struct { + Slug string `json:"slug"` + Voice string `json:"voice"` + FromChapter int `json:"from_chapter"` // 0 = chapter 1 + ToChapter int `json:"to_chapter"` // 0 = all chapters +} + +// handleAdminPodcast handles POST /api/admin/podcast. +// Creates an ai_job of kind="podcast" and spawns a background goroutine that +// generates TTS for any chapters that do not yet have audio. +func (s *Server) handleAdminPodcast(w http.ResponseWriter, r *http.Request) { + if s.deps.AIJobStore == nil { + jsonError(w, http.StatusServiceUnavailable, "ai job store not configured") + return + } + + var req podcastJobRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if strings.TrimSpace(req.Voice) == "" { + req.Voice = s.cfg.DefaultVoice + } + + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", req.Slug)) + return + } + + chapters, err := s.deps.BookReader.ListChapters(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "list chapters: "+err.Error()) + return + } + if len(chapters) == 0 { + jsonError(w, http.StatusBadRequest, "book has no chapters") + return + } + + from := req.FromChapter + if from <= 0 { + from = 1 + } + to := req.ToChapter + if to <= 0 || to > chapters[len(chapters)-1].Number { + to = chapters[len(chapters)-1].Number + } + if from > to { + jsonError(w, http.StatusBadRequest, "from_chapter must be ≤ to_chapter") + return + } + + paramsJSON, _ := json.Marshal(map[string]any{ + "voice": req.Voice, + "from_chapter": from, + "to_chapter": to, + }) + + jobID, createErr := s.deps.AIJobStore.CreateAIJob(r.Context(), domain.AIJob{ + Kind: "podcast", + Slug: req.Slug, + Status: domain.TaskStatusPending, + Model: req.Voice, + FromItem: from, + ToItem: to, + ItemsTotal: to - from + 1, + Payload: string(paramsJSON), + Started: time.Now(), + }) + if createErr != nil { + jsonError(w, http.StatusInternalServerError, "create job: "+createErr.Error()) + return + } + + jobCtx, jobCancel := context.WithCancel(context.Background()) + registerCancelJob(jobID, jobCancel) + + _ = s.deps.AIJobStore.UpdateAIJob(r.Context(), jobID, map[string]any{ + "status": string(domain.TaskStatusRunning), + }) + + s.deps.Log.Info("admin: podcast job started", + "job_id", jobID, "slug", req.Slug, "voice", req.Voice, + "from", from, "to", to) + + // Capture dependencies for goroutine. + store := s.deps.AIJobStore + bookReader := s.deps.BookReader + audioStore := s.deps.AudioStore + kokoroClient := s.deps.Kokoro + pocketTTSClient := s.deps.PocketTTS + cfaiClient := s.deps.CFAI + logger := s.deps.Log + capturedMeta := meta + capturedChapters := chapters + capturedReq := req + + go func() { + defer deregisterCancelJob(jobID) + defer jobCancel() + + done := 0 + var lastErr string + + for _, ch := range capturedChapters { + if jobCtx.Err() != nil { + _ = store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": string(domain.TaskStatusCancelled), + "items_done": done, + "error_message": "cancelled", + "finished": time.Now().Format(time.RFC3339), + }) + return + } + + if ch.Number < from || ch.Number > to { + continue + } + + audioKey := audioStore.AudioObjectKey(capturedMeta.Slug, ch.Number, capturedReq.Voice) + if audioStore.AudioExists(jobCtx, audioKey) { + // Already generated — count it and move on. + done++ + _ = store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "items_done": done, + }) + continue + } + + raw, readErr := bookReader.ReadChapter(jobCtx, capturedMeta.Slug, ch.Number) + if readErr != nil { + lastErr = fmt.Sprintf("ch.%d read: %v", ch.Number, readErr) + logger.Warn("podcast: read chapter failed", + "slug", capturedMeta.Slug, "chapter", ch.Number, "err", readErr) + continue + } + text := stripMarkdown(raw) + if text == "" { + continue + } + + audioData, genErr := podcastGenerateAudio(jobCtx, text, capturedReq.Voice, + kokoroClient, pocketTTSClient, cfaiClient, logger) + if genErr != nil { + lastErr = fmt.Sprintf("ch.%d tts: %v", ch.Number, genErr) + logger.Warn("podcast: tts failed", + "slug", capturedMeta.Slug, "chapter", ch.Number, + "voice", capturedReq.Voice, "err", genErr) + continue + } + + if putErr := audioStore.PutAudio(jobCtx, audioKey, audioData); putErr != nil { + lastErr = fmt.Sprintf("ch.%d put: %v", ch.Number, putErr) + logger.Warn("podcast: put audio failed", + "slug", capturedMeta.Slug, "chapter", ch.Number, "err", putErr) + continue + } + + done++ + _ = store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "items_done": done, + }) + logger.Info("podcast: chapter audio ready", + "slug", capturedMeta.Slug, "chapter", ch.Number, "voice", capturedReq.Voice) + } + + finalStatus := string(domain.TaskStatusDone) + if lastErr != "" && done == 0 { + finalStatus = string(domain.TaskStatusFailed) + } + _ = store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": finalStatus, + "items_done": done, + "error_message": lastErr, + "finished": time.Now().Format(time.RFC3339), + }) + logger.Info("podcast: job finished", + "slug", capturedMeta.Slug, "voice", capturedReq.Voice, + "done", done, "status", finalStatus) + }() + + writeJSON(w, http.StatusAccepted, map[string]any{ + "job_id": jobID, + "slug": req.Slug, + "voice": req.Voice, + "from": from, + "to": to, + "items_total": to - from + 1, + }) +} + +// handleAdminPodcastStatus handles GET /api/admin/podcast/{slug}. +// Returns all podcast ai_jobs for slug, newest first. +func (s *Server) handleAdminPodcastStatus(w http.ResponseWriter, r *http.Request) { + if s.deps.AIJobStore == nil { + jsonError(w, http.StatusServiceUnavailable, "ai job store not configured") + return + } + slug := r.PathValue("slug") + + all, err := s.deps.AIJobStore.ListAIJobs(r.Context()) + if err != nil { + jsonError(w, http.StatusInternalServerError, "list jobs: "+err.Error()) + return + } + + var jobs []domain.AIJob + for _, j := range all { + if j.Kind == "podcast" && j.Slug == slug { + jobs = append(jobs, j) + } + } + writeJSON(w, 0, map[string]any{"jobs": jobs}) +} + +// ── TTS helper ───────────────────────────────────────────────────────────────── + +// podcastGenerateAudio generates audio for a full chapter text using the +// appropriate TTS engine for voice. For Kokoro it splits text into ~1 000-char +// sentence-boundary chunks and concatenates the resulting MP3 frames, matching +// the runner's behaviour (see runner.kokoroGenerateChunked). +func podcastGenerateAudio( + ctx context.Context, + text, voice string, + kokoroClient kokoro.Client, + pocketTTSClient pockettts.Client, + cfaiClient cfai.Client, + logger interface{ Warn(string, ...any); Info(string, ...any) }, +) ([]byte, error) { + switch { + case pockettts.IsPocketTTSVoice(voice): + if pocketTTSClient == nil { + return nil, fmt.Errorf("pocket-tts client not configured") + } + return pocketTTSClient.GenerateAudio(ctx, text, voice) + + case cfai.IsCFAIVoice(voice): + if cfaiClient == nil { + return nil, fmt.Errorf("cloudflare AI client not configured") + } + return cfaiClient.GenerateAudio(ctx, text, voice) + + default: // Kokoro + if kokoroClient == nil { + return nil, fmt.Errorf("kokoro client not configured") + } + return podcastKokoroChunked(ctx, kokoroClient, text, voice, logger) + } +} + +// podcastKokoroChunked mirrors runner.kokoroGenerateChunked: splits text into +// ~1 000-character sentence-boundary chunks and concatenates raw MP3 bytes. +func podcastKokoroChunked( + ctx context.Context, + k kokoro.Client, + text, voice string, + logger interface{ Warn(string, ...any); Info(string, ...any) }, +) ([]byte, error) { + const chunkSize = 1000 + chunks := podcastChunkText(text, chunkSize) + logger.Info("podcast: kokoro chunked generation", "chunks", len(chunks), "total_chars", len(text)) + + var combined []byte + for i, chunk := range chunks { + data, err := k.GenerateAudio(ctx, chunk, voice) + if err != nil { + return nil, fmt.Errorf("chunk %d/%d: %w", i+1, len(chunks), err) + } + combined = append(combined, data...) + } + return combined, nil +} + +// podcastChunkText mirrors runner.chunkText. +func podcastChunkText(text string, maxChars int) []string { + if len(text) <= maxChars { + return []string{text} + } + delimiters := []string{".\n", "!\n", "?\n", ". ", "! ", "? ", "\n\n", "\n"} + var chunks []string + remaining := text + for len(remaining) > 0 { + if len(remaining) <= maxChars { + chunks = append(chunks, strings.TrimSpace(remaining)) + break + } + window := remaining[:maxChars] + cutAt := -1 + for _, d := range delimiters { + idx := strings.LastIndex(window, d) + if idx > 0 && idx+len(d) > cutAt { + cutAt = idx + len(d) + } + } + if cutAt <= 0 { + cutAt = maxChars + } + if chunk := strings.TrimSpace(remaining[:cutAt]); chunk != "" { + chunks = append(chunks, chunk) + } + remaining = strings.TrimSpace(remaining[cutAt:]) + } + return chunks +} diff --git a/backend/internal/backend/handlers_push.go b/backend/internal/backend/handlers_push.go new file mode 100644 index 0000000..7fe7324 --- /dev/null +++ b/backend/internal/backend/handlers_push.go @@ -0,0 +1,87 @@ +package backend + +import ( + "encoding/json" + "net/http" + "os" + + "github.com/libnovel/backend/internal/storage" +) + +// handleGetVAPIDPublicKey handles GET /api/push-subscriptions/vapid-public-key. +// Returns the VAPID public key so the SvelteKit frontend can subscribe browsers. +func (s *Server) handleGetVAPIDPublicKey(w http.ResponseWriter, r *http.Request) { + key := os.Getenv("VAPID_PUBLIC_KEY") + if key == "" { + jsonError(w, http.StatusServiceUnavailable, "push notifications not configured") + return + } + writeJSON(w, 0, map[string]string{"public_key": key}) +} + +// handleSavePushSubscription handles POST /api/push-subscriptions. +// Registers a new browser push subscription for the authenticated user. +func (s *Server) handleSavePushSubscription(w http.ResponseWriter, r *http.Request) { + store, ok := s.deps.Producer.(*storage.Store) + if !ok { + jsonError(w, http.StatusInternalServerError, "storage not available") + return + } + + var body struct { + UserID string `json:"user_id"` + Endpoint string `json:"endpoint"` + P256DH string `json:"p256dh"` + Auth string `json:"auth"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + jsonError(w, http.StatusBadRequest, "invalid request body") + return + } + if body.UserID == "" || body.Endpoint == "" || body.P256DH == "" || body.Auth == "" { + jsonError(w, http.StatusBadRequest, "user_id, endpoint, p256dh and auth are required") + return + } + + if err := store.SavePushSubscription(r.Context(), storage.PushSubscription{ + UserID: body.UserID, + Endpoint: body.Endpoint, + P256DH: body.P256DH, + Auth: body.Auth, + }); err != nil { + jsonError(w, http.StatusInternalServerError, "save push subscription: "+err.Error()) + return + } + + writeJSON(w, 0, map[string]any{"success": true}) +} + +// handleDeletePushSubscription handles DELETE /api/push-subscriptions. +// Removes a push subscription by endpoint for the given user. +func (s *Server) handleDeletePushSubscription(w http.ResponseWriter, r *http.Request) { + store, ok := s.deps.Producer.(*storage.Store) + if !ok { + jsonError(w, http.StatusInternalServerError, "storage not available") + return + } + + var body struct { + UserID string `json:"user_id"` + Endpoint string `json:"endpoint"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + jsonError(w, http.StatusBadRequest, "invalid request body") + return + } + if body.UserID == "" || body.Endpoint == "" { + jsonError(w, http.StatusBadRequest, "user_id and endpoint are required") + return + } + + if err := store.DeletePushSubscription(r.Context(), body.UserID, body.Endpoint); err != nil { + jsonError(w, http.StatusInternalServerError, "delete push subscription: "+err.Error()) + return + } + + writeJSON(w, 0, map[string]any{"success": true}) +} diff --git a/backend/internal/backend/handlers_split.go b/backend/internal/backend/handlers_split.go new file mode 100644 index 0000000..2de2d26 --- /dev/null +++ b/backend/internal/backend/handlers_split.go @@ -0,0 +1,141 @@ +package backend + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/domain" +) + +// handleAdminSplitChapters handles POST /api/admin/books/{slug}/split-chapters. +// +// Request body (JSON): +// +// { "text": "<full text with --- dividers and optional ## Title lines>" } +// +// The text is split on lines containing only "---". Each segment may start with +// a "## Title" line which becomes the chapter title; remaining lines are the +// chapter content. Sequential chapter numbers 1..N are assigned. +// +// All existing chapters for the book are replaced: WriteChapter is called for +// each new chapter (upsert by number), so chapters beyond N are not deleted — +// use the dedup endpoint afterwards if needed. +func (s *Server) handleAdminSplitChapters(w http.ResponseWriter, r *http.Request) { + if s.deps.BookWriter == nil { + jsonError(w, http.StatusServiceUnavailable, "book writer not configured") + return + } + + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + + var req struct { + Text string `json:"text"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Text) == "" { + jsonError(w, http.StatusBadRequest, "text is required") + return + } + + chapters := splitChapterText(req.Text) + if len(chapters) == 0 { + jsonError(w, http.StatusUnprocessableEntity, "no chapters produced from text") + return + } + + for _, ch := range chapters { + var mdContent string + if ch.Title != "" && ch.Title != fmt.Sprintf("Chapter %d", ch.Number) { + mdContent = fmt.Sprintf("# %s\n\n%s", ch.Title, ch.Content) + } else { + mdContent = fmt.Sprintf("# Chapter %d\n\n%s", ch.Number, ch.Content) + } + domainCh := domain.Chapter{ + Ref: domain.ChapterRef{Number: ch.Number, Title: ch.Title}, + Text: mdContent, + } + if err := s.deps.BookWriter.WriteChapter(r.Context(), slug, domainCh); err != nil { + jsonError(w, http.StatusInternalServerError, fmt.Sprintf("write chapter %d: %s", ch.Number, err.Error())) + return + } + } + + writeJSON(w, 0, map[string]any{ + "chapters": len(chapters), + "slug": slug, + }) +} + +// splitChapterText splits text on "---" divider lines into bookstore.Chapter +// slices. Each segment may optionally start with a "## Title" header line. +func splitChapterText(text string) []bookstore.Chapter { + lines := strings.Split(text, "\n") + + // Collect raw segments split on "---" dividers. + var segments [][]string + cur := []string{} + for _, line := range lines { + if strings.TrimSpace(line) == "---" { + segments = append(segments, cur) + cur = []string{} + } else { + cur = append(cur, line) + } + } + segments = append(segments, cur) // last segment + + var chapters []bookstore.Chapter + chNum := 0 + for _, seg := range segments { + // Trim leading/trailing blank lines from the segment. + start, end := 0, len(seg) + for start < end && strings.TrimSpace(seg[start]) == "" { + start++ + } + for end > start && strings.TrimSpace(seg[end-1]) == "" { + end-- + } + seg = seg[start:end] + if len(seg) == 0 { + continue + } + + // Check for a "## Title" header on the first line. + title := "" + contentStart := 0 + if strings.HasPrefix(strings.TrimSpace(seg[0]), "## ") { + title = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(seg[0]), "## ")) + contentStart = 1 + // Skip blank lines after the title. + for contentStart < len(seg) && strings.TrimSpace(seg[contentStart]) == "" { + contentStart++ + } + } + + content := strings.TrimSpace(strings.Join(seg[contentStart:], "\n")) + if content == "" { + continue + } + + chNum++ + if title == "" { + title = fmt.Sprintf("Chapter %d", chNum) + } + chapters = append(chapters, bookstore.Chapter{ + Number: chNum, + Title: title, + Content: content, + }) + } + return chapters +} diff --git a/backend/internal/backend/handlers_submit.go b/backend/internal/backend/handlers_submit.go new file mode 100644 index 0000000..69f6ca2 --- /dev/null +++ b/backend/internal/backend/handlers_submit.go @@ -0,0 +1,161 @@ +package backend + +import ( + "encoding/json" + "errors" + "net/http" + "regexp" + "strings" + "unicode" + + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/storage" +) + +// handleAdminPublishBook handles PATCH /api/admin/books/{slug}/publish. +// Sets visibility=public so the book is visible to all users. +func (s *Server) handleAdminPublishBook(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + if s.deps.BookAdminStore == nil { + jsonError(w, http.StatusServiceUnavailable, "book admin store not configured") + return + } + if err := s.deps.BookAdminStore.PublishBook(r.Context(), slug); err != nil { + if errors.Is(err, storage.ErrNotFound) { + jsonError(w, http.StatusNotFound, "book not found") + return + } + s.deps.Log.Error("publish book failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, err.Error()) + return + } + // Sync the visibility change to Meilisearch immediately. + if meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), slug); err == nil && ok { + if upsertErr := s.deps.SearchIndex.UpsertBook(r.Context(), meta); upsertErr != nil { + s.deps.Log.Warn("publish book: meili upsert failed", "slug", slug, "err", upsertErr) + } + } + s.deps.Log.Info("book published", "slug", slug) + writeJSON(w, http.StatusOK, map[string]string{"slug": slug, "visibility": domain.VisibilityPublic}) +} + +// handleAdminUnpublishBook handles PATCH /api/admin/books/{slug}/unpublish. +// Sets visibility=admin_only, hiding the book from regular users. +func (s *Server) handleAdminUnpublishBook(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + if slug == "" { + jsonError(w, http.StatusBadRequest, "missing slug") + return + } + if s.deps.BookAdminStore == nil { + jsonError(w, http.StatusServiceUnavailable, "book admin store not configured") + return + } + if err := s.deps.BookAdminStore.UnpublishBook(r.Context(), slug); err != nil { + if errors.Is(err, storage.ErrNotFound) { + jsonError(w, http.StatusNotFound, "book not found") + return + } + s.deps.Log.Error("unpublish book failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, err.Error()) + return + } + // Sync to Meilisearch. + if meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), slug); err == nil && ok { + if upsertErr := s.deps.SearchIndex.UpsertBook(r.Context(), meta); upsertErr != nil { + s.deps.Log.Warn("unpublish book: meili upsert failed", "slug", slug, "err", upsertErr) + } + } + s.deps.Log.Info("book unpublished", "slug", slug) + writeJSON(w, http.StatusOK, map[string]string{"slug": slug, "visibility": domain.VisibilityAdminOnly}) +} + +// handleAdminSubmitBook handles POST /api/admin/books/submit. +// Creates a new author-submitted book with visibility=public. +// The book starts with zero chapters; chapters are added via the import pipeline. +func (s *Server) handleAdminSubmitBook(w http.ResponseWriter, r *http.Request) { + var req struct { + Title string `json:"title"` + Author string `json:"author"` + Cover string `json:"cover"` + Summary string `json:"summary"` + Genres []string `json:"genres"` + Status string `json:"status"` + SubmittedBy string `json:"submitted_by"` // app_users ID of submitting author + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "invalid request body") + return + } + req.Title = strings.TrimSpace(req.Title) + if req.Title == "" { + jsonError(w, http.StatusBadRequest, "title is required") + return + } + if req.Status == "" { + req.Status = "ongoing" + } + + slug := slugifyTitle(req.Title) + if slug == "" { + jsonError(w, http.StatusBadRequest, "could not derive a slug from title") + return + } + + if s.deps.BookAdminStore == nil { + jsonError(w, http.StatusServiceUnavailable, "book admin store not configured") + return + } + + meta := domain.BookMeta{ + Slug: slug, + Title: req.Title, + Author: req.Author, + Cover: req.Cover, + Summary: req.Summary, + Genres: req.Genres, + Status: req.Status, + Visibility: domain.VisibilityPublic, + SubmittedBy: req.SubmittedBy, + } + if err := s.deps.BookAdminStore.CreateSubmittedBook(r.Context(), meta); err != nil { + s.deps.Log.Error("submit book: create failed", "slug", slug, "err", err) + jsonError(w, http.StatusInternalServerError, "failed to create book") + return + } + + // Index in Meilisearch immediately so it appears in search/catalogue. + if upsertErr := s.deps.SearchIndex.UpsertBook(r.Context(), meta); upsertErr != nil { + s.deps.Log.Warn("submit book: meili upsert failed", "slug", slug, "err", upsertErr) + } + + s.deps.Log.Info("book submitted", "slug", slug, "title", req.Title, "by", req.SubmittedBy) + writeJSON(w, http.StatusCreated, map[string]string{"slug": slug}) +} + +// slugifyTitle converts a book title into a URL-safe slug. +// e.g. "The Wandering Sword" → "the-wandering-sword" +var nonAlnum = regexp.MustCompile(`[^a-z0-9]+`) + +func slugifyTitle(title string) string { + // Fold to lower-case ASCII, replace non-alphanum runs with hyphens. + var b strings.Builder + for _, r := range strings.ToLower(title) { + if r <= unicode.MaxASCII && (unicode.IsLetter(r) || unicode.IsDigit(r)) { + b.WriteRune(r) + } else { + b.WriteRune('-') + } + } + slug := nonAlnum.ReplaceAllString(b.String(), "-") + slug = strings.Trim(slug, "-") + if len(slug) > 80 { + slug = slug[:80] + slug = strings.TrimRight(slug, "-") + } + return slug +} diff --git a/backend/internal/backend/handlers_textgen.go b/backend/internal/backend/handlers_textgen.go new file mode 100644 index 0000000..f5f2be4 --- /dev/null +++ b/backend/internal/backend/handlers_textgen.go @@ -0,0 +1,991 @@ +package backend + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/libnovel/backend/internal/cfai" + "github.com/libnovel/backend/internal/domain" +) + +// chapterNamesBatchSize is the number of chapters sent per LLM request. +// Keeps output well within the 4096-token response limit (~30 tokens/title). +const chapterNamesBatchSize = 100 + +// handleAdminTextGenModels handles GET /api/admin/text-gen/models. +// Returns the list of supported Cloudflare AI text generation models. +func (s *Server) handleAdminTextGenModels(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil { + jsonError(w, http.StatusServiceUnavailable, "text generation not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN missing)") + return + } + models := s.deps.TextGen.Models() + writeJSON(w, 0, map[string]any{"models": models}) +} + +// ── Chapter names ───────────────────────────────────────────────────────────── + +// textGenChapterNamesRequest is the JSON body for POST /api/admin/text-gen/chapter-names. +type textGenChapterNamesRequest struct { + // Slug is the book slug whose chapters to process. + Slug string `json:"slug"` + // Pattern is a free-text description of the desired naming convention, + // e.g. "Chapter {n}: {brief scene description}". + Pattern string `json:"pattern"` + // Model is the CF Workers AI model ID. Defaults to the recommended model when empty. + Model string `json:"model"` + // MaxTokens limits response length (0 = model default). + MaxTokens int `json:"max_tokens"` + // FromChapter is the first chapter to process (1-based). 0 = start from chapter 1. + FromChapter int `json:"from_chapter"` + // ToChapter is the last chapter to process (inclusive). 0 = process all. + ToChapter int `json:"to_chapter"` + // JobID is an optional existing ai_job ID for resuming a previous run. + // If set, the handler resumes from items_done instead of starting from scratch. + JobID string `json:"job_id"` +} + +// proposedChapterTitle is a single chapter with its AI-proposed title. +type proposedChapterTitle struct { + Number int `json:"number"` + // OldTitle is the current title stored in the database. + OldTitle string `json:"old_title"` + // NewTitle is the AI-proposed replacement. + NewTitle string `json:"new_title"` +} + +// chapterNamesBatchEvent is one SSE event emitted per processed batch. +type chapterNamesBatchEvent struct { + // JobID is the PB ai_job ID for this run (emitted on the first event only). + JobID string `json:"job_id,omitempty"` + // Batch is the 1-based batch index. + Batch int `json:"batch"` + // TotalBatches is the total number of batches. + TotalBatches int `json:"total_batches"` + // ChaptersDone is the cumulative count of chapters processed so far. + ChaptersDone int `json:"chapters_done"` + // TotalChapters is the total chapter count for this book. + TotalChapters int `json:"total_chapters"` + // Model is the CF AI model used. + Model string `json:"model"` + // Chapters contains the proposed titles for this batch. + Chapters []proposedChapterTitle `json:"chapters"` + // Error is non-empty if this batch failed. + Error string `json:"error,omitempty"` + // Done is true on the final sentinel event (no Chapters). + Done bool `json:"done,omitempty"` +} + +// handleAdminTextGenChapterNames handles POST /api/admin/text-gen/chapter-names. +// +// Splits all chapters into batches of chapterNamesBatchSize, sends each batch +// to the LLM sequentially, and streams results back as Server-Sent Events so +// the frontend can show live progress. Each SSE data line is a JSON-encoded +// chapterNamesBatchEvent. The final event has Done=true. +// +// Does NOT persist anything — the frontend shows a diff and the user must +// confirm via POST /api/admin/text-gen/chapter-names/apply. +func (s *Server) handleAdminTextGenChapterNames(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil { + jsonError(w, http.StatusServiceUnavailable, "text generation not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN missing)") + return + } + + var req textGenChapterNamesRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if strings.TrimSpace(req.Pattern) == "" { + jsonError(w, http.StatusBadRequest, "pattern is required") + return + } + + // Load existing chapter list. + allChapters, err := s.deps.BookReader.ListChapters(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "list chapters: "+err.Error()) + return + } + if len(allChapters) == 0 { + jsonError(w, http.StatusNotFound, fmt.Sprintf("no chapters found for slug %q", req.Slug)) + return + } + + // Apply chapter range filter. + chapters := allChapters + if req.FromChapter > 0 || req.ToChapter > 0 { + filtered := chapters[:0] + for _, ch := range allChapters { + if req.FromChapter > 0 && ch.Number < req.FromChapter { + continue + } + if req.ToChapter > 0 && ch.Number > req.ToChapter { + break + } + filtered = append(filtered, ch) + } + chapters = filtered + } + if len(chapters) == 0 { + jsonError(w, http.StatusBadRequest, "no chapters in the specified range") + return + } + + model := cfai.TextModel(req.Model) + if model == "" { + model = cfai.DefaultTextModel + } + // 4096 tokens comfortably fits 100 chapter titles (~30 tokens each). + maxTokens := req.MaxTokens + if maxTokens <= 0 { + maxTokens = 4096 + } + + // Index existing titles for old/new diff. + existing := make(map[int]string, len(chapters)) + for _, ch := range chapters { + existing[ch.Number] = ch.Title + } + + // Partition chapters into batches. + batches := chunkChapters(chapters, chapterNamesBatchSize) + totalBatches := len(batches) + + s.deps.Log.Info("admin: text-gen chapter-names requested", + "slug", req.Slug, "chapters", len(chapters), + "batches", totalBatches, "model", model, "max_tokens", maxTokens) + + systemPrompt := `You are a chapter title editor for a web novel platform. ` + + `The user provides a list of chapter numbers with their current titles, ` + + `and a naming pattern template. ` + + `Your job: produce one new title for every chapter, following the pattern exactly. ` + + `Pattern placeholders: {n} = the chapter number (integer), {scene} = a very short (2–5 word) scene hint derived from the existing title. ` + + `RULES: ` + + `1. Do NOT include the chapter number inside the title text — the {n} placeholder is already in the pattern. ` + + `2. Do NOT include any prefix like "Chapter X -" or "Chapter X:" inside the title field itself. ` + + `3. The "title" field in your JSON must be the fully-rendered string (e.g. if pattern is "Chapter {n}: {scene}", output "Chapter 3: The Bet"). ` + + `4. Respond ONLY with a raw JSON array — no prose, no markdown fences, no explanation. ` + + `5. Each element: {"number": <int>, "title": <string>}. ` + + `6. Output every chapter in the input list, in order. Do not skip any.` + + // Switch to SSE before writing anything. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("X-Accel-Buffering", "no") // disable nginx/caddy buffering + flusher, canFlush := w.(http.Flusher) + + sseWrite := func(evt chapterNamesBatchEvent) { + b, _ := json.Marshal(evt) + fmt.Fprintf(w, "data: %s\n\n", b) + if canFlush { + flusher.Flush() + } + } + + // Create or resume an ai_job record for tracking. + var jobID string + resumeFrom := 0 + jobCtx := r.Context() + var jobCancel context.CancelFunc + + if s.deps.AIJobStore != nil { + if req.JobID != "" { + if existingJob, ok, _ := s.deps.AIJobStore.GetAIJob(r.Context(), req.JobID); ok { + jobID = req.JobID + resumeFrom = existingJob.ItemsDone + _ = s.deps.AIJobStore.UpdateAIJob(r.Context(), jobID, map[string]any{ + "status": string(domain.TaskStatusRunning), + "items_total": len(chapters), + }) + } + } + if jobID == "" { + jobPayload := fmt.Sprintf(`{"pattern":%q}`, req.Pattern) + id, createErr := s.deps.AIJobStore.CreateAIJob(r.Context(), domain.AIJob{ + Kind: "chapter-names", + Slug: req.Slug, + Status: domain.TaskStatusRunning, + FromItem: req.FromChapter, + ToItem: req.ToChapter, + ItemsTotal: len(chapters), + Model: string(model), + Payload: jobPayload, + Started: time.Now(), + }) + if createErr == nil { + jobID = id + } + } + if jobID != "" { + jobCtx, jobCancel = context.WithCancel(r.Context()) + registerCancelJob(jobID, jobCancel) + defer deregisterCancelJob(jobID) + defer jobCancel() + } + } + + var allResults []proposedChapterTitle + chaptersDone := resumeFrom + firstEvent := true + for i, batch := range batches { + if jobCtx.Err() != nil { + return // client disconnected or cancelled + } + // Skip batches already processed in a previous run. + batchEnd := (i + 1) * chapterNamesBatchSize + if batchEnd <= resumeFrom { + continue + } + + var chapterListSB strings.Builder + for _, ch := range batch { + chapterListSB.WriteString(fmt.Sprintf("%d: %s\n", ch.Number, ch.Title)) + } + userPrompt := fmt.Sprintf("Naming pattern: %s\n\nChapters:\n%s", req.Pattern, chapterListSB.String()) + + raw, genErr := s.deps.TextGen.Generate(jobCtx, cfai.TextRequest{ + Model: model, + Messages: []cfai.TextMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + }, + MaxTokens: maxTokens, + }) + if genErr != nil { + s.deps.Log.Error("admin: text-gen chapter-names batch failed", + "batch", i+1, "err", genErr) + evt := chapterNamesBatchEvent{ + Batch: i + 1, + TotalBatches: totalBatches, + ChaptersDone: chaptersDone, + TotalChapters: len(chapters), + Model: string(model), + Error: genErr.Error(), + } + if firstEvent { + evt.JobID = jobID + firstEvent = false + } + sseWrite(evt) + continue + } + + proposed := parseChapterTitlesJSON(raw) + result := make([]proposedChapterTitle, 0, len(proposed)) + for _, p := range proposed { + result = append(result, proposedChapterTitle{ + Number: p.Number, + OldTitle: existing[p.Number], + NewTitle: p.Title, + }) + } + allResults = append(allResults, result...) + chaptersDone += len(batch) + + if jobID != "" && s.deps.AIJobStore != nil { + _ = s.deps.AIJobStore.UpdateAIJob(r.Context(), jobID, map[string]any{ + "items_done": chaptersDone, + }) + } + + evt := chapterNamesBatchEvent{ + Batch: i + 1, + TotalBatches: totalBatches, + ChaptersDone: chaptersDone, + TotalChapters: len(chapters), + Model: string(model), + Chapters: result, + } + if firstEvent { + evt.JobID = jobID + firstEvent = false + } + sseWrite(evt) + } + + // Mark job as done in PB, persisting results so the Review button works. + // Use context.Background() — r.Context() may be cancelled if the SSE client + // disconnected before processing finished, which would silently drop results. + if jobID != "" && s.deps.AIJobStore != nil { + status := domain.TaskStatusDone + if jobCtx.Err() != nil { + status = domain.TaskStatusCancelled + } + resultsJSON, _ := json.Marshal(allResults) + finalPayload := fmt.Sprintf(`{"pattern":%q,"slug":%q,"results":%s}`, + req.Pattern, req.Slug, string(resultsJSON)) + _ = s.deps.AIJobStore.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": string(status), + "items_done": chaptersDone, + "finished": time.Now().Format(time.RFC3339), + "payload": finalPayload, + }) + } + + // Final sentinel event. + sseWrite(chapterNamesBatchEvent{Done: true, TotalChapters: len(chapters), Model: string(model)}) +} + +// chunkChapters splits a chapter slice into batches of at most size n. +func chunkChapters(chapters []domain.ChapterInfo, n int) [][]domain.ChapterInfo { + var batches [][]domain.ChapterInfo + for len(chapters) > 0 { + end := n + if end > len(chapters) { + end = len(chapters) + } + batches = append(batches, chapters[:end]) + chapters = chapters[end:] + } + return batches +} + +// parseChapterTitlesJSON extracts the JSON array from a model response. +// It tolerates markdown fences and surrounding prose. +type rawChapterTitle struct { + Number int `json:"number"` + Title string `json:"title"` +} + +func parseChapterTitlesJSON(raw string) []rawChapterTitle { + // Strip markdown fences if present. + s := raw + if idx := strings.Index(s, "```json"); idx >= 0 { + s = s[idx+7:] + } else if idx := strings.Index(s, "```"); idx >= 0 { + s = s[idx+3:] + } + if idx := strings.LastIndex(s, "```"); idx >= 0 { + s = s[:idx] + } + // Find the JSON array boundaries. + start := strings.Index(s, "[") + end := strings.LastIndex(s, "]") + if start < 0 || end <= start { + return nil + } + s = s[start : end+1] + var out []rawChapterTitle + json.Unmarshal([]byte(s), &out) //nolint:errcheck + return out +} + +// handleAdminTextGenChapterNamesAsync handles POST /api/admin/text-gen/chapter-names/async. +// +// Fire-and-forget variant: validates inputs, creates an ai_job record, spawns a +// background goroutine, and returns HTTP 202 with {job_id} immediately. The +// goroutine runs all batches, stores the proposed titles in the job payload, and +// marks the job done/failed/cancelled when finished. +// +// The client can poll GET /api/admin/ai-jobs/{id} for progress, then call +// POST /api/admin/text-gen/chapter-names/apply once the job is "done". +func (s *Server) handleAdminTextGenChapterNamesAsync(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil { + jsonError(w, http.StatusServiceUnavailable, "text generation not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN missing)") + return + } + + var req textGenChapterNamesRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if strings.TrimSpace(req.Pattern) == "" { + jsonError(w, http.StatusBadRequest, "pattern is required") + return + } + + // Load existing chapter list (use request context — just for validation). + allChapters, err := s.deps.BookReader.ListChapters(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "list chapters: "+err.Error()) + return + } + if len(allChapters) == 0 { + jsonError(w, http.StatusNotFound, fmt.Sprintf("no chapters found for slug %q", req.Slug)) + return + } + + // Apply chapter range filter. + chapters := allChapters + if req.FromChapter > 0 || req.ToChapter > 0 { + filtered := chapters[:0] + for _, ch := range allChapters { + if req.FromChapter > 0 && ch.Number < req.FromChapter { + continue + } + if req.ToChapter > 0 && ch.Number > req.ToChapter { + break + } + filtered = append(filtered, ch) + } + chapters = filtered + } + if len(chapters) == 0 { + jsonError(w, http.StatusBadRequest, "no chapters in the specified range") + return + } + + model := cfai.TextModel(req.Model) + if model == "" { + model = cfai.DefaultTextModel + } + maxTokens := req.MaxTokens + if maxTokens <= 0 { + maxTokens = 4096 + } + + // Index existing titles for old/new diff. + existing := make(map[int]string, len(chapters)) + for _, ch := range chapters { + existing[ch.Number] = ch.Title + } + + batches := chunkChapters(chapters, chapterNamesBatchSize) + totalBatches := len(batches) + + if s.deps.AIJobStore == nil { + jsonError(w, http.StatusServiceUnavailable, "ai job store not configured") + return + } + + jobPayload := fmt.Sprintf(`{"pattern":%q}`, req.Pattern) + jobID, createErr := s.deps.AIJobStore.CreateAIJob(r.Context(), domain.AIJob{ + Kind: "chapter-names", + Slug: req.Slug, + Status: domain.TaskStatusPending, + FromItem: req.FromChapter, + ToItem: req.ToChapter, + ItemsTotal: len(chapters), + Model: string(model), + Payload: jobPayload, + Started: time.Now(), + }) + if createErr != nil { + jsonError(w, http.StatusInternalServerError, "create ai job: "+createErr.Error()) + return + } + + jobCtx, jobCancel := context.WithCancel(context.Background()) + registerCancelJob(jobID, jobCancel) + + s.deps.Log.Info("admin: text-gen chapter-names async started", + "job_id", jobID, "slug", req.Slug, + "chapters", len(chapters), "batches", totalBatches, "model", model) + + // Mark running before returning so the UI sees it immediately. + _ = s.deps.AIJobStore.UpdateAIJob(r.Context(), jobID, map[string]any{ + "status": string(domain.TaskStatusRunning), + }) + + systemPrompt := `You are a chapter title editor for a web novel platform. ` + + `The user provides a list of chapter numbers with their current titles, ` + + `and a naming pattern template. ` + + `Your job: produce one new title for every chapter, following the pattern exactly. ` + + `Pattern placeholders: {n} = the chapter number (integer), {scene} = a very short (2–5 word) scene hint derived from the existing title. ` + + `RULES: ` + + `1. Do NOT include the chapter number inside the title text — the {n} placeholder is already in the pattern. ` + + `2. Do NOT include any prefix like "Chapter X -" or "Chapter X:" inside the title field itself. ` + + `3. The "title" field in your JSON must be the fully-rendered string (e.g. if pattern is "Chapter {n}: {scene}", output "Chapter 3: The Bet"). ` + + `4. Respond ONLY with a raw JSON array — no prose, no markdown fences, no explanation. ` + + `5. Each element: {"number": <int>, "title": <string>}. ` + + `6. Output every chapter in the input list, in order. Do not skip any.` + + // Capture all locals needed in the goroutine. + store := s.deps.AIJobStore + textGen := s.deps.TextGen + logger := s.deps.Log + capturedModel := model + capturedMaxTokens := maxTokens + capturedPattern := req.Pattern + capturedSlug := req.Slug + + go func() { + defer deregisterCancelJob(jobID) + defer jobCancel() + defer func() { + if r := recover(); r != nil { + logger.Error("admin: text-gen chapter-names goroutine panic", "job_id", jobID, "recover", r) + } + }() + + var allResults []proposedChapterTitle + chaptersDone := 0 + + for i, batch := range batches { + if jobCtx.Err() != nil { + _ = store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": string(domain.TaskStatusCancelled), + "finished": time.Now().Format(time.RFC3339), + }) + return + } + + var chapterListSB strings.Builder + for _, ch := range batch { + chapterListSB.WriteString(fmt.Sprintf("%d: %s\n", ch.Number, ch.Title)) + } + userPrompt := fmt.Sprintf("Naming pattern: %s\n\nChapters:\n%s", capturedPattern, chapterListSB.String()) + + raw, genErr := textGen.Generate(jobCtx, cfai.TextRequest{ + Model: capturedModel, + Messages: []cfai.TextMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + }, + MaxTokens: capturedMaxTokens, + }) + if genErr != nil { + logger.Error("admin: text-gen chapter-names async batch failed", + "job_id", jobID, "batch", i+1, "err", genErr) + // Continue — skip errored batch rather than aborting. + continue + } + + proposed := parseChapterTitlesJSON(raw) + for _, p := range proposed { + allResults = append(allResults, proposedChapterTitle{ + Number: p.Number, + OldTitle: existing[p.Number], + NewTitle: p.Title, + }) + } + chaptersDone += len(batch) + + _ = store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "items_done": chaptersDone, + }) + } + + // Persist results into payload so the UI can load them for review. + resultsJSON, _ := json.Marshal(allResults) + finalPayload := fmt.Sprintf(`{"pattern":%q,"slug":%q,"results":%s}`, + capturedPattern, capturedSlug, string(resultsJSON)) + + status := domain.TaskStatusDone + if jobCtx.Err() != nil { + status = domain.TaskStatusCancelled + } + if err := store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": string(status), + "items_done": chaptersDone, + "finished": time.Now().Format(time.RFC3339), + }); err != nil { + logger.Error("admin: text-gen chapter-names failed to mark job done", "job_id", jobID, "err", err) + } + if err := store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "payload": finalPayload, + }); err != nil { + logger.Error("admin: text-gen chapter-names failed to write job payload", "job_id", jobID, "err", err) + } + logger.Info("admin: text-gen chapter-names async done", + "job_id", jobID, "slug", capturedSlug, + "results", len(allResults), "status", string(status)) + }() + + writeJSON(w, http.StatusAccepted, map[string]any{"job_id": jobID}) +} + +// ── Apply chapter names ─────────────────────────────────────────────────────── + +// applyChapterNamesRequest is the JSON body for POST /api/admin/text-gen/chapter-names/apply. +type applyChapterNamesRequest struct { + // Slug is the book slug to update. + Slug string `json:"slug"` + // Chapters is the list of chapters to save (number + new_title pairs). + // The UI may modify individual titles before confirming. + Chapters []applyChapterEntry `json:"chapters"` +} + +type applyChapterEntry struct { + Number int `json:"number"` + Title string `json:"title"` +} + +// handleAdminTextGenApplyChapterNames handles POST /api/admin/text-gen/chapter-names/apply. +// +// Persists the confirmed chapter titles to PocketBase chapters_idx. +func (s *Server) handleAdminTextGenApplyChapterNames(w http.ResponseWriter, r *http.Request) { + if s.deps.BookWriter == nil { + jsonError(w, http.StatusServiceUnavailable, "book writer not configured") + return + } + + var req applyChapterNamesRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if len(req.Chapters) == 0 { + jsonError(w, http.StatusBadRequest, "chapters is required") + return + } + + refs := make([]domain.ChapterRef, 0, len(req.Chapters)) + for _, ch := range req.Chapters { + if ch.Number <= 0 { + continue + } + refs = append(refs, domain.ChapterRef{ + Number: ch.Number, + Title: strings.TrimSpace(ch.Title), + }) + } + + if err := s.deps.BookWriter.WriteChapterRefs(r.Context(), req.Slug, refs); err != nil { + s.deps.Log.Error("admin: apply chapter names failed", "slug", req.Slug, "err", err) + jsonError(w, http.StatusInternalServerError, "write chapter refs: "+err.Error()) + return + } + + s.deps.Log.Info("admin: chapter names applied", "slug", req.Slug, "count", len(refs)) + writeJSON(w, 0, map[string]any{"updated": len(refs)}) +} + +// ── Book description ────────────────────────────────────────────────────────── + +// textGenDescriptionRequest is the JSON body for POST /api/admin/text-gen/description. +type textGenDescriptionRequest struct { + // Slug is the book slug whose description to regenerate. + Slug string `json:"slug"` + // Instructions is an optional free-text hint for the AI, + // e.g. "Write a 3-sentence blurb, avoid spoilers, dramatic tone." + Instructions string `json:"instructions"` + // Model is the CF Workers AI model ID. Defaults to recommended when empty. + Model string `json:"model"` + // MaxTokens limits response length (0 = model default). + MaxTokens int `json:"max_tokens"` +} + +// textGenDescriptionResponse is the JSON body returned by POST /api/admin/text-gen/description. +type textGenDescriptionResponse struct { + // OldDescription is the current summary stored in the database. + OldDescription string `json:"old_description"` + // NewDescription is the AI-proposed replacement. + NewDescription string `json:"new_description"` + // Model is the model that was used. + Model string `json:"model"` +} + +// handleAdminTextGenDescription handles POST /api/admin/text-gen/description. +// +// Reads the current book metadata, sends it to the LLM, and returns a proposed +// new description. Does NOT persist anything — the user must confirm via +// POST /api/admin/text-gen/description/apply. +func (s *Server) handleAdminTextGenDescription(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil { + jsonError(w, http.StatusServiceUnavailable, "text generation not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN missing)") + return + } + + var req textGenDescriptionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + + // Load current book metadata. + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", req.Slug)) + return + } + + systemPrompt := `You are a book description writer for a web novel platform. ` + + `Given a book's title, author, genres, and current description, write an improved ` + + `description that accurately captures the story. ` + + `Respond with ONLY the new description text — no title, no labels, no markdown, no quotes.` + + instructions := strings.TrimSpace(req.Instructions) + if instructions == "" { + instructions = "Write a compelling 2–4 sentence description. Keep it spoiler-free and engaging." + } + + userPrompt := fmt.Sprintf( + "Title: %s\nAuthor: %s\nGenres: %s\nStatus: %s\n\nCurrent description:\n%s\n\nInstructions: %s", + meta.Title, + meta.Author, + strings.Join(meta.Genres, ", "), + meta.Status, + meta.Summary, + instructions, + ) + + model := cfai.TextModel(req.Model) + if model == "" { + model = cfai.DefaultTextModel + } + + s.deps.Log.Info("admin: text-gen description requested", + "slug", req.Slug, "model", model) + + newDesc, genErr := s.deps.TextGen.Generate(r.Context(), cfai.TextRequest{ + Model: model, + Messages: []cfai.TextMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + }, + MaxTokens: req.MaxTokens, + }) + if genErr != nil { + s.deps.Log.Error("admin: text-gen description failed", "err", genErr) + jsonError(w, http.StatusBadGateway, "text generation failed: "+genErr.Error()) + return + } + + writeJSON(w, 0, textGenDescriptionResponse{ + OldDescription: meta.Summary, + NewDescription: strings.TrimSpace(newDesc), + Model: string(model), + }) +} + +// ── Apply description ───────────────────────────────────────────────────────── + +// applyDescriptionRequest is the JSON body for POST /api/admin/text-gen/description/apply. +type applyDescriptionRequest struct { + // Slug is the book slug to update. + Slug string `json:"slug"` + // Description is the new summary text to persist. + Description string `json:"description"` +} + +// handleAdminTextGenApplyDescription handles POST /api/admin/text-gen/description/apply. +// +// Updates only the summary field in PocketBase, leaving all other book metadata +// unchanged. +func (s *Server) handleAdminTextGenApplyDescription(w http.ResponseWriter, r *http.Request) { + if s.deps.BookWriter == nil { + jsonError(w, http.StatusServiceUnavailable, "book writer not configured") + return + } + + var req applyDescriptionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + if strings.TrimSpace(req.Description) == "" { + jsonError(w, http.StatusBadRequest, "description is required") + return + } + + // Read existing metadata so we can write it back with only summary changed. + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", req.Slug)) + return + } + + meta.Summary = strings.TrimSpace(req.Description) + if err := s.deps.BookWriter.WriteMetadata(r.Context(), meta); err != nil { + s.deps.Log.Error("admin: apply description failed", "slug", req.Slug, "err", err) + jsonError(w, http.StatusInternalServerError, "write metadata: "+err.Error()) + return + } + + s.deps.Log.Info("admin: book description applied", "slug", req.Slug) + writeJSON(w, 0, map[string]any{"updated": true}) +} + +// handleAdminTextGenDescriptionAsync handles POST /api/admin/text-gen/description/async. +// +// Fire-and-forget variant: validates inputs, creates an ai_job record of kind +// "description", spawns a background goroutine that calls the LLM, stores the +// old/new description in the job payload, and marks the job done/failed. +// Returns HTTP 202 with {job_id} immediately. +func (s *Server) handleAdminTextGenDescriptionAsync(w http.ResponseWriter, r *http.Request) { + if s.deps.TextGen == nil { + jsonError(w, http.StatusServiceUnavailable, "text generation not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN missing)") + return + } + if s.deps.AIJobStore == nil { + jsonError(w, http.StatusServiceUnavailable, "ai job store not configured") + return + } + + var req textGenDescriptionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, http.StatusBadRequest, "parse body: "+err.Error()) + return + } + if strings.TrimSpace(req.Slug) == "" { + jsonError(w, http.StatusBadRequest, "slug is required") + return + } + + // Load current metadata eagerly so we can fail fast if the book is missing. + meta, ok, err := s.deps.BookReader.ReadMetadata(r.Context(), req.Slug) + if err != nil { + jsonError(w, http.StatusInternalServerError, "read metadata: "+err.Error()) + return + } + if !ok { + jsonError(w, http.StatusNotFound, fmt.Sprintf("book %q not found", req.Slug)) + return + } + + model := cfai.TextModel(req.Model) + if model == "" { + model = cfai.DefaultTextModel + } + + instructions := strings.TrimSpace(req.Instructions) + if instructions == "" { + instructions = "Write a compelling 2–4 sentence description. Keep it spoiler-free and engaging." + } + + // Encode the initial params (without result) as the starting payload. + type initPayload struct { + Instructions string `json:"instructions"` + OldDescription string `json:"old_description"` + } + initJSON, _ := json.Marshal(initPayload{ + Instructions: instructions, + OldDescription: meta.Summary, + }) + + jobID, createErr := s.deps.AIJobStore.CreateAIJob(r.Context(), domain.AIJob{ + Kind: "description", + Slug: req.Slug, + Status: domain.TaskStatusPending, + Model: string(model), + Payload: string(initJSON), + Started: time.Now(), + }) + if createErr != nil { + jsonError(w, http.StatusInternalServerError, "create ai job: "+createErr.Error()) + return + } + + jobCtx, jobCancel := context.WithCancel(context.Background()) + registerCancelJob(jobID, jobCancel) + + _ = s.deps.AIJobStore.UpdateAIJob(r.Context(), jobID, map[string]any{ + "status": string(domain.TaskStatusRunning), + }) + + s.deps.Log.Info("admin: text-gen description async started", + "job_id", jobID, "slug", req.Slug, "model", model) + + // Capture locals. + store := s.deps.AIJobStore + textGen := s.deps.TextGen + logger := s.deps.Log + capturedMeta := meta + capturedModel := model + capturedInstructions := instructions + capturedMaxTokens := req.MaxTokens + + go func() { + defer deregisterCancelJob(jobID) + defer jobCancel() + defer func() { + if r := recover(); r != nil { + logger.Error("admin: text-gen description goroutine panic", "job_id", jobID, "recover", r) + } + }() + + if jobCtx.Err() != nil { + _ = store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": string(domain.TaskStatusCancelled), + "finished": time.Now().Format(time.RFC3339), + }) + return + } + + systemPrompt := `You are a book description writer for a web novel platform. ` + + `Given a book's title, author, genres, and current description, write an improved ` + + `description that accurately captures the story. ` + + `Respond with ONLY the new description text — no title, no labels, no markdown, no quotes.` + + userPrompt := fmt.Sprintf( + "Title: %s\nAuthor: %s\nGenres: %s\nStatus: %s\n\nCurrent description:\n%s\n\nInstructions: %s", + capturedMeta.Title, + capturedMeta.Author, + strings.Join(capturedMeta.Genres, ", "), + capturedMeta.Status, + capturedMeta.Summary, + capturedInstructions, + ) + + newDesc, genErr := textGen.Generate(jobCtx, cfai.TextRequest{ + Model: capturedModel, + Messages: []cfai.TextMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + }, + MaxTokens: capturedMaxTokens, + }) + if genErr != nil { + logger.Error("admin: text-gen description async failed", "job_id", jobID, "err", genErr) + _ = store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": string(domain.TaskStatusFailed), + "error_message": genErr.Error(), + "finished": time.Now().Format(time.RFC3339), + }) + return + } + + type resultPayload struct { + Instructions string `json:"instructions"` + OldDescription string `json:"old_description"` + NewDescription string `json:"new_description"` + } + resultJSON, _ := json.Marshal(resultPayload{ + Instructions: capturedInstructions, + OldDescription: capturedMeta.Summary, + NewDescription: strings.TrimSpace(newDesc), + }) + + if err := store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "status": string(domain.TaskStatusDone), + "items_done": 1, + "items_total": 1, + "finished": time.Now().Format(time.RFC3339), + }); err != nil { + logger.Error("admin: text-gen description failed to mark job done", "job_id", jobID, "err", err) + } + if err := store.UpdateAIJob(context.Background(), jobID, map[string]any{ + "payload": string(resultJSON), + }); err != nil { + logger.Error("admin: text-gen description failed to write job payload", "job_id", jobID, "err", err) + } + logger.Info("admin: text-gen description async done", "job_id", jobID, "slug", capturedMeta.Slug) + }() + + writeJSON(w, http.StatusAccepted, map[string]any{"job_id": jobID}) +} diff --git a/backend/internal/backend/server.go b/backend/internal/backend/server.go new file mode 100644 index 0000000..befc065 --- /dev/null +++ b/backend/internal/backend/server.go @@ -0,0 +1,564 @@ +// Package backend implements the HTTP API server for the LibNovel backend. +// +// The server exposes all endpoints consumed by the SvelteKit UI: +// - Book/chapter reads from PocketBase/MinIO via bookstore interfaces +// - Task creation (scrape + audio) via taskqueue.Producer — the runner binary +// picks up and executes those tasks asynchronously +// - Presigned MinIO URLs for media playback/upload +// - Session-scoped reading progress +// - Live novelfire.net search (no scraper interface needed; direct HTTP) +// - Kokoro voice list +// +// The backend never scrapes directly. All scraping (metadata, chapter list, +// chapter text, audio TTS) is delegated to the runner binary via PocketBase +// task records. GET /api/book-preview enqueues a task when the book is absent. +// +// All external dependencies are injected as interfaces; concrete types live in +// internal/storage and are wired by cmd/backend/main.go. +package backend + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "sync" + "time" + + sentryhttp "github.com/getsentry/sentry-go/http" + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/cfai" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/meili" + "github.com/libnovel/backend/internal/pockettts" + "github.com/libnovel/backend/internal/taskqueue" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +// Dependencies holds all external services the backend server depends on. +// Every field is an interface so test doubles can be injected freely. +type Dependencies struct { + // BookReader reads book metadata and chapter text from PocketBase/MinIO. + BookReader bookstore.BookReader + // RankingStore reads ranking data from PocketBase. + RankingStore bookstore.RankingStore + // AudioStore checks audio object existence and computes MinIO keys. + AudioStore bookstore.AudioStore + // TranslationStore checks translation existence and reads/writes translated markdown. + TranslationStore bookstore.TranslationStore + // PresignStore generates short-lived MinIO URLs. + PresignStore bookstore.PresignStore + // ProgressStore reads/writes per-session reading progress. + ProgressStore bookstore.ProgressStore + // CoverStore reads and writes book cover images from MinIO. + // If nil, the cover endpoint falls back to a CDN redirect. + CoverStore bookstore.CoverStore + // ChapterImageStore reads and writes per-chapter illustration images from MinIO. + // If nil, chapter image endpoints return 404/503. + ChapterImageStore bookstore.ChapterImageStore + // Producer creates scrape/audio tasks in PocketBase. + Producer taskqueue.Producer + // TaskReader reads scrape/audio task records from PocketBase. + TaskReader taskqueue.Reader + // SearchIndex provides full-text book search via Meilisearch. + // If nil, the local-only fallback search is used. + SearchIndex meili.Client + // Kokoro is the Kokoro TTS client (used for voice list only in the backend; + // audio generation is done by the runner). + Kokoro kokoro.Client + // PocketTTS is the pocket-tts client (used for voice list only in the backend; + // audio generation is done by the runner). + PocketTTS pockettts.Client + // CFAI is the Cloudflare Workers AI TTS client (used for voice sample + // generation and audio-stream live TTS; audio task generation is done by the runner). + CFAI cfai.Client + // ImageGen is the Cloudflare Workers AI image generation client. + // If nil, image generation endpoints return 503. + ImageGen cfai.ImageGenClient + // TextGen is the Cloudflare Workers AI text generation client. + // If nil, text generation endpoints return 503. + TextGen cfai.TextGenClient + // BookWriter writes book metadata and chapter refs to PocketBase. + // Used by admin text-gen apply endpoints. + BookWriter bookstore.BookWriter + // ImportFileStore uploads raw PDF/EPUB files to MinIO for the runner to process. + // Always wired to the concrete *storage.Store (not the Asynq wrapper). + ImportFileStore bookstore.ImportFileStore + // AIJobStore tracks long-running AI generation jobs in PocketBase. + // If nil, job persistence is disabled (jobs still run but are not recorded). + AIJobStore bookstore.AIJobStore + // BookAdminStore provides admin-only operations: archive, unarchive, hard-delete. + // If nil, the admin book management endpoints return 503. + BookAdminStore bookstore.BookAdminStore + // NotificationStore manages per-user in-app notifications. + // Always wired directly to *storage.Store (not the Asynq wrapper) so + // notification endpoints work regardless of whether Redis/Asynq is in use. + NotificationStore bookstore.NotificationStore + // Log is the structured logger. + Log *slog.Logger +} + +// Config holds HTTP server tuning parameters. +type Config struct { + // Addr is the listen address, e.g. ":8080". + Addr string + // DefaultVoice is used when no voice is specified in audio requests. + DefaultVoice string + // Version and Commit are embedded in /health and /api/version responses. + Version string + Commit string + // AdminToken is the bearer token required for all /api/admin/* endpoints. + // When empty a startup warning is logged and admin routes are unprotected. + AdminToken string +} + +// Server is the HTTP API server. +type Server struct { + cfg Config + deps Dependencies + + // voiceMu guards cachedVoices. Populated lazily on first GET /api/voices. + voiceMu sync.RWMutex + cachedVoices []domain.Voice +} + +// New creates a Server from cfg and deps. +func New(cfg Config, deps Dependencies) *Server { + if cfg.DefaultVoice == "" { + cfg.DefaultVoice = "af_bella" + } + if deps.Log == nil { + deps.Log = slog.Default() + } + if deps.SearchIndex == nil { + deps.SearchIndex = meili.NoopClient{} + } + return &Server{cfg: cfg, deps: deps} +} + +// requireAdmin returns a handler that enforces Bearer token authentication. +// When AdminToken is empty all requests are allowed through (with a warning logged +// once at startup via ListenAndServe). +func (s *Server) requireAdmin(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if s.cfg.AdminToken == "" { + next(w, r) + return + } + if r.Header.Get("Authorization") != "Bearer "+s.cfg.AdminToken { + jsonError(w, http.StatusUnauthorized, "unauthorized") + return + } + next(w, r) + } +} + +// ListenAndServe registers all routes and starts the HTTP server. +// It blocks until ctx is cancelled, then performs a graceful shutdown. +func (s *Server) ListenAndServe(ctx context.Context) error { + if s.cfg.AdminToken == "" { + s.deps.Log.Warn("backend: BACKEND_ADMIN_TOKEN is not set — /api/admin/* endpoints are unprotected") + } + + mux := http.NewServeMux() + + // Health / version + mux.HandleFunc("GET /health", s.handleHealth) + mux.HandleFunc("GET /api/version", s.handleVersion) + + // Scrape task creation (202 Accepted — runner executes asynchronously) + mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) + mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) + mux.HandleFunc("POST /scrape/book/range", s.handleScrapeBookRange) + + // Scrape task status / history + mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus) + mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks) + + // Cancel a pending task (scrape or audio) + mux.HandleFunc("POST /api/cancel-task/{id}", s.handleCancelTask) + + // Browse & search + mux.HandleFunc("GET /api/search", s.handleSearch) + + // Catalogue (Meilisearch-backed browse + search — preferred path for UI) + mux.HandleFunc("GET /api/catalogue", s.handleCatalogue) + + // Ranking (from PocketBase) + mux.HandleFunc("GET /api/ranking", s.handleGetRanking) + + // Cover proxy (live URL redirect) + mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover) + + // Book preview (enqueues scrape task if not in library; returns stored data if already scraped) + mux.HandleFunc("GET /api/book-preview/{slug}", s.handleBookPreview) + + // Chapter text (served from MinIO via PocketBase index) + mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText) + // Raw markdown chapter content — served directly from MinIO by the backend. + // Use this instead of presign+fetch to avoid SvelteKit→MinIO network path. + mux.HandleFunc("GET /api/chapter-markdown/{slug}/{n}", s.handleChapterMarkdown) + + // Chapter text preview — live scrape from novelfire.net, no store writes. + // Used when the chapter is not yet in the library (preview mode). + mux.HandleFunc("GET /api/chapter-text-preview/{slug}/{n}", s.handleChapterTextPreview) + + // Reindex chapters_idx from MinIO + mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex) + + // Audio task creation (backend creates task; runner executes) + mux.HandleFunc("POST /api/audio/{slug}/{n}", s.handleAudioGenerate) + mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus) + mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy) + // Streaming audio: serves from MinIO if cached, else streams live TTS + // while simultaneously uploading to MinIO for future requests. + mux.HandleFunc("GET /api/audio-stream/{slug}/{n}", s.handleAudioStream) + // TTS for arbitrary short text (chapter announcements) — no MinIO caching. + mux.HandleFunc("GET /api/tts-announce", s.handleTTSAnnounce) + // CF AI preview: generates only the first ~1 800-char chunk so the client + // can start playing immediately while the full audio is generated by the runner. + mux.HandleFunc("GET /api/audio-preview/{slug}/{n}", s.handleAudioPreview) + + // Translation task creation (backend creates task; runner executes via LibreTranslate) + mux.HandleFunc("POST /api/translation/{slug}/{n}", s.handleTranslationGenerate) + mux.HandleFunc("GET /api/translation/status/{slug}/{n}", s.handleTranslationStatus) + mux.HandleFunc("GET /api/translation/{slug}/{n}", s.handleTranslationRead) + + // admin is a shorthand that wraps every /api/admin/* handler with bearer-token auth. + admin := func(pattern string, h http.HandlerFunc) { + mux.HandleFunc(pattern, s.requireAdmin(h)) + } + + // Admin translation endpoints + admin("GET /api/admin/translation/jobs", s.handleAdminTranslationJobs) + admin("POST /api/admin/translation/bulk", s.handleAdminTranslationBulk) + + // Admin audio endpoints + admin("GET /api/admin/audio/jobs", s.handleAdminAudioJobs) + admin("POST /api/admin/audio/bulk", s.handleAdminAudioBulk) + admin("POST /api/admin/audio/cancel-bulk", s.handleAdminAudioCancelBulk) + + // Admin image generation endpoints + admin("GET /api/admin/image-gen/models", s.handleAdminImageGenModels) + admin("POST /api/admin/image-gen", s.handleAdminImageGen) + admin("POST /api/admin/image-gen/async", s.handleAdminImageGenAsync) + admin("POST /api/admin/image-gen/save-cover", s.handleAdminImageGenSaveCover) + admin("POST /api/admin/image-gen/save-chapter-image", s.handleAdminImageGenSaveChapterImage) + + // Chapter image serving + mux.HandleFunc("GET /api/chapter-image/{domain}/{slug}/{n}", s.handleGetChapterImage) + mux.HandleFunc("HEAD /api/chapter-image/{domain}/{slug}/{n}", s.handleHeadChapterImage) + + // Admin text generation endpoints (chapter names + book description) + admin("GET /api/admin/text-gen/models", s.handleAdminTextGenModels) + admin("POST /api/admin/text-gen/chapter-names", s.handleAdminTextGenChapterNames) + admin("POST /api/admin/text-gen/chapter-names/async", s.handleAdminTextGenChapterNamesAsync) + admin("POST /api/admin/text-gen/chapter-names/apply", s.handleAdminTextGenApplyChapterNames) + admin("POST /api/admin/text-gen/description", s.handleAdminTextGenDescription) + admin("POST /api/admin/text-gen/description/async", s.handleAdminTextGenDescriptionAsync) + admin("POST /api/admin/text-gen/description/apply", s.handleAdminTextGenApplyDescription) + + // Admin catalogue enrichment endpoints + admin("POST /api/admin/text-gen/tagline", s.handleAdminTextGenTagline) + admin("POST /api/admin/text-gen/genres", s.handleAdminTextGenGenres) + admin("POST /api/admin/text-gen/genres/apply", s.handleAdminTextGenApplyGenres) + admin("POST /api/admin/text-gen/content-warnings", s.handleAdminTextGenContentWarnings) + admin("POST /api/admin/text-gen/quality-score", s.handleAdminTextGenQualityScore) + admin("POST /api/admin/catalogue/batch-covers", s.handleAdminBatchCovers) + admin("POST /api/admin/catalogue/batch-covers/cancel", s.handleAdminBatchCoversCancel) + admin("POST /api/admin/catalogue/refresh-metadata/{slug}", s.handleAdminRefreshMetadata) + + // Admin AI job tracking endpoints + admin("GET /api/admin/ai-jobs", s.handleAdminListAIJobs) + admin("GET /api/admin/ai-jobs/{id}", s.handleAdminGetAIJob) + admin("POST /api/admin/ai-jobs/{id}/cancel", s.handleAdminCancelAIJob) + + // Auto-prompt generation from book/chapter content + admin("POST /api/admin/image-gen/auto-prompt", s.handleAdminImageGenAutoPrompt) + + // Admin data repair endpoints + admin("POST /api/admin/dedup-chapters/{slug}", s.handleDedupChapters) + + // Admin book management (soft-delete / hard-delete / publish visibility) + admin("PATCH /api/admin/books/{slug}/archive", s.handleAdminArchiveBook) + admin("PATCH /api/admin/books/{slug}/unarchive", s.handleAdminUnarchiveBook) + admin("DELETE /api/admin/books/{slug}", s.handleAdminDeleteBook) + admin("PATCH /api/admin/books/{slug}/publish", s.handleAdminPublishBook) + admin("PATCH /api/admin/books/{slug}/unpublish", s.handleAdminUnpublishBook) + + // Author book submission (creates a public book with no scraped content) + admin("POST /api/admin/books/submit", s.handleAdminSubmitBook) + + // Admin chapter split (imported books) + admin("POST /api/admin/books/{slug}/split-chapters", s.handleAdminSplitChapters) + + // Import (PDF/EPUB) + admin("POST /api/admin/import", s.handleAdminImport) + admin("GET /api/admin/import", s.handleAdminImportList) + admin("GET /api/admin/import/{id}", s.handleAdminImportStatus) + + // Podcast RSS feed generation and serving + // Public endpoints — no auth; podcast clients subscribe to these URLs. + mux.HandleFunc("GET /podcast/audio/{slug}/{n}/{voice}", s.handlePodcastAudio) + mux.HandleFunc("GET /podcast/{slug}", s.handlePodcastFeed) + // Admin endpoints — trigger generation job and check status. + admin("POST /api/admin/podcast", s.handleAdminPodcast) + admin("GET /api/admin/podcast/{slug}", s.handleAdminPodcastStatus) + + // Notifications + mux.HandleFunc("GET /api/notifications", s.handleListNotifications) + mux.HandleFunc("PATCH /api/notifications", s.handleMarkAllNotificationsRead) + mux.HandleFunc("PATCH /api/notifications/{id}", s.handleMarkNotificationRead) + mux.HandleFunc("DELETE /api/notifications", s.handleClearAllNotifications) + mux.HandleFunc("DELETE /api/notifications/{id}", s.handleDismissNotification) + + // Web Push subscriptions + mux.HandleFunc("GET /api/push-subscriptions/vapid-public-key", s.handleGetVAPIDPublicKey) + mux.HandleFunc("POST /api/push-subscriptions", s.handleSavePushSubscription) + mux.HandleFunc("DELETE /api/push-subscriptions", s.handleDeletePushSubscription) + + // Voices list + mux.HandleFunc("GET /api/voices", s.handleVoices) + + // Presigned URLs + mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter) + mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio) + mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample) + mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload) + mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar) + mux.HandleFunc("PUT /api/avatar-upload/{userId}", s.handleAvatarUpload) + + // EPUB export + mux.HandleFunc("GET /api/export/{slug}", s.handleExportEPUB) + + // Reading progress + mux.HandleFunc("GET /api/progress", s.handleGetProgress) + mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress) + mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress) + + // Wrap mux with OTel tracing (no-op when no TracerProvider is set), + // then with Sentry for panic recovery and error reporting. + var handler http.Handler = mux + handler = otelhttp.NewHandler(handler, "libnovel.backend", + otelhttp.WithMessageEvents(otelhttp.ReadEvents, otelhttp.WriteEvents), + ) + handler = sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle(handler) + + srv := &http.Server{ + Addr: s.cfg.Addr, + Handler: handler, + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Minute, // audio-stream can take several minutes for a full chapter + IdleTimeout: 60 * time.Second, + } + + errCh := make(chan error, 1) + go func() { errCh <- srv.ListenAndServe() }() + s.deps.Log.Info("backend: HTTP server listening", "addr", s.cfg.Addr) + + select { + case <-ctx.Done(): + s.deps.Log.Info("backend: context cancelled, starting graceful shutdown") + shutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := srv.Shutdown(shutCtx); err != nil { + s.deps.Log.Error("backend: graceful shutdown failed", "err", err) + return err + } + s.deps.Log.Info("backend: shutdown complete") + return nil + case err := <-errCh: + return err + } +} + +// ── Session cookie helpers ───────────────────────────────────────────────────── + +const sessionCookieName = "libnovel_session" + +func sessionID(r *http.Request) string { + c, err := r.Cookie(sessionCookieName) + if err != nil { + return "" + } + return c.Value +} + +func newSessionID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +func ensureSession(w http.ResponseWriter, r *http.Request) string { + if id := sessionID(r); id != "" { + return id + } + id, err := newSessionID() + if err != nil { + id = fmt.Sprintf("fallback-%d", time.Now().UnixNano()) + } + http.SetCookie(w, &http.Cookie{ + Name: sessionCookieName, + Value: id, + Path: "/", + HttpOnly: true, + SameSite: http.SameSiteLaxMode, + MaxAge: 365 * 24 * 60 * 60, + }) + return id +} + +// ── Utility helpers ──────────────────────────────────────────────────────────── + +// writeJSON writes v as a JSON response with status code. Status 0 → 200. +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + if status != 0 { + w.WriteHeader(status) + } + _ = json.NewEncoder(w).Encode(v) +} + +// jsonError writes a JSON error body and the given status code. +func jsonError(w http.ResponseWriter, status int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]string{"error": msg}) +} + +// voices returns the merged list of available voices from Kokoro and pocket-tts. +// On the first call it fetches from both services and caches the result. +// Falls back to the hardcoded Kokoro list on error. +func (s *Server) voices(ctx context.Context) []domain.Voice { + s.voiceMu.RLock() + cached := s.cachedVoices + s.voiceMu.RUnlock() + if len(cached) > 0 { + return cached + } + + fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var result []domain.Voice + + // ── Kokoro voices ───────────────────────────────────────────────────────── + var kokoroIDs []string + if s.deps.Kokoro != nil { + ids, err := s.deps.Kokoro.ListVoices(fetchCtx) + if err != nil || len(ids) == 0 { + s.deps.Log.Warn("backend: could not fetch kokoro voices, using built-in list", "err", err) + ids = kokoroVoiceIDs + } else { + s.deps.Log.Info("backend: fetched kokoro voices", "count", len(ids)) + } + kokoroIDs = ids + } else { + kokoroIDs = kokoroVoiceIDs + } + for _, id := range kokoroIDs { + result = append(result, kokoroVoice(id)) + } + + // ── Pocket-TTS voices ───────────────────────────────────────────────────── + if s.deps.PocketTTS != nil { + ids, err := s.deps.PocketTTS.ListVoices(fetchCtx) + if err != nil { + s.deps.Log.Warn("backend: could not fetch pocket-tts voices", "err", err) + } else { + for _, id := range ids { + result = append(result, pocketTTSVoice(id)) + } + s.deps.Log.Info("backend: fetched pocket-tts voices", "count", len(ids)) + } + } + + // ── Cloudflare AI voices ────────────────────────────────────────────────── + if s.deps.CFAI != nil { + for _, speaker := range cfai.Speakers() { + gender := "m" + if cfai.IsFemale(speaker) { + gender = "f" + } + result = append(result, domain.Voice{ + ID: cfai.VoiceID(speaker), + Engine: "cfai", + Lang: "en", + Gender: gender, + }) + } + s.deps.Log.Info("backend: loaded CF AI voices", "count", len(cfai.Speakers())) + } + + s.voiceMu.Lock() + s.cachedVoices = result + s.voiceMu.Unlock() + return result +} + +// kokoroVoice builds a domain.Voice for a Kokoro voice ID. +// The two-character prefix encodes language and gender: +// +// af/am → en-us f/m | bf/bm → en-gb f/m +// ef/em → es f/m | ff → fr f +// hf/hm → hi f/m | if/im → it f/m +// jf/jm → ja f/m | pf/pm → pt f/m +// zf/zm → zh f/m +func kokoroVoice(id string) domain.Voice { + type meta struct{ lang, gender string } + prefixMap := map[string]meta{ + "af": {"en-us", "f"}, "am": {"en-us", "m"}, + "bf": {"en-gb", "f"}, "bm": {"en-gb", "m"}, + "ef": {"es", "f"}, "em": {"es", "m"}, + "ff": {"fr", "f"}, + "hf": {"hi", "f"}, "hm": {"hi", "m"}, + "if": {"it", "f"}, "im": {"it", "m"}, + "jf": {"ja", "f"}, "jm": {"ja", "m"}, + "pf": {"pt", "f"}, "pm": {"pt", "m"}, + "zf": {"zh", "f"}, "zm": {"zh", "m"}, + } + if len(id) >= 2 { + if m, ok := prefixMap[id[:2]]; ok { + return domain.Voice{ID: id, Engine: "kokoro", Lang: m.lang, Gender: m.gender} + } + } + return domain.Voice{ID: id, Engine: "kokoro", Lang: "en", Gender: ""} +} + +// pocketTTSVoice builds a domain.Voice for a pocket-tts voice ID. +// All pocket-tts voices are English audiobook narrators. +func pocketTTSVoice(id string) domain.Voice { + femaleVoices := map[string]struct{}{ + "alba": {}, "fantine": {}, "cosette": {}, "eponine": {}, + "azelma": {}, "anna": {}, "vera": {}, "mary": {}, "jane": {}, "eve": {}, + } + gender := "m" + if _, ok := femaleVoices[id]; ok { + gender = "f" + } + return domain.Voice{ID: id, Engine: "pocket-tts", Lang: "en", Gender: gender} +} + +// handleHealth handles GET /health. +func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, 0, map[string]string{ + "status": "ok", + "version": s.cfg.Version, + "commit": s.cfg.Commit, + }) +} + +// handleVersion handles GET /api/version. +func (s *Server) handleVersion(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, 0, map[string]string{ + "version": s.cfg.Version, + "commit": s.cfg.Commit, + }) +} diff --git a/backend/internal/bookstore/bookstore.go b/backend/internal/bookstore/bookstore.go new file mode 100644 index 0000000..8c31cad --- /dev/null +++ b/backend/internal/bookstore/bookstore.go @@ -0,0 +1,273 @@ +// Package bookstore defines the segregated read/write interfaces for book, +// chapter, ranking, progress, audio, and presign data. +// +// Interface segregation: +// - BookWriter — used by the runner to persist scraped data. +// - BookReader — used by the backend to serve book/chapter data. +// - RankingStore — used by both runner (write) and backend (read). +// - PresignStore — used only by the backend for URL signing. +// - AudioStore — used by the runner to store audio; backend for presign. +// - ProgressStore— used only by the backend for reading progress. +// +// Concrete implementations live in internal/storage. +package bookstore + +import ( + "context" + "io" + "time" + + "github.com/libnovel/backend/internal/domain" +) + +// BookWriter is the write side used by the runner after scraping a book. +type BookWriter interface { + // WriteMetadata upserts all bibliographic fields for a book. + WriteMetadata(ctx context.Context, meta domain.BookMeta) error + + // WriteChapter stores a fully-scraped chapter's text in MinIO and + // updates the chapters_idx record in PocketBase. + WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error + + // WriteChapterRefs persists chapter metadata (number + title) into + // chapters_idx without fetching or storing chapter text. + WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error + + // ChapterExists returns true if the markdown object for ref already exists. + ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool + + // DeduplicateChapters removes duplicate chapters_idx records for slug, + // keeping only one record per chapter number (the one with the latest + // updated timestamp). Returns the number of duplicate records deleted. + DeduplicateChapters(ctx context.Context, slug string) (int, error) +} + +// BookReader is the read side used by the backend to serve content. +type BookReader interface { + // ReadMetadata returns the metadata for slug. + // Returns (zero, false, nil) when not found. + ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error) + + // ListBooks returns all books sorted alphabetically by title. + ListBooks(ctx context.Context) ([]domain.BookMeta, error) + + // LocalSlugs returns the set of slugs that have metadata stored. + LocalSlugs(ctx context.Context) (map[string]bool, error) + + // MetadataMtime returns the Unix-second mtime of the metadata record, or 0. + MetadataMtime(ctx context.Context, slug string) int64 + + // ReadChapter returns the raw markdown for chapter number n. + ReadChapter(ctx context.Context, slug string, n int) (string, error) + + // ListChapters returns all stored chapters for slug, sorted by number. + ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error) + + // CountChapters returns the count of stored chapters. + CountChapters(ctx context.Context, slug string) int + + // ReindexChapters rebuilds chapters_idx from MinIO objects for slug. + ReindexChapters(ctx context.Context, slug string) (int, error) +} + +// RankingStore covers ranking reads and writes. +type RankingStore interface { + // WriteRankingItem upserts a single ranking entry (keyed on Slug). + WriteRankingItem(ctx context.Context, item domain.RankingItem) error + + // ReadRankingItems returns all ranking items sorted by rank ascending. + ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error) + + // RankingFreshEnough returns true when ranking rows exist and the most + // recent Updated timestamp is within maxAge. + RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) +} + +// AudioStore covers audio object storage (runner writes; backend reads). +type AudioStore interface { + // AudioObjectKey returns the MinIO object key for a cached MP3 audio file. + // Format: {slug}/{n}/{voice}.mp3 + AudioObjectKey(slug string, n int, voice string) string + + // AudioObjectKeyExt returns the MinIO object key for a cached audio file + // with a custom extension (e.g. "mp3" or "wav"). + AudioObjectKeyExt(slug string, n int, voice, ext string) string + + // AudioExists returns true when the audio object is present in MinIO. + AudioExists(ctx context.Context, key string) bool + + // PutAudio stores raw audio bytes under the given MinIO object key. + PutAudio(ctx context.Context, key string, data []byte) error + + // AudioObjectSize returns the byte size of the audio object at key, + // or -1 if the object does not exist or size cannot be determined. + AudioObjectSize(ctx context.Context, key string) int64 + + // PutAudioStream uploads audio from r to MinIO under key. + // size must be the exact byte length of r, or -1 to use multipart upload. + // contentType should be "audio/mpeg" or "audio/wav". + PutAudioStream(ctx context.Context, key string, r io.Reader, size int64, contentType string) error +} + +// PresignStore generates short-lived URLs — used exclusively by the backend. +type PresignStore interface { + // PresignChapter returns a presigned GET URL for a chapter markdown object. + PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) + + // PresignAudio returns a presigned GET URL for an audio object. + PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) + + // PresignAvatarUpload returns a short-lived presigned PUT URL for uploading + // an avatar image. ext should be "jpg", "png", or "webp". + PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error) + + // PresignAvatarURL returns a presigned GET URL for a user's avatar. + // Returns ("", false, nil) when no avatar exists. + PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) + + // PutAvatar stores raw image bytes for a user avatar directly in MinIO. + // ext should be "jpg", "png", or "webp". Returns the object key. + PutAvatar(ctx context.Context, userID, ext, contentType string, data []byte) (key string, err error) + + // DeleteAvatar removes all avatar objects for a user. + DeleteAvatar(ctx context.Context, userID string) error +} + +// ProgressStore covers per-session reading progress — backend only. +type ProgressStore interface { + // GetProgress returns the reading progress for the given session + slug. + GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool) + + // SetProgress saves or updates reading progress. + SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error + + // AllProgress returns all progress entries for a session. + AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error) + + // DeleteProgress removes progress for a specific slug. + DeleteProgress(ctx context.Context, sessionID, slug string) error +} + +// CoverStore covers book cover image storage in MinIO. +// The runner writes covers during catalogue refresh; the backend reads them. +type CoverStore interface { + // PutCover stores a raw cover image for a book identified by slug. + PutCover(ctx context.Context, slug string, data []byte, contentType string) error + + // GetCover retrieves the cover image for a book. Returns (nil, false, nil) + // when no cover exists for the given slug. + GetCover(ctx context.Context, slug string) ([]byte, string, bool, error) + + // CoverExists returns true when a cover image is stored for slug. + CoverExists(ctx context.Context, slug string) bool +} + +// AIJobStore manages AI generation jobs tracked in PocketBase. +type AIJobStore interface { + // CreateAIJob inserts a new ai_job record with status=running and returns its ID. + CreateAIJob(ctx context.Context, job domain.AIJob) (string, error) + // GetAIJob retrieves a single ai_job by ID. + // Returns (zero, false, nil) when not found. + GetAIJob(ctx context.Context, id string) (domain.AIJob, bool, error) + // UpdateAIJob patches an existing ai_job record with the given fields. + UpdateAIJob(ctx context.Context, id string, fields map[string]any) error + // ListAIJobs returns all ai_job records sorted by started descending. + ListAIJobs(ctx context.Context) ([]domain.AIJob, error) +} + +// ChapterImageStore covers per-chapter illustration images stored in MinIO. +// The backend admin writes them; the backend serves them. +type ChapterImageStore interface { + // PutChapterImage stores a raw image for chapter n of slug in MinIO. + PutChapterImage(ctx context.Context, slug string, n int, data []byte, contentType string) error + + // GetChapterImage retrieves the image for chapter n of slug. + // Returns (nil, "", false, nil) when no image exists. + GetChapterImage(ctx context.Context, slug string, n int) ([]byte, string, bool, error) + + // ChapterImageExists returns true when an image is stored for slug/n. + ChapterImageExists(ctx context.Context, slug string, n int) bool +} + +// TranslationStore covers machine-translated chapter storage in MinIO. +// The runner writes translations; the backend reads them. +type TranslationStore interface { + // TranslationObjectKey returns the MinIO object key for a cached translation. + TranslationObjectKey(lang, slug string, n int) string + + // TranslationExists returns true when the translation object is present in MinIO. + TranslationExists(ctx context.Context, key string) bool + + // PutTranslation stores raw translated markdown under the given MinIO object key. + PutTranslation(ctx context.Context, key string, data []byte) error + + // GetTranslation retrieves translated markdown from MinIO. + GetTranslation(ctx context.Context, key string) (string, error) +} + +// Chapter represents a single chapter extracted from PDF/EPUB. +type Chapter struct { + Number int // 1-based chapter number + Title string // chapter title (may be empty) + Content string // plain text content +} + +// BookImporter handles PDF/EPUB file parsing and chapter extraction. +// Used by the runner to import books from uploaded files. +type BookImporter interface { + // Import extracts chapters from a PDF or EPUB file stored in MinIO. + // Returns the extracted chapters or an error. + Import(ctx context.Context, objectKey, fileType string) ([]Chapter, error) +} + +// BookAdminStore covers admin-only operations for managing books in the catalogue. +// All methods require admin authorisation at the HTTP handler level. +type BookAdminStore interface { + // ArchiveBook sets archived=true on a book record, hiding it from all + // public search and catalogue responses. Returns ErrNotFound when the + // slug does not exist. + ArchiveBook(ctx context.Context, slug string) error + + // UnarchiveBook clears archived on a book record, making it publicly + // visible again. Returns ErrNotFound when the slug does not exist. + UnarchiveBook(ctx context.Context, slug string) error + + // DeleteBook permanently removes all data for a book: + // - PocketBase books record + // - All PocketBase chapters_idx records + // - All MinIO chapter markdown objects ({slug}/chapter-*.md) + // - MinIO cover image (covers/{slug}.jpg) + // The caller is responsible for also deleting the Meilisearch document. + DeleteBook(ctx context.Context, slug string) error + + // PublishBook sets visibility=public, making the book visible to all users. + PublishBook(ctx context.Context, slug string) error + + // UnpublishBook sets visibility=admin_only, hiding the book from regular users. + UnpublishBook(ctx context.Context, slug string) error + + // CreateSubmittedBook creates a new author-submitted book with visibility=public. + CreateSubmittedBook(ctx context.Context, meta domain.BookMeta) error +} + +// ImportFileStore uploads raw import files to object storage. +// Kept separate from BookImporter so the HTTP handler can upload the file +// without a concrete type assertion, regardless of which Producer is wired. +type ImportFileStore interface { + PutImportFile(ctx context.Context, objectKey string, data []byte) error + // PutImportChapters stores the pre-parsed chapters JSON under the given key. + PutImportChapters(ctx context.Context, key string, data []byte) error + // GetImportChapters retrieves the pre-parsed chapters JSON. + GetImportChapters(ctx context.Context, key string) ([]byte, error) +} + +// NotificationStore manages per-user in-app notifications. +// Always wired directly to the concrete *storage.Store so it works +// regardless of whether the Asynq task-queue wrapper is in use. +type NotificationStore interface { + ListNotifications(ctx context.Context, userID string, limit int) ([]map[string]any, error) + MarkNotificationRead(ctx context.Context, id string) error + MarkAllNotificationsRead(ctx context.Context, userID string) error + DeleteNotification(ctx context.Context, id string) error + ClearAllNotifications(ctx context.Context, userID string) error +} diff --git a/backend/internal/bookstore/bookstore_test.go b/backend/internal/bookstore/bookstore_test.go new file mode 100644 index 0000000..9192cbb --- /dev/null +++ b/backend/internal/bookstore/bookstore_test.go @@ -0,0 +1,148 @@ +package bookstore_test + +import ( + "context" + "io" + "testing" + "time" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/domain" +) + +// ── Mock that satisfies all bookstore interfaces ────────────────────────────── + +type mockStore struct{} + +// BookWriter +func (m *mockStore) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil } +func (m *mockStore) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error { return nil } +func (m *mockStore) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error { + return nil +} +func (m *mockStore) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool { + return false +} + +// BookReader +func (m *mockStore) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) { + return domain.BookMeta{}, false, nil +} +func (m *mockStore) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil } +func (m *mockStore) LocalSlugs(_ context.Context) (map[string]bool, error) { + return map[string]bool{}, nil +} +func (m *mockStore) MetadataMtime(_ context.Context, _ string) int64 { return 0 } +func (m *mockStore) ReadChapter(_ context.Context, _ string, _ int) (string, error) { + return "", nil +} +func (m *mockStore) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) { + return nil, nil +} +func (m *mockStore) CountChapters(_ context.Context, _ string) int { return 0 } +func (m *mockStore) ReindexChapters(_ context.Context, _ string) (int, error) { return 0, nil } +func (m *mockStore) DeduplicateChapters(_ context.Context, _ string) (int, error) { return 0, nil } + +// RankingStore +func (m *mockStore) WriteRankingItem(_ context.Context, _ domain.RankingItem) error { return nil } +func (m *mockStore) ReadRankingItems(_ context.Context) ([]domain.RankingItem, error) { + return nil, nil +} +func (m *mockStore) RankingFreshEnough(_ context.Context, _ time.Duration) (bool, error) { + return false, nil +} + +// AudioStore +func (m *mockStore) AudioObjectKey(_ string, _ int, _ string) string { return "" } +func (m *mockStore) AudioObjectKeyExt(_ string, _ int, _, _ string) string { return "" } +func (m *mockStore) AudioExists(_ context.Context, _ string) bool { return false } +func (m *mockStore) PutAudio(_ context.Context, _ string, _ []byte) error { return nil } +func (m *mockStore) AudioObjectSize(_ context.Context, _ string) int64 { return -1 } +func (m *mockStore) PutAudioStream(_ context.Context, _ string, _ io.Reader, _ int64, _ string) error { + return nil +} + +// PresignStore +func (m *mockStore) PresignChapter(_ context.Context, _ string, _ int, _ time.Duration) (string, error) { + return "", nil +} +func (m *mockStore) PresignAudio(_ context.Context, _ string, _ time.Duration) (string, error) { + return "", nil +} +func (m *mockStore) PresignAvatarUpload(_ context.Context, _, _ string) (string, string, error) { + return "", "", nil +} +func (m *mockStore) PresignAvatarURL(_ context.Context, _ string) (string, bool, error) { + return "", false, nil +} +func (m *mockStore) PutAvatar(_ context.Context, _, _, _ string, _ []byte) (string, error) { + return "", nil +} +func (m *mockStore) DeleteAvatar(_ context.Context, _ string) error { return nil } + +// ProgressStore +func (m *mockStore) GetProgress(_ context.Context, _, _ string) (domain.ReadingProgress, bool) { + return domain.ReadingProgress{}, false +} +func (m *mockStore) SetProgress(_ context.Context, _ string, _ domain.ReadingProgress) error { + return nil +} +func (m *mockStore) AllProgress(_ context.Context, _ string) ([]domain.ReadingProgress, error) { + return nil, nil +} +func (m *mockStore) DeleteProgress(_ context.Context, _, _ string) error { return nil } + +// ── Compile-time interface satisfaction ─────────────────────────────────────── + +var _ bookstore.BookWriter = (*mockStore)(nil) +var _ bookstore.BookReader = (*mockStore)(nil) +var _ bookstore.RankingStore = (*mockStore)(nil) +var _ bookstore.AudioStore = (*mockStore)(nil) +var _ bookstore.PresignStore = (*mockStore)(nil) +var _ bookstore.ProgressStore = (*mockStore)(nil) + +// ── Behavioural tests ───────────────────────────────────────────────────────── + +func TestBookWriter_WriteMetadata_ReturnsNilError(t *testing.T) { + var w bookstore.BookWriter = &mockStore{} + if err := w.WriteMetadata(context.Background(), domain.BookMeta{Slug: "test"}); err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +func TestBookReader_ReadMetadata_NotFound(t *testing.T) { + var r bookstore.BookReader = &mockStore{} + _, found, err := r.ReadMetadata(context.Background(), "unknown") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if found { + t.Error("expected not found") + } +} + +func TestRankingStore_RankingFreshEnough_ReturnsFalse(t *testing.T) { + var s bookstore.RankingStore = &mockStore{} + fresh, err := s.RankingFreshEnough(context.Background(), time.Hour) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fresh { + t.Error("expected false") + } +} + +func TestAudioStore_AudioExists_ReturnsFalse(t *testing.T) { + var s bookstore.AudioStore = &mockStore{} + if s.AudioExists(context.Background(), "audio/slug/1/af_bella.mp3") { + t.Error("expected false") + } +} + +func TestProgressStore_GetProgress_NotFound(t *testing.T) { + var s bookstore.ProgressStore = &mockStore{} + _, found := s.GetProgress(context.Background(), "session-1", "slug") + if found { + t.Error("expected not found") + } +} diff --git a/backend/internal/browser/browser.go b/backend/internal/browser/browser.go new file mode 100644 index 0000000..9c4d669 --- /dev/null +++ b/backend/internal/browser/browser.go @@ -0,0 +1,191 @@ +// Package browser provides a rate-limited HTTP client for web scraping. +package browser + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "sync" + "time" +) + +// ErrRateLimit is returned by GetContent when the server responds with 429. +// It carries the suggested retry delay (from Retry-After header, or a default). +var ErrRateLimit = errors.New("rate limited (429)") + +// RateLimitError wraps ErrRateLimit and carries the suggested wait duration. +type RateLimitError struct { + // RetryAfter is how long the caller should wait before retrying. + // Derived from the Retry-After response header when present; otherwise a default. + RetryAfter time.Duration +} + +func (e *RateLimitError) Error() string { + return fmt.Sprintf("rate limited (429): retry after %s", e.RetryAfter) +} + +func (e *RateLimitError) Is(target error) bool { return target == ErrRateLimit } + +// defaultRateLimitDelay is used when the server returns 429 with no Retry-After header. +const defaultRateLimitDelay = 60 * time.Second + +// Client is the interface used by scrapers to fetch raw page HTML. +// Implementations must be safe for concurrent use. +type Client interface { + // GetContent fetches the URL and returns the full response body as a string. + // It should respect the provided context for cancellation and timeouts. + GetContent(ctx context.Context, pageURL string) (string, error) +} + +// Config holds tunable parameters for the direct HTTP client. +type Config struct { + // MaxConcurrent limits the number of simultaneous in-flight requests. + // Defaults to 5 when 0. + MaxConcurrent int + // Timeout is the per-request deadline. Defaults to 90s when 0. + Timeout time.Duration +} + +// DirectClient is a plain net/http-based Client with a concurrency semaphore. +type DirectClient struct { + http *http.Client + semaphore chan struct{} +} + +// NewDirectClient returns a DirectClient configured by cfg. +func NewDirectClient(cfg Config) *DirectClient { + if cfg.MaxConcurrent <= 0 { + cfg.MaxConcurrent = 5 + } + if cfg.Timeout <= 0 { + cfg.Timeout = 90 * time.Second + } + + transport := &http.Transport{ + MaxIdleConnsPerHost: cfg.MaxConcurrent * 2, + DisableCompression: false, + } + + return &DirectClient{ + http: &http.Client{ + Transport: transport, + Timeout: cfg.Timeout, + }, + semaphore: make(chan struct{}, cfg.MaxConcurrent), + } +} + +// GetContent fetches pageURL respecting the concurrency limit. +func (c *DirectClient) GetContent(ctx context.Context, pageURL string) (string, error) { + // Acquire semaphore slot. + select { + case c.semaphore <- struct{}{}: + case <-ctx.Done(): + return "", ctx.Err() + } + defer func() { <-c.semaphore }() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) + if err != nil { + return "", fmt.Errorf("browser: build request %s: %w", pageURL, err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.5") + + resp, err := c.http.Do(req) + if err != nil { + return "", fmt.Errorf("browser: GET %s: %w", pageURL, err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusTooManyRequests { + delay := defaultRateLimitDelay + if ra := resp.Header.Get("Retry-After"); ra != "" { + if secs, err := strconv.Atoi(ra); err == nil && secs > 0 { + delay = time.Duration(secs) * time.Second + } + } + return "", &RateLimitError{RetryAfter: delay} + } + + if resp.StatusCode >= 400 { + return "", fmt.Errorf("browser: GET %s returned %d", pageURL, resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("browser: read body %s: %w", pageURL, err) + } + return string(body), nil +} + +// Do implements httputil.Client so DirectClient can be passed to RetryGet. +func (c *DirectClient) Do(req *http.Request) (*http.Response, error) { + select { + case c.semaphore <- struct{}{}: + case <-req.Context().Done(): + return nil, req.Context().Err() + } + defer func() { <-c.semaphore }() + return c.http.Do(req) +} + +// ── Stub for testing ────────────────────────────────────────────────────────── + +// StubClient is a test double for Client. It returns pre-configured responses +// keyed on URL. Calls to unknown URLs return an error. +type StubClient struct { + mu sync.Mutex + pages map[string]string + errors map[string]error + callLog []string +} + +// NewStub creates a StubClient with no pages pre-loaded. +func NewStub() *StubClient { + return &StubClient{ + pages: make(map[string]string), + errors: make(map[string]error), + } +} + +// SetPage registers a URL → HTML body mapping. +func (s *StubClient) SetPage(u, html string) { + s.mu.Lock() + s.pages[u] = html + s.mu.Unlock() +} + +// SetError registers a URL → error mapping (returned instead of a body). +func (s *StubClient) SetError(u string, err error) { + s.mu.Lock() + s.errors[u] = err + s.mu.Unlock() +} + +// CallLog returns the ordered list of URLs that were requested. +func (s *StubClient) CallLog() []string { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]string, len(s.callLog)) + copy(out, s.callLog) + return out +} + +// GetContent returns the registered page or an error for the URL. +func (s *StubClient) GetContent(_ context.Context, pageURL string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.callLog = append(s.callLog, pageURL) + if err, ok := s.errors[pageURL]; ok { + return "", err + } + if html, ok := s.pages[pageURL]; ok { + return html, nil + } + return "", fmt.Errorf("stub: no page registered for %q", pageURL) +} diff --git a/backend/internal/browser/browser_test.go b/backend/internal/browser/browser_test.go new file mode 100644 index 0000000..c5dd55d --- /dev/null +++ b/backend/internal/browser/browser_test.go @@ -0,0 +1,141 @@ +package browser_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/libnovel/backend/internal/browser" +) + +func TestDirectClient_GetContent_Success(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("<html>hello</html>")) + })) + defer srv.Close() + + c := browser.NewDirectClient(browser.Config{MaxConcurrent: 2, Timeout: 5 * time.Second}) + body, err := c.GetContent(context.Background(), srv.URL) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body != "<html>hello</html>" { + t.Errorf("want <html>hello</html>, got %q", body) + } +} + +func TestDirectClient_GetContent_4xxReturnsError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + c := browser.NewDirectClient(browser.Config{}) + _, err := c.GetContent(context.Background(), srv.URL) + if err == nil { + t.Fatal("expected error for 404") + } +} + +func TestDirectClient_SemaphoreBlocksConcurrency(t *testing.T) { + const maxConcurrent = 2 + var inflight atomic.Int32 + var peak atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := inflight.Add(1) + if int(n) > int(peak.Load()) { + peak.Store(n) + } + time.Sleep(20 * time.Millisecond) + inflight.Add(-1) + w.Write([]byte("ok")) + })) + defer srv.Close() + + c := browser.NewDirectClient(browser.Config{MaxConcurrent: maxConcurrent, Timeout: 5 * time.Second}) + + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + c.GetContent(context.Background(), srv.URL) + }() + } + wg.Wait() + + if int(peak.Load()) > maxConcurrent { + t.Errorf("concurrent requests exceeded limit: peak=%d, limit=%d", peak.Load(), maxConcurrent) + } +} + +func TestDirectClient_ContextCancel(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(200 * time.Millisecond) + w.Write([]byte("ok")) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel before making the request + + c := browser.NewDirectClient(browser.Config{}) + _, err := c.GetContent(ctx, srv.URL) + if err == nil { + t.Fatal("expected context cancellation error") + } +} + +// ── StubClient ──────────────────────────────────────────────────────────────── + +func TestStubClient_ReturnsRegisteredPage(t *testing.T) { + stub := browser.NewStub() + stub.SetPage("http://example.com/page1", "<html>page1</html>") + + body, err := stub.GetContent(context.Background(), "http://example.com/page1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body != "<html>page1</html>" { + t.Errorf("want page1 html, got %q", body) + } +} + +func TestStubClient_ReturnsRegisteredError(t *testing.T) { + stub := browser.NewStub() + want := errors.New("network failure") + stub.SetError("http://example.com/bad", want) + + _, err := stub.GetContent(context.Background(), "http://example.com/bad") + if err == nil { + t.Fatal("expected error") + } +} + +func TestStubClient_UnknownURLReturnsError(t *testing.T) { + stub := browser.NewStub() + _, err := stub.GetContent(context.Background(), "http://unknown.example.com/") + if err == nil { + t.Fatal("expected error for unknown URL") + } +} + +func TestStubClient_CallLog(t *testing.T) { + stub := browser.NewStub() + stub.SetPage("http://example.com/a", "a") + stub.SetPage("http://example.com/b", "b") + + stub.GetContent(context.Background(), "http://example.com/a") + stub.GetContent(context.Background(), "http://example.com/b") + + log := stub.CallLog() + if len(log) != 2 || log[0] != "http://example.com/a" || log[1] != "http://example.com/b" { + t.Errorf("unexpected call log: %v", log) + } +} diff --git a/backend/internal/cfai/client.go b/backend/internal/cfai/client.go new file mode 100644 index 0000000..078432a --- /dev/null +++ b/backend/internal/cfai/client.go @@ -0,0 +1,315 @@ +// Package cfai provides a client for Cloudflare Workers AI Text-to-Speech models. +// +// The Cloudflare Workers AI REST API is used to run TTS models: +// +// POST https://api.cloudflare.com/client/v4/accounts/{accountID}/ai/run/{model} +// Authorization: Bearer {apiToken} +// Content-Type: application/json +// { "text": "...", "speaker": "luna" } +// +// → 200 audio/mpeg — raw MP3 bytes +// +// Currently supported model: @cf/deepgram/aura-2-en (40 English speakers). +// Voice IDs are prefixed with "cfai:" to distinguish them from Kokoro/pocket-tts +// voices (e.g. "cfai:luna", "cfai:orion"). +// +// The API is batch-only (no streaming), so GenerateAudio waits for the full +// response. There is no 100-second Cloudflare proxy timeout because we are +// calling the Cloudflare API directly, not routing through a Cloudflare-proxied +// homelab tunnel. +// +// The aura-2-en model enforces a hard 2 000-character limit per request. +// GenerateAudio transparently splits longer texts into sentence-boundary chunks +// and concatenates the resulting MP3 frames. +package cfai + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +const ( + // DefaultModel is the Cloudflare Workers AI TTS model used by default. + DefaultModel = "@cf/deepgram/aura-2-en" + + // voicePrefix is the prefix used to namespace CF AI voice IDs. + voicePrefix = "cfai:" +) + +// aura2Speakers is the exhaustive list of speakers supported by aura-2-en. +var aura2Speakers = []string{ + "amalthea", "andromeda", "apollo", "arcas", "aries", "asteria", + "athena", "atlas", "aurora", "callista", "cora", "cordelia", + "delia", "draco", "electra", "harmonia", "helena", "hera", + "hermes", "hyperion", "iris", "janus", "juno", "jupiter", + "luna", "mars", "minerva", "neptune", "odysseus", "ophelia", + "orion", "orpheus", "pandora", "phoebe", "pluto", "saturn", + "thalia", "theia", "vesta", "zeus", +} + +// femaleSpeakers is the set of aura-2-en speaker names that are female voices. +var femaleSpeakers = map[string]struct{}{ + "amalthea": {}, "andromeda": {}, "aries": {}, "asteria": {}, + "athena": {}, "aurora": {}, "callista": {}, "cora": {}, + "cordelia": {}, "delia": {}, "electra": {}, "harmonia": {}, + "helena": {}, "hera": {}, "iris": {}, "juno": {}, + "luna": {}, "minerva": {}, "ophelia": {}, "pandora": {}, + "phoebe": {}, "thalia": {}, "theia": {}, "vesta": {}, +} + +// IsCFAIVoice reports whether voice is served by the Cloudflare AI client. +// CF AI voices use the "cfai:" prefix, e.g. "cfai:luna". +func IsCFAIVoice(voice string) bool { + return strings.HasPrefix(voice, voicePrefix) +} + +// SpeakerName strips the "cfai:" prefix and returns the bare speaker name. +// If voice is not a CF AI voice the original string is returned unchanged. +func SpeakerName(voice string) string { + return strings.TrimPrefix(voice, voicePrefix) +} + +// VoiceID returns the full voice ID (with prefix) for a bare speaker name. +func VoiceID(speaker string) string { + return voicePrefix + speaker +} + +// VoiceSampleKey returns the MinIO object key for a CF AI voice sample MP3. +func VoiceSampleKey(voice string) string { + safe := strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || r == '_' || r == '-' { + return r + } + return '_' + }, voice) + return fmt.Sprintf("_voice-samples/%s.mp3", safe) +} + +// IsFemale reports whether the given CF AI voice ID (with or without prefix) +// is a female speaker. +func IsFemale(voice string) bool { + speaker := SpeakerName(voice) + _, ok := femaleSpeakers[speaker] + return ok +} + +// Speakers returns all available bare speaker names for aura-2-en. +func Speakers() []string { + out := make([]string, len(aura2Speakers)) + copy(out, aura2Speakers) + return out +} + +// Client is the interface for interacting with Cloudflare Workers AI TTS. +type Client interface { + // GenerateAudio synthesises text using the given voice (e.g. "cfai:luna") + // and returns raw MP3 bytes. + GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) + + // StreamAudioMP3 is not natively supported by the CF AI batch API. + // It buffers the full response and returns an io.ReadCloser over the bytes, + // so callers can use it like a stream without special-casing. + StreamAudioMP3(ctx context.Context, text, voice string) (io.ReadCloser, error) + + // StreamAudioWAV is not natively supported; the CF AI model returns MP3. + // This method returns the same MP3 bytes wrapped as an io.ReadCloser. + StreamAudioWAV(ctx context.Context, text, voice string) (io.ReadCloser, error) + + // ListVoices returns all available voice IDs (with the "cfai:" prefix). + ListVoices(ctx context.Context) ([]string, error) +} + +// httpClient is the concrete CF AI HTTP client. +type httpClient struct { + accountID string + apiToken string + model string + http *http.Client +} + +// New returns a Client for the given Cloudflare account and API token. +// model defaults to DefaultModel when empty. +func New(accountID, apiToken, model string) Client { + if model == "" { + model = DefaultModel + } + return &httpClient{ + accountID: accountID, + apiToken: apiToken, + model: model, + http: &http.Client{Timeout: 5 * time.Minute}, + } +} + +// GenerateAudio calls the Cloudflare Workers AI TTS endpoint and returns MP3 bytes. +// The aura-2-en model rejects inputs longer than 2 000 characters, so this method +// splits the text into sentence-bounded chunks and concatenates the MP3 responses. +func (c *httpClient) GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) { + if text == "" { + return nil, fmt.Errorf("cfai: empty text") + } + speaker := SpeakerName(voice) + if speaker == "" { + speaker = "luna" + } + + chunks := splitText(text, 1800) // stay comfortably under the 2 000-char limit + var combined []byte + for _, chunk := range chunks { + part, err := c.generateChunk(ctx, chunk, speaker) + if err != nil { + return nil, err + } + combined = append(combined, part...) + } + return combined, nil +} + +// generateChunk sends a single ≤2 000-character request and returns MP3 bytes. +func (c *httpClient) generateChunk(ctx context.Context, text, speaker string) ([]byte, error) { + body, err := json.Marshal(map[string]any{ + "text": text, + "speaker": speaker, + }) + if err != nil { + return nil, fmt.Errorf("cfai: marshal request: %w", err) + } + + url := fmt.Sprintf("https://api.cloudflare.com/client/v4/accounts/%s/ai/run/%s", + c.accountID, c.model) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("cfai: build request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+c.apiToken) + req.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("cfai: request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("cfai: server returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + mp3, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("cfai: read response: %w", err) + } + return mp3, nil +} + +// splitText splits src into chunks of at most maxChars characters each. +// It tries to break at paragraph boundaries first, then at sentence-ending +// punctuation (. ! ?), and falls back to the nearest space. +func splitText(src string, maxChars int) []string { + if len(src) <= maxChars { + return []string{src} + } + + var chunks []string + remaining := src + + for len(remaining) > 0 { + if len(remaining) <= maxChars { + chunks = append(chunks, strings.TrimSpace(remaining)) + break + } + + // Search window: the first maxChars bytes of remaining. + // Use byte length here because the API limit is in bytes/chars for ASCII; + // for safety we operate on rune-aware slices. + window := remaining + if len(window) > maxChars { + // Trim to maxChars runes (not bytes), ensuring we don't split a multi-byte char. + window = runeSlice(remaining, maxChars) + } + + cut := -1 + + // 1. Prefer paragraph break (\n\n or \n). + if i := strings.LastIndex(window, "\n\n"); i > 0 { + cut = i + 2 + } else if i := strings.LastIndex(window, "\n"); i > 0 { + cut = i + 1 + } + + // 2. Fall back to sentence-ending punctuation followed by a space. + if cut < 0 { + for _, punct := range []string{". ", "! ", "? ", ".\n", "!\n", "?\n"} { + if i := strings.LastIndex(window, punct); i > 0 { + candidate := i + len(punct) + if cut < 0 || candidate > cut { + cut = candidate + } + } + } + } + + // 3. Last resort: nearest space. + if cut < 0 { + if i := strings.LastIndex(window, " "); i > 0 { + cut = i + 1 + } + } + + // 4. Hard cut at maxChars runes if no boundary found. + if cut < 0 { + cut = len(window) + } + + chunk := strings.TrimSpace(remaining[:cut]) + if chunk != "" { + chunks = append(chunks, chunk) + } + remaining = remaining[cut:] + } + + return chunks +} + +// runeSlice returns the first n runes of s as a string. +func runeSlice(s string, n int) string { + count := 0 + for i := range s { + if count == n { + return s[:i] + } + count++ + } + return s +} + +// StreamAudioMP3 generates audio and wraps the MP3 bytes as an io.ReadCloser. +func (c *httpClient) StreamAudioMP3(ctx context.Context, text, voice string) (io.ReadCloser, error) { + mp3, err := c.GenerateAudio(ctx, text, voice) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(mp3)), nil +} + +// StreamAudioWAV generates audio (MP3) and wraps it as an io.ReadCloser. +// Note: the CF AI aura-2-en model returns MP3 regardless of the method name. +func (c *httpClient) StreamAudioWAV(ctx context.Context, text, voice string) (io.ReadCloser, error) { + return c.StreamAudioMP3(ctx, text, voice) +} + +// ListVoices returns all available CF AI voice IDs (with the "cfai:" prefix). +func (c *httpClient) ListVoices(_ context.Context) ([]string, error) { + ids := make([]string, len(aura2Speakers)) + for i, s := range aura2Speakers { + ids[i] = VoiceID(s) + } + return ids, nil +} diff --git a/backend/internal/cfai/image.go b/backend/internal/cfai/image.go new file mode 100644 index 0000000..713b95f --- /dev/null +++ b/backend/internal/cfai/image.go @@ -0,0 +1,475 @@ +// Image generation via Cloudflare Workers AI text-to-image models. +// +// API reference: +// +// POST https://api.cloudflare.com/client/v4/accounts/{accountID}/ai/run/{model} +// Authorization: Bearer {apiToken} +// +// FLUX.2 models (flux-2-dev, flux-2-klein-4b, flux-2-klein-9b): +// +// Content-Type: multipart/form-data +// Fields: prompt, num_steps, width, height, guidance, image_b64 (optional) +// Response: { "image": "<base64 JPEG>" } +// +// Other models (flux-1-schnell, SDXL, SD 1.5): +// +// Content-Type: application/json +// Body: { "prompt": "...", "num_steps": 20 } +// Response: { "image": "<base64>" } or raw bytes depending on model +// +// Reference-image request (FLUX.2): +// +// Same multipart form; include image_b64 field with base64-encoded reference. +// +// Reference-image request (SD img2img): +// +// JSON body: { "prompt": "...", "image": [r,g,b,a,...], "strength": 0.75 } +// +// Recommended models for LibNovel: +// - Book covers (no reference): flux-2-dev, flux-2-klein-9b, lucid-origin +// - Chapter images (speed): flux-2-klein-4b, flux-1-schnell +// - With reference image: flux-2-dev, flux-2-klein-9b, sd-v1-5-img2img +package cfai + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "image" + "image/draw" + "image/jpeg" + _ "image/jpeg" // register JPEG decoder + "image/png" + _ "image/png" // register PNG decoder + "io" + "mime/multipart" + "net/http" + "strings" + "time" +) + +// ImageModel identifies a Cloudflare Workers AI text-to-image model. +type ImageModel string + +const ( + // ImageModelFlux2Dev — best quality, multi-reference. Recommended for covers. + ImageModelFlux2Dev ImageModel = "@cf/black-forest-labs/flux-2-dev" + // ImageModelFlux2Klein9B — 9B params, multi-reference. Good for covers. + ImageModelFlux2Klein9B ImageModel = "@cf/black-forest-labs/flux-2-klein-9b" + // ImageModelFlux2Klein4B — ultra-fast, unified gen+edit. Recommended for chapters. + ImageModelFlux2Klein4B ImageModel = "@cf/black-forest-labs/flux-2-klein-4b" + // ImageModelFlux1Schnell — fastest, text-only. Good for quick illustrations. + ImageModelFlux1Schnell ImageModel = "@cf/black-forest-labs/flux-1-schnell" + // ImageModelSDXLLightning — fast 1024px generation. + ImageModelSDXLLightning ImageModel = "@cf/bytedance/stable-diffusion-xl-lightning" + // ImageModelSD15Img2Img — explicit img2img with flat RGBA reference. + ImageModelSD15Img2Img ImageModel = "@cf/runwayml/stable-diffusion-v1-5-img2img" + // ImageModelSDXLBase — Stability AI SDXL base. + ImageModelSDXLBase ImageModel = "@cf/stabilityai/stable-diffusion-xl-base-1.0" + // ImageModelLucidOrigin — Leonardo AI; strong prompt adherence. + ImageModelLucidOrigin ImageModel = "@cf/leonardo/lucid-origin" + // ImageModelPhoenix10 — Leonardo AI; accurate text rendering. + ImageModelPhoenix10 ImageModel = "@cf/leonardo/phoenix-1.0" + + // DefaultImageModel is the default model for book-cover generation. + DefaultImageModel = ImageModelFlux2Dev +) + +// ImageModelInfo describes a single image generation model. +type ImageModelInfo struct { + ID string `json:"id"` + Label string `json:"label"` + Provider string `json:"provider"` + SupportsRef bool `json:"supports_ref"` + RecommendedFor []string `json:"recommended_for"` // "cover" and/or "chapter" + Description string `json:"description"` +} + +// AllImageModels returns metadata about every supported image model. +func AllImageModels() []ImageModelInfo { + return []ImageModelInfo{ + { + ID: string(ImageModelFlux2Dev), Label: "FLUX.2 Dev", Provider: "Black Forest Labs", + SupportsRef: true, RecommendedFor: []string{"cover"}, + Description: "Best quality; multi-reference editing. Recommended for book covers.", + }, + { + ID: string(ImageModelFlux2Klein9B), Label: "FLUX.2 Klein 9B", Provider: "Black Forest Labs", + SupportsRef: true, RecommendedFor: []string{"cover"}, + Description: "9B parameters with multi-reference support.", + }, + { + ID: string(ImageModelFlux2Klein4B), Label: "FLUX.2 Klein 4B", Provider: "Black Forest Labs", + SupportsRef: true, RecommendedFor: []string{"chapter"}, + Description: "Ultra-fast unified gen+edit. Recommended for chapter images.", + }, + { + ID: string(ImageModelFlux1Schnell), Label: "FLUX.1 Schnell", Provider: "Black Forest Labs", + SupportsRef: false, RecommendedFor: []string{"chapter"}, + Description: "Fastest inference. Good for quick chapter illustrations.", + }, + { + ID: string(ImageModelSDXLLightning), Label: "SDXL Lightning", Provider: "ByteDance", + SupportsRef: false, RecommendedFor: []string{"chapter"}, + Description: "Lightning-fast 1024px images in a few steps.", + }, + { + ID: string(ImageModelSD15Img2Img), Label: "SD 1.5 img2img", Provider: "RunwayML", + SupportsRef: true, RecommendedFor: []string{"cover", "chapter"}, + Description: "Explicit img2img: generates from a reference image + prompt.", + }, + { + ID: string(ImageModelSDXLBase), Label: "SDXL Base 1.0", Provider: "Stability AI", + SupportsRef: false, RecommendedFor: []string{"cover"}, + Description: "Stable Diffusion XL base model.", + }, + { + ID: string(ImageModelLucidOrigin), Label: "Lucid Origin", Provider: "Leonardo AI", + SupportsRef: false, RecommendedFor: []string{"cover"}, + Description: "Highly prompt-responsive; strong graphic design and HD renders.", + }, + { + ID: string(ImageModelPhoenix10), Label: "Phoenix 1.0", Provider: "Leonardo AI", + SupportsRef: false, RecommendedFor: []string{"cover"}, + Description: "Exceptional prompt adherence; accurate text rendering.", + }, + } +} + +// ImageRequest is the input to GenerateImage / GenerateImageFromReference. +type ImageRequest struct { + // Prompt is the text description of the desired image. + Prompt string + // Model is the CF Workers AI model. Defaults to DefaultImageModel when empty. + Model ImageModel + // NumSteps controls inference quality (default 20). Range: 1–20. + NumSteps int + // Width and Height in pixels. 0 = model default (typically 1024x1024). + Width, Height int + // Guidance controls prompt adherence (default 7.5). + Guidance float64 + // Strength for img2img: 0.0 = copy reference, 1.0 = ignore reference (default 0.75). + Strength float64 +} + +// ImageGenClient generates images via Cloudflare Workers AI. +type ImageGenClient interface { + // GenerateImage creates an image from a text prompt only. + // Returns raw PNG bytes. + GenerateImage(ctx context.Context, req ImageRequest) ([]byte, error) + + // GenerateImageFromReference creates an image from a text prompt + reference image. + // refImage should be PNG or JPEG bytes. Returns raw PNG bytes. + GenerateImageFromReference(ctx context.Context, req ImageRequest, refImage []byte) ([]byte, error) + + // Models returns metadata about all supported image models. + Models() []ImageModelInfo +} + +// imageGenHTTPClient is the concrete CF AI image generation client. +type imageGenHTTPClient struct { + accountID string + apiToken string + http *http.Client +} + +// NewImageGen returns an ImageGenClient for the given Cloudflare account. +func NewImageGen(accountID, apiToken string) ImageGenClient { + return &imageGenHTTPClient{ + accountID: accountID, + apiToken: apiToken, + http: &http.Client{Timeout: 5 * time.Minute}, + } +} + +// requiresMultipart reports whether the model requires a multipart/form-data +// request body instead of JSON. FLUX.2 models on Cloudflare Workers AI changed +// their API to require multipart and return {"image":"<base64>"} instead of +// raw image bytes. +func requiresMultipart(model ImageModel) bool { + switch model { + case ImageModelFlux2Dev, ImageModelFlux2Klein4B, ImageModelFlux2Klein9B: + return true + default: + return false + } +} + +// GenerateImage generates an image from text only. +func (c *imageGenHTTPClient) GenerateImage(ctx context.Context, req ImageRequest) ([]byte, error) { + req = applyImageDefaults(req) + + // FLUX.2 multipart models use "steps"; JSON models use "num_steps". + stepsKey := "num_steps" + if requiresMultipart(req.Model) { + stepsKey = "steps" + } + + fields := map[string]any{ + "prompt": req.Prompt, + stepsKey: req.NumSteps, + } + if req.Width > 0 { + fields["width"] = req.Width + } + if req.Height > 0 { + fields["height"] = req.Height + } + if req.Guidance > 0 { + fields["guidance"] = req.Guidance + } + return c.callImageAPI(ctx, req.Model, fields, nil) +} + +// refImageMaxDim is the maximum dimension (width or height) for reference images +// sent to Cloudflare Workers AI. CF's JSON body limit is ~4 MB; a 768px JPEG +// stays well under that while preserving enough detail for img2img guidance. +const refImageMaxDim = 768 + +// GenerateImageFromReference generates an image from a text prompt + reference image. +func (c *imageGenHTTPClient) GenerateImageFromReference(ctx context.Context, req ImageRequest, refImage []byte) ([]byte, error) { + if len(refImage) == 0 { + return c.GenerateImage(ctx, req) + } + req = applyImageDefaults(req) + + // Shrink the reference image if it exceeds the safe payload size. + refImage = resizeRefImage(refImage, refImageMaxDim) + + // FLUX.2 multipart models use "steps"; JSON models use "num_steps". + stepsKey := "num_steps" + if requiresMultipart(req.Model) { + stepsKey = "steps" + } + + fields := map[string]any{ + "prompt": req.Prompt, + stepsKey: req.NumSteps, + } + if req.Width > 0 { + fields["width"] = req.Width + } + if req.Height > 0 { + fields["height"] = req.Height + } + if req.Guidance > 0 { + fields["guidance"] = req.Guidance + } + + if requiresMultipart(req.Model) { + // FLUX.2: reference image sent as base64 form field "image_b64". + fields["image_b64"] = base64.StdEncoding.EncodeToString(refImage) + if req.Strength > 0 { + fields["strength"] = req.Strength + } + return c.callImageAPI(ctx, req.Model, fields, nil) + } + + if req.Model == ImageModelSD15Img2Img { + pixels, err := decodeImageToRGBA(refImage) + if err != nil { + return nil, fmt.Errorf("cfai/image: decode reference: %w", err) + } + strength := req.Strength + if strength <= 0 { + strength = 0.75 + } + fields["image"] = pixels + fields["strength"] = strength + return c.callImageAPI(ctx, req.Model, fields, nil) + } + + // Other FLUX models: image_b64 JSON field. + fields["image_b64"] = base64.StdEncoding.EncodeToString(refImage) + if req.Strength > 0 { + fields["strength"] = req.Strength + } + return c.callImageAPI(ctx, req.Model, fields, nil) +} + +// Models returns all supported image model metadata. +func (c *imageGenHTTPClient) Models() []ImageModelInfo { + return AllImageModels() +} + +func (c *imageGenHTTPClient) callImageAPI(ctx context.Context, model ImageModel, fields map[string]any, _ []byte) ([]byte, error) { + cfURL := fmt.Sprintf("https://api.cloudflare.com/client/v4/accounts/%s/ai/run/%s", + c.accountID, string(model)) + + var ( + bodyReader io.Reader + contentType string + ) + + if requiresMultipart(model) { + // Build a multipart/form-data body from the fields map. + // All values are serialised to their string representation. + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + for k, v := range fields { + var strVal string + switch tv := v.(type) { + case string: + strVal = tv + default: + encoded, merr := json.Marshal(tv) + if merr != nil { + return nil, fmt.Errorf("cfai/image: marshal field %q: %w", k, merr) + } + strVal = strings.Trim(string(encoded), `"`) + } + if werr := mw.WriteField(k, strVal); werr != nil { + return nil, fmt.Errorf("cfai/image: write field %q: %w", k, werr) + } + } + if cerr := mw.Close(); cerr != nil { + return nil, fmt.Errorf("cfai/image: close multipart writer: %w", cerr) + } + bodyReader = &buf + contentType = mw.FormDataContentType() + } else { + encoded, merr := json.Marshal(fields) + if merr != nil { + return nil, fmt.Errorf("cfai/image: marshal: %w", merr) + } + bodyReader = bytes.NewReader(encoded) + contentType = "application/json" + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, cfURL, bodyReader) + if err != nil { + return nil, fmt.Errorf("cfai/image: build request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+c.apiToken) + req.Header.Set("Content-Type", contentType) + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("cfai/image: http: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("cfai/image: read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + msg := string(respBody) + if len(msg) > 300 { + msg = msg[:300] + } + return nil, fmt.Errorf("cfai/image: model %s returned %d: %s", model, resp.StatusCode, msg) + } + + // Try to parse as {"image": "<base64>"} first (FLUX.2 and newer models). + // Fall back to treating the body as raw image bytes for legacy models. + var jsonResp struct { + Image string `json:"image"` + } + if jerr := json.Unmarshal(respBody, &jsonResp); jerr == nil && jsonResp.Image != "" { + imgBytes, decErr := base64.StdEncoding.DecodeString(jsonResp.Image) + if decErr != nil { + // Try raw (no padding) base64 + imgBytes, decErr = base64.RawStdEncoding.DecodeString(jsonResp.Image) + if decErr != nil { + return nil, fmt.Errorf("cfai/image: decode base64 response: %w", decErr) + } + } + return imgBytes, nil + } + + // Legacy: model returned raw image bytes directly. + return respBody, nil +} + +func applyImageDefaults(req ImageRequest) ImageRequest { + if req.Model == "" { + req.Model = DefaultImageModel + } + if req.NumSteps <= 0 { + req.NumSteps = 20 + } + return req +} + +// resizeRefImage down-scales an image so that its longest side is at most maxDim +// pixels, then re-encodes it as JPEG (quality 85). If the image is already small +// enough, or if decoding fails, the original bytes are returned unchanged. +// This keeps the JSON payload well under Cloudflare Workers AI's 4 MB body limit. +func resizeRefImage(data []byte, maxDim int) []byte { + src, format, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return data + } + b := src.Bounds() + w, h := b.Dx(), b.Dy() + + longest := w + if h > longest { + longest = h + } + if longest <= maxDim { + return data // already fits + } + + // Compute target dimensions preserving aspect ratio. + scale := float64(maxDim) / float64(longest) + newW := int(float64(w)*scale + 0.5) + newH := int(float64(h)*scale + 0.5) + if newW < 1 { + newW = 1 + } + if newH < 1 { + newH = 1 + } + + // Nearest-neighbour downsample (no extra deps, sufficient for reference guidance). + dst := image.NewRGBA(image.Rect(0, 0, newW, newH)) + for y := 0; y < newH; y++ { + for x := 0; x < newW; x++ { + srcX := b.Min.X + int(float64(x)/scale) + srcY := b.Min.Y + int(float64(y)/scale) + draw.Draw(dst, image.Rect(x, y, x+1, y+1), src, image.Pt(srcX, srcY), draw.Src) + } + } + + var buf bytes.Buffer + if format == "jpeg" { + if encErr := jpeg.Encode(&buf, dst, &jpeg.Options{Quality: 85}); encErr != nil { + return data + } + } else { + if encErr := png.Encode(&buf, dst); encErr != nil { + return data + } + } + return buf.Bytes() +} + +// decodeImageToRGBA decodes PNG/JPEG bytes to a flat []uint8 RGBA pixel array +// required by the stable-diffusion-v1-5-img2img model. +func decodeImageToRGBA(data []byte) ([]uint8, error) { + img, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("decode image: %w", err) + } + bounds := img.Bounds() + w := bounds.Max.X - bounds.Min.X + h := bounds.Max.Y - bounds.Min.Y + pixels := make([]uint8, w*h*4) + idx := 0 + for y := bounds.Min.Y; y < bounds.Max.Y; y++ { + for x := bounds.Min.X; x < bounds.Max.X; x++ { + r, g, b, a := img.At(x, y).RGBA() + pixels[idx] = uint8(r >> 8) + pixels[idx+1] = uint8(g >> 8) + pixels[idx+2] = uint8(b >> 8) + pixels[idx+3] = uint8(a >> 8) + idx += 4 + } + } + return pixels, nil +} diff --git a/backend/internal/cfai/text.go b/backend/internal/cfai/text.go new file mode 100644 index 0000000..c62bf10 --- /dev/null +++ b/backend/internal/cfai/text.go @@ -0,0 +1,257 @@ +// Text generation via Cloudflare Workers AI LLM models. +// +// API reference: +// +// POST https://api.cloudflare.com/client/v4/accounts/{accountID}/ai/run/{model} +// Authorization: Bearer {apiToken} +// Content-Type: application/json +// +// Request body (all models): +// +// { "messages": [{"role":"system","content":"..."},{"role":"user","content":"..."}] } +// +// Response (wrapped): +// +// { "result": { "response": "..." }, "success": true } +package cfai + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +// TextModel identifies a Cloudflare Workers AI text generation model. +type TextModel string + +const ( + // TextModelGemma4 — Google Gemma 4, 256k context. + TextModelGemma4 TextModel = "@cf/google/gemma-4-26b-a4b-it" + // TextModelLlama4Scout — Meta Llama 4 Scout 17B, multimodal. + TextModelLlama4Scout TextModel = "@cf/meta/llama-4-scout-17b-16e-instruct" + // TextModelLlama33_70B — Meta Llama 3.3 70B, fast fp8. + TextModelLlama33_70B TextModel = "@cf/meta/llama-3.3-70b-instruct-fp8-fast" + // TextModelQwen3_30B — Qwen3 30B MoE, function calling. + TextModelQwen3_30B TextModel = "@cf/qwen/qwen3-30b-a3b-fp8" + // TextModelMistralSmall — Mistral Small 3.1 24B, 128k context. + TextModelMistralSmall TextModel = "@cf/mistralai/mistral-small-3.1-24b-instruct" + // TextModelQwQ32B — Qwen QwQ 32B reasoning model. + TextModelQwQ32B TextModel = "@cf/qwen/qwq-32b" + // TextModelDeepSeekR1 — DeepSeek R1 distill Qwen 32B. + TextModelDeepSeekR1 TextModel = "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b" + // TextModelGemma3_12B — Google Gemma 3 12B, 80k context. + TextModelGemma3_12B TextModel = "@cf/google/gemma-3-12b-it" + // TextModelGPTOSS120B — OpenAI gpt-oss-120b, high reasoning. + TextModelGPTOSS120B TextModel = "@cf/openai/gpt-oss-120b" + // TextModelGPTOSS20B — OpenAI gpt-oss-20b, lower latency. + TextModelGPTOSS20B TextModel = "@cf/openai/gpt-oss-20b" + // TextModelNemotron3 — NVIDIA Nemotron 3 120B, agentic. + TextModelNemotron3 TextModel = "@cf/nvidia/nemotron-3-120b-a12b" + // TextModelLlama32_3B — Meta Llama 3.2 3B, lightweight. + TextModelLlama32_3B TextModel = "@cf/meta/llama-3.2-3b-instruct" + + // DefaultTextModel is the default model used when none is specified. + DefaultTextModel = TextModelLlama4Scout +) + +// TextModelInfo describes a single text generation model. +type TextModelInfo struct { + ID string `json:"id"` + Label string `json:"label"` + Provider string `json:"provider"` + ContextSize int `json:"context_size"` // max context in tokens + Description string `json:"description"` +} + +// AllTextModels returns metadata about every supported text generation model. +func AllTextModels() []TextModelInfo { + return []TextModelInfo{ + { + ID: string(TextModelGemma4), Label: "Gemma 4 26B", Provider: "Google", + ContextSize: 256000, + Description: "Google's most intelligent open model family. 256k context, function calling.", + }, + { + ID: string(TextModelLlama4Scout), Label: "Llama 4 Scout 17B", Provider: "Meta", + ContextSize: 131000, + Description: "Natively multimodal, 16 experts. Good all-purpose model with function calling.", + }, + { + ID: string(TextModelLlama33_70B), Label: "Llama 3.3 70B (fp8 fast)", Provider: "Meta", + ContextSize: 24000, + Description: "Llama 3.3 70B quantized to fp8 for speed. Excellent instruction following.", + }, + { + ID: string(TextModelQwen3_30B), Label: "Qwen3 30B MoE", Provider: "Qwen", + ContextSize: 32768, + Description: "MoE architecture with strong reasoning and instruction following.", + }, + { + ID: string(TextModelMistralSmall), Label: "Mistral Small 3.1 24B", Provider: "MistralAI", + ContextSize: 128000, + Description: "Strong text performance with 128k context and function calling.", + }, + { + ID: string(TextModelQwQ32B), Label: "QwQ 32B (reasoning)", Provider: "Qwen", + ContextSize: 24000, + Description: "Reasoning model — thinks before answering. Slower but more accurate.", + }, + { + ID: string(TextModelDeepSeekR1), Label: "DeepSeek R1 32B", Provider: "DeepSeek", + ContextSize: 80000, + Description: "R1-distilled reasoning model. Outperforms o1-mini on many benchmarks.", + }, + { + ID: string(TextModelGemma3_12B), Label: "Gemma 3 12B", Provider: "Google", + ContextSize: 80000, + Description: "Multimodal, 128k context, multilingual (140+ languages).", + }, + { + ID: string(TextModelGPTOSS120B), Label: "GPT-OSS 120B", Provider: "OpenAI", + ContextSize: 128000, + Description: "OpenAI open-weight model for production, general purpose, high reasoning.", + }, + { + ID: string(TextModelGPTOSS20B), Label: "GPT-OSS 20B", Provider: "OpenAI", + ContextSize: 128000, + Description: "OpenAI open-weight model for lower latency and specialized use cases.", + }, + { + ID: string(TextModelNemotron3), Label: "Nemotron 3 120B", Provider: "NVIDIA", + ContextSize: 256000, + Description: "Hybrid MoE with leading accuracy for multi-agent applications.", + }, + { + ID: string(TextModelLlama32_3B), Label: "Llama 3.2 3B", Provider: "Meta", + ContextSize: 80000, + Description: "Lightweight model for simple tasks. Fast and cheap.", + }, + } +} + +// TextMessage is a single message in a chat conversation. +type TextMessage struct { + Role string `json:"role"` // "system" or "user" + Content string `json:"content"` // message text +} + +// TextRequest is the input to Generate. +type TextRequest struct { + // Model is the CF Workers AI model ID. Defaults to DefaultTextModel when empty. + Model TextModel + // Messages is the conversation history (system + user messages). + Messages []TextMessage + // MaxTokens limits the output length (0 = model default). + MaxTokens int +} + +// TextGenClient generates text via Cloudflare Workers AI LLM models. +type TextGenClient interface { + // Generate sends a chat-style request and returns the model's response text. + Generate(ctx context.Context, req TextRequest) (string, error) + + // Models returns metadata about all supported text generation models. + Models() []TextModelInfo +} + +// textGenHTTPClient is the concrete CF AI text generation client. +type textGenHTTPClient struct { + accountID string + apiToken string + http *http.Client +} + +// NewTextGen returns a TextGenClient for the given Cloudflare account. +func NewTextGen(accountID, apiToken string) TextGenClient { + return &textGenHTTPClient{ + accountID: accountID, + apiToken: apiToken, + http: &http.Client{Timeout: 5 * time.Minute}, + } +} + +// Generate sends messages to the model and returns the response text. +func (c *textGenHTTPClient) Generate(ctx context.Context, req TextRequest) (string, error) { + if req.Model == "" { + req.Model = DefaultTextModel + } + + body := map[string]any{ + "messages": req.Messages, + } + if req.MaxTokens > 0 { + body["max_tokens"] = req.MaxTokens + } + + encoded, err := json.Marshal(body) + if err != nil { + return "", fmt.Errorf("cfai/text: marshal: %w", err) + } + + url := fmt.Sprintf("https://api.cloudflare.com/client/v4/accounts/%s/ai/run/%s", + c.accountID, string(req.Model)) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(encoded)) + if err != nil { + return "", fmt.Errorf("cfai/text: build request: %w", err) + } + httpReq.Header.Set("Authorization", "Bearer "+c.apiToken) + httpReq.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(httpReq) + if err != nil { + return "", fmt.Errorf("cfai/text: http: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errBody, _ := io.ReadAll(resp.Body) + msg := string(errBody) + if len(msg) > 300 { + msg = msg[:300] + } + return "", fmt.Errorf("cfai/text: model %s returned %d: %s", req.Model, resp.StatusCode, msg) + } + + // CF AI wraps responses: { "result": { "response": "..." }, "success": true } + // Some models (e.g. Llama 4 Scout) return response as an array: + // { "result": { "response": [{"generated_text":"..."}] } } + var wrapper struct { + Result struct { + Response json.RawMessage `json:"response"` + } `json:"result"` + Success bool `json:"success"` + Errors []string `json:"errors"` + } + if err := json.NewDecoder(resp.Body).Decode(&wrapper); err != nil { + return "", fmt.Errorf("cfai/text: decode response: %w", err) + } + if !wrapper.Success { + return "", fmt.Errorf("cfai/text: model %s error: %v", req.Model, wrapper.Errors) + } + // Try plain string first. + var text string + if err := json.Unmarshal(wrapper.Result.Response, &text); err == nil { + return text, nil + } + // Fall back: array of objects with a "generated_text" field + // (older CF AI models return [{"generated_text":"..."}]). + var arr []struct { + GeneratedText string `json:"generated_text"` + } + if err := json.Unmarshal(wrapper.Result.Response, &arr); err == nil && len(arr) > 0 && arr[0].GeneratedText != "" { + return arr[0].GeneratedText, nil + } + // Final fallback: model returned the result directly as a JSON value + // (e.g. Llama 4 Scout returns [{"number":1,"title":"..."},...] directly). + // Return the raw JSON bytes as a string so callers can parse it themselves. + return string(wrapper.Result.Response), nil +} + +// Models returns all supported text generation model metadata. +func (c *textGenHTTPClient) Models() []TextModelInfo { + return AllTextModels() +} diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go new file mode 100644 index 0000000..4266518 --- /dev/null +++ b/backend/internal/config/config.go @@ -0,0 +1,328 @@ +// Package config loads all service configuration from environment variables. +// Both the runner and backend binaries call config.Load() at startup; each +// uses only the sub-struct relevant to it. +// +// Every field has a documented default so the service starts sensibly without +// any environment configuration (useful for local development). +package config + +import ( + "os" + "strconv" + "strings" + "time" +) + +// PocketBase holds connection settings for the remote PocketBase instance. +type PocketBase struct { + // URL is the base URL of the PocketBase instance, e.g. https://pb.libnovel.cc + URL string + // AdminEmail is the admin account email used for API authentication. + AdminEmail string + // AdminPassword is the admin account password. + AdminPassword string +} + +// MinIO holds connection settings for the remote MinIO / S3-compatible store. +type MinIO struct { + // Endpoint is the host:port of the MinIO S3 API, e.g. storage.libnovel.cc:443 + Endpoint string + // PublicEndpoint is the browser-visible endpoint used for presigned URLs. + // Falls back to Endpoint when empty. + PublicEndpoint string + // AccessKey is the MinIO access key. + AccessKey string + // SecretKey is the MinIO secret key. + SecretKey string + // UseSSL enables TLS for the internal MinIO connection. + UseSSL bool + // PublicUseSSL enables TLS for presigned URL generation. + PublicUseSSL bool + // BucketChapters is the bucket that holds chapter markdown objects. + BucketChapters string + // BucketAudio is the bucket that holds generated audio MP3 objects. + BucketAudio string + // BucketAvatars is the bucket that holds user avatar images. + BucketAvatars string + // BucketBrowse is the bucket that holds cached browse page snapshots (JSON). + BucketBrowse string + // BucketTranslations is the bucket that holds machine-translated chapter markdown. + BucketTranslations string +} + +// Kokoro holds connection settings for the Kokoro-FastAPI TTS service. +type Kokoro struct { + // URL is the base URL of the Kokoro service, e.g. https://tts.libnovel.cc + // An empty string disables Kokoro TTS generation. + URL string + // DefaultVoice is the voice used when none is specified. + DefaultVoice string +} + +// PocketTTS holds connection settings for the kyutai-labs/pocket-tts service. +type PocketTTS struct { + // URL is the base URL of the pocket-tts service, e.g. https://pocket-tts.libnovel.cc + // An empty string disables pocket-tts generation. + URL string +} + +// CFAI holds credentials for Cloudflare Workers AI TTS. +type CFAI struct { + // AccountID is the Cloudflare account ID. + // An empty string disables CF AI generation. + AccountID string + // APIToken is a Workers AI API token with Workers AI Read+Edit permissions. + APIToken string + // Model is the Workers AI TTS model ID. + // Defaults to "@cf/deepgram/aura-2-en" when empty. + Model string +} + +// LibreTranslate holds connection settings for a self-hosted LibreTranslate instance. +type LibreTranslate struct { + // URL is the base URL of the LibreTranslate instance, e.g. https://translate.libnovel.cc + // An empty string disables machine translation entirely. + URL string + // APIKey is the optional API key for the LibreTranslate instance. + // Leave empty if the instance runs without authentication. + APIKey string +} + +// HTTP holds settings for the HTTP server (backend only). +type HTTP struct { + // Addr is the listen address, e.g. ":8080" + Addr string + // AdminToken is the bearer token required for all /api/admin/* endpoints. + // Set via BACKEND_ADMIN_TOKEN. When empty, admin endpoints are unprotected — + // only acceptable when the backend is unreachable from the public internet. + AdminToken string +} + +// Meilisearch holds connection settings for the Meilisearch full-text search service. +type Meilisearch struct { + // URL is the base URL of the Meilisearch instance, e.g. http://localhost:7700 + // An empty string disables Meilisearch indexing and search. + URL string + // APIKey is the Meilisearch master/search API key. + APIKey string +} + +// Valkey holds connection settings for the Valkey/Redis presign URL cache. +type Valkey struct { + // Addr is the host:port of the Valkey instance, e.g. localhost:6379 + // An empty string disables the Valkey cache (falls through to MinIO directly). + Addr string +} + +// Redis holds connection settings for the Asynq task queue Redis instance. +// This is separate from Valkey (presign cache) — it may point to the same +// Redis or a dedicated one. An empty Addr falls back to PocketBase polling. +type Redis struct { + // Addr is the host:port (or rediss://... URL) of the Redis instance. + // Use rediss:// scheme for TLS (e.g. rediss://:password@redis.libnovel.cc:6380). + // An empty string disables Asynq and falls back to PocketBase polling. + Addr string + // Password is the Redis AUTH password. + // Not needed when Addr is a full rediss:// URL that includes the password. + Password string +} + +// VAPID holds Web Push VAPID key pair for browser push notifications. +// Generate a pair once with: go run ./cmd/genkeys (or use the web-push CLI). +// The public key is exposed via GET /api/push-subscriptions/vapid-public-key +// and embedded in the SvelteKit app via PUBLIC_VAPID_PUBLIC_KEY. +type VAPID struct { + // PublicKey is the base64url-encoded VAPID public key (65 bytes, uncompressed EC P-256). + PublicKey string + // PrivateKey is the base64url-encoded VAPID private key (32 bytes). + PrivateKey string + // Subject is the mailto: or https: URL used as the VAPID subscriber contact. + Subject string +} + +// Runner holds settings specific to the runner/worker binary. +type Runner struct { + // PollInterval is how often the runner checks PocketBase for pending tasks. + PollInterval time.Duration + // MaxConcurrentScrape limits simultaneous book-scrape goroutines. + MaxConcurrentScrape int + // MaxConcurrentAudio limits simultaneous audio-generation goroutines. + MaxConcurrentAudio int + // MaxConcurrentTranslation limits simultaneous translation goroutines. + MaxConcurrentTranslation int + // WorkerID is a unique identifier for this runner instance. + // Defaults to the system hostname. + WorkerID string + // Workers is the number of chapter-scraping goroutines per book. + Workers int + // Timeout is the per-request HTTP timeout for scraping. + Timeout time.Duration + // MetricsAddr is the listen address for the runner /metrics HTTP endpoint. + // Defaults to ":9091". Set to "" to disable. + MetricsAddr string + // CatalogueRefreshInterval is how often the runner walks the full catalogue, + // scrapes per-book metadata, downloads covers, and re-indexes in Meilisearch. + // Defaults to 24h. Set to 0 to use the default. + CatalogueRefreshInterval time.Duration + // SkipInitialCatalogueRefresh prevents the runner from running a full + // catalogue walk on startup. Useful for quick restarts where the catalogue + // is already indexed and a 24h walk would be wasteful. + // Controlled by RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true. + SkipInitialCatalogueRefresh bool + // CatalogueRequestDelay is the base delay inserted between per-book metadata + // requests during a catalogue refresh. A random jitter of up to 50% is added + // on top. Defaults to 2s. Increase to reduce 429 pressure on novelfire.net. + // Controlled by RUNNER_CATALOGUE_REQUEST_DELAY (e.g. "3s", "500ms"). + CatalogueRequestDelay time.Duration +} + +// Config is the top-level configuration struct consumed by both binaries. +type Config struct { + PocketBase PocketBase + MinIO MinIO + Kokoro Kokoro + PocketTTS PocketTTS + CFAI CFAI + LibreTranslate LibreTranslate + HTTP HTTP + Runner Runner + Meilisearch Meilisearch + Valkey Valkey + Redis Redis + VAPID VAPID + // LogLevel is one of "debug", "info", "warn", "error". + LogLevel string +} + +// Load reads all configuration from environment variables and returns a +// populated Config. Missing variables fall back to documented defaults. +func Load() Config { + workerID, _ := os.Hostname() + if workerID == "" { + workerID = "runner-default" + } + + return Config{ + LogLevel: envOr("LOG_LEVEL", "info"), + + PocketBase: PocketBase{ + URL: envOr("POCKETBASE_URL", "http://localhost:8090"), + AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"), + AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"), + }, + + MinIO: MinIO{ + Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"), + PublicEndpoint: envOr("MINIO_PUBLIC_ENDPOINT", ""), + AccessKey: envOr("MINIO_ACCESS_KEY", "admin"), + SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"), + UseSSL: envBool("MINIO_USE_SSL", false), + PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true), + BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "chapters"), + BucketAudio: envOr("MINIO_BUCKET_AUDIO", "audio"), + BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "avatars"), + BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "catalogue"), + BucketTranslations: envOr("MINIO_BUCKET_TRANSLATIONS", "translations"), + }, + + Kokoro: Kokoro{ + URL: envOr("KOKORO_URL", ""), + DefaultVoice: envOr("KOKORO_VOICE", "af_bella"), + }, + + PocketTTS: PocketTTS{ + URL: envOr("POCKET_TTS_URL", ""), + }, + + CFAI: CFAI{ + AccountID: envOr("CFAI_ACCOUNT_ID", ""), + APIToken: envOr("CFAI_API_TOKEN", ""), + Model: envOr("CFAI_TTS_MODEL", ""), + }, + + LibreTranslate: LibreTranslate{ + URL: envOr("LIBRETRANSLATE_URL", ""), + APIKey: envOr("LIBRETRANSLATE_API_KEY", ""), + }, + + HTTP: HTTP{ + Addr: envOr("BACKEND_HTTP_ADDR", ":8080"), + AdminToken: envOr("BACKEND_ADMIN_TOKEN", ""), + }, + + Runner: Runner{ + PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second), + MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1), + MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1), + MaxConcurrentTranslation: envInt("RUNNER_MAX_CONCURRENT_TRANSLATION", 1), + WorkerID: envOr("RUNNER_WORKER_ID", workerID), + Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU() + Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second), + MetricsAddr: envOr("RUNNER_METRICS_ADDR", ":9091"), + CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0), + SkipInitialCatalogueRefresh: envBool("RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH", false), + CatalogueRequestDelay: envDuration("RUNNER_CATALOGUE_REQUEST_DELAY", 2*time.Second), + }, + + Meilisearch: Meilisearch{ + URL: envOr("MEILI_URL", ""), + APIKey: envOr("MEILI_API_KEY", ""), + }, + + Valkey: Valkey{ + Addr: envOr("VALKEY_ADDR", ""), + }, + + Redis: Redis{ + Addr: envOr("REDIS_ADDR", ""), + Password: envOr("REDIS_PASSWORD", ""), + }, + + VAPID: VAPID{ + PublicKey: envOr("VAPID_PUBLIC_KEY", ""), + PrivateKey: envOr("VAPID_PRIVATE_KEY", ""), + Subject: envOr("VAPID_SUBJECT", "mailto:admin@libnovel.cc"), + }, + } +} + +// ── helpers ─────────────────────────────────────────────────────────────────── + +func envOr(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func envBool(key string, fallback bool) bool { + v := os.Getenv(key) + if v == "" { + return fallback + } + return strings.ToLower(v) == "true" +} + +func envInt(key string, fallback int) int { + v := os.Getenv(key) + if v == "" { + return fallback + } + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return fallback + } + return n +} + +func envDuration(key string, fallback time.Duration) time.Duration { + v := os.Getenv(key) + if v == "" { + return fallback + } + d, err := time.ParseDuration(v) + if err != nil { + return fallback + } + return d +} diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go new file mode 100644 index 0000000..d281b44 --- /dev/null +++ b/backend/internal/config/config_test.go @@ -0,0 +1,127 @@ +package config_test + +import ( + "os" + "testing" + "time" + + "github.com/libnovel/backend/internal/config" +) + +func TestLoad_Defaults(t *testing.T) { + // Unset all relevant vars so we test pure defaults. + unset := []string{ + "LOG_LEVEL", + "POCKETBASE_URL", "POCKETBASE_ADMIN_EMAIL", "POCKETBASE_ADMIN_PASSWORD", + "MINIO_ENDPOINT", "MINIO_PUBLIC_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", + "MINIO_USE_SSL", "MINIO_PUBLIC_USE_SSL", + "MINIO_BUCKET_CHAPTERS", "MINIO_BUCKET_AUDIO", "MINIO_BUCKET_AVATARS", + "KOKORO_URL", "KOKORO_VOICE", + "BACKEND_HTTP_ADDR", + "RUNNER_POLL_INTERVAL", "RUNNER_MAX_CONCURRENT_SCRAPE", "RUNNER_MAX_CONCURRENT_AUDIO", + "RUNNER_WORKER_ID", "RUNNER_WORKERS", "RUNNER_TIMEOUT", + } + for _, k := range unset { + t.Setenv(k, "") + } + + cfg := config.Load() + + if cfg.LogLevel != "info" { + t.Errorf("LogLevel: want info, got %q", cfg.LogLevel) + } + if cfg.PocketBase.URL != "http://localhost:8090" { + t.Errorf("PocketBase.URL: want http://localhost:8090, got %q", cfg.PocketBase.URL) + } + if cfg.MinIO.BucketChapters != "chapters" { + t.Errorf("MinIO.BucketChapters: want chapters, got %q", cfg.MinIO.BucketChapters) + } + if cfg.MinIO.UseSSL != false { + t.Errorf("MinIO.UseSSL: want false, got %v", cfg.MinIO.UseSSL) + } + if cfg.MinIO.PublicUseSSL != true { + t.Errorf("MinIO.PublicUseSSL: want true, got %v", cfg.MinIO.PublicUseSSL) + } + if cfg.Kokoro.DefaultVoice != "af_bella" { + t.Errorf("Kokoro.DefaultVoice: want af_bella, got %q", cfg.Kokoro.DefaultVoice) + } + if cfg.HTTP.Addr != ":8080" { + t.Errorf("HTTP.Addr: want :8080, got %q", cfg.HTTP.Addr) + } + if cfg.Runner.PollInterval != 30*time.Second { + t.Errorf("Runner.PollInterval: want 30s, got %v", cfg.Runner.PollInterval) + } + if cfg.Runner.MaxConcurrentScrape != 1 { + t.Errorf("Runner.MaxConcurrentScrape: want 1, got %d", cfg.Runner.MaxConcurrentScrape) + } + if cfg.Runner.MaxConcurrentAudio != 1 { + t.Errorf("Runner.MaxConcurrentAudio: want 1, got %d", cfg.Runner.MaxConcurrentAudio) + } +} + +func TestLoad_EnvOverride(t *testing.T) { + t.Setenv("LOG_LEVEL", "debug") + t.Setenv("POCKETBASE_URL", "https://pb.libnovel.cc") + t.Setenv("MINIO_USE_SSL", "true") + t.Setenv("MINIO_PUBLIC_USE_SSL", "false") + t.Setenv("RUNNER_POLL_INTERVAL", "1m") + t.Setenv("RUNNER_MAX_CONCURRENT_SCRAPE", "5") + t.Setenv("RUNNER_WORKER_ID", "homelab-01") + t.Setenv("BACKEND_HTTP_ADDR", ":9090") + t.Setenv("KOKORO_URL", "https://kokoro.libnovel.cc") + + cfg := config.Load() + + if cfg.LogLevel != "debug" { + t.Errorf("LogLevel: want debug, got %q", cfg.LogLevel) + } + if cfg.PocketBase.URL != "https://pb.libnovel.cc" { + t.Errorf("PocketBase.URL: want https://pb.libnovel.cc, got %q", cfg.PocketBase.URL) + } + if !cfg.MinIO.UseSSL { + t.Error("MinIO.UseSSL: want true") + } + if cfg.MinIO.PublicUseSSL { + t.Error("MinIO.PublicUseSSL: want false") + } + if cfg.Runner.PollInterval != time.Minute { + t.Errorf("Runner.PollInterval: want 1m, got %v", cfg.Runner.PollInterval) + } + if cfg.Runner.MaxConcurrentScrape != 5 { + t.Errorf("Runner.MaxConcurrentScrape: want 5, got %d", cfg.Runner.MaxConcurrentScrape) + } + if cfg.Runner.WorkerID != "homelab-01" { + t.Errorf("Runner.WorkerID: want homelab-01, got %q", cfg.Runner.WorkerID) + } + if cfg.HTTP.Addr != ":9090" { + t.Errorf("HTTP.Addr: want :9090, got %q", cfg.HTTP.Addr) + } + if cfg.Kokoro.URL != "https://kokoro.libnovel.cc" { + t.Errorf("Kokoro.URL: want https://kokoro.libnovel.cc, got %q", cfg.Kokoro.URL) + } +} + +func TestLoad_InvalidInt_FallsToDefault(t *testing.T) { + t.Setenv("RUNNER_MAX_CONCURRENT_SCRAPE", "notanumber") + cfg := config.Load() + if cfg.Runner.MaxConcurrentScrape != 1 { + t.Errorf("want default 1, got %d", cfg.Runner.MaxConcurrentScrape) + } +} + +func TestLoad_InvalidDuration_FallsToDefault(t *testing.T) { + t.Setenv("RUNNER_POLL_INTERVAL", "notaduration") + cfg := config.Load() + if cfg.Runner.PollInterval != 30*time.Second { + t.Errorf("want default 30s, got %v", cfg.Runner.PollInterval) + } +} + +func TestLoad_WorkerID_FallsToHostname(t *testing.T) { + t.Setenv("RUNNER_WORKER_ID", "") + cfg := config.Load() + host, _ := os.Hostname() + if host != "" && cfg.Runner.WorkerID != host { + t.Errorf("want hostname %q, got %q", host, cfg.Runner.WorkerID) + } +} diff --git a/backend/internal/domain/domain.go b/backend/internal/domain/domain.go new file mode 100644 index 0000000..c093dbd --- /dev/null +++ b/backend/internal/domain/domain.go @@ -0,0 +1,246 @@ +// Package domain contains the core value types shared across all packages +// in this module. It has zero internal imports — only the standard library. +// Every other package imports domain; domain imports nothing from this module. +package domain + +import "time" + +// ── Book types ──────────────────────────────────────────────────────────────── + +// Visibility values for BookMeta.Visibility. +const ( + VisibilityPublic = "public" // visible to all users + VisibilityAdminOnly = "admin_only" // visible only to admin users (e.g. scraped content) +) + +// BookMeta carries all bibliographic information about a novel. +type BookMeta struct { + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author"` + Cover string `json:"cover,omitempty"` + Status string `json:"status,omitempty"` + Genres []string `json:"genres,omitempty"` + Summary string `json:"summary,omitempty"` + TotalChapters int `json:"total_chapters,omitempty"` + SourceURL string `json:"source_url"` + Ranking int `json:"ranking,omitempty"` + Rating float64 `json:"rating,omitempty"` + // MetaUpdated is the Unix timestamp (seconds) when the book record was last + // updated in PocketBase. Populated on read; not sent on write (PocketBase + // manages its own updated field). + MetaUpdated int64 `json:"meta_updated,omitempty"` + // Archived is true when the book has been soft-deleted by an admin. + // Archived books are excluded from all public search and catalogue responses. + Archived bool `json:"archived,omitempty"` + // Visibility controls who can see this book. + // "public" = all users; "admin_only" = admin only (default for scraped content). + Visibility string `json:"visibility,omitempty"` + // SubmittedBy is the app_users ID of the author who submitted this book, + // or empty for scraped books. + SubmittedBy string `json:"submitted_by,omitempty"` +} + +// CatalogueEntry is a lightweight book reference returned by catalogue pages. +type CatalogueEntry struct { + Slug string `json:"slug"` + Title string `json:"title"` + URL string `json:"url"` +} + +// ChapterRef is a reference to a single chapter returned by chapter-list pages. +type ChapterRef struct { + Number int `json:"number"` + Title string `json:"title"` + URL string `json:"url"` + Volume int `json:"volume,omitempty"` +} + +// Chapter contains the fully-extracted text of a single chapter. +type Chapter struct { + Ref ChapterRef `json:"ref"` + Text string `json:"text"` +} + +// RankingItem represents a single entry in the novel ranking list. +type RankingItem struct { + Rank int `json:"rank"` + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author,omitempty"` + Cover string `json:"cover,omitempty"` + Status string `json:"status,omitempty"` + Genres []string `json:"genres,omitempty"` + SourceURL string `json:"source_url,omitempty"` + Updated time.Time `json:"updated,omitempty"` +} + +// ── Voice types ─────────────────────────────────────────────────────────────── + +// Voice describes a single text-to-speech voice available in the system. +type Voice struct { + // ID is the voice identifier passed to TTS clients (e.g. "af_bella", "alba"). + ID string `json:"id"` + // Engine is "kokoro" or "pocket-tts". + Engine string `json:"engine"` + // Lang is the primary language tag (e.g. "en-us", "en-gb", "en", "es", "fr"). + Lang string `json:"lang"` + // Gender is "f" or "m". + Gender string `json:"gender"` +} + +// ── Storage record types ────────────────────────────────────────────────────── + +// ChapterInfo is a lightweight chapter descriptor stored in the index. +type ChapterInfo struct { + Number int `json:"number"` + Title string `json:"title"` + Date string `json:"date,omitempty"` +} + +// ReadingProgress holds a single user's reading position for one book. +type ReadingProgress struct { + Slug string `json:"slug"` + Chapter int `json:"chapter"` + UpdatedAt time.Time `json:"updated_at"` +} + +// ── Task record types ───────────────────────────────────────────────────────── + +// TaskStatus enumerates the lifecycle states of any task. +type TaskStatus string + +const ( + TaskStatusPending TaskStatus = "pending" + TaskStatusRunning TaskStatus = "running" + TaskStatusDone TaskStatus = "done" + TaskStatusFailed TaskStatus = "failed" + TaskStatusCancelled TaskStatus = "cancelled" +) + +// ScrapeTask represents a book-scraping job stored in PocketBase. +type ScrapeTask struct { + ID string `json:"id"` + Kind string `json:"kind"` // "catalogue" | "book" | "book_range" + TargetURL string `json:"target_url"` // non-empty for single-book tasks + FromChapter int `json:"from_chapter,omitempty"` + ToChapter int `json:"to_chapter,omitempty"` + WorkerID string `json:"worker_id,omitempty"` + Status TaskStatus `json:"status"` + BooksFound int `json:"books_found"` + ChaptersScraped int `json:"chapters_scraped"` + ChaptersSkipped int `json:"chapters_skipped"` + Errors int `json:"errors"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` +} + +// ScrapeResult is the outcome reported by the runner after finishing a ScrapeTask. +type ScrapeResult struct { + // Slug is the book slug that was scraped. Empty for catalogue tasks. + Slug string `json:"slug,omitempty"` + BooksFound int `json:"books_found"` + ChaptersScraped int `json:"chapters_scraped"` + ChaptersSkipped int `json:"chapters_skipped"` + Errors int `json:"errors"` + ErrorMessage string `json:"error_message,omitempty"` +} + +// AudioTask represents an audio-generation job stored in PocketBase. +type AudioTask struct { + ID string `json:"id"` + CacheKey string `json:"cache_key"` // "slug/chapter/voice" + Slug string `json:"slug"` + Chapter int `json:"chapter"` + Voice string `json:"voice"` + WorkerID string `json:"worker_id,omitempty"` + Status TaskStatus `json:"status"` + ErrorMessage string `json:"error_message,omitempty"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished,omitempty"` +} + +// AudioResult is the outcome reported by the runner after finishing an AudioTask. +type AudioResult struct { + ObjectKey string `json:"object_key,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` +} + +// TranslationTask represents a machine-translation job stored in PocketBase. +type TranslationTask struct { + ID string `json:"id"` + CacheKey string `json:"cache_key"` // "{slug}/{chapter}/{lang}" + Slug string `json:"slug"` + Chapter int `json:"chapter"` + Lang string `json:"lang"` + WorkerID string `json:"worker_id,omitempty"` + Status TaskStatus `json:"status"` + ErrorMessage string `json:"error_message,omitempty"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished,omitempty"` +} + +// TranslationResult is the outcome reported by the runner after finishing a TranslationTask. +type TranslationResult struct { + ObjectKey string `json:"object_key,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` +} + +// ImportTask represents a PDF/EPUB import job stored in PocketBase. +type ImportTask struct { + ID string `json:"id"` + Slug string `json:"slug"` // derived from filename + Title string `json:"title"` + FileName string `json:"file_name"` + FileType string `json:"file_type"` // "pdf" or "epub" + ObjectKey string `json:"object_key,omitempty"` // MinIO path to uploaded file + ChaptersKey string `json:"chapters_key,omitempty"` // MinIO path to pre-parsed chapters JSON + Author string `json:"author,omitempty"` + CoverURL string `json:"cover_url,omitempty"` + Genres []string `json:"genres,omitempty"` + Summary string `json:"summary,omitempty"` + BookStatus string `json:"book_status,omitempty"` // "ongoing" | "completed" | "hiatus" + WorkerID string `json:"worker_id,omitempty"` + InitiatorUserID string `json:"initiator_user_id,omitempty"` // PocketBase user ID who submitted the import + Status TaskStatus `json:"status"` + ChaptersDone int `json:"chapters_done"` + ChaptersTotal int `json:"chapters_total"` + ErrorMessage string `json:"error_message,omitempty"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished,omitempty"` +} + +// ImportResult is the outcome reported by the runner after finishing an ImportTask. +type ImportResult struct { + Slug string `json:"slug,omitempty"` + ChaptersImported int `json:"chapters_imported"` + ErrorMessage string `json:"error_message,omitempty"` +} + +// AIJob represents an AI generation task tracked in PocketBase (ai_jobs collection). +type AIJob struct { + ID string `json:"id"` + // Kind is one of: "chapter-names", "batch-covers", "chapter-covers", "refresh-metadata". + Kind string `json:"kind"` + // Slug is the book slug for per-book jobs; empty for catalogue-wide jobs. + Slug string `json:"slug"` + Status TaskStatus `json:"status"` + // FromItem is the first item to process (chapter number, or 0-based book index). + // 0 = start from the beginning. + FromItem int `json:"from_item"` + // ToItem is the last item to process (inclusive). 0 = process all. + ToItem int `json:"to_item"` + // ItemsDone is the cumulative count of successfully processed items. + ItemsDone int `json:"items_done"` + // ItemsTotal is the total number of items in this job. + ItemsTotal int `json:"items_total"` + Model string `json:"model"` + // Payload is a JSON-encoded string with job-specific parameters + // (e.g. naming pattern for chapter-names, num_steps for batch-covers). + Payload string `json:"payload"` + ErrorMessage string `json:"error_message,omitempty"` + Started time.Time `json:"started,omitempty"` + Finished time.Time `json:"finished,omitempty"` + HeartbeatAt time.Time `json:"heartbeat_at,omitempty"` +} diff --git a/backend/internal/domain/domain_test.go b/backend/internal/domain/domain_test.go new file mode 100644 index 0000000..c364657 --- /dev/null +++ b/backend/internal/domain/domain_test.go @@ -0,0 +1,104 @@ +package domain_test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/libnovel/backend/internal/domain" +) + +func TestBookMeta_JSONRoundtrip(t *testing.T) { + orig := domain.BookMeta{ + Slug: "a-great-novel", + Title: "A Great Novel", + Author: "Jane Doe", + Cover: "https://example.com/cover.jpg", + Status: "Ongoing", + Genres: []string{"Fantasy", "Action"}, + Summary: "A thrilling tale.", + TotalChapters: 120, + SourceURL: "https://novelfire.net/book/a-great-novel", + Ranking: 3, + } + + b, err := json.Marshal(orig) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var got domain.BookMeta + if err := json.Unmarshal(b, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got.Slug != orig.Slug { + t.Errorf("Slug: want %q, got %q", orig.Slug, got.Slug) + } + if got.TotalChapters != orig.TotalChapters { + t.Errorf("TotalChapters: want %d, got %d", orig.TotalChapters, got.TotalChapters) + } + if len(got.Genres) != len(orig.Genres) { + t.Errorf("Genres len: want %d, got %d", len(orig.Genres), len(got.Genres)) + } +} + +func TestChapterRef_JSONRoundtrip(t *testing.T) { + orig := domain.ChapterRef{Number: 42, Title: "The Battle", URL: "https://example.com/ch-42", Volume: 2} + b, _ := json.Marshal(orig) + var got domain.ChapterRef + json.Unmarshal(b, &got) + if got != orig { + t.Errorf("want %+v, got %+v", orig, got) + } +} + +func TestRankingItem_JSONRoundtrip(t *testing.T) { + now := time.Now().Truncate(time.Second) + orig := domain.RankingItem{ + Rank: 1, + Slug: "top-novel", + Title: "Top Novel", + SourceURL: "https://novelfire.net/book/top-novel", + Updated: now, + } + b, _ := json.Marshal(orig) + var got domain.RankingItem + json.Unmarshal(b, &got) + if got.Rank != orig.Rank || got.Slug != orig.Slug { + t.Errorf("want %+v, got %+v", orig, got) + } +} + +func TestScrapeResult_JSONRoundtrip(t *testing.T) { + orig := domain.ScrapeResult{BooksFound: 10, ChaptersScraped: 200, ChaptersSkipped: 5, Errors: 1, ErrorMessage: "one error"} + b, _ := json.Marshal(orig) + var got domain.ScrapeResult + json.Unmarshal(b, &got) + if got != orig { + t.Errorf("want %+v, got %+v", orig, got) + } +} + +func TestAudioResult_JSONRoundtrip(t *testing.T) { + orig := domain.AudioResult{ObjectKey: "audio/slug/1/af_bella.mp3"} + b, _ := json.Marshal(orig) + var got domain.AudioResult + json.Unmarshal(b, &got) + if got != orig { + t.Errorf("want %+v, got %+v", orig, got) + } +} + +func TestTaskStatus_Values(t *testing.T) { + cases := []domain.TaskStatus{ + domain.TaskStatusPending, + domain.TaskStatusRunning, + domain.TaskStatusDone, + domain.TaskStatusFailed, + domain.TaskStatusCancelled, + } + for _, s := range cases { + if s == "" { + t.Errorf("TaskStatus constant must not be empty") + } + } +} diff --git a/backend/internal/httputil/httputil.go b/backend/internal/httputil/httputil.go new file mode 100644 index 0000000..f36359e --- /dev/null +++ b/backend/internal/httputil/httputil.go @@ -0,0 +1,124 @@ +// Package httputil provides shared HTTP helpers used by both the runner and +// backend binaries. It has no imports from this module — only the standard +// library — so it is safe to import from anywhere in the dependency graph. +package httputil + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" +) + +// Client is the minimal interface for making HTTP GET requests. +// *http.Client satisfies this interface. +type Client interface { + Do(req *http.Request) (*http.Response, error) +} + +// ErrMaxRetries is returned when RetryGet exhausts all attempts. +var ErrMaxRetries = errors.New("httputil: max retries exceeded") + +// errClientError is returned by doGet for 4xx responses; it signals that the +// request should NOT be retried (the client is at fault). +var errClientError = errors.New("httputil: client error") + +// RetryGet fetches url using client, retrying on network errors or 5xx +// responses with exponential backoff. It returns the full response body as a +// string on success. +// +// - maxAttempts: total number of attempts (must be >= 1) +// - baseDelay: initial wait before the second attempt; doubles each retry +func RetryGet(ctx context.Context, client Client, url string, maxAttempts int, baseDelay time.Duration) (string, error) { + if maxAttempts < 1 { + maxAttempts = 1 + } + delay := baseDelay + + var lastErr error + for attempt := 0; attempt < maxAttempts; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(delay): + } + delay *= 2 + } + + body, err := doGet(ctx, client, url) + if err == nil { + return body, nil + } + lastErr = err + + // Do not retry on context cancellation. + if ctx.Err() != nil { + return "", ctx.Err() + } + // Do not retry on 4xx — the client is at fault. + if errors.Is(err, errClientError) { + return "", err + } + } + + return "", fmt.Errorf("%w after %d attempts: %w", ErrMaxRetries, maxAttempts, lastErr) +} + +func doGet(ctx context.Context, client Client, url string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", fmt.Errorf("build request: %w", err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)") + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("GET %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 500 { + return "", fmt.Errorf("GET %s: server error %d", url, resp.StatusCode) + } + if resp.StatusCode >= 400 { + return "", fmt.Errorf("%w: GET %s: client error %d", errClientError, url, resp.StatusCode) + } + + raw, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("read body %s: %w", url, err) + } + return string(raw), nil +} + +// WriteJSON writes v as JSON to w with the given HTTP status code and sets the +// Content-Type header to application/json. +func WriteJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +// WriteError writes a JSON error object {"error": msg} with the given status. +func WriteError(w http.ResponseWriter, status int, msg string) { + WriteJSON(w, status, map[string]string{"error": msg}) +} + +// maxBodyBytes is the limit applied by DecodeJSON to prevent unbounded reads. +const maxBodyBytes = 1 << 20 // 1 MiB + +// DecodeJSON decodes a JSON request body into v. It enforces a 1 MiB size +// limit and returns a descriptive error on any failure. +func DecodeJSON(r *http.Request, v any) error { + r.Body = http.MaxBytesReader(nil, r.Body, maxBodyBytes) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil { + return fmt.Errorf("decode JSON body: %w", err) + } + return nil +} diff --git a/backend/internal/httputil/httputil_test.go b/backend/internal/httputil/httputil_test.go new file mode 100644 index 0000000..2af3bba --- /dev/null +++ b/backend/internal/httputil/httputil_test.go @@ -0,0 +1,181 @@ +package httputil_test + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/libnovel/backend/internal/httputil" +) + +// ── RetryGet ────────────────────────────────────────────────────────────────── + +func TestRetryGet_ImmediateSuccess(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hello")) + })) + defer srv.Close() + + body, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 3, time.Millisecond) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body != "hello" { + t.Errorf("want hello, got %q", body) + } +} + +func TestRetryGet_RetriesOn5xx(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Write([]byte("ok")) + })) + defer srv.Close() + + body, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 5, time.Millisecond) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if body != "ok" { + t.Errorf("want ok, got %q", body) + } + if calls != 3 { + t.Errorf("want 3 calls, got %d", calls) + } +} + +func TestRetryGet_MaxAttemptsExceeded(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + _, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 3, time.Millisecond) + if err == nil { + t.Fatal("expected error, got nil") + } +} + +func TestRetryGet_ContextCancelDuringBackoff(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + + // Cancel after first failed attempt hits the backoff wait. + go func() { time.Sleep(5 * time.Millisecond); cancel() }() + + _, err := httputil.RetryGet(ctx, srv.Client(), srv.URL, 10, 500*time.Millisecond) + if err == nil { + t.Fatal("expected context cancellation error") + } +} + +func TestRetryGet_NoRetryOn4xx(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + _, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 5, time.Millisecond) + if err == nil { + t.Fatal("expected error for 404") + } + // 4xx is NOT retried — should be exactly 1 call. + if calls != 1 { + t.Errorf("want 1 call for 4xx, got %d", calls) + } +} + +// ── WriteJSON ───────────────────────────────────────────────────────────────── + +func TestWriteJSON_SetsHeadersAndStatus(t *testing.T) { + rr := httptest.NewRecorder() + httputil.WriteJSON(rr, http.StatusCreated, map[string]string{"key": "val"}) + + if rr.Code != http.StatusCreated { + t.Errorf("status: want 201, got %d", rr.Code) + } + if ct := rr.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("Content-Type: want application/json, got %q", ct) + } + var got map[string]string + if err := json.NewDecoder(rr.Body).Decode(&got); err != nil { + t.Fatalf("decode body: %v", err) + } + if got["key"] != "val" { + t.Errorf("body key: want val, got %q", got["key"]) + } +} + +// ── WriteError ──────────────────────────────────────────────────────────────── + +func TestWriteError_Format(t *testing.T) { + rr := httptest.NewRecorder() + httputil.WriteError(rr, http.StatusBadRequest, "bad input") + + if rr.Code != http.StatusBadRequest { + t.Errorf("status: want 400, got %d", rr.Code) + } + var got map[string]string + json.NewDecoder(rr.Body).Decode(&got) + if got["error"] != "bad input" { + t.Errorf("error field: want bad input, got %q", got["error"]) + } +} + +// ── DecodeJSON ──────────────────────────────────────────────────────────────── + +func TestDecodeJSON_HappyPath(t *testing.T) { + body := `{"name":"test","value":42}` + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + + var payload struct { + Name string `json:"name"` + Value int `json:"value"` + } + if err := httputil.DecodeJSON(req, &payload); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if payload.Name != "test" || payload.Value != 42 { + t.Errorf("unexpected payload: %+v", payload) + } +} + +func TestDecodeJSON_UnknownFieldReturnsError(t *testing.T) { + body := `{"name":"test","unknown_field":"boom"}` + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + + var payload struct { + Name string `json:"name"` + } + if err := httputil.DecodeJSON(req, &payload); err == nil { + t.Fatal("expected error for unknown field, got nil") + } +} + +func TestDecodeJSON_BodyTooLarge(t *testing.T) { + // Build a body > 1 MiB. + big := bytes.Repeat([]byte("a"), 2<<20) + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(big)) + + var payload map[string]any + if err := httputil.DecodeJSON(req, &payload); err == nil { + t.Fatal("expected error for oversized body, got nil") + } +} diff --git a/backend/internal/kokoro/client.go b/backend/internal/kokoro/client.go new file mode 100644 index 0000000..6895385 --- /dev/null +++ b/backend/internal/kokoro/client.go @@ -0,0 +1,255 @@ +// Package kokoro provides a client for the Kokoro-FastAPI TTS service. +// +// The Kokoro API is an OpenAI-compatible audio speech API that returns a +// download link (X-Download-Path header) instead of streaming audio directly. +// GenerateAudio handles the two-step flow: POST /v1/audio/speech → GET /v1/download/{file}. +package kokoro + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// Client is the interface for interacting with the Kokoro TTS service. +type Client interface { + // GenerateAudio synthesises text using voice and returns raw MP3 bytes. + GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) + + // StreamAudioMP3 synthesises text and returns an io.ReadCloser that streams + // MP3-encoded audio incrementally. Uses the kokoro-fastapi streaming mode + // (stream:true), which delivers MP3 frames as they are generated without + // waiting for the full output. The caller must always close the ReadCloser. + StreamAudioMP3(ctx context.Context, text, voice string) (io.ReadCloser, error) + + // StreamAudioWAV synthesises text and returns an io.ReadCloser that streams + // WAV-encoded audio incrementally using kokoro-fastapi's streaming mode with + // response_format:"wav". The caller must always close the ReadCloser. + StreamAudioWAV(ctx context.Context, text, voice string) (io.ReadCloser, error) + + // ListVoices returns the available voice IDs. Falls back to an empty slice + // on error — callers should treat an empty list as "service unavailable". + ListVoices(ctx context.Context) ([]string, error) +} + +// httpClient is the concrete Kokoro HTTP client. +type httpClient struct { + baseURL string + http *http.Client +} + +// New returns a Kokoro Client targeting baseURL (e.g. "https://kokoro.example.com"). +func New(baseURL string) Client { + return &httpClient{ + baseURL: strings.TrimRight(baseURL, "/"), + http: &http.Client{Timeout: 10 * time.Minute}, + } +} + +// GenerateAudio calls POST /v1/audio/speech (return_download_link=true) and then +// downloads the resulting MP3 from GET /v1/download/{filename}. +func (c *httpClient) GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) { + if text == "" { + return nil, fmt.Errorf("kokoro: empty text") + } + if voice == "" { + voice = "af_bella" + } + + // ── Step 1: request generation ──────────────────────────────────────────── + reqBody, err := json.Marshal(map[string]any{ + "model": "kokoro", + "input": text, + "voice": voice, + "response_format": "mp3", + "speed": 1.0, + "stream": false, + "return_download_link": true, + }) + if err != nil { + return nil, fmt.Errorf("kokoro: marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/v1/audio/speech", bytes.NewReader(reqBody)) + if err != nil { + return nil, fmt.Errorf("kokoro: build speech request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("kokoro: speech request: %w", err) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("kokoro: speech returned %d", resp.StatusCode) + } + + dlPath := resp.Header.Get("X-Download-Path") + if dlPath == "" { + return nil, fmt.Errorf("kokoro: no X-Download-Path header in response") + } + filename := dlPath + if idx := strings.LastIndex(dlPath, "/"); idx >= 0 { + filename = dlPath[idx+1:] + } + if filename == "" { + return nil, fmt.Errorf("kokoro: empty filename in X-Download-Path: %q", dlPath) + } + + // ── Step 2: download the generated file ─────────────────────────────────── + dlURL := c.baseURL + "/v1/download/" + filename + dlReq, err := http.NewRequestWithContext(ctx, http.MethodGet, dlURL, nil) + if err != nil { + return nil, fmt.Errorf("kokoro: build download request: %w", err) + } + + dlResp, err := c.http.Do(dlReq) + if err != nil { + return nil, fmt.Errorf("kokoro: download request: %w", err) + } + defer dlResp.Body.Close() + + if dlResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("kokoro: download returned %d", dlResp.StatusCode) + } + + data, err := io.ReadAll(dlResp.Body) + if err != nil { + return nil, fmt.Errorf("kokoro: read download body: %w", err) + } + return data, nil +} + +// StreamAudioMP3 calls POST /v1/audio/speech with stream:true and returns an +// io.ReadCloser that delivers MP3 frames as kokoro generates them. +// kokoro-fastapi emits raw MP3 bytes when stream mode is enabled — no download +// redirect; the response body IS the audio stream. +func (c *httpClient) StreamAudioMP3(ctx context.Context, text, voice string) (io.ReadCloser, error) { + if text == "" { + return nil, fmt.Errorf("kokoro: empty text") + } + if voice == "" { + voice = "af_bella" + } + + reqBody, err := json.Marshal(map[string]any{ + "model": "kokoro", + "input": text, + "voice": voice, + "response_format": "mp3", + "speed": 1.0, + "stream": true, + }) + if err != nil { + return nil, fmt.Errorf("kokoro: marshal stream request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/v1/audio/speech", bytes.NewReader(reqBody)) + if err != nil { + return nil, fmt.Errorf("kokoro: build stream request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("kokoro: stream request: %w", err) + } + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("kokoro: stream returned %d", resp.StatusCode) + } + return resp.Body, nil +} + +// StreamAudioWAV calls POST /v1/audio/speech with stream:true and response_format:wav, +// returning an io.ReadCloser that delivers WAV bytes as kokoro generates them. +func (c *httpClient) StreamAudioWAV(ctx context.Context, text, voice string) (io.ReadCloser, error) { + if text == "" { + return nil, fmt.Errorf("kokoro: empty text") + } + if voice == "" { + voice = "af_bella" + } + + reqBody, err := json.Marshal(map[string]any{ + "model": "kokoro", + "input": text, + "voice": voice, + "response_format": "wav", + "speed": 1.0, + "stream": true, + }) + if err != nil { + return nil, fmt.Errorf("kokoro: marshal wav stream request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/v1/audio/speech", bytes.NewReader(reqBody)) + if err != nil { + return nil, fmt.Errorf("kokoro: build wav stream request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("kokoro: wav stream request: %w", err) + } + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("kokoro: wav stream returned %d", resp.StatusCode) + } + return resp.Body, nil +} + +// ListVoices calls GET /v1/audio/voices and returns the list of voice IDs. +func (c *httpClient) ListVoices(ctx context.Context) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + c.baseURL+"/v1/audio/voices", nil) + if err != nil { + return nil, fmt.Errorf("kokoro: build voices request: %w", err) + } + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("kokoro: voices request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("kokoro: voices returned %d", resp.StatusCode) + } + + var result struct { + Voices []string `json:"voices"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("kokoro: decode voices response: %w", err) + } + return result.Voices, nil +} + +// VoiceSampleKey returns the MinIO object key for a voice sample MP3. +// Key: _voice-samples/{voice}.mp3 (sanitised). +func VoiceSampleKey(voice string) string { + safe := strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || r == '_' || r == '-' { + return r + } + return '_' + }, voice) + return fmt.Sprintf("_voice-samples/%s.mp3", safe) +} diff --git a/backend/internal/kokoro/client_test.go b/backend/internal/kokoro/client_test.go new file mode 100644 index 0000000..9d63bbc --- /dev/null +++ b/backend/internal/kokoro/client_test.go @@ -0,0 +1,291 @@ +package kokoro_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/libnovel/backend/internal/kokoro" +) + +// ── VoiceSampleKey ──────────────────────────────────────────────────────────── + +func TestVoiceSampleKey(t *testing.T) { + tests := []struct { + voice string + want string + }{ + {"af_bella", "_voice-samples/af_bella.mp3"}, + {"am_echo", "_voice-samples/am_echo.mp3"}, + {"voice with spaces", "_voice-samples/voice_with_spaces.mp3"}, + {"special!@#chars", "_voice-samples/special___chars.mp3"}, + {"", "_voice-samples/.mp3"}, + } + for _, tt := range tests { + t.Run(tt.voice, func(t *testing.T) { + got := kokoro.VoiceSampleKey(tt.voice) + if got != tt.want { + t.Errorf("VoiceSampleKey(%q) = %q, want %q", tt.voice, got, tt.want) + } + }) + } +} + +// ── GenerateAudio ───────────────────────────────────────────────────────────── + +func TestGenerateAudio_EmptyText(t *testing.T) { + srv := httptest.NewServer(http.NotFoundHandler()) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.GenerateAudio(context.Background(), "", "af_bella") + if err == nil { + t.Fatal("expected error for empty text, got nil") + } + if !strings.Contains(err.Error(), "empty text") { + t.Errorf("expected 'empty text' in error, got: %v", err) + } +} + +func TestGenerateAudio_DefaultVoice(t *testing.T) { + // Tracks that the voice defaults to af_bella when empty. + var capturedBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/speech" { + buf := make([]byte, 512) + n, _ := r.Body.Read(buf) + capturedBody = string(buf[:n]) + w.Header().Set("X-Download-Path", "/download/test_file.mp3") + w.WriteHeader(http.StatusOK) + return + } + if strings.HasPrefix(r.URL.Path, "/v1/download/") { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("fake-mp3-data")) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + data, err := c.GenerateAudio(context.Background(), "hello world", "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(data) != "fake-mp3-data" { + t.Errorf("unexpected data: %q", string(data)) + } + if !strings.Contains(capturedBody, `"af_bella"`) { + t.Errorf("expected default voice af_bella in request body, got: %s", capturedBody) + } +} + +func TestGenerateAudio_SpeechNon200(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/speech" { + w.WriteHeader(http.StatusInternalServerError) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.GenerateAudio(context.Background(), "text", "af_bella") + if err == nil { + t.Fatal("expected error for non-200 speech response") + } + if !strings.Contains(err.Error(), "500") { + t.Errorf("expected 500 in error, got: %v", err) + } +} + +func TestGenerateAudio_NoDownloadPathHeader(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/speech" { + // No X-Download-Path header + w.WriteHeader(http.StatusOK) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.GenerateAudio(context.Background(), "text", "af_bella") + if err == nil { + t.Fatal("expected error for missing X-Download-Path") + } + if !strings.Contains(err.Error(), "X-Download-Path") { + t.Errorf("expected X-Download-Path in error, got: %v", err) + } +} + +func TestGenerateAudio_DownloadFails(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/speech" { + w.Header().Set("X-Download-Path", "/v1/download/speech.mp3") + w.WriteHeader(http.StatusOK) + return + } + if strings.HasPrefix(r.URL.Path, "/v1/download/") { + w.WriteHeader(http.StatusNotFound) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.GenerateAudio(context.Background(), "text", "af_bella") + if err == nil { + t.Fatal("expected error for failed download") + } + if !strings.Contains(err.Error(), "404") { + t.Errorf("expected 404 in error, got: %v", err) + } +} + +func TestGenerateAudio_FullPath(t *testing.T) { + // X-Download-Path with a full path: extract just filename. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/speech" { + w.Header().Set("X-Download-Path", "/some/nested/path/audio_abc123.mp3") + w.WriteHeader(http.StatusOK) + return + } + if r.URL.Path == "/v1/download/audio_abc123.mp3" { + _, _ = w.Write([]byte("audio-bytes")) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + data, err := c.GenerateAudio(context.Background(), "text", "af_bella") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(data) != "audio-bytes" { + t.Errorf("unexpected data: %q", string(data)) + } +} + +func TestGenerateAudio_ContextCancelled(t *testing.T) { + // Server that hangs — context should cancel before we get a response. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Never respond. + select {} + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + c := kokoro.New(srv.URL) + _, err := c.GenerateAudio(ctx, "text", "af_bella") + if err == nil { + t.Fatal("expected error for cancelled context") + } +} + +// ── ListVoices ──────────────────────────────────────────────────────────────── + +func TestListVoices_Success(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/voices" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"voices":["af_bella","am_adam","bf_emma"]}`)) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + voices, err := c.ListVoices(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(voices) != 3 { + t.Errorf("expected 3 voices, got %d: %v", len(voices), voices) + } + if voices[0] != "af_bella" { + t.Errorf("expected first voice to be af_bella, got %q", voices[0]) + } +} + +func TestListVoices_Non200(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.ListVoices(context.Background()) + if err == nil { + t.Fatal("expected error for non-200 response") + } + if !strings.Contains(err.Error(), "503") { + t.Errorf("expected 503 in error, got: %v", err) + } +} + +func TestListVoices_MalformedJSON(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`not-json`)) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + _, err := c.ListVoices(context.Background()) + if err == nil { + t.Fatal("expected error for malformed JSON") + } +} + +func TestListVoices_EmptyVoices(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"voices":[]}`)) + })) + defer srv.Close() + + c := kokoro.New(srv.URL) + voices, err := c.ListVoices(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(voices) != 0 { + t.Errorf("expected 0 voices, got %d", len(voices)) + } +} + +// ── New ─────────────────────────────────────────────────────────────────────── + +func TestNew_TrailingSlashStripped(t *testing.T) { + // Verify that a trailing slash on baseURL doesn't produce double-slash paths. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/audio/voices" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"voices":["af_bella"]}`)) + return + } + http.NotFound(w, r) + })) + defer srv.Close() + + c := kokoro.New(srv.URL + "/") // trailing slash + voices, err := c.ListVoices(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(voices) == 0 { + t.Error("expected at least one voice") + } +} diff --git a/backend/internal/libretranslate/client.go b/backend/internal/libretranslate/client.go new file mode 100644 index 0000000..3417864 --- /dev/null +++ b/backend/internal/libretranslate/client.go @@ -0,0 +1,181 @@ +// Package libretranslate provides an HTTP client for a self-hosted +// LibreTranslate instance. It handles text chunking, concurrent translation, +// and reassembly so callers can pass arbitrarily long markdown strings. +package libretranslate + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "time" +) + +const ( + // maxChunkBytes is the target maximum size of each chunk sent to + // LibreTranslate. LibreTranslate's default limit is 5000 characters; + // we stay comfortably below that. + maxChunkBytes = 4500 + // concurrency is the number of simultaneous translation requests per chapter. + concurrency = 3 +) + +// Client translates text via LibreTranslate. +// A nil Client is valid — all calls return the original text unchanged. +type Client interface { + // Translate translates text from sourceLang to targetLang. + // text is a raw markdown string. The returned string is the translated + // markdown, reassembled in original paragraph order. + Translate(ctx context.Context, text, sourceLang, targetLang string) (string, error) +} + +// New returns a Client for the given LibreTranslate URL. +// Returns nil when url is empty, which disables translation. +func New(url, apiKey string) Client { + if url == "" { + return nil + } + return &httpClient{ + url: strings.TrimRight(url, "/"), + apiKey: apiKey, + http: &http.Client{Timeout: 60 * time.Second}, + } +} + +type httpClient struct { + url string + apiKey string + http *http.Client +} + +// Translate splits text into paragraph chunks, translates them concurrently +// (up to concurrency goroutines), and reassembles in order. +func (c *httpClient) Translate(ctx context.Context, text, sourceLang, targetLang string) (string, error) { + paragraphs := splitParagraphs(text) + if len(paragraphs) == 0 { + return text, nil + } + chunks := binChunks(paragraphs, maxChunkBytes) + + translated := make([]string, len(chunks)) + errs := make([]error, len(chunks)) + + sem := make(chan struct{}, concurrency) + var wg sync.WaitGroup + + for i, chunk := range chunks { + wg.Add(1) + sem <- struct{}{} + go func(idx int, chunkText string) { + defer wg.Done() + defer func() { <-sem }() + result, err := c.translateChunk(ctx, chunkText, sourceLang, targetLang) + translated[idx] = result + errs[idx] = err + }(i, chunk) + } + wg.Wait() + + for _, err := range errs { + if err != nil { + return "", err + } + } + + return strings.Join(translated, "\n\n"), nil +} + +// translateChunk sends a single POST /translate request. +func (c *httpClient) translateChunk(ctx context.Context, text, sourceLang, targetLang string) (string, error) { + reqBody := map[string]string{ + "q": text, + "source": sourceLang, + "target": targetLang, + "format": "html", + } + if c.apiKey != "" { + reqBody["api_key"] = c.apiKey + } + + b, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("libretranslate: marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url+"/translate", bytes.NewReader(b)) + if err != nil { + return "", fmt.Errorf("libretranslate: build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.http.Do(req) + if err != nil { + return "", fmt.Errorf("libretranslate: request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + var errBody struct { + Error string `json:"error"` + } + _ = json.NewDecoder(resp.Body).Decode(&errBody) + return "", fmt.Errorf("libretranslate: status %d: %s", resp.StatusCode, errBody.Error) + } + + var result struct { + TranslatedText string `json:"translatedText"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("libretranslate: decode response: %w", err) + } + return result.TranslatedText, nil +} + +// splitParagraphs splits markdown text on blank lines, preserving non-empty paragraphs. +func splitParagraphs(text string) []string { + // Normalise line endings. + text = strings.ReplaceAll(text, "\r\n", "\n") + // Split on double newlines (blank lines between paragraphs). + parts := strings.Split(text, "\n\n") + var paragraphs []string + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + paragraphs = append(paragraphs, p) + } + } + return paragraphs +} + +// binChunks groups paragraphs into chunks each at most maxBytes in length. +// Each chunk is a single string with paragraphs joined by "\n\n". +func binChunks(paragraphs []string, maxBytes int) []string { + var chunks []string + var current strings.Builder + + for _, p := range paragraphs { + needed := len(p) + if current.Len() > 0 { + needed += 2 // for the "\n\n" separator + } + + if current.Len()+needed > maxBytes && current.Len() > 0 { + // Flush current chunk. + chunks = append(chunks, current.String()) + current.Reset() + } + + if current.Len() > 0 { + current.WriteString("\n\n") + } + current.WriteString(p) + } + + if current.Len() > 0 { + chunks = append(chunks, current.String()) + } + return chunks +} diff --git a/backend/internal/meili/client.go b/backend/internal/meili/client.go new file mode 100644 index 0000000..1554558 --- /dev/null +++ b/backend/internal/meili/client.go @@ -0,0 +1,357 @@ +// Package meili provides a thin Meilisearch client for indexing and searching +// locally scraped books. +// +// Index: +// - Name: "books" +// - Primary key: "slug" +// - Searchable attributes: title, author, genres, summary +// - Filterable attributes: status, genres +// - Sortable attributes: rank, rating, total_chapters, meta_updated +// +// The client is intentionally simple: UpsertBook and Search only. All +// Meilisearch-specific details (index management, attribute configuration) +// are handled once in Configure(), called at startup. +package meili + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/libnovel/backend/internal/domain" + "github.com/meilisearch/meilisearch-go" +) + +const indexName = "books" + +// Client is the interface for Meilisearch operations used by runner and backend. +type Client interface { + // UpsertBook adds or updates a book document in the search index. + UpsertBook(ctx context.Context, book domain.BookMeta) error + // BookExists reports whether a book with the given slug is already in the + // index. Used by the catalogue refresh to skip re-indexing known books. + BookExists(ctx context.Context, slug string) bool + // DeleteBook removes a book document from the search index by slug. + DeleteBook(ctx context.Context, slug string) error + // Search returns up to limit books matching query. + // Archived books are always excluded. + Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error) + // Catalogue queries books with optional filters, sort, and pagination. + // Returns books, the total hit count for pagination, and a FacetResult + // with available genre and status values from the index. + // Archived books are always excluded. + Catalogue(ctx context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) +} + +// CatalogueQuery holds parameters for the /api/catalogue endpoint. +type CatalogueQuery struct { + Q string // full-text query (may be empty for browse) + Genre string // genre filter, e.g. "fantasy" or "all" + Status string // status filter, e.g. "ongoing", "completed", or "all" + Sort string // sort field: "popular", "new", "update", "top-rated", "rank", "" + Page int // 1-indexed + Limit int // items per page, default 20 + // AdminAll disables the visibility filter so admin users see all non-archived + // books including those marked admin_only. + AdminAll bool +} + +// FacetResult holds the available filter values discovered from the index. +// Values are sorted alphabetically and include only those present in the index. +type FacetResult struct { + Genres []string // distinct genre values + Statuses []string // distinct status values +} + +// MeiliClient wraps the meilisearch-go SDK. +type MeiliClient struct { + idx meilisearch.IndexManager +} + +// New creates a MeiliClient. Call Configure() once at startup to ensure the +// index exists and has the correct attribute settings. +func New(host, apiKey string) *MeiliClient { + cli := meilisearch.New(host, meilisearch.WithAPIKey(apiKey)) + return &MeiliClient{idx: cli.Index(indexName)} +} + +// Configure creates the index if absent and sets searchable/filterable +// attributes. It is idempotent — safe to call on every startup. +func Configure(host, apiKey string) error { + cli := meilisearch.New(host, meilisearch.WithAPIKey(apiKey)) + + // Create index with primary key. Returns 202 if exists — ignore. + task, err := cli.CreateIndex(&meilisearch.IndexConfig{ + Uid: indexName, + PrimaryKey: "slug", + }) + if err != nil { + // 400 "index_already_exists" is not an error here; the SDK returns + // an error with Code "index_already_exists" which we can ignore. + // Any other error is fatal. + if apiErr, ok := err.(*meilisearch.Error); ok && apiErr.MeilisearchApiError.Code == "index_already_exists" { + // already exists — continue + } else { + return fmt.Errorf("meili: create index: %w", err) + } + } else { + _ = task // task is async; we don't wait for it + } + + idx := cli.Index(indexName) + + searchable := []string{"title", "author", "genres", "summary"} + if _, err := idx.UpdateSearchableAttributes(&searchable); err != nil { + return fmt.Errorf("meili: update searchable attributes: %w", err) + } + + filterable := []interface{}{"status", "genres", "archived", "visibility"} + if _, err := idx.UpdateFilterableAttributes(&filterable); err != nil { + return fmt.Errorf("meili: update filterable attributes: %w", err) + } + + sortable := []string{"rank", "rating", "total_chapters", "meta_updated"} + if _, err := idx.UpdateSortableAttributes(&sortable); err != nil { + return fmt.Errorf("meili: update sortable attributes: %w", err) + } + + return nil +} + +// bookDoc is the Meilisearch document shape for a book. +type bookDoc struct { + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author"` + Cover string `json:"cover"` + Status string `json:"status"` + Genres []string `json:"genres"` + Summary string `json:"summary"` + TotalChapters int `json:"total_chapters"` + SourceURL string `json:"source_url"` + Rank int `json:"rank"` + Rating float64 `json:"rating"` + // MetaUpdated is the Unix timestamp (seconds) of the last PocketBase update. + // Used for sort=update ("recently updated" ordering). + MetaUpdated int64 `json:"meta_updated"` + // Archived is true when the book has been soft-deleted by an admin. + // Used as a filter to exclude archived books from all search results. + Archived bool `json:"archived"` + // Visibility is "public" or "admin_only". Only public books are shown to + // non-admin users. Empty string is treated as admin_only for safety. + Visibility string `json:"visibility"` +} + +func toDoc(b domain.BookMeta) bookDoc { + return bookDoc{ + Slug: b.Slug, + Title: b.Title, + Author: b.Author, + Cover: b.Cover, + Status: b.Status, + Genres: b.Genres, + Summary: b.Summary, + TotalChapters: b.TotalChapters, + SourceURL: b.SourceURL, + Rank: b.Ranking, + Rating: b.Rating, + MetaUpdated: b.MetaUpdated, + Archived: b.Archived, + Visibility: b.Visibility, + } +} + +func fromDoc(d bookDoc) domain.BookMeta { + return domain.BookMeta{ + Slug: d.Slug, + Title: d.Title, + Author: d.Author, + Cover: d.Cover, + Status: d.Status, + Genres: d.Genres, + Summary: d.Summary, + TotalChapters: d.TotalChapters, + SourceURL: d.SourceURL, + Ranking: d.Rank, + Rating: d.Rating, + MetaUpdated: d.MetaUpdated, + Archived: d.Archived, + Visibility: d.Visibility, + } +} + +// UpsertBook adds or replaces the book document in Meilisearch. The operation +// is fire-and-forget (Meilisearch processes tasks asynchronously). +func (c *MeiliClient) UpsertBook(_ context.Context, book domain.BookMeta) error { + docs := []bookDoc{toDoc(book)} + pk := "slug" + if _, err := c.idx.AddDocuments(docs, &meilisearch.DocumentOptions{PrimaryKey: &pk}); err != nil { + return fmt.Errorf("meili: upsert book %q: %w", book.Slug, err) + } + return nil +} + +// BookExists reports whether the slug is already present in the index. +// It fetches the document by primary key; a 404 or any error is treated as +// "not present" (safe default: re-index rather than silently skip). +func (c *MeiliClient) BookExists(_ context.Context, slug string) bool { + var doc bookDoc + err := c.idx.GetDocument(slug, nil, &doc) + return err == nil && doc.Slug != "" +} + +// DeleteBook removes a book document from the index by slug. +// The operation is fire-and-forget (Meilisearch processes tasks asynchronously). +func (c *MeiliClient) DeleteBook(_ context.Context, slug string) error { + if _, err := c.idx.DeleteDocument(slug, nil); err != nil { + return fmt.Errorf("meili: delete book %q: %w", slug, err) + } + return nil +} + +// Search returns books matching query, up to limit results. +// Archived books are always excluded. +func (c *MeiliClient) Search(_ context.Context, query string, limit int) ([]domain.BookMeta, error) { + if limit <= 0 { + limit = 20 + } + res, err := c.idx.Search(query, &meilisearch.SearchRequest{ + Limit: int64(limit), + Filter: `archived = false AND visibility = "public"`, + }) + if err != nil { + return nil, fmt.Errorf("meili: search %q: %w", query, err) + } + + books := make([]domain.BookMeta, 0, len(res.Hits)) + for _, hit := range res.Hits { + // Hit is map[string]json.RawMessage — unmarshal directly into bookDoc. + var doc bookDoc + raw, err := json.Marshal(hit) + if err != nil { + continue + } + if err := json.Unmarshal(raw, &doc); err != nil { + continue + } + books = append(books, fromDoc(doc)) + } + return books, nil +} + +// Catalogue queries books with optional full-text search, genre/status filters, +// sort order, and pagination. Returns matching books, the total estimate, and +// a FacetResult containing available genre and status values from the index. +func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) { + if q.Limit <= 0 { + q.Limit = 20 + } + if q.Page <= 0 { + q.Page = 1 + } + + req := &meilisearch.SearchRequest{ + Limit: int64(q.Limit), + Offset: int64((q.Page - 1) * q.Limit), + // Request facet distribution so the UI can build filter options + // dynamically without hardcoding genre/status lists. + Facets: []string{"genres", "status"}, + } + + // Build filter — always exclude archived books; restrict to public unless admin. + filters := []string{"archived = false"} + if !q.AdminAll { + filters = append(filters, `visibility = "public"`) + } + if q.Genre != "" && q.Genre != "all" { + filters = append(filters, fmt.Sprintf("genres = %q", q.Genre)) + } + if q.Status != "" && q.Status != "all" { + filters = append(filters, fmt.Sprintf("status = %q", q.Status)) + } + req.Filter = strings.Join(filters, " AND ") + + // Map UI sort tokens to Meilisearch sort expressions. + switch q.Sort { + case "rank": + req.Sort = []string{"rank:asc"} + case "top-rated": + req.Sort = []string{"rating:desc"} + case "new": + req.Sort = []string{"total_chapters:desc"} + case "update": + req.Sort = []string{"meta_updated:desc"} + // "popular" and "" → relevance (no explicit sort) + } + + res, err := c.idx.Search(q.Q, req) + if err != nil { + return nil, 0, FacetResult{}, fmt.Errorf("meili: catalogue query: %w", err) + } + + books := make([]domain.BookMeta, 0, len(res.Hits)) + for _, hit := range res.Hits { + var doc bookDoc + raw, err := json.Marshal(hit) + if err != nil { + continue + } + if err := json.Unmarshal(raw, &doc); err != nil { + continue + } + books = append(books, fromDoc(doc)) + } + + facets := parseFacets(res.FacetDistribution) + return books, res.EstimatedTotalHits, facets, nil +} + +// parseFacets extracts sorted genre and status slices from a Meilisearch +// facetDistribution raw JSON value. +// The JSON shape is: {"genres":{"fantasy":12,"action":5},"status":{"ongoing":7}} +func parseFacets(raw json.RawMessage) FacetResult { + var result FacetResult + if len(raw) == 0 { + return result + } + var dist map[string]map[string]int64 + if err := json.Unmarshal(raw, &dist); err != nil { + return result + } + if m, ok := dist["genres"]; ok { + for k := range m { + result.Genres = append(result.Genres, k) + } + sortStrings(result.Genres) + } + if m, ok := dist["status"]; ok { + for k := range m { + result.Statuses = append(result.Statuses, k) + } + sortStrings(result.Statuses) + } + return result +} + +// sortStrings sorts a slice of strings in place. +func sortStrings(s []string) { + for i := 1; i < len(s); i++ { + for j := i; j > 0 && s[j] < s[j-1]; j-- { + s[j], s[j-1] = s[j-1], s[j] + } + } +} + +// NoopClient is a no-op Client used when Meilisearch is not configured. +type NoopClient struct{} + +func (NoopClient) UpsertBook(_ context.Context, _ domain.BookMeta) error { return nil } +func (NoopClient) BookExists(_ context.Context, _ string) bool { return false } +func (NoopClient) DeleteBook(_ context.Context, _ string) error { return nil } +func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) { + return nil, nil +} +func (NoopClient) Catalogue(_ context.Context, _ CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) { + return nil, 0, FacetResult{}, nil +} diff --git a/scraper/internal/scraper/htmlutil/htmlutil.go b/backend/internal/novelfire/htmlutil/htmlutil.go similarity index 83% rename from scraper/internal/scraper/htmlutil/htmlutil.go rename to backend/internal/novelfire/htmlutil/htmlutil.go index 7a25980..5f4b0ef 100644 --- a/scraper/internal/scraper/htmlutil/htmlutil.go +++ b/backend/internal/novelfire/htmlutil/htmlutil.go @@ -3,13 +3,31 @@ package htmlutil import ( + "net/url" "regexp" "strings" - "github.com/libnovel/scraper/internal/scraper" + "github.com/libnovel/backend/internal/scraper" "golang.org/x/net/html" ) +// ResolveURL returns an absolute URL. If href is already absolute it is +// returned unchanged. Otherwise it is resolved against base. +func ResolveURL(base, href string) string { + if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") { + return href + } + b, err := url.Parse(base) + if err != nil { + return base + href + } + ref, err := url.Parse(href) + if err != nil { + return base + href + } + return b.ResolveReference(ref).String() +} + // ParseHTML parses raw HTML and returns the root node. func ParseHTML(raw string) (*html.Node, error) { return html.Parse(strings.NewReader(raw)) @@ -48,8 +66,8 @@ matched: return true } -// attrVal returns the value of attribute key from node n. -func attrVal(n *html.Node, key string) string { +// AttrVal returns the value of attribute key from node n. +func AttrVal(n *html.Node, key string) string { for _, a := range n.Attr { if a.Key == key { return a.Val @@ -58,8 +76,8 @@ func attrVal(n *html.Node, key string) string { return "" } -// textContent returns the concatenated text content of all descendant text nodes. -func textContent(n *html.Node) string { +// TextContent returns the concatenated text content of all descendant text nodes. +func TextContent(n *html.Node) string { var sb strings.Builder var walk func(*html.Node) walk = func(cur *html.Node) { @@ -114,9 +132,9 @@ func FindAll(root *html.Node, sel scraper.Selector) []*html.Node { // If sel.Attr is set the attribute value is returned; otherwise the inner text. func ExtractText(n *html.Node, sel scraper.Selector) string { if sel.Attr != "" { - return attrVal(n, sel.Attr) + return AttrVal(n, sel.Attr) } - return textContent(n) + return TextContent(n) } // ExtractFirst locates the first match in root and returns its text/attr value. @@ -140,29 +158,15 @@ func ExtractAll(root *html.Node, sel scraper.Selector) []string { return out } -// InnerHTML returns the serialized inner HTML of node n. -func InnerHTML(n *html.Node) string { - var sb strings.Builder - for c := n.FirstChild; c != nil; c = c.NextSibling { - _ = html.Render(&sb, c) - } - return sb.String() -} - // NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown -// representation suitable for chapter storage. Block elements become newlines; -// inline elements are inlined. Runs of more than one blank line are collapsed -// to a single blank line. +// representation suitable for chapter storage. func NodeToMarkdown(n *html.Node) string { var sb strings.Builder nodeToMD(n, &sb) - // Collapse 3+ consecutive newlines (i.e. more than one blank line) to 2. out := multiBlankLine.ReplaceAllString(sb.String(), "\n\n") return strings.TrimSpace(out) } -// multiBlankLine matches three or more consecutive newline characters -// (any mix of \n and surrounding whitespace-only lines). var multiBlankLine = regexp.MustCompile(`\n(\s*\n){2,}`) var blockElements = map[string]bool{ diff --git a/backend/internal/novelfire/scraper.go b/backend/internal/novelfire/scraper.go new file mode 100644 index 0000000..411ec21 --- /dev/null +++ b/backend/internal/novelfire/scraper.go @@ -0,0 +1,538 @@ +// Package novelfire provides a NovelScraper implementation for novelfire.net. +// +// Site structure (as of 2025): +// +// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N +// Book page : https://novelfire.net/book/{slug} +// Chapters : https://novelfire.net/book/{slug}/chapters?page=N +// Chapter : https://novelfire.net/book/{slug}/{chapter-slug} +package novelfire + +import ( + "context" + "errors" + "fmt" + "log/slog" + "math/rand" + "net/url" + "path" + "strconv" + "strings" + "time" + + "github.com/libnovel/backend/internal/browser" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/novelfire/htmlutil" + "github.com/libnovel/backend/internal/scraper" + "golang.org/x/net/html" +) + +const ( + baseURL = "https://novelfire.net" + cataloguePath = "/genre-all/sort-new/status-all/all-novel" + rankingPath = "/genre-all/sort-popular/status-all/all-novel" +) + +// Scraper is the novelfire.net implementation of scraper.NovelScraper. +type Scraper struct { + client browser.Client + log *slog.Logger +} + +// Compile-time interface check. +var _ scraper.NovelScraper = (*Scraper)(nil) + +// New returns a new novelfire Scraper backed by client. +func New(client browser.Client, log *slog.Logger) *Scraper { + if log == nil { + log = slog.Default() + } + return &Scraper{client: client, log: log} +} + +// SourceName implements NovelScraper. +func (s *Scraper) SourceName() string { return "novelfire.net" } + +// ── CatalogueProvider ───────────────────────────────────────────────────────── + +// ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages. +// Each page fetch uses retryGet with 429-aware exponential backoff. +// A small inter-page delay (cataloguePageDelay) is inserted between requests to +// avoid hammering the server when paging through hundreds of catalogue pages. +func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) { + entries := make(chan domain.CatalogueEntry, 64) + errs := make(chan error, 16) + + go func() { + defer close(entries) + defer close(errs) + + pageURL := baseURL + cataloguePath + page := 1 + + for pageURL != "" { + select { + case <-ctx.Done(): + return + default: + } + + // Polite inter-page delay — skipped on the very first page. + if page > 1 { + jitter := time.Duration(500+rand.Intn(1000)) * time.Millisecond + select { + case <-ctx.Done(): + return + case <-time.After(jitter): + } + } + + s.log.Info("scraping catalogue page", "page", page, "url", pageURL) + raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 10*time.Second) + if err != nil { + errs <- fmt.Errorf("catalogue page %d: %w", page, err) + return + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + errs <- fmt.Errorf("catalogue page %d parse: %w", page, err) + return + } + + cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true}) + if len(cards) == 0 { + s.log.Warn("no novel cards found, stopping pagination", "page", page) + return + } + + for _, card := range cards { + linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"}) + titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}) + + var title, href string + if linkNode != nil { + href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) + } + if titleNode != nil { + title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{})) + } + if href == "" || title == "" { + continue + } + + bookURL := resolveURL(baseURL, href) + select { + case <-ctx.Done(): + return + case entries <- domain.CatalogueEntry{Slug: slugFromURL(bookURL), Title: title, URL: bookURL}: + } + } + + if !hasNextPageLink(root) { + break + } + nextHref := "" + for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) { + if htmlutil.AttrVal(a, "rel") == "next" { + nextHref = htmlutil.AttrVal(a, "href") + break + } + } + if nextHref == "" { + break + } + pageURL = resolveURL(baseURL, nextHref) + page++ + } + }() + + return entries, errs +} + +// ── MetadataProvider ────────────────────────────────────────────────────────── + +// ScrapeMetadata fetches and parses book metadata from the book's landing page. +// Uses retryGet with 429-aware exponential backoff (up to 9 attempts). +func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) { + s.log.Debug("metadata fetch starting", "url", bookURL) + + raw, err := retryGet(ctx, s.log, s.client, bookURL, 9, 10*time.Second) + if err != nil { + return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err) + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + return domain.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err) + } + + title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"}) + author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"}) + + var cover string + if fig := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil { + cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"}) + if cover != "" && !strings.HasPrefix(cover, "http") { + cover = baseURL + cover + } + } + + // Status: novelfire renders <strong class="ongoing">Ongoing</strong> (or + // "completed", "hiatus") inside the .header-stats block. We take the text + // content and lowercase it so the index value is always canonical lowercase. + var status string + for _, cls := range []string{"ongoing", "completed", "hiatus"} { + if v := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "strong", Class: cls}); v != "" { + status = strings.ToLower(strings.TrimSpace(v)) + break + } + } + + // Genres: novelfire renders <div class="categories"><ul><li><a class="property-item">Genre</a> + // Each <a class="property-item"> is one genre tag. Lowercase for index consistency. + var genres []string + if categoriesNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "categories"}); categoriesNode != nil { + for _, v := range htmlutil.ExtractAll(categoriesNode, scraper.Selector{Tag: "a", Class: "property-item", Multiple: true}) { + if v != "" { + genres = append(genres, strings.ToLower(strings.TrimSpace(v))) + } + } + } + + summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"}) + totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"}) + totalChapters := parseChapterCount(totalStr) + + slug := slugFromURL(bookURL) + + meta := domain.BookMeta{ + Slug: slug, + Title: title, + Author: author, + Cover: cover, + Status: status, + Genres: genres, + Summary: summary, + TotalChapters: totalChapters, + SourceURL: bookURL, + } + s.log.Debug("metadata parsed", "slug", meta.Slug, "title", meta.Title) + return meta, nil +} + +// ── ChapterListProvider ─────────────────────────────────────────────────────── + +// ScrapeChapterList returns chapter references for a book, ordered ascending. +// upTo > 0 stops pagination as soon as at least upTo chapter numbers have been +// collected — use this for range scrapes so we don't paginate 100 pages just +// to discover refs we'll never scrape. upTo == 0 fetches all pages. +// Each page fetch uses retryGet with 429-aware exponential backoff. +func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error) { + var refs []domain.ChapterRef + baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters" + page := 1 + + for { + select { + case <-ctx.Done(): + return refs, ctx.Err() + default: + } + + pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page) + s.log.Debug("scraping chapter list", "page", page, "url", pageURL) + + raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second) + if err != nil { + return refs, fmt.Errorf("chapter list page %d: %w", page, err) + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + return refs, fmt.Errorf("chapter list page %d parse: %w", page, err) + } + + chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"}) + if chapterList == nil { + s.log.Debug("chapter list container not found, stopping pagination", "page", page) + break + } + + items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"}) + if len(items) == 0 { + break + } + + for _, item := range items { + linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"}) + if linkNode == nil { + continue + } + href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"}) + chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{}) + if href == "" { + continue + } + chURL := resolveURL(baseURL, href) + num := chapterNumberFromURL(chURL) + if num <= 0 { + num = len(refs) + 1 + s.log.Warn("chapter number not parseable from URL, falling back to position", + "url", chURL, "position", num) + } + refs = append(refs, domain.ChapterRef{ + Number: num, + Title: strings.TrimSpace(chTitle), + URL: chURL, + }) + } + + // Early-stop: if we have seen at least upTo chapter numbers, we have + // enough refs to cover the requested range — no need to paginate further. + if upTo > 0 && len(refs) > 0 && refs[len(refs)-1].Number >= upTo { + s.log.Debug("chapter list early-stop reached", "upTo", upTo, "collected", len(refs)) + break + } + + page++ + } + + return refs, nil +} + +// ── ChapterTextProvider ─────────────────────────────────────────────────────── + +// ScrapeChapterText fetches and parses a single chapter page. +func (s *Scraper) ScrapeChapterText(ctx context.Context, ref domain.ChapterRef) (domain.Chapter, error) { + s.log.Debug("chapter text fetch starting", "chapter", ref.Number, "url", ref.URL) + + raw, err := retryGet(ctx, s.log, s.client, ref.URL, 9, 6*time.Second) + if err != nil { + return domain.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err) + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + return domain.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err) + } + + container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"}) + if container == nil { + return domain.Chapter{}, fmt.Errorf("chapter %d: #content container not found in %s", ref.Number, ref.URL) + } + + text := htmlutil.NodeToMarkdown(container) + + s.log.Debug("chapter text parsed", "chapter", ref.Number, "text_bytes", len(text)) + + return domain.Chapter{Ref: ref, Text: text}, nil +} + +// ── RankingProvider ─────────────────────────────────────────────────────────── + +// ScrapeRanking pages through up to maxPages pages of the popular-novels listing. +// maxPages <= 0 means all pages. The caller decides whether to persist items. +func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) { + entries := make(chan domain.BookMeta, 32) + errs := make(chan error, 16) + + go func() { + defer close(entries) + defer close(errs) + + rank := 1 + + for page := 1; maxPages <= 0 || page <= maxPages; page++ { + select { + case <-ctx.Done(): + return + default: + } + + pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page) + s.log.Info("scraping popular ranking page", "page", page, "url", pageURL) + + raw, err := s.client.GetContent(ctx, pageURL) + if err != nil { + errs <- fmt.Errorf("ranking page %d: %w", page, err) + return + } + + root, err := htmlutil.ParseHTML(raw) + if err != nil { + errs <- fmt.Errorf("ranking page %d parse: %w", page, err) + return + } + + cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true}) + if len(cards) == 0 { + break + } + + for _, card := range cards { + linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"}) + if linkNode == nil { + continue + } + href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) + bookURL := resolveURL(baseURL, href) + if bookURL == "" { + continue + } + + title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})) + if title == "" { + title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"})) + } + if title == "" { + continue + } + + var cover string + if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil { + cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"}) + if cover == "" { + cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"}) + } + if strings.HasPrefix(cover, "data:") { + cover = "" + } + if cover != "" && !strings.HasPrefix(cover, "http") { + cover = baseURL + cover + } + } + + meta := domain.BookMeta{ + Slug: slugFromURL(bookURL), + Title: title, + Cover: cover, + SourceURL: bookURL, + Ranking: rank, + } + rank++ + + select { + case <-ctx.Done(): + return + case entries <- meta: + } + } + + if !hasNextPageLink(root) { + break + } + } + }() + + return entries, errs +} + +// ── helpers ─────────────────────────────────────────────────────────────────── + +func resolveURL(base, href string) string { return htmlutil.ResolveURL(base, href) } + +func hasNextPageLink(root *html.Node) bool { + links := htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) + for _, a := range links { + for _, attr := range a.Attr { + if attr.Key == "rel" && attr.Val == "next" { + return true + } + } + } + return false +} + +func slugFromURL(bookURL string) string { + u, err := url.Parse(bookURL) + if err != nil { + return bookURL + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(parts) >= 2 && parts[0] == "book" { + return parts[1] + } + if len(parts) > 0 { + return parts[len(parts)-1] + } + return "" +} + +func parseChapterCount(s string) int { + s = strings.ReplaceAll(s, ",", "") + fields := strings.Fields(s) + if len(fields) == 0 { + return 0 + } + n, _ := strconv.Atoi(fields[0]) + return n +} + +func chapterNumberFromURL(chapterURL string) int { + u, err := url.Parse(chapterURL) + if err != nil { + return 0 + } + seg := path.Base(u.Path) + seg = strings.TrimPrefix(seg, "chapter-") + seg = strings.TrimPrefix(seg, "chap-") + seg = strings.TrimPrefix(seg, "ch-") + digits := strings.FieldsFunc(seg, func(r rune) bool { + return r < '0' || r > '9' + }) + if len(digits) == 0 { + return 0 + } + n, _ := strconv.Atoi(digits[0]) + return n +} + +// retryGet calls client.GetContent up to maxAttempts times with exponential backoff. +// If the server returns 429 (ErrRateLimit), the suggested Retry-After delay is used +// instead of the geometric backoff delay. +func retryGet( + ctx context.Context, + log *slog.Logger, + client browser.Client, + pageURL string, + maxAttempts int, + baseDelay time.Duration, +) (string, error) { + var lastErr error + delay := baseDelay + for attempt := 1; attempt <= maxAttempts; attempt++ { + raw, err := client.GetContent(ctx, pageURL) + if err == nil { + return raw, nil + } + lastErr = err + if ctx.Err() != nil { + return "", err + } + if attempt < maxAttempts { + // If the server is rate-limiting us, honour its Retry-After delay. + waitFor := delay + var rlErr *browser.RateLimitError + if errors.As(err, &rlErr) { + waitFor = rlErr.RetryAfter + if log != nil { + log.Warn("rate limited, backing off", + "url", pageURL, "attempt", attempt, "retry_in", waitFor) + } + } else { + if log != nil { + log.Warn("fetch failed, retrying", + "url", pageURL, "attempt", attempt, "retry_in", delay, "err", err) + } + delay *= 2 + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(waitFor): + } + } + } + return "", lastErr +} diff --git a/backend/internal/novelfire/scraper_test.go b/backend/internal/novelfire/scraper_test.go new file mode 100644 index 0000000..6c29a3b --- /dev/null +++ b/backend/internal/novelfire/scraper_test.go @@ -0,0 +1,180 @@ +package novelfire + +import ( + "context" + "log/slog" + "testing" +) + +func TestSlugFromURL(t *testing.T) { + cases := []struct { + url string + want string + }{ + {"https://novelfire.net/book/shadow-slave", "shadow-slave"}, + {"https://novelfire.net/book/a-dragon-against-the-whole-world", "a-dragon-against-the-whole-world"}, + {"https://novelfire.net/book/foo/chapter-1", "foo"}, + {"https://novelfire.net/", ""}, + {"not-a-url", "not-a-url"}, + } + for _, c := range cases { + got := slugFromURL(c.url) + if got != c.want { + t.Errorf("slugFromURL(%q) = %q, want %q", c.url, got, c.want) + } + } +} + +func TestChapterNumberFromURL(t *testing.T) { + cases := []struct { + url string + want int + }{ + {"https://novelfire.net/book/shadow-slave/chapter-42", 42}, + {"https://novelfire.net/book/shadow-slave/chapter-1000", 1000}, + {"https://novelfire.net/book/shadow-slave/chap-7", 7}, + {"https://novelfire.net/book/shadow-slave/ch-3", 3}, + {"https://novelfire.net/book/shadow-slave/42", 42}, + {"https://novelfire.net/book/shadow-slave/no-number-here", 0}, + {"not-a-url", 0}, + } + for _, c := range cases { + got := chapterNumberFromURL(c.url) + if got != c.want { + t.Errorf("chapterNumberFromURL(%q) = %d, want %d", c.url, got, c.want) + } + } +} + +func TestParseChapterCount(t *testing.T) { + cases := []struct { + in string + want int + }{ + {"123 Chapters", 123}, + {"1,234 Chapters", 1234}, + {"0", 0}, + {"", 0}, + {"500", 500}, + } + for _, c := range cases { + got := parseChapterCount(c.in) + if got != c.want { + t.Errorf("parseChapterCount(%q) = %d, want %d", c.in, got, c.want) + } + } +} + +func TestRetryGet_ContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel immediately + + stub := newStubClient() + stub.setError("https://example.com/page", context.Canceled) + + _, err := retryGet(ctx, nil, stub, "https://example.com/page", 3, 0) + if err == nil { + t.Fatal("expected error on cancelled context") + } +} + +func TestRetryGet_EventualSuccess(t *testing.T) { + stub := newStubClient() + calls := 0 + stub.setFn("https://example.com/page", func() (string, error) { + calls++ + if calls < 3 { + return "", context.DeadlineExceeded + } + return "<html>ok</html>", nil + }) + + got, err := retryGet(context.Background(), nil, stub, "https://example.com/page", 5, 0) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != "<html>ok</html>" { + t.Errorf("got %q, want html", got) + } + if calls != 3 { + t.Errorf("expected 3 calls, got %d", calls) + } +} + +// TestParseMetadataSelectors verifies that the status and genres selectors +// match the current novelfire.net HTML structure. +func TestParseMetadataSelectors(t *testing.T) { + // Minimal HTML reproducing the relevant novelfire.net book page structure. + const html = `<!DOCTYPE html> +<html><body> +<h1 class="novel-title">Shadow Slave</h1> +<span class="author">Guiltythree</span> +<figure class="cover"><img src="https://cdn.example.com/cover.jpg"></figure> +<div class="header-stats"> + <span><strong>123</strong><small>Chapters</small></span> + <span> <strong class="ongoing">Ongoing</strong> <small>Status</small></span> +</div> +<div class="categories"> + <h4>Genres</h4> + <ul> + <li><a href="/genre-fantasy/..." class="property-item">Fantasy</a></li> + <li><a href="/genre-action/..." class="property-item">Action</a></li> + <li><a href="/genre-adventure/..." class="property-item">Adventure</a></li> + </ul> +</div> +<span class="chapter-count">123 Chapters</span> +</body></html>` + + stub := newStubClient() + stub.setFn("https://novelfire.net/book/shadow-slave", func() (string, error) { + return html, nil + }) + + s := &Scraper{client: stub, log: slog.Default()} + meta, err := s.ScrapeMetadata(t.Context(), "https://novelfire.net/book/shadow-slave") + if err != nil { + t.Fatalf("ScrapeMetadata: %v", err) + } + + if meta.Status != "ongoing" { + t.Errorf("status = %q, want %q", meta.Status, "ongoing") + } + + wantGenres := []string{"fantasy", "action", "adventure"} + if len(meta.Genres) != len(wantGenres) { + t.Fatalf("genres = %v, want %v", meta.Genres, wantGenres) + } + for i, g := range meta.Genres { + if g != wantGenres[i] { + t.Errorf("genres[%d] = %q, want %q", i, g, wantGenres[i]) + } + } +} + +// ── minimal stub client for tests ───────────────────────────────────────────── + +type stubClient struct { + errors map[string]error + fns map[string]func() (string, error) +} + +func newStubClient() *stubClient { + return &stubClient{ + errors: make(map[string]error), + fns: make(map[string]func() (string, error)), + } +} + +func (s *stubClient) setError(u string, err error) { s.errors[u] = err } + +func (s *stubClient) setFn(u string, fn func() (string, error)) { s.fns[u] = fn } + +func (s *stubClient) GetContent(_ context.Context, pageURL string) (string, error) { + if fn, ok := s.fns[pageURL]; ok { + return fn() + } + if err, ok := s.errors[pageURL]; ok { + return "", err + } + return "", context.DeadlineExceeded +} diff --git a/backend/internal/orchestrator/orchestrator.go b/backend/internal/orchestrator/orchestrator.go new file mode 100644 index 0000000..a97917a --- /dev/null +++ b/backend/internal/orchestrator/orchestrator.go @@ -0,0 +1,224 @@ +// Package orchestrator coordinates metadata extraction, chapter-list fetching, +// and parallel chapter scraping for a single book. +// +// Design: +// - RunBook scrapes one book (metadata + chapter list + chapter texts) end-to-end. +// - N worker goroutines pull chapter refs from a shared queue and call ScrapeChapterText. +// - The caller (runner poll loop) owns the outer task-claim / finish cycle. +// - An optional PostMetadata hook (set in Config) is called after WriteMetadata +// succeeds. The runner uses this to upsert books into Meilisearch. +package orchestrator + +import ( + "context" + "fmt" + "log/slog" + "runtime" + "sync" + "sync/atomic" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/scraper" +) + +// Config holds tunable parameters for the orchestrator. +type Config struct { + // Workers is the number of goroutines used to scrape chapters in parallel. + // Defaults to runtime.NumCPU() when 0. + Workers int + // PostMetadata is an optional hook called with the scraped BookMeta after + // WriteMetadata succeeds. Errors from the hook are logged but not fatal. + // Used by the runner to index books in Meilisearch. + PostMetadata func(ctx context.Context, meta domain.BookMeta) +} + +// Orchestrator runs a single-book scrape pipeline. +type Orchestrator struct { + novel scraper.NovelScraper + store bookstore.BookWriter + log *slog.Logger + workers int + postMetadata func(ctx context.Context, meta domain.BookMeta) +} + +// New returns a new Orchestrator. +func New(cfg Config, novel scraper.NovelScraper, store bookstore.BookWriter, log *slog.Logger) *Orchestrator { + if log == nil { + log = slog.Default() + } + workers := cfg.Workers + if workers <= 0 { + workers = runtime.NumCPU() + } + return &Orchestrator{ + novel: novel, + store: store, + log: log, + workers: workers, + postMetadata: cfg.PostMetadata, + } +} + +// RunBook scrapes a single book described by task. It handles: +// 1. Metadata scrape + write +// 2. Chapter list scrape + write +// 3. Parallel chapter text scrape + write (worker pool) +// +// Returns a ScrapeResult with counters. The result's ErrorMessage is non-empty +// if the run failed at the metadata or chapter-list level. +func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) domain.ScrapeResult { + o.log.Debug("orchestrator: RunBook starting", + "task_id", task.ID, + "kind", task.Kind, + "url", task.TargetURL, + "workers", o.workers, + ) + + var result domain.ScrapeResult + + if task.TargetURL == "" { + result.ErrorMessage = "task has no target URL" + return result + } + + // ── Step 1: Metadata ────────────────────────────────────────────────────── + meta, err := o.novel.ScrapeMetadata(ctx, task.TargetURL) + if err != nil { + o.log.Error("metadata scrape failed", "url", task.TargetURL, "err", err) + result.ErrorMessage = fmt.Sprintf("metadata: %v", err) + result.Errors++ + return result + } + result.Slug = meta.Slug + + if err := o.store.WriteMetadata(ctx, meta); err != nil { + o.log.Error("metadata write failed", "slug", meta.Slug, "err", err) + // non-fatal: continue to chapters + result.Errors++ + } else { + result.BooksFound = 1 + result.Slug = meta.Slug + // Fire optional post-metadata hook (e.g. Meilisearch indexing). + if o.postMetadata != nil { + o.postMetadata(ctx, meta) + } + } + + o.log.Debug("metadata saved", "slug", meta.Slug, "title", meta.Title) + + // ── Step 2: Chapter list ────────────────────────────────────────────────── + refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL, task.ToChapter) + if err != nil { + o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err) + result.ErrorMessage = fmt.Sprintf("chapter list: %v", err) + result.Errors++ + return result + } + + o.log.Debug("chapter list fetched", "slug", meta.Slug, "chapters", len(refs)) + + // Persist chapter refs (without text) so the index exists early. + if wErr := o.store.WriteChapterRefs(ctx, meta.Slug, refs); wErr != nil { + o.log.Warn("chapter refs write failed", "slug", meta.Slug, "err", wErr) + } + + // ── Step 3: Chapter texts (worker pool) ─────────────────────────────────── + type chapterJob struct { + slug string + ref domain.ChapterRef + total int // total chapters to scrape (for progress logging) + } + work := make(chan chapterJob, o.workers*4) + + var scraped, skipped, errors atomic.Int64 + var wg sync.WaitGroup + + for i := 0; i < o.workers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + for job := range work { + select { + case <-ctx.Done(): + return + default: + } + + if o.store.ChapterExists(ctx, job.slug, job.ref) { + o.log.Debug("chapter already exists, skipping", + "slug", job.slug, "chapter", job.ref.Number) + skipped.Add(1) + continue + } + + ch, err := o.novel.ScrapeChapterText(ctx, job.ref) + if err != nil { + o.log.Error("chapter scrape failed", + "slug", job.slug, "chapter", job.ref.Number, "err", err) + errors.Add(1) + continue + } + + if err := o.store.WriteChapter(ctx, job.slug, ch); err != nil { + o.log.Error("chapter write failed", + "slug", job.slug, "chapter", job.ref.Number, "err", err) + errors.Add(1) + continue + } + + n := scraped.Add(1) + // Log a progress summary every 25 chapters scraped. + if n%25 == 0 { + o.log.Info("scraping chapters", + "slug", job.slug, "scraped", n, "total", job.total) + } + } + }(i) + } + + // Count how many chapters will actually be enqueued (for progress logging). + toScrape := 0 + for _, ref := range refs { + if task.FromChapter > 0 && ref.Number < task.FromChapter { + continue + } + if task.ToChapter > 0 && ref.Number > task.ToChapter { + continue + } + toScrape++ + } + + // Enqueue chapter jobs respecting the optional range filter from the task. + for _, ref := range refs { + if task.FromChapter > 0 && ref.Number < task.FromChapter { + skipped.Add(1) + continue + } + if task.ToChapter > 0 && ref.Number > task.ToChapter { + skipped.Add(1) + continue + } + select { + case <-ctx.Done(): + goto drain + case work <- chapterJob{slug: meta.Slug, ref: ref, total: toScrape}: + } + } + +drain: + close(work) + wg.Wait() + + result.ChaptersScraped = int(scraped.Load()) + result.ChaptersSkipped = int(skipped.Load()) + result.Errors += int(errors.Load()) + + o.log.Info("book scrape finished", + "slug", meta.Slug, + "scraped", result.ChaptersScraped, + "skipped", result.ChaptersSkipped, + "errors", result.Errors, + ) + return result +} diff --git a/backend/internal/orchestrator/orchestrator_test.go b/backend/internal/orchestrator/orchestrator_test.go new file mode 100644 index 0000000..d84aa98 --- /dev/null +++ b/backend/internal/orchestrator/orchestrator_test.go @@ -0,0 +1,212 @@ +package orchestrator + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/libnovel/backend/internal/domain" +) + +// ── stubs ───────────────────────────────────────────────────────────────────── + +type stubScraper struct { + meta domain.BookMeta + metaErr error + refs []domain.ChapterRef + refsErr error + chapters map[int]domain.Chapter + chapErr map[int]error +} + +func (s *stubScraper) SourceName() string { return "stub" } + +func (s *stubScraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) { + ch := make(chan domain.CatalogueEntry) + errs := make(chan error) + close(ch) + close(errs) + return ch, errs +} + +func (s *stubScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) { + return s.meta, s.metaErr +} + +func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) { + return s.refs, s.refsErr +} + +func (s *stubScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) { + if s.chapErr != nil { + if err, ok := s.chapErr[ref.Number]; ok { + return domain.Chapter{}, err + } + } + if s.chapters != nil { + if ch, ok := s.chapters[ref.Number]; ok { + return ch, nil + } + } + return domain.Chapter{Ref: ref, Text: "text"}, nil +} + +func (s *stubScraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) { + ch := make(chan domain.BookMeta) + errs := make(chan error) + close(ch) + close(errs) + return ch, errs +} + +type stubStore struct { + mu sync.Mutex + metaWritten []domain.BookMeta + chaptersWritten []domain.Chapter + existing map[string]bool // "slug:N" → exists + writeMetaErr error +} + +func (s *stubStore) WriteMetadata(_ context.Context, meta domain.BookMeta) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.writeMetaErr != nil { + return s.writeMetaErr + } + s.metaWritten = append(s.metaWritten, meta) + return nil +} + +func (s *stubStore) WriteChapter(_ context.Context, slug string, ch domain.Chapter) error { + s.mu.Lock() + defer s.mu.Unlock() + s.chaptersWritten = append(s.chaptersWritten, ch) + return nil +} + +func (s *stubStore) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error { + return nil +} + +func (s *stubStore) DeduplicateChapters(_ context.Context, _ string) (int, error) { return 0, nil } + +func (s *stubStore) ChapterExists(_ context.Context, slug string, ref domain.ChapterRef) bool { + s.mu.Lock() + defer s.mu.Unlock() + key := slug + ":" + string(rune('0'+ref.Number)) + return s.existing[key] +} + +// ── tests ────────────────────────────────────────────────────────────────────── + +func TestRunBook_HappyPath(t *testing.T) { + sc := &stubScraper{ + meta: domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, + refs: []domain.ChapterRef{ + {Number: 1, Title: "Ch 1", URL: "https://example.com/book/test-book/chapter-1"}, + {Number: 2, Title: "Ch 2", URL: "https://example.com/book/test-book/chapter-2"}, + {Number: 3, Title: "Ch 3", URL: "https://example.com/book/test-book/chapter-3"}, + }, + } + st := &stubStore{} + o := New(Config{Workers: 2}, sc, st, nil) + + task := domain.ScrapeTask{ + ID: "t1", + Kind: "book", + TargetURL: "https://example.com/book/test-book", + } + + result := o.RunBook(context.Background(), task) + + if result.ErrorMessage != "" { + t.Fatalf("unexpected error: %s", result.ErrorMessage) + } + if result.BooksFound != 1 { + t.Errorf("BooksFound = %d, want 1", result.BooksFound) + } + if result.ChaptersScraped != 3 { + t.Errorf("ChaptersScraped = %d, want 3", result.ChaptersScraped) + } +} + +func TestRunBook_MetadataError(t *testing.T) { + sc := &stubScraper{metaErr: errors.New("404 not found")} + st := &stubStore{} + o := New(Config{Workers: 1}, sc, st, nil) + + result := o.RunBook(context.Background(), domain.ScrapeTask{ + ID: "t2", + TargetURL: "https://example.com/book/missing", + }) + + if result.ErrorMessage == "" { + t.Fatal("expected ErrorMessage to be set") + } + if result.Errors != 1 { + t.Errorf("Errors = %d, want 1", result.Errors) + } +} + +func TestRunBook_ChapterRange(t *testing.T) { + sc := &stubScraper{ + meta: domain.BookMeta{Slug: "range-book", SourceURL: "https://example.com/book/range-book"}, + refs: func() []domain.ChapterRef { + var refs []domain.ChapterRef + for i := 1; i <= 10; i++ { + refs = append(refs, domain.ChapterRef{Number: i, URL: "https://example.com/book/range-book/chapter-" + string(rune('0'+i))}) + } + return refs + }(), + } + st := &stubStore{} + o := New(Config{Workers: 2}, sc, st, nil) + + result := o.RunBook(context.Background(), domain.ScrapeTask{ + ID: "t3", + TargetURL: "https://example.com/book/range-book", + FromChapter: 3, + ToChapter: 7, + }) + + if result.ErrorMessage != "" { + t.Fatalf("unexpected error: %s", result.ErrorMessage) + } + // chapters 3–7 = 5 scraped, chapters 1-2 and 8-10 = 5 skipped + if result.ChaptersScraped != 5 { + t.Errorf("ChaptersScraped = %d, want 5", result.ChaptersScraped) + } + if result.ChaptersSkipped != 5 { + t.Errorf("ChaptersSkipped = %d, want 5", result.ChaptersSkipped) + } +} + +func TestRunBook_ContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + sc := &stubScraper{ + meta: domain.BookMeta{Slug: "ctx-book", SourceURL: "https://example.com/book/ctx-book"}, + refs: []domain.ChapterRef{ + {Number: 1, URL: "https://example.com/book/ctx-book/chapter-1"}, + }, + } + st := &stubStore{} + o := New(Config{Workers: 1}, sc, st, nil) + + // Should not panic; result may have errors or zero chapters. + result := o.RunBook(ctx, domain.ScrapeTask{ + ID: "t4", + TargetURL: "https://example.com/book/ctx-book", + }) + _ = result +} + +func TestRunBook_EmptyTargetURL(t *testing.T) { + o := New(Config{Workers: 1}, &stubScraper{}, &stubStore{}, nil) + result := o.RunBook(context.Background(), domain.ScrapeTask{ID: "t5"}) + if result.ErrorMessage == "" { + t.Fatal("expected ErrorMessage for empty target URL") + } +} diff --git a/backend/internal/otelsetup/otelsetup.go b/backend/internal/otelsetup/otelsetup.go new file mode 100644 index 0000000..09d58cb --- /dev/null +++ b/backend/internal/otelsetup/otelsetup.go @@ -0,0 +1,120 @@ +// Package otelsetup initialises the OpenTelemetry SDK for the LibNovel backend. +// +// It reads two environment variables: +// +// OTEL_EXPORTER_OTLP_ENDPOINT — OTLP/HTTP endpoint; accepts either a full +// URL ("https://otel.example.com") or a bare +// host[:port] ("otel-collector:4318"). +// TLS is used when the value starts with "https://". +// OTEL_SERVICE_NAME — service name reported in traces (default: "backend") +// +// When OTEL_EXPORTER_OTLP_ENDPOINT is empty the function is a no-op: it +// returns a nil shutdown func and the default slog.Logger, so callers never +// need to branch on it. +// +// Usage in main.go: +// +// shutdown, log, err := otelsetup.Init(ctx, version) +// if err != nil { return err } +// if shutdown != nil { defer shutdown() } +package otelsetup + +import ( + "context" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "go.opentelemetry.io/contrib/bridges/otelslog" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + otellog "go.opentelemetry.io/otel/log/global" + "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" +) + +// Init sets up TracerProvider and LoggerProvider that export via OTLP/HTTP. +// +// Returns: +// - shutdown: flushes and stops both providers (nil when OTel is disabled). +// - logger: an slog.Logger bridged to OTel logs (falls back to default when disabled). +// - err: non-nil only on SDK initialisation failure. +func Init(ctx context.Context, version string) (shutdown func(), logger *slog.Logger, err error) { + rawEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + if rawEndpoint == "" { + return nil, slog.Default(), nil // OTel disabled — not an error + } + + // WithEndpoint expects a host[:port] value — no scheme. + // Support both "https://otel.example.com" and "otel-collector:4318". + useTLS := strings.HasPrefix(rawEndpoint, "https://") + endpoint := strings.TrimPrefix(rawEndpoint, "https://") + endpoint = strings.TrimPrefix(endpoint, "http://") + + serviceName := os.Getenv("OTEL_SERVICE_NAME") + if serviceName == "" { + serviceName = "backend" + } + + // ── Shared resource ─────────────────────────────────────────────────────── + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceName(serviceName), + semconv.ServiceVersion(version), + ), + ) + if err != nil { + return nil, slog.Default(), fmt.Errorf("otelsetup: create resource: %w", err) + } + + // ── Trace provider ──────────────────────────────────────────────────────── + traceOpts := []otlptracehttp.Option{otlptracehttp.WithEndpoint(endpoint)} + if !useTLS { + traceOpts = append(traceOpts, otlptracehttp.WithInsecure()) + } + traceExp, err := otlptracehttp.New(ctx, traceOpts...) + if err != nil { + return nil, slog.Default(), fmt.Errorf("otelsetup: create OTLP trace exporter: %w", err) + } + + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(traceExp), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.2))), + ) + otel.SetTracerProvider(tp) + + // ── Log provider ────────────────────────────────────────────────────────── + logOpts := []otlploghttp.Option{otlploghttp.WithEndpoint(endpoint)} + if !useTLS { + logOpts = append(logOpts, otlploghttp.WithInsecure()) + } + logExp, err := otlploghttp.New(ctx, logOpts...) + if err != nil { + return nil, slog.Default(), fmt.Errorf("otelsetup: create OTLP log exporter: %w", err) + } + + lp := log.NewLoggerProvider( + log.WithProcessor(log.NewBatchProcessor(logExp)), + log.WithResource(res), + ) + otellog.SetLoggerProvider(lp) + + // Bridge slog → OTel logs. Structured fields and trace IDs are forwarded + // automatically; Grafana can correlate log lines with Tempo traces. + otelLogger := otelslog.NewLogger(serviceName) + + shutdown = func() { + shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = tp.Shutdown(shutCtx) + _ = lp.Shutdown(shutCtx) + } + + return shutdown, otelLogger, nil +} diff --git a/backend/internal/pockettts/client.go b/backend/internal/pockettts/client.go new file mode 100644 index 0000000..52bab1f --- /dev/null +++ b/backend/internal/pockettts/client.go @@ -0,0 +1,254 @@ +// Package pockettts provides a client for the kyutai-labs/pocket-tts TTS service. +// +// pocket-tts exposes a non-OpenAI API: +// +// POST /tts (multipart form: text, voice_url) → streaming WAV +// GET /health → {"status":"healthy"} +// +// GenerateAudio streams the WAV response and transcodes it to MP3 using ffmpeg, +// so callers receive MP3 bytes — the same format as the kokoro client — and the +// rest of the pipeline does not need to care which TTS engine was used. +// +// StreamAudioMP3 is the streaming variant: it returns an io.ReadCloser that +// yields MP3-encoded audio incrementally as pocket-tts generates it, without +// buffering the full output. +// +// Predefined voices (pass the bare name as the voice parameter): +// +// alba, marius, javert, jean, fantine, cosette, eponine, azelma, +// anna, vera, charles, paul, george, mary, jane, michael, eve, +// bill_boerst, peter_yearsley, stuart_bell +package pockettts + +import ( + "bytes" + "context" + "fmt" + "io" + "mime/multipart" + "net/http" + "os/exec" + "strings" + "time" +) + +// PredefinedVoices is the set of voice names built into pocket-tts. +// The runner uses this to decide which TTS engine to route a task to. +var PredefinedVoices = map[string]struct{}{ + "alba": {}, "marius": {}, "javert": {}, "jean": {}, + "fantine": {}, "cosette": {}, "eponine": {}, "azelma": {}, + "anna": {}, "vera": {}, "charles": {}, "paul": {}, + "george": {}, "mary": {}, "jane": {}, "michael": {}, + "eve": {}, "bill_boerst": {}, "peter_yearsley": {}, "stuart_bell": {}, +} + +// IsPocketTTSVoice reports whether voice is served by pocket-tts. +func IsPocketTTSVoice(voice string) bool { + _, ok := PredefinedVoices[voice] + return ok +} + +// Client is the interface for interacting with the pocket-tts service. +type Client interface { + // GenerateAudio synthesises text using the given voice and returns MP3 bytes. + // Voice must be one of the predefined pocket-tts voice names. + GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) + + // StreamAudioMP3 synthesises text and returns an io.ReadCloser that streams + // MP3-encoded audio incrementally via a live ffmpeg transcode pipe. + // The caller must always close the returned ReadCloser. + StreamAudioMP3(ctx context.Context, text, voice string) (io.ReadCloser, error) + + // StreamAudioWAV synthesises text and returns an io.ReadCloser that streams + // raw WAV audio directly from pocket-tts without any transcoding. + // The stream begins with a WAV header followed by 16-bit PCM frames at 16 kHz. + // The caller must always close the returned ReadCloser. + StreamAudioWAV(ctx context.Context, text, voice string) (io.ReadCloser, error) + + // ListVoices returns the available predefined voice names. + ListVoices(ctx context.Context) ([]string, error) +} + +// httpClient is the concrete pocket-tts HTTP client. +type httpClient struct { + baseURL string + http *http.Client +} + +// New returns a Client targeting baseURL (e.g. "https://pocket-tts.libnovel.cc"). +func New(baseURL string) Client { + return &httpClient{ + baseURL: strings.TrimRight(baseURL, "/"), + http: &http.Client{Timeout: 10 * time.Minute}, + } +} + +// GenerateAudio posts to POST /tts and transcodes the WAV response to MP3 +// using the system ffmpeg binary. Requires ffmpeg to be on PATH (available in +// the runner Docker image via Alpine's ffmpeg package). +func (c *httpClient) GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) { + if text == "" { + return nil, fmt.Errorf("pockettts: empty text") + } + if voice == "" { + voice = "alba" + } + + resp, err := c.postTTS(ctx, text, voice) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + wavData, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("pockettts: read response body: %w", err) + } + + // ── Transcode WAV → MP3 via ffmpeg ──────────────────────────────────────── + mp3Data, err := wavToMP3(ctx, wavData) + if err != nil { + return nil, fmt.Errorf("pockettts: transcode to mp3: %w", err) + } + return mp3Data, nil +} + +// StreamAudioMP3 posts to POST /tts and returns an io.ReadCloser that delivers +// MP3 bytes as pocket-tts generates WAV frames. ffmpeg runs as a subprocess +// with stdin connected to the live WAV stream and stdout piped to the caller. +// The caller must always close the returned ReadCloser. +func (c *httpClient) StreamAudioMP3(ctx context.Context, text, voice string) (io.ReadCloser, error) { + if text == "" { + return nil, fmt.Errorf("pockettts: empty text") + } + if voice == "" { + voice = "alba" + } + + resp, err := c.postTTS(ctx, text, voice) + if err != nil { + return nil, err + } + + // Start ffmpeg: read WAV from stdin (the live HTTP body), write MP3 to stdout. + cmd := exec.CommandContext(ctx, + "ffmpeg", + "-hide_banner", "-loglevel", "error", + "-i", "pipe:0", // WAV from stdin + "-f", "mp3", // output format + "-q:a", "2", // VBR ~190 kbps + "pipe:1", // MP3 to stdout + ) + cmd.Stdin = resp.Body + + pr, pw := io.Pipe() + cmd.Stdout = pw + + var stderrBuf bytes.Buffer + cmd.Stderr = &stderrBuf + + if err := cmd.Start(); err != nil { + resp.Body.Close() + return nil, fmt.Errorf("pockettts: start ffmpeg: %w", err) + } + + // Close the write end of the pipe when ffmpeg exits, propagating any error. + go func() { + waitErr := cmd.Wait() + resp.Body.Close() + if waitErr != nil { + pw.CloseWithError(fmt.Errorf("ffmpeg: %w (stderr: %s)", waitErr, stderrBuf.String())) + } else { + pw.Close() + } + }() + + return pr, nil +} + +// StreamAudioWAV posts to POST /tts and returns an io.ReadCloser that delivers +// raw WAV bytes directly from pocket-tts — no ffmpeg transcoding required. +// The first bytes will be a WAV header (RIFF/fmt chunk) followed by PCM frames. +// The caller must always close the returned ReadCloser. +func (c *httpClient) StreamAudioWAV(ctx context.Context, text, voice string) (io.ReadCloser, error) { + if text == "" { + return nil, fmt.Errorf("pockettts: empty text") + } + if voice == "" { + voice = "alba" + } + + resp, err := c.postTTS(ctx, text, voice) + if err != nil { + return nil, err + } + return resp.Body, nil +} + +// ListVoices returns the statically known predefined voice names. +// pocket-tts has no REST endpoint for listing voices. +func (c *httpClient) ListVoices(_ context.Context) ([]string, error) { + voices := make([]string, 0, len(PredefinedVoices)) + for v := range PredefinedVoices { + voices = append(voices, v) + } + return voices, nil +} + +// postTTS sends a multipart POST /tts request and returns the raw response. +// The caller is responsible for closing resp.Body. +func (c *httpClient) postTTS(ctx context.Context, text, voice string) (*http.Response, error) { + var body bytes.Buffer + mw := multipart.NewWriter(&body) + + if err := mw.WriteField("text", text); err != nil { + return nil, fmt.Errorf("pockettts: write text field: %w", err) + } + if err := mw.WriteField("voice_url", voice); err != nil { + return nil, fmt.Errorf("pockettts: write voice_url field: %w", err) + } + if err := mw.Close(); err != nil { + return nil, fmt.Errorf("pockettts: close multipart writer: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/tts", &body) + if err != nil { + return nil, fmt.Errorf("pockettts: build request: %w", err) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("pockettts: request: %w", err) + } + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + return nil, fmt.Errorf("pockettts: server returned %d", resp.StatusCode) + } + return resp, nil +} + +// wavToMP3 converts raw WAV bytes to MP3 using ffmpeg. +// ffmpeg reads from stdin (pipe:0) and writes to stdout (pipe:1). +func wavToMP3(ctx context.Context, wav []byte) ([]byte, error) { + cmd := exec.CommandContext(ctx, + "ffmpeg", + "-hide_banner", "-loglevel", "error", + "-i", "pipe:0", // read WAV from stdin + "-f", "mp3", // output format + "-q:a", "2", // VBR quality ~190 kbps + "pipe:1", // write MP3 to stdout + ) + cmd.Stdin = bytes.NewReader(wav) + + var out, stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("ffmpeg: %w (stderr: %s)", err, stderr.String()) + } + return out.Bytes(), nil +} diff --git a/backend/internal/presigncache/cache.go b/backend/internal/presigncache/cache.go new file mode 100644 index 0000000..a721728 --- /dev/null +++ b/backend/internal/presigncache/cache.go @@ -0,0 +1,96 @@ +// Package presigncache provides a Valkey (Redis-compatible) backed cache for +// MinIO presigned URLs. The backend generates presigned URLs and stores them +// here with a TTL; subsequent requests for the same key return the cached URL +// without re-contacting MinIO. +// +// Design: +// - Cache is intentionally best-effort: Get returns ("", false, nil) on any +// Valkey error, so callers always have a fallback path to regenerate. +// - Set silently drops errors — a miss on the next request is acceptable. +// - TTL should be set shorter than the actual presigned URL lifetime so that +// cached URLs are always valid when served. Recommended: 55 minutes for a +// 1-hour presigned URL. +package presigncache + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +// Cache is the interface for presign URL caching. +// Implementations must be safe for concurrent use. +type Cache interface { + // Get returns the cached URL for key. ok is false on cache miss or error. + Get(ctx context.Context, key string) (url string, ok bool, err error) + // Set stores url under key with the given TTL. + Set(ctx context.Context, key, url string, ttl time.Duration) error + // Delete removes key from the cache. + Delete(ctx context.Context, key string) error +} + +// ValkeyCache is a Cache backed by Valkey / Redis via go-redis. +type ValkeyCache struct { + rdb *redis.Client +} + +// New creates a ValkeyCache connecting to addr (e.g. "valkey:6379"). +// The connection is not established until the first command; use Ping to +// verify connectivity at startup. +func New(addr string) *ValkeyCache { + rdb := redis.NewClient(&redis.Options{ + Addr: addr, + DialTimeout: 2 * time.Second, + ReadTimeout: 1 * time.Second, + WriteTimeout: 1 * time.Second, + }) + return &ValkeyCache{rdb: rdb} +} + +// Ping checks connectivity. Call once at startup. +func (c *ValkeyCache) Ping(ctx context.Context) error { + if err := c.rdb.Ping(ctx).Err(); err != nil { + return fmt.Errorf("presigncache: ping valkey: %w", err) + } + return nil +} + +// Get returns (url, true, nil) on hit, ("", false, nil) on miss, and +// ("", false, err) only on unexpected errors (not redis.Nil). +func (c *ValkeyCache) Get(ctx context.Context, key string) (string, bool, error) { + val, err := c.rdb.Get(ctx, key).Result() + if err == redis.Nil { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("presigncache: get %q: %w", key, err) + } + return val, true, nil +} + +// Set stores url under key with ttl. Errors are returned but are non-fatal +// for callers — a Set failure means the next request will miss and regenerate. +func (c *ValkeyCache) Set(ctx context.Context, key, url string, ttl time.Duration) error { + if err := c.rdb.Set(ctx, key, url, ttl).Err(); err != nil { + return fmt.Errorf("presigncache: set %q: %w", key, err) + } + return nil +} + +// Delete removes key from the cache. It is not an error if the key does not exist. +func (c *ValkeyCache) Delete(ctx context.Context, key string) error { + if err := c.rdb.Del(ctx, key).Err(); err != nil { + return fmt.Errorf("presigncache: delete %q: %w", key, err) + } + return nil +} + +// NoopCache is a no-op Cache that always returns a miss. Used when Valkey is +// not configured (e.g. local development without Docker). +type NoopCache struct{} + +func (NoopCache) Get(_ context.Context, _ string) (string, bool, error) { return "", false, nil } +func (NoopCache) Set(_ context.Context, _, _ string, _ time.Duration) error { return nil } +func (NoopCache) Delete(_ context.Context, _ string) error { return nil } diff --git a/backend/internal/runner/asynq_runner.go b/backend/internal/runner/asynq_runner.go new file mode 100644 index 0000000..643fcf6 --- /dev/null +++ b/backend/internal/runner/asynq_runner.go @@ -0,0 +1,256 @@ +package runner + +// asynq_runner.go — Asynq-based task dispatch for the runner. +// +// When cfg.RedisAddr is set, Run() calls runAsynq() instead of runPoll(). +// The Asynq server replaces the polling loop: it listens on Redis for tasks +// enqueued by the backend Producer and delivers them immediately. +// +// Handlers in this file decode Asynq job payloads and call the existing +// runScrapeTask / runAudioTask methods, keeping all execution logic in one place. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sync" + "time" + + "github.com/hibiken/asynq" + asynqmetrics "github.com/hibiken/asynq/x/metrics" + "github.com/libnovel/backend/internal/asynqqueue" + "github.com/libnovel/backend/internal/domain" +) + +// runAsynq starts an Asynq server that replaces the PocketBase poll loop. +// It also starts the periodic catalogue refresh ticker. +// Blocks until ctx is cancelled. +func (r *Runner) runAsynq(ctx context.Context) error { + redisOpt, err := r.redisConnOpt() + if err != nil { + return fmt.Errorf("runner: parse redis addr: %w", err) + } + + srv := asynq.NewServer(redisOpt, asynq.Config{ + // Allocate concurrency slots for each task type. + // Total concurrency = scrape + audio slots. + Concurrency: r.cfg.MaxConcurrentScrape + r.cfg.MaxConcurrentAudio, + Queues: map[string]int{ + asynqqueue.QueueDefault: 1, + }, + // Let Asynq handle retries with exponential back-off. + RetryDelayFunc: asynq.DefaultRetryDelayFunc, + // Log errors from handlers via the existing structured logger. + ErrorHandler: asynq.ErrorHandlerFunc(func(_ context.Context, task *asynq.Task, err error) { + r.deps.Log.Error("runner: asynq task failed", + "type", task.Type(), + "err", err, + ) + }), + }) + + mux := asynq.NewServeMux() + mux.HandleFunc(asynqqueue.TypeAudioGenerate, r.handleAudioTask) + mux.HandleFunc(asynqqueue.TypeScrapeBook, r.handleScrapeTask) + mux.HandleFunc(asynqqueue.TypeScrapeCatalogue, r.handleScrapeTask) + mux.HandleFunc(asynqqueue.TypeImportBook, r.handleImportTask) + + // Register Asynq queue metrics with the default Prometheus registry so + // the /metrics endpoint (metrics.go) can expose them. + inspector := asynq.NewInspector(redisOpt) + collector := asynqmetrics.NewQueueMetricsCollector(inspector) + if err := r.metricsRegistry.Register(collector); err != nil { + r.deps.Log.Warn("runner: could not register asynq prometheus collector", "err", err) + } + + // Start the periodic catalogue refresh. + catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval) + defer catalogueTick.Stop() + if !r.cfg.SkipInitialCatalogueRefresh { + go r.runCatalogueRefresh(ctx) + } else { + r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)") + } + + r.deps.Log.Info("runner: asynq mode active", "redis_addr", r.cfg.RedisAddr) + + // ── Heartbeat goroutine ────────────────────────────────────────────── + // Write /tmp/runner.alive every 30s so Docker healthcheck passes in asynq mode. + // This mirrors the heartbeat file behavior from the poll() loop. + go func() { + heartbeatTick := time.NewTicker(r.cfg.StaleTaskThreshold / 2) + defer heartbeatTick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-heartbeatTick.C: + if f, err := os.Create("/tmp/runner.alive"); err != nil { + r.deps.Log.Warn("runner: could not write heartbeat file", "err", err) + } else { + f.Close() + } + } + } + }() + + // ── Translation polling goroutine ──────────────────────────────────── + // Translation tasks live in PocketBase (not Redis), so we need a separate + // poll loop to claim and dispatch them. This runs alongside the Asynq server. + translationSem := make(chan struct{}, r.cfg.MaxConcurrentTranslation) + var translationWg sync.WaitGroup + go func() { + tick := time.NewTicker(r.cfg.PollInterval) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + r.pollTranslationTasks(ctx, translationSem, &translationWg) + } + } + }() + + // Run catalogue refresh ticker in the background. + go func() { + for { + select { + case <-ctx.Done(): + return + case <-catalogueTick.C: + go r.runCatalogueRefresh(ctx) + } + } + }() + + // Start Asynq server (non-blocking). + if err := srv.Start(mux); err != nil { + return fmt.Errorf("runner: asynq server start: %w", err) + } + + // Block until context is cancelled, then gracefully stop. + <-ctx.Done() + r.deps.Log.Info("runner: context cancelled, shutting down asynq server") + srv.Shutdown() + + // Wait for translation tasks to complete. + translationWg.Wait() + return nil +} + +// redisConnOpt parses cfg.RedisAddr into an asynq.RedisConnOpt. +// Supports full "redis://" / "rediss://" URLs and plain "host:port". +func (r *Runner) redisConnOpt() (asynq.RedisConnOpt, error) { + addr := r.cfg.RedisAddr + // ParseRedisURI handles redis:// and rediss:// schemes. + if len(addr) > 7 && (addr[:8] == "redis://" || addr[:9] == "rediss://") { + return asynq.ParseRedisURI(addr) + } + // Plain "host:port" — use RedisClientOpt directly. + return asynq.RedisClientOpt{ + Addr: addr, + Password: r.cfg.RedisPassword, + }, nil +} + +// handleScrapeTask is the Asynq handler for TypeScrapeBook and TypeScrapeCatalogue. +func (r *Runner) handleScrapeTask(ctx context.Context, t *asynq.Task) error { + var p asynqqueue.ScrapePayload + if err := json.Unmarshal(t.Payload(), &p); err != nil { + return fmt.Errorf("unmarshal scrape payload: %w", err) + } + task := domain.ScrapeTask{ + ID: p.PBTaskID, + Kind: p.Kind, + TargetURL: p.TargetURL, + FromChapter: p.FromChapter, + ToChapter: p.ToChapter, + } + r.tasksRunning.Add(1) + defer r.tasksRunning.Add(-1) + r.runScrapeTask(ctx, task) + return nil +} + +// handleAudioTask is the Asynq handler for TypeAudioGenerate. +func (r *Runner) handleAudioTask(ctx context.Context, t *asynq.Task) error { + var p asynqqueue.AudioPayload + if err := json.Unmarshal(t.Payload(), &p); err != nil { + return fmt.Errorf("unmarshal audio payload: %w", err) + } + task := domain.AudioTask{ + ID: p.PBTaskID, + Slug: p.Slug, + Chapter: p.Chapter, + Voice: p.Voice, + } + r.tasksRunning.Add(1) + defer r.tasksRunning.Add(-1) + r.runAudioTask(ctx, task) + return nil +} + +// handleImportTask is the Asynq handler for TypeImportBook (PDF/EPUB import). +func (r *Runner) handleImportTask(ctx context.Context, t *asynq.Task) error { + var p asynqqueue.ImportPayload + if err := json.Unmarshal(t.Payload(), &p); err != nil { + return fmt.Errorf("unmarshal import payload: %w", err) + } + task := domain.ImportTask{ + ID: p.PBTaskID, + Slug: p.Slug, + Title: p.Title, + FileType: p.FileType, + ChaptersKey: p.ChaptersKey, + } + r.tasksRunning.Add(1) + defer r.tasksRunning.Add(-1) + r.runImportTask(ctx, task, p.ObjectKey) + return nil +} + +// pollTranslationTasks claims all available translation tasks from PocketBase +// and dispatches them to goroutines. Translation tasks don't go through Redis/Asynq +// because they're stored in PocketBase, so we need this separate poll loop. +func (r *Runner) pollTranslationTasks(ctx context.Context, translationSem chan struct{}, wg *sync.WaitGroup) { + // Reap orphaned tasks (same logic as poll() in runner.go). + if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil { + r.deps.Log.Warn("runner: reap stale translation tasks failed", "err", err) + } else if n > 0 { + r.deps.Log.Info("runner: reaped stale translation tasks", "count", n) + } + +translationLoop: + for { + if ctx.Err() != nil { + return + } + select { + case translationSem <- struct{}{}: + // Slot acquired — proceed to claim a task. + default: + // All slots busy; leave remaining pending tasks for next tick. + break translationLoop + } + task, ok, err := r.deps.Consumer.ClaimNextTranslationTask(ctx, r.cfg.WorkerID) + if err != nil { + <-translationSem + r.deps.Log.Error("runner: ClaimNextTranslationTask failed", "err", err) + break + } + if !ok { + <-translationSem + break + } + r.tasksRunning.Add(1) + wg.Add(1) + go func(t domain.TranslationTask) { + defer wg.Done() + defer func() { <-translationSem }() + defer r.tasksRunning.Add(-1) + r.runTranslationTask(ctx, t) + }(task) + } +} diff --git a/backend/internal/runner/catalogue_refresh.go b/backend/internal/runner/catalogue_refresh.go new file mode 100644 index 0000000..6e2bea5 --- /dev/null +++ b/backend/internal/runner/catalogue_refresh.go @@ -0,0 +1,186 @@ +package runner + +// catalogue_refresh.go — independent loop that walks the full novelfire.net +// catalogue, scrapes per-book metadata, downloads cover images to MinIO, and +// indexes every book in Meilisearch. +// +// Design: +// - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run(). +// - Also fires once on startup (unless SkipInitialCatalogueRefresh is set). +// - ScrapeCatalogue streams CatalogueEntry values over a channel — already has +// its own inter-page jitter + retryGet (see scraper.go). +// - Per-book: only metadata is scraped here (not chapters). Chapters are scraped +// on-demand when a user opens a book or via an explicit scrape task. +// - Between each metadata request a configurable base delay plus up to 50% +// random jitter is applied (CatalogueRequestDelay, default 2s). This keeps +// the request rate well below novelfire.net's rate limit even for ~15k books. +// - ScrapeMetadata itself uses retryGet with 429-aware exponential backoff +// (up to 9 attempts), so transient rate limits are handled gracefully. +// - Cover images are fetched and stored in MinIO on first sight; subsequent +// refreshes skip covers that already exist (CoverExists check). +// - Books already present in Meilisearch are skipped entirely (fast path). +// - Errors for individual books are logged and skipped; the loop never aborts. + +import ( + "context" + "fmt" + "io" + "math/rand" + "net/http" + "time" +) + +// runCatalogueRefresh performs one full catalogue walk: scrapes metadata for +// every book on novelfire.net, downloads covers to MinIO, and upserts to +// Meilisearch. Individual book failures are logged and skipped. +func (r *Runner) runCatalogueRefresh(ctx context.Context) { + if r.deps.Novel == nil { + r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured") + return + } + if r.deps.BookWriter == nil { + r.deps.Log.Warn("runner: catalogue refresh skipped — BookWriter not configured") + return + } + + delay := r.cfg.CatalogueRequestDelay + log := r.deps.Log.With("op", "catalogue_refresh") + log.Info("runner: catalogue refresh starting", "request_delay", delay) + + entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx) + + ok, skipped, errCount := 0, 0, 0 + for entry := range entries { + if ctx.Err() != nil { + break + } + + // Fast path: skip books already indexed in Meilisearch. + if r.deps.SearchIndex.BookExists(ctx, entry.Slug) { + skipped++ + continue + } + + // Polite delay between metadata requests: base + up to 50% jitter. + // This applies before every fetch so we never fire bursts. + jitter := time.Duration(rand.Int63n(int64(delay / 2))) + select { + case <-ctx.Done(): + break + case <-time.After(delay + jitter): + } + + // ScrapeMetadata internally retries on 429 with exponential back-off. + meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL) + if err != nil { + log.Warn("runner: catalogue refresh: metadata scrape failed — skipping book", + "slug", entry.Slug, "url", entry.URL, "err", err) + errCount++ + continue + } + + // Rewrite cover URL to backend proxy path so UI never hits CDN directly. + originalCover := meta.Cover + meta.Cover = fmt.Sprintf("/api/cover/novelfire.net/%s", meta.Slug) + + // Persist to PocketBase. + if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil { + log.Warn("runner: catalogue refresh: WriteMetadata failed — skipping book", + "slug", meta.Slug, "err", err) + errCount++ + continue + } + + // Index in Meilisearch (non-fatal). + if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil { + log.Warn("runner: catalogue refresh: UpsertBook failed", + "slug", meta.Slug, "err", err) + } + + // Download cover to MinIO if not already cached (non-fatal). + if r.deps.CoverStore != nil && originalCover != "" { + if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) { + if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil { + log.Warn("runner: catalogue refresh: cover download failed", + "slug", meta.Slug, "url", originalCover, "err", err) + } + } + } + + ok++ + if ok%50 == 0 { + log.Info("runner: catalogue refresh progress", + "scraped", ok, "skipped", skipped, "errors", errCount) + } + } + + if err := <-errCh; err != nil { + log.Warn("runner: catalogue refresh: catalogue stream error", "err", err) + } + + log.Info("runner: catalogue refresh finished", + "ok", ok, "skipped", skipped, "errors", errCount) +} + +// downloadCover fetches the cover image from coverURL and stores it in MinIO +// under covers/{slug}.jpg. It retries up to 3 times with exponential backoff +// on transient errors (5xx, network failures). +func (r *Runner) downloadCover(ctx context.Context, slug, coverURL string) error { + const maxRetries = 3 + delay := 2 * time.Second + + var lastErr error + for attempt := 0; attempt < maxRetries; attempt++ { + if ctx.Err() != nil { + return ctx.Err() + } + if attempt > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(delay): + } + delay *= 2 + } + + data, err := fetchCoverBytes(ctx, coverURL) + if err != nil { + lastErr = err + continue + } + + if err := r.deps.CoverStore.PutCover(ctx, slug, data, ""); err != nil { + return fmt.Errorf("put cover: %w", err) + } + return nil + } + return fmt.Errorf("download cover after %d retries: %w", maxRetries, lastErr) +} + +// fetchCoverBytes performs a single HTTP GET for coverURL and returns the body. +func fetchCoverBytes(ctx context.Context, coverURL string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, coverURL, nil) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)") + req.Header.Set("Referer", "https://novelfire.net/") + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("http get: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 500 { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("upstream %d for %s", resp.StatusCode, coverURL) + } + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, resp.Body) + return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, coverURL) + } + + return io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5 MiB cap +} diff --git a/backend/internal/runner/helpers.go b/backend/internal/runner/helpers.go new file mode 100644 index 0000000..3842c39 --- /dev/null +++ b/backend/internal/runner/helpers.go @@ -0,0 +1,71 @@ +package runner + +import ( + "regexp" + "strings" +) + +// stripMarkdown removes common markdown syntax from src, returning plain text +// suitable for TTS. Mirrors the helper in the scraper's server package. +func stripMarkdown(src string) string { + src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "") + src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "") + src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "") + src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "") + src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1") + src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "") + src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "") + src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "") + src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n") + return strings.TrimSpace(src) +} + +// chunkText splits text into chunks of at most maxChars characters, breaking +// at sentence boundaries (". ", "! ", "? ", "\n") so that the TTS service +// receives natural prose fragments rather than mid-sentence cuts. +// +// If a single sentence exceeds maxChars it is included as its own chunk — +// never silently truncated. +func chunkText(text string, maxChars int) []string { + if len(text) <= maxChars { + return []string{text} + } + + // Sentence-boundary delimiters — we split AFTER these sequences. + // Order matters: longer sequences first. + delimiters := []string{".\n", "!\n", "?\n", ". ", "! ", "? ", "\n\n", "\n"} + + var chunks []string + remaining := text + + for len(remaining) > 0 { + if len(remaining) <= maxChars { + chunks = append(chunks, strings.TrimSpace(remaining)) + break + } + + // Find the last sentence boundary within the maxChars window. + window := remaining[:maxChars] + cutAt := -1 + for _, delim := range delimiters { + idx := strings.LastIndex(window, delim) + if idx > 0 && idx+len(delim) > cutAt { + cutAt = idx + len(delim) + } + } + + if cutAt <= 0 { + // No boundary found — hard-break at maxChars to avoid infinite loop. + cutAt = maxChars + } + + chunk := strings.TrimSpace(remaining[:cutAt]) + if chunk != "" { + chunks = append(chunks, chunk) + } + remaining = strings.TrimSpace(remaining[cutAt:]) + } + + return chunks +} + diff --git a/backend/internal/runner/metrics.go b/backend/internal/runner/metrics.go new file mode 100644 index 0000000..659b113 --- /dev/null +++ b/backend/internal/runner/metrics.go @@ -0,0 +1,113 @@ +package runner + +// metrics.go — Prometheus metrics HTTP endpoint for the runner. +// +// GET /metrics returns a Prometheus text/plain scrape response. +// Exposes: +// - Standard Go runtime metrics (via promhttp) +// - Runner task counters (tasks_running, tasks_completed, tasks_failed) +// - Asynq queue metrics (registered in asynq_runner.go when Redis is enabled) +// +// GET /health — simple liveness probe. + +import ( + "context" + "fmt" + "log/slog" + "net" + "net/http" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// metricsServer serves GET /metrics and GET /health for the runner process. +type metricsServer struct { + addr string + r *Runner + log *slog.Logger +} + +func newMetricsServer(addr string, r *Runner, log *slog.Logger) *metricsServer { + ms := &metricsServer{addr: addr, r: r, log: log} + ms.registerCollectors() + return ms +} + +// registerCollectors registers runner-specific Prometheus collectors. +// Called once at construction; Asynq queue collector is registered separately +// in asynq_runner.go after the Redis connection is established. +func (ms *metricsServer) registerCollectors() { + // Runner task gauges / counters backed by the atomic fields on Runner. + ms.r.metricsRegistry.MustRegister(prometheus.NewGaugeFunc( + prometheus.GaugeOpts{ + Namespace: "runner", + Name: "tasks_running", + Help: "Number of tasks currently being processed.", + }, + func() float64 { return float64(ms.r.tasksRunning.Load()) }, + )) + ms.r.metricsRegistry.MustRegister(prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: "runner", + Name: "tasks_completed_total", + Help: "Total number of tasks completed successfully since startup.", + }, + func() float64 { return float64(ms.r.tasksCompleted.Load()) }, + )) + ms.r.metricsRegistry.MustRegister(prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: "runner", + Name: "tasks_failed_total", + Help: "Total number of tasks that ended in failure since startup.", + }, + func() float64 { return float64(ms.r.tasksFailed.Load()) }, + )) + ms.r.metricsRegistry.MustRegister(prometheus.NewGaugeFunc( + prometheus.GaugeOpts{ + Namespace: "runner", + Name: "uptime_seconds", + Help: "Seconds since the runner process started.", + }, + func() float64 { return time.Since(ms.r.startedAt).Seconds() }, + )) +} + +// ListenAndServe starts the HTTP server and blocks until ctx is cancelled or +// a fatal listen error occurs. +func (ms *metricsServer) ListenAndServe(ctx context.Context) error { + mux := http.NewServeMux() + mux.Handle("GET /metrics", promhttp.HandlerFor(ms.r.metricsRegistry, promhttp.HandlerOpts{})) + mux.HandleFunc("GET /health", ms.handleHealth) + + srv := &http.Server{ + Addr: ms.addr, + Handler: mux, + ReadTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + BaseContext: func(_ net.Listener) context.Context { return ctx }, + } + + errCh := make(chan error, 1) + go func() { + ms.log.Info("runner: metrics server listening", "addr", ms.addr) + errCh <- srv.ListenAndServe() + }() + + select { + case <-ctx.Done(): + shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = srv.Shutdown(shutCtx) + return nil + case err := <-errCh: + return fmt.Errorf("runner: metrics server: %w", err) + } +} + +// handleHealth handles GET /health — simple liveness probe. +func (ms *metricsServer) handleHealth(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok"}`)) +} diff --git a/backend/internal/runner/runner.go b/backend/internal/runner/runner.go new file mode 100644 index 0000000..f279cba --- /dev/null +++ b/backend/internal/runner/runner.go @@ -0,0 +1,884 @@ +// Package runner implements the worker loop that polls PocketBase for pending +// scrape and audio tasks, executes them, and reports results back. +// +// Design: +// - Run(ctx) loops on a ticker; each tick claims and dispatches pending tasks. +// - Scrape tasks are dispatched to the Orchestrator (one goroutine per task, +// up to MaxConcurrentScrape). +// - Audio tasks fetch chapter text, call Kokoro, upload to MinIO, and report +// the result back (up to MaxConcurrentAudio goroutines). +// - The runner is stateless between ticks; all state lives in PocketBase. +// - Atomic task counters are exposed via /metrics (see metrics.go). +// - Books are indexed in Meilisearch via an orchestrator.Config.PostMetadata +// hook injected at construction time. +package runner + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "sync" + "sync/atomic" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/cfai" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/kokoro" + "github.com/libnovel/backend/internal/libretranslate" + "github.com/libnovel/backend/internal/meili" + "github.com/libnovel/backend/internal/orchestrator" + "github.com/libnovel/backend/internal/pockettts" + "github.com/libnovel/backend/internal/scraper" + "github.com/libnovel/backend/internal/storage" + "github.com/libnovel/backend/internal/taskqueue" + "github.com/libnovel/backend/internal/webpush" + "github.com/prometheus/client_golang/prometheus" +) + +// Notifier creates notifications for users. +type Notifier interface { + CreateNotification(ctx context.Context, userID, title, message, link string) error +} + +// ChapterIngester persists imported chapters for a book. +type ChapterIngester interface { + IngestChapters(ctx context.Context, slug string, chapters []bookstore.Chapter) error +} + +// ImportChapterStore retrieves pre-parsed chapter JSON blobs from object storage. +type ImportChapterStore interface { + GetImportChapters(ctx context.Context, key string) ([]byte, error) +} + +// Config tunes the runner behaviour. +type Config struct { + // WorkerID uniquely identifies this runner instance in PocketBase records. + WorkerID string + // PollInterval is how often the runner checks for new tasks. + // Only used in PocketBase-polling mode (RedisAddr == ""). + PollInterval time.Duration + // MaxConcurrentScrape limits simultaneous book-scrape goroutines. + MaxConcurrentScrape int + // MaxConcurrentAudio limits simultaneous audio-generation goroutines. + MaxConcurrentAudio int + // MaxConcurrentTranslation limits simultaneous translation goroutines. + MaxConcurrentTranslation int + // OrchestratorWorkers is the chapter-scraping parallelism inside each book run. + OrchestratorWorkers int + // HeartbeatInterval is how often active tasks PATCH their heartbeat_at + // timestamp to signal they are still alive. Defaults to 30s when 0. + // Only used in PocketBase-polling mode. + HeartbeatInterval time.Duration + // StaleTaskThreshold is how old a heartbeat must be (or absent) before the + // task is considered orphaned and reset to pending. Defaults to 2m when 0. + // Only used in PocketBase-polling mode. + StaleTaskThreshold time.Duration + // CatalogueRefreshInterval is how often the runner walks the full catalogue, + // scrapes per-book metadata, downloads covers, and re-indexes everything in + // Meilisearch. Defaults to 24h (expensive — full catalogue walk). + CatalogueRefreshInterval time.Duration + // CatalogueRequestDelay is the base inter-request pause during a catalogue + // refresh metadata walk. Jitter of up to 50% is added on top. + // Defaults to 2s. Set via RUNNER_CATALOGUE_REQUEST_DELAY. + CatalogueRequestDelay time.Duration + // SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that + // otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval) + // still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for + // quick restarts where the catalogue is already up to date. + SkipInitialCatalogueRefresh bool + // MetricsAddr is the HTTP listen address for the /metrics endpoint. + // Defaults to ":9091". Set to "" to disable. + MetricsAddr string + // RedisAddr is the address of the Redis instance used for Asynq task + // dispatch. When set the runner switches from PocketBase-polling mode to + // Asynq ServeMux mode (immediate task delivery, no polling). + // Supports plain "host:port" or a full "rediss://..." URL. + // When empty the runner falls back to PocketBase polling. + RedisAddr string + // RedisPassword is the Redis AUTH password. + // Not required when RedisAddr is a full URL that includes credentials. + RedisPassword string +} + +// Dependencies are the external services the runner depends on. +type Dependencies struct { + // Consumer claims tasks from PocketBase. + Consumer taskqueue.Consumer + // BookWriter persists scraped data (used by orchestrator). + BookWriter bookstore.BookWriter + // BookReader reads chapter text for audio generation. + BookReader bookstore.BookReader + // AudioStore persists generated audio and checks key existence. + AudioStore bookstore.AudioStore + // TranslationStore persists translated markdown and checks key existence. + TranslationStore bookstore.TranslationStore + // CoverStore stores book cover images in MinIO. + CoverStore bookstore.CoverStore + // BookImport handles PDF/EPUB file parsing and chapter extraction. + // Kept for backward compatibility when ChaptersKey is not set. + BookImport bookstore.BookImporter + // ImportChapterStore retrieves pre-parsed chapter JSON blobs from MinIO. + // When set and the task has a ChaptersKey, the runner reads from here + // instead of calling BookImport.Import() (the new preferred path). + ImportChapterStore ImportChapterStore + // ChapterIngester persists extracted chapters into MinIO/PocketBase. + ChapterIngester ChapterIngester + // Notifier creates notifications for users. + Notifier Notifier + // WebPush sends browser push notifications to subscribed users. + // If nil, push notifications are disabled. + WebPush *webpush.Sender + // Store is the underlying *storage.Store; used for push subscription lookups. + // Only needed when WebPush is non-nil. + Store *storage.Store + // SearchIndex indexes books in Meilisearch after scraping. + // If nil a no-op is used. + SearchIndex meili.Client + // Novel is the scraper implementation. + Novel scraper.NovelScraper + // Kokoro is the Kokoro-FastAPI TTS client (GPU, OpenAI-compatible voices). + Kokoro kokoro.Client + // PocketTTS is the pocket-tts client (CPU, kyutai voices: alba, marius, etc.). + // If nil, pocket-tts voice tasks will fail with a clear error. + PocketTTS pockettts.Client + // CFAI is the Cloudflare Workers AI TTS client (cfai:* prefixed voices). + // If nil, CF AI voice tasks will fail with a clear error. + CFAI cfai.Client + // LibreTranslate is the machine translation client. + // If nil, translation tasks will fail with a clear error. + LibreTranslate libretranslate.Client + // Log is the structured logger. + Log *slog.Logger +} + +// Runner is the main worker process. +type Runner struct { + cfg Config + deps Dependencies + + metricsRegistry *prometheus.Registry + + // Atomic task counters — read by /metrics without locking. + tasksRunning atomic.Int64 + tasksCompleted atomic.Int64 + tasksFailed atomic.Int64 + + startedAt time.Time +} + +// New creates a Runner from cfg and deps. +func New(cfg Config, deps Dependencies) *Runner { + if cfg.PollInterval <= 0 { + cfg.PollInterval = 30 * time.Second + } + if cfg.MaxConcurrentScrape <= 0 { + cfg.MaxConcurrentScrape = 2 + } + if cfg.MaxConcurrentAudio <= 0 { + cfg.MaxConcurrentAudio = 1 + } + if cfg.MaxConcurrentTranslation <= 0 { + cfg.MaxConcurrentTranslation = 1 + } + if cfg.WorkerID == "" { + cfg.WorkerID = "runner" + } + if cfg.HeartbeatInterval <= 0 { + cfg.HeartbeatInterval = 30 * time.Second + } + if cfg.StaleTaskThreshold <= 0 { + cfg.StaleTaskThreshold = 2 * time.Minute + } + if cfg.CatalogueRefreshInterval <= 0 { + cfg.CatalogueRefreshInterval = 24 * time.Hour + } + if cfg.CatalogueRequestDelay <= 0 { + cfg.CatalogueRequestDelay = 2 * time.Second + } + if cfg.MetricsAddr == "" { + cfg.MetricsAddr = ":9091" + } + if deps.Log == nil { + deps.Log = slog.Default() + } + if deps.SearchIndex == nil { + deps.SearchIndex = meili.NoopClient{} + } + return &Runner{cfg: cfg, deps: deps, startedAt: time.Now(), metricsRegistry: prometheus.NewRegistry()} +} + +// Run starts the worker loop and the metrics HTTP server, blocking until ctx +// is cancelled. +// +// When cfg.RedisAddr is set the runner uses Asynq (immediate task delivery). +// Otherwise it falls back to PocketBase polling (legacy mode). +func (r *Runner) Run(ctx context.Context) error { + r.deps.Log.Info("runner: starting", + "worker_id", r.cfg.WorkerID, + "mode", r.mode(), + "max_scrape", r.cfg.MaxConcurrentScrape, + "max_audio", r.cfg.MaxConcurrentAudio, + "max_translation", r.cfg.MaxConcurrentTranslation, + "catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval, + "metrics_addr", r.cfg.MetricsAddr, + ) + + // Start metrics HTTP server in background if configured. + if r.cfg.MetricsAddr != "" { + ms := newMetricsServer(r.cfg.MetricsAddr, r, r.deps.Log) + go func() { + if err := ms.ListenAndServe(ctx); err != nil { + r.deps.Log.Error("runner: metrics server error", "err", err) + } + }() + } + + if r.cfg.RedisAddr != "" { + return r.runAsynq(ctx) + } + return r.runPoll(ctx) +} + +// mode returns a short string describing the active dispatch mode. +func (r *Runner) mode() string { + if r.cfg.RedisAddr != "" { + return "asynq" + } + return "poll" +} + +// runPoll is the legacy PocketBase-polling dispatch loop. +// Used when cfg.RedisAddr is empty. +func (r *Runner) runPoll(ctx context.Context) error { + scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape) + audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio) + translationSem := make(chan struct{}, r.cfg.MaxConcurrentTranslation) + importSem := make(chan struct{}, 1) // Limit concurrent imports + var wg sync.WaitGroup + + tick := time.NewTicker(r.cfg.PollInterval) + defer tick.Stop() + + catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval) + defer catalogueTick.Stop() + + // Run one catalogue refresh immediately on startup (unless skipped by flag). + if !r.cfg.SkipInitialCatalogueRefresh { + go r.runCatalogueRefresh(ctx) + } else { + r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)") + } + + r.deps.Log.Info("runner: poll mode active", "poll_interval", r.cfg.PollInterval) + + // Run one poll immediately on startup, then on each tick. + for { + r.poll(ctx, scrapeSem, audioSem, translationSem, importSem, &wg) + + select { + case <-ctx.Done(): + r.deps.Log.Info("runner: context cancelled, draining active tasks") + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + r.deps.Log.Info("runner: all tasks drained, exiting") + case <-time.After(2 * time.Minute): + r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit") + } + return nil + case <-catalogueTick.C: + go r.runCatalogueRefresh(ctx) + case <-tick.C: + } + } +} + +// poll claims all available pending tasks and dispatches them to goroutines. +func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem, translationSem, importSem chan struct{}, wg *sync.WaitGroup) { + // ── Heartbeat file ──────────────────────────────────────────────────── + // Touch /tmp/runner.alive so the Docker health check can confirm the + // runner is actively polling. Failure is non-fatal — just log it. + if f, err := os.Create("/tmp/runner.alive"); err != nil { + r.deps.Log.Warn("runner: could not write heartbeat file", "err", err) + } else { + f.Close() + } + + // ── Reap orphaned tasks ─────────────────────────────────────────────── + if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil { + r.deps.Log.Warn("runner: reap stale tasks failed", "err", err) + } else if n > 0 { + r.deps.Log.Info("runner: reaped stale tasks", "count", n) + } + + // ── Scrape tasks ────────────────────────────────────────────────────── + for { + if ctx.Err() != nil { + return + } + task, ok, err := r.deps.Consumer.ClaimNextScrapeTask(ctx, r.cfg.WorkerID) + if err != nil { + r.deps.Log.Error("runner: ClaimNextScrapeTask failed", "err", err) + break + } + if !ok { + break + } + select { + case scrapeSem <- struct{}{}: + default: + r.deps.Log.Warn("runner: scrape semaphore full, will retry next tick", + "task_id", task.ID) + break + } + r.tasksRunning.Add(1) + wg.Add(1) + go func(t domain.ScrapeTask) { + defer wg.Done() + defer func() { <-scrapeSem }() + defer r.tasksRunning.Add(-1) + r.runScrapeTask(ctx, t) + }(task) + } + + // ── Audio tasks ─────────────────────────────────────────────────────── + // Only claim tasks when there is a free slot in the semaphore. + // This avoids the old bug where we claimed (status→running) a task and + // then couldn't dispatch it, leaving it orphaned until the reaper fired. +audioLoop: + for { + if ctx.Err() != nil { + return + } + // Check capacity before claiming to avoid orphaning tasks. + select { + case audioSem <- struct{}{}: + // Slot acquired — proceed to claim a task. + default: + // All slots busy; leave remaining pending tasks for next tick. + break audioLoop + } + task, ok, err := r.deps.Consumer.ClaimNextAudioTask(ctx, r.cfg.WorkerID) + if err != nil { + <-audioSem // release the pre-acquired slot + r.deps.Log.Error("runner: ClaimNextAudioTask failed", "err", err) + break + } + if !ok { + <-audioSem // release the pre-acquired slot; queue empty + break + } + r.tasksRunning.Add(1) + wg.Add(1) + go func(t domain.AudioTask) { + defer wg.Done() + defer func() { <-audioSem }() + defer r.tasksRunning.Add(-1) + r.runAudioTask(ctx, t) + }(task) + } + + // ── Translation tasks ───────────────────────────────────────────────── +translationLoop: + for { + if ctx.Err() != nil { + return + } + select { + case translationSem <- struct{}{}: + // Slot acquired — proceed to claim a task. + default: + // All slots busy; leave remaining pending tasks for next tick. + break translationLoop + } + task, ok, err := r.deps.Consumer.ClaimNextTranslationTask(ctx, r.cfg.WorkerID) + if err != nil { + <-translationSem + r.deps.Log.Error("runner: ClaimNextTranslationTask failed", "err", err) + break + } + if !ok { + <-translationSem + break + } + r.tasksRunning.Add(1) + wg.Add(1) + go func(t domain.TranslationTask) { + defer wg.Done() + defer func() { <-translationSem }() + defer r.tasksRunning.Add(-1) + r.runTranslationTask(ctx, t) + }(task) + } + + // ── Import tasks ───────────────────────────────────────────────────── +importLoop: + for { + if ctx.Err() != nil { + return + } + select { + case importSem <- struct{}{}: + // Slot acquired — proceed to claim a task. + default: + // All slots busy; leave remaining pending tasks for next tick. + break importLoop + } + task, ok, err := r.deps.Consumer.ClaimNextImportTask(ctx, r.cfg.WorkerID) + if err != nil { + <-importSem + r.deps.Log.Error("runner: ClaimNextImportTask failed", "err", err) + break + } + if !ok { + <-importSem + break + } + r.tasksRunning.Add(1) + wg.Add(1) + go func(t domain.ImportTask) { + defer wg.Done() + defer func() { <-importSem }() + defer r.tasksRunning.Add(-1) + r.runImportTask(ctx, t, t.ObjectKey) + }(task) + } +} + +// newOrchestrator builds an orchestrator with the Meilisearch post-hook wired in. +func (r *Runner) newOrchestrator() *orchestrator.Orchestrator { + oCfg := orchestrator.Config{ + Workers: r.cfg.OrchestratorWorkers, + PostMetadata: func(ctx context.Context, meta domain.BookMeta) { + if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil { + r.deps.Log.Warn("runner: meilisearch upsert failed", + "slug", meta.Slug, "err", err) + } + }, + } + return orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log) +} + +// runScrapeTask executes one scrape task end-to-end and reports the result. +func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) { + ctx, span := otel.Tracer("runner").Start(ctx, "runner.scrape_task") + defer span.End() + span.SetAttributes( + attribute.String("task.id", task.ID), + attribute.String("task.kind", task.Kind), + attribute.String("task.url", task.TargetURL), + ) + + log := r.deps.Log.With("task_id", task.ID, "kind", task.Kind, "url", task.TargetURL) + log.Info("runner: scrape task starting") + + hbCtx, hbCancel := context.WithCancel(ctx) + defer hbCancel() + go func() { + tick := time.NewTicker(r.cfg.HeartbeatInterval) + defer tick.Stop() + for { + select { + case <-hbCtx.Done(): + return + case <-tick.C: + if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil { + log.Warn("runner: heartbeat failed", "err", err) + } + } + } + }() + + o := r.newOrchestrator() + var result domain.ScrapeResult + + switch task.Kind { + case "catalogue": + result = r.runCatalogueTask(ctx, task, o, log) + case "book", "book_range": + result = o.RunBook(ctx, task) + default: + result.ErrorMessage = fmt.Sprintf("unknown task kind: %q", task.Kind) + log.Warn("runner: unknown task kind") + } + + // Use a fresh context for the final write so a cancelled task context doesn't + // prevent the result counters from being persisted to PocketBase. + finishCtx, finishCancel := context.WithTimeout(context.Background(), 15*time.Second) + defer finishCancel() + if err := r.deps.Consumer.FinishScrapeTask(finishCtx, task.ID, result); err != nil { + log.Error("runner: FinishScrapeTask failed", "err", err) + } + + if result.ErrorMessage != "" { + r.tasksFailed.Add(1) + span.SetStatus(codes.Error, result.ErrorMessage) + if r.deps.Notifier != nil { + _ = r.deps.Notifier.CreateNotification(ctx, "admin", + "Scrape Failed", + fmt.Sprintf("Scrape task (%s) failed: %s", task.Kind, result.ErrorMessage), + "/admin/tasks") + } + } else { + r.tasksCompleted.Add(1) + span.SetStatus(codes.Ok, "") + if r.deps.Notifier != nil { + _ = r.deps.Notifier.CreateNotification(ctx, "admin", + "Scrape Complete", + fmt.Sprintf("Scraped %d chapters, skipped %d (%s)", result.ChaptersScraped, result.ChaptersSkipped, task.Kind), + "/admin/tasks") + } + // Fan-out in-app new-chapter notification to all users who have this book + // in their library. Runs in background so it doesn't block the task loop. + if r.deps.Store != nil && result.ChaptersScraped > 0 && + result.Slug != "" && task.Kind != "catalogue" { + go func() { + notifyCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + title := result.Slug + _ = r.deps.Store.NotifyUsersWithBook(notifyCtx, result.Slug, + "New chapters available", + fmt.Sprintf("%d new chapter(s) added to %s", result.ChaptersScraped, title), + "/books/"+result.Slug) + }() + } + // Send Web Push notifications to subscribed browsers. + if r.deps.WebPush != nil && r.deps.Store != nil && + result.ChaptersScraped > 0 && result.Slug != "" && task.Kind != "catalogue" { + go r.deps.WebPush.SendToBook(context.Background(), r.deps.Store, result.Slug, webpush.Payload{ + Title: "New chapter available", + Body: fmt.Sprintf("%d new chapter(s) added", result.ChaptersScraped), + URL: "/books/" + result.Slug, + Icon: "/icon-192.png", + }) + } + } + + log.Info("runner: scrape task finished", + "scraped", result.ChaptersScraped, + "skipped", result.ChaptersSkipped, + "errors", result.Errors, + ) +} + +// runCatalogueTask runs a full catalogue scrape. +func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o *orchestrator.Orchestrator, log *slog.Logger) domain.ScrapeResult { + entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx) + var result domain.ScrapeResult + + for entry := range entries { + if ctx.Err() != nil { + break + } + bookTask := domain.ScrapeTask{ + ID: task.ID, + Kind: "book", + TargetURL: entry.URL, + } + bookResult := o.RunBook(ctx, bookTask) + result.BooksFound += bookResult.BooksFound + result.ChaptersScraped += bookResult.ChaptersScraped + result.ChaptersSkipped += bookResult.ChaptersSkipped + result.Errors += bookResult.Errors + } + + if err := <-errCh; err != nil { + log.Warn("runner: catalogue scrape finished with error", "err", err) + result.Errors++ + if result.ErrorMessage == "" { + result.ErrorMessage = err.Error() + } + } + return result +} + +// runAudioTask executes one audio-generation task. +func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) { + ctx, span := otel.Tracer("runner").Start(ctx, "runner.audio_task") + defer span.End() + span.SetAttributes( + attribute.String("task.id", task.ID), + attribute.String("book.slug", task.Slug), + attribute.Int("chapter.number", task.Chapter), + attribute.String("audio.voice", task.Voice), + ) + + log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "voice", task.Voice) + log.Info("runner: audio task starting") + + hbCtx, hbCancel := context.WithCancel(ctx) + defer hbCancel() + go func() { + tick := time.NewTicker(r.cfg.HeartbeatInterval) + defer tick.Stop() + for { + select { + case <-hbCtx.Done(): + return + case <-tick.C: + if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil { + log.Warn("runner: heartbeat failed", "err", err) + } + } + } + }() + + fail := func(msg string) { + log.Error("runner: audio task failed", "reason", msg) + r.tasksFailed.Add(1) + span.SetStatus(codes.Error, msg) + result := domain.AudioResult{ErrorMessage: msg} + if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil { + log.Error("runner: FinishAudioTask failed", "err", err) + } + if r.deps.Notifier != nil { + _ = r.deps.Notifier.CreateNotification(ctx, "admin", + "Audio Failed", + fmt.Sprintf("Ch.%d of %s (%s): %s", task.Chapter, task.Slug, task.Voice, msg), + fmt.Sprintf("/books/%s", task.Slug)) + } + } + + raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter) + if err != nil { + fail(fmt.Sprintf("read chapter: %v", err)) + return + } + text := stripMarkdown(raw) + if text == "" { + fail("chapter text is empty after stripping markdown") + return + } + + var audioData []byte + if pockettts.IsPocketTTSVoice(task.Voice) { + if r.deps.PocketTTS == nil { + fail("pocket-tts client not configured (POCKET_TTS_URL is empty)") + return + } + var genErr error + audioData, genErr = r.deps.PocketTTS.GenerateAudio(ctx, text, task.Voice) + if genErr != nil { + fail(fmt.Sprintf("pocket-tts generate: %v", genErr)) + return + } + log.Info("runner: audio generated via pocket-tts", "voice", task.Voice) + } else if cfai.IsCFAIVoice(task.Voice) { + if r.deps.CFAI == nil { + fail("cloudflare AI client not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN empty)") + return + } + var genErr error + audioData, genErr = r.deps.CFAI.GenerateAudio(ctx, text, task.Voice) + if genErr != nil { + fail(fmt.Sprintf("cfai generate: %v", genErr)) + return + } + log.Info("runner: audio generated via cloudflare AI", "voice", task.Voice) + } else { + if r.deps.Kokoro == nil { + fail("kokoro client not configured (KOKORO_URL is empty)") + return + } + var genErr error + audioData, genErr = kokoroGenerateChunked(ctx, r.deps.Kokoro, text, task.Voice, log) + if genErr != nil { + fail(fmt.Sprintf("kokoro generate: %v", genErr)) + return + } + log.Info("runner: audio generated via kokoro-fastapi", "voice", task.Voice) + } + + key := r.deps.AudioStore.AudioObjectKey(task.Slug, task.Chapter, task.Voice) + if err := r.deps.AudioStore.PutAudio(ctx, key, audioData); err != nil { + fail(fmt.Sprintf("put audio: %v", err)) + return + } + + r.tasksCompleted.Add(1) + span.SetStatus(codes.Ok, "") + result := domain.AudioResult{ObjectKey: key} + if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil { + log.Error("runner: FinishAudioTask failed", "err", err) + } + if r.deps.Notifier != nil { + _ = r.deps.Notifier.CreateNotification(ctx, "admin", + "Audio Ready", + fmt.Sprintf("Ch.%d of %s (%s) is ready", task.Chapter, task.Slug, task.Voice), + fmt.Sprintf("/books/%s", task.Slug)) + } + log.Info("runner: audio task finished", "key", key) +} + +// kokoroGenerateChunked splits text into ~1 000-character sentence-boundary +// chunks, calls Kokoro.GenerateAudio for each, and concatenates the raw MP3 +// bytes. This avoids EOF / timeout failures that occur when the Kokoro +// FastAPI server receives very large inputs (e.g. a full imported PDF chapter). +// +// Concatenating raw MP3 frames is valid — MP3 is a frame-based format and +// standard players handle multi-segment files correctly. +func kokoroGenerateChunked(ctx context.Context, k kokoro.Client, text, voice string, log *slog.Logger) ([]byte, error) { + const chunkSize = 1000 + + chunks := chunkText(text, chunkSize) + log.Info("runner: kokoro chunked generation", "chunks", len(chunks), "total_chars", len(text)) + + var combined []byte + for i, chunk := range chunks { + data, err := k.GenerateAudio(ctx, chunk, voice) + if err != nil { + return nil, fmt.Errorf("chunk %d/%d: %w", i+1, len(chunks), err) + } + combined = append(combined, data...) + log.Info("runner: kokoro chunk done", "chunk", i+1, "of", len(chunks), "bytes", len(data)) + } + return combined, nil +} + +// runImportTask executes one PDF/EPUB import task. +// Preferred path: when task.ChaptersKey is set, it reads pre-parsed chapters +// JSON from MinIO (written by the backend at upload time) and ingests them. +// Fallback path: when ChaptersKey is empty, calls BookImport.Import() to +// parse the raw file on the runner (legacy behaviour, not used for new tasks). +func (r *Runner) runImportTask(ctx context.Context, task domain.ImportTask, objectKey string) { + ctx, span := otel.Tracer("runner").Start(ctx, "runner.import_task") + defer span.End() + span.SetAttributes( + attribute.String("task.id", task.ID), + attribute.String("book.slug", task.Slug), + attribute.String("file.type", task.FileType), + attribute.String("chapters_key", task.ChaptersKey), + ) + + log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "file_type", task.FileType) + log.Info("runner: import task starting", "chapters_key", task.ChaptersKey) + + hbCtx, hbCancel := context.WithCancel(ctx) + defer hbCancel() + go func() { + tick := time.NewTicker(r.cfg.HeartbeatInterval) + defer tick.Stop() + for { + select { + case <-hbCtx.Done(): + return + case <-tick.C: + if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil { + log.Warn("runner: heartbeat failed", "err", err) + } + } + } + }() + + fail := func(msg string) { + log.Error("runner: import task failed", "reason", msg) + r.tasksFailed.Add(1) + span.SetStatus(codes.Error, msg) + result := domain.ImportResult{ErrorMessage: msg} + if err := r.deps.Consumer.FinishImportTask(ctx, task.ID, result); err != nil { + log.Error("runner: FinishImportTask failed", "err", err) + } + } + + var chapters []bookstore.Chapter + + if task.ChaptersKey != "" && r.deps.ImportChapterStore != nil { + // New path: read pre-parsed chapters JSON uploaded by the backend. + raw, err := r.deps.ImportChapterStore.GetImportChapters(ctx, task.ChaptersKey) + if err != nil { + fail(fmt.Sprintf("get chapters JSON: %v", err)) + return + } + if err := json.Unmarshal(raw, &chapters); err != nil { + fail(fmt.Sprintf("unmarshal chapters JSON: %v", err)) + return + } + log.Info("runner: loaded pre-parsed chapters", "count", len(chapters)) + } else { + // Legacy path: parse the raw file on the runner. + if r.deps.BookImport == nil { + fail("book import not configured (BookImport dependency missing)") + return + } + var err error + chapters, err = r.deps.BookImport.Import(ctx, objectKey, task.FileType) + if err != nil { + fail(fmt.Sprintf("import file: %v", err)) + return + } + log.Info("runner: parsed chapters from file (legacy path)", "count", len(chapters)) + } + + if len(chapters) == 0 { + fail("no chapters extracted from file") + return + } + + // Persist chapters via ChapterIngester. + if r.deps.ChapterIngester == nil { + fail("chapter ingester not configured") + return + } + if err := r.deps.ChapterIngester.IngestChapters(ctx, task.Slug, chapters); err != nil { + fail(fmt.Sprintf("store chapters: %v", err)) + return + } + + // Write book metadata so the book appears in PocketBase catalogue. + if r.deps.BookWriter != nil { + meta := domain.BookMeta{ + Slug: task.Slug, + Title: task.Title, + Author: task.Author, + Cover: task.CoverURL, + Status: task.BookStatus, + Genres: task.Genres, + Summary: task.Summary, + TotalChapters: len(chapters), + } + if meta.Status == "" { + meta.Status = "completed" + } + if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil { + log.Warn("runner: import task WriteMetadata failed (non-fatal)", "err", err) + } else { + // Index in Meilisearch so the book is searchable. + if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil { + log.Warn("runner: import task meilisearch upsert failed (non-fatal)", "err", err) + } + } + } + + r.tasksCompleted.Add(1) + span.SetStatus(codes.Ok, "") + result := domain.ImportResult{ + Slug: task.Slug, + ChaptersImported: len(chapters), + } + if err := r.deps.Consumer.FinishImportTask(ctx, task.ID, result); err != nil { + log.Error("runner: FinishImportTask failed", "err", err) + } + + // Notify the user who initiated the import. + if r.deps.Notifier != nil { + msg := fmt.Sprintf("Import completed: %d chapters from %s", len(chapters), task.Title) + targetUser := task.InitiatorUserID + if targetUser == "" { + targetUser = "admin" + } + _ = r.deps.Notifier.CreateNotification(ctx, targetUser, "Import Complete", msg, "/admin/import") + } + + log.Info("runner: import task finished", "chapters", len(chapters)) +} diff --git a/backend/internal/runner/runner_test.go b/backend/internal/runner/runner_test.go new file mode 100644 index 0000000..8dd3531 --- /dev/null +++ b/backend/internal/runner/runner_test.go @@ -0,0 +1,413 @@ +package runner_test + +import ( + "bytes" + "context" + "errors" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/runner" +) + +// ── Stub types ──────────────────────────────────────────────────────────────── + +// stubConsumer is a test double for taskqueue.Consumer. +type stubConsumer struct { + scrapeQueue []domain.ScrapeTask + audioQueue []domain.AudioTask + scrapeIdx int + audioIdx int + finished []string + failCalled []string + claimErr error +} + +func (s *stubConsumer) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) { + if s.claimErr != nil { + return domain.ScrapeTask{}, false, s.claimErr + } + if s.scrapeIdx >= len(s.scrapeQueue) { + return domain.ScrapeTask{}, false, nil + } + t := s.scrapeQueue[s.scrapeIdx] + s.scrapeIdx++ + return t, true, nil +} + +func (s *stubConsumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) { + if s.claimErr != nil { + return domain.AudioTask{}, false, s.claimErr + } + if s.audioIdx >= len(s.audioQueue) { + return domain.AudioTask{}, false, nil + } + t := s.audioQueue[s.audioIdx] + s.audioIdx++ + return t, true, nil +} + +func (s *stubConsumer) ClaimNextTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) { + return domain.TranslationTask{}, false, nil +} + +func (s *stubConsumer) ClaimNextImportTask(_ context.Context, _ string) (domain.ImportTask, bool, error) { + return domain.ImportTask{}, false, nil +} + +func (s *stubConsumer) FinishScrapeTask(_ context.Context, id string, _ domain.ScrapeResult) error { + s.finished = append(s.finished, id) + return nil +} + +func (s *stubConsumer) FinishAudioTask(_ context.Context, id string, _ domain.AudioResult) error { + s.finished = append(s.finished, id) + return nil +} + +func (s *stubConsumer) FinishTranslationTask(_ context.Context, id string, _ domain.TranslationResult) error { + s.finished = append(s.finished, id) + return nil +} + +func (s *stubConsumer) FinishImportTask(_ context.Context, id string, _ domain.ImportResult) error { + s.finished = append(s.finished, id) + return nil +} + +func (s *stubConsumer) FailTask(_ context.Context, id, _ string) error { + s.failCalled = append(s.failCalled, id) + return nil +} + +func (s *stubConsumer) HeartbeatTask(_ context.Context, _ string) error { return nil } + +func (s *stubConsumer) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) { + return 0, nil +} + +// stubBookWriter satisfies bookstore.BookWriter (no-op). +type stubBookWriter struct{} + +func (s *stubBookWriter) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil } +func (s *stubBookWriter) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error { + return nil +} +func (s *stubBookWriter) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error { + return nil +} +func (s *stubBookWriter) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool { + return false +} + +func (s *stubBookWriter) DeduplicateChapters(_ context.Context, _ string) (int, error) { + return 0, nil +} + +// stubBookReader satisfies bookstore.BookReader — returns a single chapter. +type stubBookReader struct { + text string + readErr error +} + +func (s *stubBookReader) ReadChapter(_ context.Context, _ string, _ int) (string, error) { + return s.text, s.readErr +} +func (s *stubBookReader) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) { + return domain.BookMeta{}, false, nil +} +func (s *stubBookReader) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil } +func (s *stubBookReader) LocalSlugs(_ context.Context) (map[string]bool, error) { return nil, nil } +func (s *stubBookReader) MetadataMtime(_ context.Context, _ string) int64 { return 0 } +func (s *stubBookReader) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) { + return nil, nil +} +func (s *stubBookReader) CountChapters(_ context.Context, _ string) int { return 0 } +func (s *stubBookReader) ReindexChapters(_ context.Context, _ string) (int, error) { + return 0, nil +} + +// stubAudioStore satisfies bookstore.AudioStore. +type stubAudioStore struct { + putCalled atomic.Int32 + putErr error +} + +func (s *stubAudioStore) AudioObjectKey(slug string, n int, voice string) string { + return slug + "/" + string(rune('0'+n)) + "/" + voice + ".mp3" +} +func (s *stubAudioStore) AudioObjectKeyExt(slug string, n int, voice, ext string) string { + return slug + "/" + string(rune('0'+n)) + "/" + voice + "." + ext +} +func (s *stubAudioStore) AudioExists(_ context.Context, _ string) bool { return false } +func (s *stubAudioStore) AudioObjectSize(_ context.Context, _ string) int64 { return -1 } +func (s *stubAudioStore) PutAudio(_ context.Context, _ string, _ []byte) error { + s.putCalled.Add(1) + return s.putErr +} +func (s *stubAudioStore) PutAudioStream(_ context.Context, _ string, _ io.Reader, _ int64, _ string) error { + s.putCalled.Add(1) + return s.putErr +} + +// stubNovelScraper satisfies scraper.NovelScraper minimally. +type stubNovelScraper struct { + entries []domain.CatalogueEntry + metaErr error + chapters []domain.ChapterRef +} + +func (s *stubNovelScraper) ScrapeCatalogue(_ context.Context) (<-chan domain.CatalogueEntry, <-chan error) { + ch := make(chan domain.CatalogueEntry, len(s.entries)) + errCh := make(chan error, 1) + for _, e := range s.entries { + ch <- e + } + close(ch) + close(errCh) + return ch, errCh +} + +func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) { + if s.metaErr != nil { + return domain.BookMeta{}, s.metaErr + } + return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil +} + +func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) { + return s.chapters, nil +} + +func (s *stubNovelScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) { + return domain.Chapter{Ref: ref, Text: "# Chapter\n\nSome text."}, nil +} + +func (s *stubNovelScraper) ScrapeRanking(_ context.Context, _ int) (<-chan domain.BookMeta, <-chan error) { + ch := make(chan domain.BookMeta) + errCh := make(chan error, 1) + close(ch) + close(errCh) + return ch, errCh +} + +func (s *stubNovelScraper) SourceName() string { return "stub" } + +// stubKokoro satisfies kokoro.Client. +type stubKokoro struct { + data []byte + genErr error + called atomic.Int32 +} + +func (s *stubKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) { + s.called.Add(1) + return s.data, s.genErr +} + +func (s *stubKokoro) StreamAudioMP3(_ context.Context, _, _ string) (io.ReadCloser, error) { + s.called.Add(1) + if s.genErr != nil { + return nil, s.genErr + } + return io.NopCloser(bytes.NewReader(s.data)), nil +} + +func (s *stubKokoro) StreamAudioWAV(_ context.Context, _, _ string) (io.ReadCloser, error) { + s.called.Add(1) + if s.genErr != nil { + return nil, s.genErr + } + return io.NopCloser(bytes.NewReader(s.data)), nil +} + +func (s *stubKokoro) ListVoices(_ context.Context) ([]string, error) { + return []string{"af_bella"}, nil +} + +// ── stripMarkdown helper ────────────────────────────────────────────────────── + +func TestStripMarkdownViaAudioTask(t *testing.T) { + // Verify markdown is stripped before sending to Kokoro. + // We inject chapter text with markdown; the kokoro stub verifies data flows. + consumer := &stubConsumer{ + audioQueue: []domain.AudioTask{ + {ID: "a1", Slug: "book", Chapter: 1, Voice: "af_bella", Status: domain.TaskStatusRunning}, + }, + } + bookReader := &stubBookReader{text: "## Chapter 1\n\nPlain **text** here."} + audioStore := &stubAudioStore{} + kokoroStub := &stubKokoro{data: []byte("mp3")} + + cfg := runner.Config{ + WorkerID: "test", + PollInterval: time.Hour, // long poll — we'll cancel manually + } + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: bookReader, + AudioStore: audioStore, + Novel: &stubNovelScraper{}, + Kokoro: kokoroStub, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = r.Run(ctx) + + if kokoroStub.called.Load() != 1 { + t.Errorf("expected Kokoro.GenerateAudio called once, got %d", kokoroStub.called.Load()) + } + if audioStore.putCalled.Load() != 1 { + t.Errorf("expected PutAudio called once, got %d", audioStore.putCalled.Load()) + } +} + +func TestAudioTask_ReadChapterError(t *testing.T) { + consumer := &stubConsumer{ + audioQueue: []domain.AudioTask{ + {ID: "a2", Slug: "book", Chapter: 2, Voice: "af_bella", Status: domain.TaskStatusRunning}, + }, + } + bookReader := &stubBookReader{readErr: errors.New("chapter not found")} + audioStore := &stubAudioStore{} + kokoroStub := &stubKokoro{data: []byte("mp3")} + + cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour} + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: bookReader, + AudioStore: audioStore, + Novel: &stubNovelScraper{}, + Kokoro: kokoroStub, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = r.Run(ctx) + + // Kokoro should not be called; FinishAudioTask should be called with error. + if kokoroStub.called.Load() != 0 { + t.Errorf("expected Kokoro not called, got %d", kokoroStub.called.Load()) + } + if len(consumer.finished) != 1 { + t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished)) + } +} + +func TestAudioTask_KokoroError(t *testing.T) { + consumer := &stubConsumer{ + audioQueue: []domain.AudioTask{ + {ID: "a3", Slug: "book", Chapter: 3, Voice: "af_bella", Status: domain.TaskStatusRunning}, + }, + } + bookReader := &stubBookReader{text: "Chapter text."} + audioStore := &stubAudioStore{} + kokoroStub := &stubKokoro{genErr: errors.New("tts failed")} + + cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour} + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: bookReader, + AudioStore: audioStore, + Novel: &stubNovelScraper{}, + Kokoro: kokoroStub, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = r.Run(ctx) + + if audioStore.putCalled.Load() != 0 { + t.Errorf("expected PutAudio not called, got %d", audioStore.putCalled.Load()) + } + if len(consumer.finished) != 1 { + t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished)) + } +} + +func TestScrapeTask_BookKind(t *testing.T) { + consumer := &stubConsumer{ + scrapeQueue: []domain.ScrapeTask{ + {ID: "s1", Kind: "book", TargetURL: "https://example.com/book/test-book", Status: domain.TaskStatusRunning}, + }, + } + + cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour} + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: &stubBookReader{}, + AudioStore: &stubAudioStore{}, + Novel: &stubNovelScraper{}, + Kokoro: &stubKokoro{}, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = r.Run(ctx) + + if len(consumer.finished) != 1 || consumer.finished[0] != "s1" { + t.Errorf("expected task s1 finished, got %v", consumer.finished) + } +} + +func TestScrapeTask_UnknownKind(t *testing.T) { + consumer := &stubConsumer{ + scrapeQueue: []domain.ScrapeTask{ + {ID: "s2", Kind: "unknown_kind", Status: domain.TaskStatusRunning}, + }, + } + + cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour} + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: &stubBookReader{}, + AudioStore: &stubAudioStore{}, + Novel: &stubNovelScraper{}, + Kokoro: &stubKokoro{}, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = r.Run(ctx) + + // Unknown kind still finishes the task (with error message in result). + if len(consumer.finished) != 1 || consumer.finished[0] != "s2" { + t.Errorf("expected task s2 finished, got %v", consumer.finished) + } +} + +func TestRun_CancelImmediately(t *testing.T) { + consumer := &stubConsumer{} + cfg := runner.Config{WorkerID: "test", PollInterval: 10 * time.Millisecond} + deps := runner.Dependencies{ + Consumer: consumer, + BookWriter: &stubBookWriter{}, + BookReader: &stubBookReader{}, + AudioStore: &stubAudioStore{}, + Novel: &stubNovelScraper{}, + Kokoro: &stubKokoro{}, + } + + r := runner.New(cfg, deps) + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cancel before Run + + err := r.Run(ctx) + if err != nil { + t.Errorf("expected nil on graceful shutdown, got %v", err) + } +} diff --git a/backend/internal/runner/translation.go b/backend/internal/runner/translation.go new file mode 100644 index 0000000..eb72391 --- /dev/null +++ b/backend/internal/runner/translation.go @@ -0,0 +1,109 @@ +package runner + +import ( + "context" + "fmt" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + + "github.com/libnovel/backend/internal/domain" +) + +// runTranslationTask executes one machine-translation task end-to-end and +// reports the result back to PocketBase. +func (r *Runner) runTranslationTask(ctx context.Context, task domain.TranslationTask) { + ctx, span := otel.Tracer("runner").Start(ctx, "runner.translation_task") + defer span.End() + span.SetAttributes( + attribute.String("task.id", task.ID), + attribute.String("book.slug", task.Slug), + attribute.Int("chapter.number", task.Chapter), + attribute.String("translation.lang", task.Lang), + ) + + log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "lang", task.Lang) + log.Info("runner: translation task starting") + + // Heartbeat goroutine — keeps the task alive while translation runs. + hbCtx, hbCancel := context.WithCancel(ctx) + defer hbCancel() + go func() { + tick := time.NewTicker(r.cfg.HeartbeatInterval) + defer tick.Stop() + for { + select { + case <-hbCtx.Done(): + return + case <-tick.C: + if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil { + log.Warn("runner: heartbeat failed", "err", err) + } + } + } + }() + + fail := func(msg string) { + log.Error("runner: translation task failed", "reason", msg) + r.tasksFailed.Add(1) + span.SetStatus(codes.Error, msg) + result := domain.TranslationResult{ErrorMessage: msg} + if err := r.deps.Consumer.FinishTranslationTask(ctx, task.ID, result); err != nil { + log.Error("runner: FinishTranslationTask failed", "err", err) + } + if r.deps.Notifier != nil { + _ = r.deps.Notifier.CreateNotification(ctx, "admin", + "Translation Failed", + fmt.Sprintf("Ch.%d of %s (%s): %s", task.Chapter, task.Slug, task.Lang, msg), + fmt.Sprintf("/books/%s", task.Slug)) + } + } + + // Guard: LibreTranslate must be configured. + if r.deps.LibreTranslate == nil { + fail("libretranslate client not configured (LIBRETRANSLATE_URL is empty)") + return + } + + // 1. Read raw markdown chapter. + raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter) + if err != nil { + fail(fmt.Sprintf("read chapter: %v", err)) + return + } + if raw == "" { + fail("chapter text is empty") + return + } + + // 2. Translate (chunked, concurrent). + translated, err := r.deps.LibreTranslate.Translate(ctx, raw, "en", task.Lang) + if err != nil { + fail(fmt.Sprintf("translate: %v", err)) + return + } + + // 3. Store translated markdown in MinIO. + key := r.deps.TranslationStore.TranslationObjectKey(task.Lang, task.Slug, task.Chapter) + if err := r.deps.TranslationStore.PutTranslation(ctx, key, []byte(translated)); err != nil { + fail(fmt.Sprintf("put translation: %v", err)) + return + } + + // 4. Report success. + r.tasksCompleted.Add(1) + span.SetStatus(codes.Ok, "") + result := domain.TranslationResult{ObjectKey: key} + if err := r.deps.Consumer.FinishTranslationTask(ctx, task.ID, result); err != nil { + log.Error("runner: FinishTranslationTask failed", "err", err) + } + if r.deps.Notifier != nil { + _ = r.deps.Notifier.CreateNotification(ctx, "admin", + "Translation Ready", + fmt.Sprintf("Ch.%d of %s translated to %s", task.Chapter, task.Slug, task.Lang), + fmt.Sprintf("/books/%s", task.Slug)) + } + log.Info("runner: translation task finished", "key", key) +} diff --git a/backend/internal/scraper/scraper.go b/backend/internal/scraper/scraper.go new file mode 100644 index 0000000..a8081f7 --- /dev/null +++ b/backend/internal/scraper/scraper.go @@ -0,0 +1,60 @@ +// Package scraper defines the NovelScraper interface and its sub-interfaces. +// Domain types live in internal/domain — this package only defines the scraping +// contract so that novelfire and any future scrapers can be swapped freely. +package scraper + +import ( + "context" + + "github.com/libnovel/backend/internal/domain" +) + +// CatalogueProvider can enumerate every novel available on a source site. +type CatalogueProvider interface { + ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) +} + +// MetadataProvider can extract structured book metadata from a novel's landing page. +type MetadataProvider interface { + ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) +} + +// ChapterListProvider can enumerate all chapters of a book. +// upTo > 0 stops pagination once at least upTo chapter numbers have been +// collected (early-exit optimisation for range scrapes). upTo == 0 fetches all pages. +type ChapterListProvider interface { + ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error) +} + +// ChapterTextProvider can extract the readable text from a single chapter page. +type ChapterTextProvider interface { + ScrapeChapterText(ctx context.Context, ref domain.ChapterRef) (domain.Chapter, error) +} + +// RankingProvider can enumerate novels from a ranking page. +type RankingProvider interface { + // ScrapeRanking pages through up to maxPages ranking pages. + // maxPages <= 0 means all pages. + ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) +} + +// NovelScraper is the full interface a concrete novel source must implement. +type NovelScraper interface { + CatalogueProvider + MetadataProvider + ChapterListProvider + ChapterTextProvider + RankingProvider + + // SourceName returns the human-readable name of this scraper, e.g. "novelfire.net". + SourceName() string +} + +// Selector describes how to locate an element in an HTML document. +type Selector struct { + Tag string + Class string + ID string + Attr string + Multiple bool +} diff --git a/backend/internal/storage/import.go b/backend/internal/storage/import.go new file mode 100644 index 0000000..ae19003 --- /dev/null +++ b/backend/internal/storage/import.go @@ -0,0 +1,857 @@ +package storage + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "os" + "sort" + "strconv" + "strings" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/domain" + minio "github.com/minio/minio-go/v7" + "github.com/pdfcpu/pdfcpu/pkg/api" + "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model" + "golang.org/x/net/html" +) + +type importer struct { + mc *minioClient +} + +// NewBookImporter creates a BookImporter that reads files from MinIO. +func NewBookImporter(s *Store) bookstore.BookImporter { + return &importer{mc: s.mc} +} + +func (i *importer) Import(ctx context.Context, objectKey, fileType string) ([]bookstore.Chapter, error) { + if fileType != "pdf" && fileType != "epub" { + return nil, fmt.Errorf("unsupported file type: %s", fileType) + } + + obj, err := i.mc.client.GetObject(ctx, "imports", objectKey, minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("get object from minio: %w", err) + } + defer obj.Close() + + data, err := io.ReadAll(obj) + if err != nil { + return nil, fmt.Errorf("read object: %w", err) + } + + if fileType == "pdf" { + return parsePDF(data) + } + return parseEPUB(data) +} + +// AnalyzeFile parses the given PDF or EPUB data and returns the detected +// chapter count and up to 3 preview lines (first non-empty line of each of +// the first 3 chapters). It is used by the analyze-only endpoint so users +// can preview chapter count before committing the import. +// Note: uses parsePDF which is backed by pdfcpu ExtractContent — fast, no hang risk. +func AnalyzeFile(data []byte, fileType string) (chapterCount int, firstLines []string, err error) { + var chapters []bookstore.Chapter + switch fileType { + case "pdf": + chapters, err = parsePDF(data) + case "epub": + chapters, err = parseEPUB(data) + default: + return 0, nil, fmt.Errorf("unsupported file type: %s", fileType) + } + if err != nil { + return 0, nil, err + } + chapterCount = len(chapters) + for i, ch := range chapters { + if i >= 3 { + break + } + line := strings.TrimSpace(ch.Content) + if nl := strings.Index(line, "\n"); nl > 0 { + line = line[:nl] + } + if len(line) > 120 { + line = line[:120] + "…" + } + firstLines = append(firstLines, line) + } + return chapterCount, firstLines, nil +} + + + +// decryptPDF strips encryption from a PDF using an empty user password. +// Returns the decrypted bytes, or an error if decryption is not possible. +// This handles the common case of "owner-only" encrypted PDFs (copy/print +// restrictions) which use an empty user password and open normally in readers. +func decryptPDF(data []byte) ([]byte, error) { + conf := model.NewDefaultConfiguration() + conf.UserPW = "" + conf.OwnerPW = "" + + var out bytes.Buffer + err := api.Decrypt(bytes.NewReader(data), &out, conf) + if err != nil { + return nil, err + } + return out.Bytes(), nil +} + +// ParseImportFile parses a PDF or EPUB and returns chapters. +// Unlike AnalyzeFile it respects ctx cancellation so callers can apply a timeout. +// For PDFs it first attempts to strip encryption with an empty password. +func ParseImportFile(ctx context.Context, data []byte, fileType string) ([]bookstore.Chapter, error) { + type result struct { + chapters []bookstore.Chapter + err error + } + ch := make(chan result, 1) + go func() { + var chapters []bookstore.Chapter + var err error + switch fileType { + case "pdf": + chapters, err = parsePDF(data) + case "epub": + chapters, err = parseEPUB(data) + default: + err = fmt.Errorf("unsupported file type: %s", fileType) + } + ch <- result{chapters, err} + }() + select { + case <-ctx.Done(): + return nil, fmt.Errorf("parse timed out: %w", ctx.Err()) + case r := <-ch: + return r.chapters, r.err + } +} + +// pdfSkipBookmarks lists bookmark titles that are front/back matter, not story chapters. +// These are skipped when building the chapter list. +var pdfSkipBookmarks = map[string]bool{ + "cover": true, "insert": true, "title page": true, "copyright": true, + "appendix": true, "color insert": true, "color illustrations": true, +} + +// parsePDF extracts text from PDF bytes and returns it as a single chapter. +// +// The full readable text is returned as one chapter so the admin can manually +// split it into chapters via the UI using --- markers. +// +// Strategy: +// 1. Decrypt owner-protected PDFs (empty user password). +// 2. Extract raw content streams for every page using pdfcpu ExtractContent. +// 3. Concatenate text from all pages in order, skipping front matter +// (cover, title page, copyright — typically the first 10 pages). +func parsePDF(data []byte) ([]bookstore.Chapter, error) { + // Decrypt owner-protected PDFs (empty user password). + decrypted, err := decryptPDF(data) + if err == nil { + data = decrypted + } + + conf := model.NewDefaultConfiguration() + conf.UserPW = "" + conf.OwnerPW = "" + + // Extract all page content streams to a temp directory. + tmpDir, err := os.MkdirTemp("", "pdf-extract-*") + if err != nil { + return nil, fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + if err := api.ExtractContent(bytes.NewReader(data), tmpDir, "out", nil, conf); err != nil { + return nil, fmt.Errorf("extract PDF content: %w", err) + } + + entries, err := os.ReadDir(tmpDir) + if err != nil || len(entries) == 0 { + return nil, fmt.Errorf("PDF has no content pages") + } + + // Parse page number from filename and build ordered text map. + pageTexts := make(map[int]string, len(entries)) + maxPage := 0 + for _, e := range entries { + pageNum := pageNumFromFilename(e.Name()) + if pageNum <= 0 { + continue + } + raw, readErr := os.ReadFile(tmpDir + "/" + e.Name()) + if readErr != nil { + continue + } + pageTexts[pageNum] = fixWin1252(extractTextFromContentStream(raw)) + if pageNum > maxPage { + maxPage = pageNum + } + } + + // Determine front-matter cutoff using bookmarks if available, + // otherwise skip the first 10 pages (cover/title/copyright). + bodyStart := 1 + bookmarks, bmErr := api.Bookmarks(bytes.NewReader(data), conf) + if bmErr == nil { + for _, bm := range bookmarks { + title := strings.ToLower(strings.TrimSpace(bm.Title)) + if !pdfSkipBookmarks[title] && bm.PageFrom > 0 { + // First non-front-matter bookmark — body starts here. + bodyStart = bm.PageFrom + break + } + } + } else if maxPage > 10 { + bodyStart = 11 + } + + // Concatenate all body pages. + var sb strings.Builder + for p := bodyStart; p <= maxPage; p++ { + t := strings.TrimSpace(pageTexts[p]) + if t == "" { + continue + } + sb.WriteString(t) + sb.WriteString("\n\n") + } + + text := strings.TrimSpace(sb.String()) + if text == "" { + return nil, fmt.Errorf("could not extract any text from PDF") + } + + return []bookstore.Chapter{{ + Number: 1, + Title: "Full Text", + Content: text, + }}, nil +} + +// pageNumFromFilename extracts the page number from a pdfcpu content-stream +// filename like "out_Content_page_42.txt". Returns 0 if not parseable. +func pageNumFromFilename(name string) int { + // Strip directory prefix and extension. + base := name + if idx := strings.LastIndex(base, "/"); idx >= 0 { + base = base[idx+1:] + } + if idx := strings.LastIndex(base, "."); idx >= 0 { + base = base[:idx] + } + // Find last "_" and parse the number after it. + if idx := strings.LastIndex(base, "_"); idx >= 0 { + n, err := strconv.Atoi(base[idx+1:]) + if err == nil && n > 0 { + return n + } + } + return 0 +} + +// win1252ToUnicode maps the Windows-1252 control range 0x80–0x9F to the +// Unicode characters they actually represent in that encoding. +// Standard Latin-1 maps these bytes to control characters; Win-1252 maps +// them to typographic symbols that appear in publisher PDFs. +var win1252ToUnicode = map[byte]rune{ + 0x80: '\u20AC', // € + 0x82: '\u201A', // ‚ + 0x83: '\u0192', // ƒ + 0x84: '\u201E', // „ + 0x85: '\u2026', // … + 0x86: '\u2020', // † + 0x87: '\u2021', // ‡ + 0x88: '\u02C6', // ˆ + 0x89: '\u2030', // ‰ + 0x8A: '\u0160', // Š + 0x8B: '\u2039', // ‹ + 0x8C: '\u0152', // Œ + 0x8E: '\u017D', // Ž + 0x91: '\u2018', // ' (left single quotation mark) + 0x92: '\u2019', // ' (right single quotation mark / apostrophe) + 0x93: '\u201C', // " (left double quotation mark) + 0x94: '\u201D', // " (right double quotation mark) + 0x95: '\u2022', // • (bullet) + 0x96: '\u2013', // – (en dash) + 0x97: '\u2014', // — (em dash) + 0x98: '\u02DC', // ˜ + 0x99: '\u2122', // ™ + 0x9A: '\u0161', // š + 0x9B: '\u203A', // › + 0x9C: '\u0153', // œ + 0x9E: '\u017E', // ž + 0x9F: '\u0178', // Ÿ +} + +// fixWin1252 replaces Windows-1252 specific bytes (0x80–0x9F) in a string +// that was decoded as raw Latin-1 bytes with their proper Unicode equivalents. +func fixWin1252(s string) string { + // Fast path: if no bytes in 0x80–0x9F range, return unchanged. + needsFix := false + for i := 0; i < len(s); i++ { + b := s[i] + if b >= 0x80 && b <= 0x9F { + needsFix = true + break + } + } + if !needsFix { + return s + } + var sb strings.Builder + sb.Grow(len(s)) + for i := 0; i < len(s); i++ { + b := s[i] + if b >= 0x80 && b <= 0x9F { + if r, ok := win1252ToUnicode[b]; ok { + sb.WriteRune(r) + continue + } + } + sb.WriteByte(b) + } + return sb.String() +} + +// extractTextFromContentStream parses a raw PDF content stream and extracts +// readable text from Tj and TJ operators. +// +// TJ arrays may contain a mix of literal strings (parenthesised) and hex glyph +// arrays. Only the literal strings are decoded — hex arrays require per-font +// ToUnicode CMaps and are skipped. Kerning adjustment numbers inside TJ arrays +// are also ignored (they're just spacing hints). +// +// Line breaks are inserted on ET / Td / TD / T* operators. +func extractTextFromContentStream(stream []byte) string { + s := string(stream) + var sb strings.Builder + i := 0 + n := len(s) + for i < n { + // TJ array: [ ... ]TJ — collect all literal strings, skip hex & numbers. + if s[i] == '[' { + j := i + 1 + for j < n && s[j] != ']' { + if s[j] == '(' { + // Literal string inside TJ array. + k := j + 1 + depth := 1 + for k < n && depth > 0 { + if s[k] == '\\' { + k += 2 + continue + } + if s[k] == '(' { + depth++ + } else if s[k] == ')' { + depth-- + } + k++ + } + lit := pdfUnescapeString(s[j+1 : k-1]) + if hasPrintableASCII(lit) { + sb.WriteString(lit) + } + j = k + continue + } + j++ + } + // Check if this is a TJ operator (skip whitespace after ']'). + end := j + 1 + for end < n && (s[end] == ' ' || s[end] == '\t' || s[end] == '\r' || s[end] == '\n') { + end++ + } + if end+2 <= n && s[end:end+2] == "TJ" && (end+2 == n || !isAlphaNum(s[end+2])) { + i = end + 2 + continue + } + i = j + 1 + continue + } + // Single string: (string) Tj + if s[i] == '(' { + j := i + 1 + depth := 1 + for j < n && depth > 0 { + if s[j] == '\\' { + j += 2 + continue + } + if s[j] == '(' { + depth++ + } else if s[j] == ')' { + depth-- + } + j++ + } + lit := pdfUnescapeString(s[i+1 : j-1]) + if hasPrintableASCII(lit) { + // Check for Tj operator. + end := j + for end < n && (s[end] == ' ' || s[end] == '\t') { + end++ + } + if end+2 <= n && s[end:end+2] == "Tj" && (end+2 == n || !isAlphaNum(s[end+2])) { + sb.WriteString(lit) + i = end + 2 + continue + } + } + i = j + continue + } + // Detect end of text object (ET) — add a newline. + if i+2 <= n && s[i:i+2] == "ET" && (i+2 == n || !isAlphaNum(s[i+2])) { + sb.WriteByte('\n') + i += 2 + continue + } + // Detect Td / TD / T* — newline within text block. + if i+2 <= n && (s[i:i+2] == "Td" || s[i:i+2] == "TD" || s[i:i+2] == "T*") && + (i+2 == n || !isAlphaNum(s[i+2])) { + sb.WriteByte('\n') + i += 2 + continue + } + i++ + } + return sb.String() +} + +func isAlphaNum(b byte) bool { + return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_' +} + +func hasPrintableASCII(s string) bool { + for _, c := range s { + if c >= 0x20 && c < 0x7F { + return true + } + } + return false +} + +// pdfUnescapeString handles PDF string escape sequences. +func pdfUnescapeString(s string) string { + if !strings.ContainsRune(s, '\\') { + return s + } + var sb strings.Builder + i := 0 + for i < len(s) { + if s[i] == '\\' && i+1 < len(s) { + switch s[i+1] { + case 'n': + sb.WriteByte('\n') + case 'r': + sb.WriteByte('\r') + case 't': + sb.WriteByte('\t') + case '(', ')', '\\': + sb.WriteByte(s[i+1]) + default: + // Octal escape \ddd + if s[i+1] >= '0' && s[i+1] <= '7' { + end := i + 2 + for end < i+5 && end < len(s) && s[end] >= '0' && s[end] <= '7' { + end++ + } + val, _ := strconv.ParseInt(s[i+1:end], 8, 16) + sb.WriteByte(byte(val)) + i = end + continue + } + sb.WriteByte(s[i+1]) + } + i += 2 + } else { + sb.WriteByte(s[i]) + i++ + } + } + return sb.String() +} + +// ── EPUB parsing ────────────────────────────────────────────────────────────── + +func parseEPUB(data []byte) ([]bookstore.Chapter, error) { + zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, fmt.Errorf("open EPUB zip: %w", err) + } + + // 1. Read META-INF/container.xml → find rootfile (content.opf path). + opfPath, err := epubRootfilePath(zr) + if err != nil { + return nil, fmt.Errorf("epub container: %w", err) + } + + // 2. Parse content.opf → spine order of chapter files. + spineFiles, titleMap, err := epubSpine(zr, opfPath) + if err != nil { + return nil, fmt.Errorf("epub spine: %w", err) + } + + if len(spineFiles) == 0 { + return nil, fmt.Errorf("EPUB spine is empty") + } + + // Base directory of the OPF file for resolving relative hrefs. + opfDir := "" + if idx := strings.LastIndex(opfPath, "/"); idx >= 0 { + opfDir = opfPath[:idx+1] + } + + var chapters []bookstore.Chapter + chNum := 0 + for i, href := range spineFiles { + fullPath := opfDir + href + content, err := epubFileContent(zr, fullPath) + if err != nil { + continue + } + text := htmlToText(content) + if strings.TrimSpace(text) == "" { + continue + } + chNum++ + title := titleMap[href] + if title == "" { + title = fmt.Sprintf("Chapter %d", chNum) + } + _ = i // spine index unused for numbering + chapters = append(chapters, bookstore.Chapter{ + Number: chNum, + Title: title, + Content: text, + }) + } + + if len(chapters) == 0 { + return nil, fmt.Errorf("no readable chapters found in EPUB") + } + return chapters, nil +} + +// epubRootfilePath parses META-INF/container.xml and returns the full-path +// of the OPF package document. +func epubRootfilePath(zr *zip.Reader) (string, error) { + f := zipFile(zr, "META-INF/container.xml") + if f == nil { + return "", fmt.Errorf("META-INF/container.xml not found") + } + rc, err := f.Open() + if err != nil { + return "", err + } + defer rc.Close() + + doc, err := html.Parse(rc) + if err != nil { + return "", err + } + + var path string + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode && strings.EqualFold(n.Data, "rootfile") { + for _, a := range n.Attr { + if strings.EqualFold(a.Key, "full-path") { + path = a.Val + return + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + + if path == "" { + return "", fmt.Errorf("rootfile full-path not found in container.xml") + } + return path, nil +} + +// epubSpine parses the OPF document and returns the spine item hrefs in order, +// plus a map from href → nav title (if available from NCX/NAV). +func epubSpine(zr *zip.Reader, opfPath string) ([]string, map[string]string, error) { + f := zipFile(zr, opfPath) + if f == nil { + return nil, nil, fmt.Errorf("OPF file %q not found in EPUB", opfPath) + } + rc, err := f.Open() + if err != nil { + return nil, nil, err + } + defer rc.Close() + + opfData, err := io.ReadAll(rc) + if err != nil { + return nil, nil, err + } + + // Build id→href map from <manifest>. + idToHref := make(map[string]string) + // Also keep a href→navTitle map (populated from NCX later). + hrefTitle := make(map[string]string) + + // Parse OPF XML with html.Parse (handles malformed XML too). + doc, _ := html.Parse(bytes.NewReader(opfData)) + + var manifestItems []struct{ id, href, mediaType string } + var spineIdrefs []string + var ncxID string + + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode { + tag := strings.ToLower(n.Data) + switch tag { + case "item": + var id, href, mt string + for _, a := range n.Attr { + switch strings.ToLower(a.Key) { + case "id": + id = a.Val + case "href": + href = a.Val + case "media-type": + mt = a.Val + } + } + if id != "" && href != "" { + manifestItems = append(manifestItems, struct{ id, href, mediaType string }{id, href, mt}) + idToHref[id] = href + } + case "itemref": + for _, a := range n.Attr { + if strings.ToLower(a.Key) == "idref" { + spineIdrefs = append(spineIdrefs, a.Val) + } + } + case "spine": + for _, a := range n.Attr { + if strings.ToLower(a.Key) == "toc" { + ncxID = a.Val + } + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + + // Build ordered spine href list. + var spineHrefs []string + for _, idref := range spineIdrefs { + if href, ok := idToHref[idref]; ok { + spineHrefs = append(spineHrefs, href) + } + } + + // If no explicit spine, fall back to all XHTML items in manifest order. + if len(spineHrefs) == 0 { + sort.Slice(manifestItems, func(i, j int) bool { + return manifestItems[i].href < manifestItems[j].href + }) + for _, it := range manifestItems { + mt := strings.ToLower(it.mediaType) + if strings.Contains(mt, "html") || strings.HasSuffix(strings.ToLower(it.href), ".html") || strings.HasSuffix(strings.ToLower(it.href), ".xhtml") { + spineHrefs = append(spineHrefs, it.href) + } + } + } + + // Try to get chapter titles from NCX (toc.ncx). + opfDir := "" + if idx := strings.LastIndex(opfPath, "/"); idx >= 0 { + opfDir = opfPath[:idx+1] + } + if ncxHref, ok := idToHref[ncxID]; ok { + ncxPath := opfDir + ncxHref + if ncxFile := zipFile(zr, ncxPath); ncxFile != nil { + if ncxRC, err := ncxFile.Open(); err == nil { + defer ncxRC.Close() + parseNCXTitles(ncxRC, hrefTitle) + } + } + } + + return spineHrefs, hrefTitle, nil +} + +// parseNCXTitles extracts navPoint label→src mappings from a toc.ncx. +func parseNCXTitles(r io.Reader, out map[string]string) { + doc, err := html.Parse(r) + if err != nil { + return + } + + // Collect navPoints: each has a <navLabel><text>…</text></navLabel> and + // a <content src="…"/> child. + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode && strings.EqualFold(n.Data, "navpoint") { + var label, src string + var inner func(*html.Node) + inner = func(c *html.Node) { + if c.Type == html.ElementNode { + if strings.EqualFold(c.Data, "text") && label == "" { + if c.FirstChild != nil && c.FirstChild.Type == html.TextNode { + label = strings.TrimSpace(c.FirstChild.Data) + } + } + if strings.EqualFold(c.Data, "content") { + for _, a := range c.Attr { + if strings.EqualFold(a.Key, "src") { + // Strip fragment identifier (#...). + src = strings.SplitN(a.Val, "#", 2)[0] + } + } + } + } + for child := c.FirstChild; child != nil; child = child.NextSibling { + inner(child) + } + } + inner(n) + if label != "" && src != "" { + out[src] = label + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) +} + +// epubFileContent returns the raw bytes of a file inside the EPUB zip. +func epubFileContent(zr *zip.Reader, path string) ([]byte, error) { + f := zipFile(zr, path) + if f == nil { + return nil, fmt.Errorf("file %q not in EPUB", path) + } + rc, err := f.Open() + if err != nil { + return nil, err + } + defer rc.Close() + return io.ReadAll(rc) +} + +// zipFile finds a file by name (case-insensitive) in a zip.Reader. +func zipFile(zr *zip.Reader, name string) *zip.File { + nameLower := strings.ToLower(name) + for _, f := range zr.File { + if strings.ToLower(f.Name) == nameLower { + return f + } + } + return nil +} + +// htmlToText converts HTML/XHTML content to plain text suitable for storage. +func htmlToText(data []byte) string { + doc, err := html.Parse(bytes.NewReader(data)) + if err != nil { + return string(data) + } + + var sb strings.Builder + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.TextNode { + text := strings.TrimSpace(n.Data) + if text != "" { + sb.WriteString(text) + sb.WriteByte(' ') + } + } + if n.Type == html.ElementNode { + switch strings.ToLower(n.Data) { + case "p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr": + // Block-level: ensure newline before content. + if sb.Len() > 0 { + s := sb.String() + if s[len(s)-1] != '\n' { + sb.WriteByte('\n') + } + } + case "script", "style", "head": + // Skip entirely. + return + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + if n.Type == html.ElementNode { + switch strings.ToLower(n.Data) { + case "p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr": + sb.WriteByte('\n') + } + } + } + walk(doc) + + // Collapse multiple blank lines. + lines := strings.Split(sb.String(), "\n") + var out []string + blanks := 0 + for _, l := range lines { + l = strings.TrimSpace(l) + if l == "" { + blanks++ + if blanks <= 1 { + out = append(out, "") + } + } else { + blanks = 0 + out = append(out, l) + } + } + return strings.TrimSpace(strings.Join(out, "\n")) +} + +// ── Chapter ingestion ───────────────────────────────────────────────────────── + +// IngestChapters stores extracted chapters for a book. +// Each chapter is written as a markdown file in the chapters MinIO bucket +// and its index record is upserted in PocketBase via WriteChapter. +func (s *Store) IngestChapters(ctx context.Context, slug string, chapters []bookstore.Chapter) error { + for _, ch := range chapters { + var mdContent string + if ch.Title != "" && ch.Title != fmt.Sprintf("Chapter %d", ch.Number) { + mdContent = fmt.Sprintf("# %s\n\n%s", ch.Title, ch.Content) + } else { + mdContent = fmt.Sprintf("# Chapter %d\n\n%s", ch.Number, ch.Content) + } + domainCh := domain.Chapter{ + Ref: domain.ChapterRef{Number: ch.Number, Title: ch.Title}, + Text: mdContent, + } + if err := s.WriteChapter(ctx, slug, domainCh); err != nil { + return fmt.Errorf("ingest chapter %d: %w", ch.Number, err) + } + } + return nil +} + +// GetImportObjectKey returns the MinIO object key for an uploaded import file. +func GetImportObjectKey(filename string) string { + return fmt.Sprintf("imports/%s", filename) +} diff --git a/backend/internal/storage/minio.go b/backend/internal/storage/minio.go new file mode 100644 index 0000000..6134bd3 --- /dev/null +++ b/backend/internal/storage/minio.go @@ -0,0 +1,306 @@ +package storage + +import ( + "context" + "fmt" + "io" + "net/url" + "path" + "strings" + "time" + + minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + + "github.com/libnovel/backend/internal/config" +) + +// minioClient wraps the official minio-go client with bucket names. +type minioClient struct { + client *minio.Client // internal — all read/write operations + pubClient *minio.Client // presign-only — initialised against the public endpoint + bucketChapters string + bucketAudio string + bucketAvatars string + bucketBrowse string + bucketTranslations string +} + +func newMinioClient(cfg config.MinIO) (*minioClient, error) { + creds := credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, "") + + internal, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: creds, + Secure: cfg.UseSSL, + }) + if err != nil { + return nil, fmt.Errorf("minio: init internal client: %w", err) + } + + // Presigned URLs must be signed with the hostname the browser will use + // (PUBLIC_MINIO_PUBLIC_URL), because AWS Signature V4 includes the Host + // header in the canonical request — a URL signed against "minio:9000" will + // return SignatureDoesNotMatch when the browser fetches it from + // "localhost:9000". + // + // However, minio-go normally makes a live BucketLocation HTTP call before + // signing, which would fail from inside the container when the public + // endpoint is externally-facing (e.g. "localhost:9000" is unreachable from + // within Docker). We prevent this by: + // 1. Setting Region: "us-east-1" — minio-go skips getBucketLocation when + // the region is already known (bucket-cache.go:49). + // 2. Setting BucketLookup: BucketLookupPath — forces path-style URLs + // (e.g. host/bucket/key), matching MinIO's default behaviour and + // avoiding any virtual-host DNS probing. + // + // When no public endpoint is configured (or it equals the internal one), + // fall back to the internal client so presigning still works. + publicEndpoint := cfg.PublicEndpoint + if u, err2 := url.Parse(publicEndpoint); err2 == nil && u.Host != "" { + publicEndpoint = u.Host // strip scheme so minio.New is happy + } + pubUseSSL := cfg.PublicUseSSL + if publicEndpoint == "" || publicEndpoint == cfg.Endpoint { + publicEndpoint = cfg.Endpoint + pubUseSSL = cfg.UseSSL + } + pub, err := minio.New(publicEndpoint, &minio.Options{ + Creds: creds, + Secure: pubUseSSL, + Region: "us-east-1", // skip live BucketLocation preflight + BucketLookup: minio.BucketLookupPath, + }) + if err != nil { + return nil, fmt.Errorf("minio: init public client: %w", err) + } + + return &minioClient{ + client: internal, + pubClient: pub, + bucketChapters: cfg.BucketChapters, + bucketAudio: cfg.BucketAudio, + bucketAvatars: cfg.BucketAvatars, + bucketBrowse: cfg.BucketBrowse, + bucketTranslations: cfg.BucketTranslations, + }, nil +} + +// ensureBuckets creates all required buckets if they don't already exist. +func (m *minioClient) ensureBuckets(ctx context.Context) error { + for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse, m.bucketTranslations} { + exists, err := m.client.BucketExists(ctx, bucket) + if err != nil { + return fmt.Errorf("minio: check bucket %q: %w", bucket, err) + } + if !exists { + if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil { + return fmt.Errorf("minio: create bucket %q: %w", bucket, err) + } + } + } + return nil +} + +// ── Key helpers ─────────────────────────────────────────────────────────────── + +// ChapterObjectKey returns the MinIO object key for a chapter markdown file. +// Format: {slug}/chapter-{n:06d}.md +func ChapterObjectKey(slug string, n int) string { + return fmt.Sprintf("%s/chapter-%06d.md", slug, n) +} + +// AudioObjectKeyExt returns the MinIO object key for a cached audio file +// with a custom extension (e.g. "mp3" or "wav"). +// Format: {slug}/{n}/{voice}.{ext} +func AudioObjectKeyExt(slug string, n int, voice, ext string) string { + return fmt.Sprintf("%s/%d/%s.%s", slug, n, voice, ext) +} + +// AudioObjectKey returns the MinIO object key for a cached MP3 audio file. +// Format: {slug}/{n}/{voice}.mp3 +func AudioObjectKey(slug string, n int, voice string) string { + return AudioObjectKeyExt(slug, n, voice, "mp3") +} + +// AvatarObjectKey returns the MinIO object key for a user avatar image. +// Format: {userID}/{ext}.{ext} +func AvatarObjectKey(userID, ext string) string { + return fmt.Sprintf("%s/%s.%s", userID, ext, ext) +} + +// CoverObjectKey returns the MinIO object key for a book cover image. +// Format: covers/{slug}.jpg +func CoverObjectKey(slug string) string { + return fmt.Sprintf("covers/%s.jpg", slug) +} + +// ChapterImageObjectKey returns the MinIO object key for a chapter illustration. +// Format: chapter-images/{slug}/{n:06d}.jpg +func ChapterImageObjectKey(slug string, n int) string { + return fmt.Sprintf("chapter-images/%s/%06d.jpg", slug, n) +} + +// TranslationObjectKey returns the MinIO object key for a translated chapter. +// Format: {lang}/{slug}/{n:06d}.md +func TranslationObjectKey(lang, slug string, n int) string { + return fmt.Sprintf("%s/%s/%06d.md", lang, slug, n) +} + +// chapterNumberFromKey extracts the chapter number from a MinIO object key. +// e.g. "my-book/chapter-000042.md" → 42 +func chapterNumberFromKey(key string) int { + base := path.Base(key) + base = strings.TrimPrefix(base, "chapter-") + base = strings.TrimSuffix(base, ".md") + var n int + fmt.Sscanf(base, "%d", &n) + return n +} + +// ── Object operations ───────────────────────────────────────────────────────── + +func (m *minioClient) putObject(ctx context.Context, bucket, key, contentType string, data []byte) error { + _, err := m.client.PutObject(ctx, bucket, key, + strings.NewReader(string(data)), + int64(len(data)), + minio.PutObjectOptions{ContentType: contentType}, + ) + return err +} + +// putObjectStream uploads from r with known size (or -1 for multipart). +func (m *minioClient) putObjectStream(ctx context.Context, bucket, key, contentType string, r io.Reader, size int64) error { + _, err := m.client.PutObject(ctx, bucket, key, r, size, + minio.PutObjectOptions{ContentType: contentType}, + ) + return err +} + +func (m *minioClient) getObject(ctx context.Context, bucket, key string) ([]byte, error) { + obj, err := m.client.GetObject(ctx, bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + return io.ReadAll(obj) +} + +func (m *minioClient) objectExists(ctx context.Context, bucket, key string) bool { + _, err := m.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + return err == nil +} + +func (m *minioClient) objectSize(ctx context.Context, bucket, key string) int64 { + info, err := m.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + if err != nil { + return -1 + } + return info.Size +} + +func (m *minioClient) presignGet(ctx context.Context, bucket, key string, expires time.Duration) (string, error) { + u, err := m.pubClient.PresignedGetObject(ctx, bucket, key, expires, nil) + if err != nil { + return "", fmt.Errorf("minio presign %s/%s: %w", bucket, key, err) + } + return u.String(), nil +} + +func (m *minioClient) presignPut(ctx context.Context, bucket, key string, expires time.Duration) (string, error) { + u, err := m.pubClient.PresignedPutObject(ctx, bucket, key, expires) + if err != nil { + return "", fmt.Errorf("minio presign PUT %s/%s: %w", bucket, key, err) + } + return u.String(), nil +} + +func (m *minioClient) deleteObjects(ctx context.Context, bucket, prefix string) error { + objCh := m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) + for obj := range objCh { + if obj.Err != nil { + return obj.Err + } + if err := m.client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}); err != nil { + return err + } + } + return nil +} + +func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string) ([]string, error) { + var keys []string + for obj := range m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) { + if obj.Err != nil { + return nil, obj.Err + } + keys = append(keys, obj.Key) + } + return keys, nil +} + +// ── Cover operations ────────────────────────────────────────────────────────── + +// putCover stores a raw cover image in the browse bucket under covers/{slug}.jpg. +func (m *minioClient) putCover(ctx context.Context, key, contentType string, data []byte) error { + return m.putObject(ctx, m.bucketBrowse, key, contentType, data) +} + +// getCover retrieves a cover image. Returns (nil, "", false, nil) when the +// object does not exist. +func (m *minioClient) getCover(ctx context.Context, key string) ([]byte, bool, error) { + if !m.objectExists(ctx, m.bucketBrowse, key) { + return nil, false, nil + } + data, err := m.getObject(ctx, m.bucketBrowse, key) + if err != nil { + return nil, false, err + } + return data, true, nil +} + +// coverExists returns true when the cover image object exists. +func (m *minioClient) coverExists(ctx context.Context, key string) bool { + return m.objectExists(ctx, m.bucketBrowse, key) +} + +// coverContentType inspects the first bytes of data to determine if it is +// a JPEG or PNG image. Falls back to "image/jpeg". +func coverContentType(data []byte) string { + if len(data) >= 4 { + // PNG magic: 0x89 0x50 0x4E 0x47 + if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47 { + return "image/png" + } + // WebP: starts with "RIFF" at 0..3 and "WEBP" at 8..11 + if len(data) >= 12 && data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' && + data[8] == 'W' && data[9] == 'E' && data[10] == 'B' && data[11] == 'P' { + return "image/webp" + } + } + return "image/jpeg" +} + +// ── Chapter image operations ─────────────────────────────────────────────────── + +// putChapterImage stores a chapter illustration in the browse bucket. +func (m *minioClient) putChapterImage(ctx context.Context, key, contentType string, data []byte) error { + return m.putObject(ctx, m.bucketBrowse, key, contentType, data) +} + +// getChapterImage retrieves a chapter illustration. Returns (nil, false, nil) +// when the object does not exist. +func (m *minioClient) getChapterImage(ctx context.Context, key string) ([]byte, bool, error) { + if !m.objectExists(ctx, m.bucketBrowse, key) { + return nil, false, nil + } + data, err := m.getObject(ctx, m.bucketBrowse, key) + if err != nil { + return nil, false, err + } + return data, true, nil +} + +// chapterImageExists returns true when the chapter image object exists. +func (m *minioClient) chapterImageExists(ctx context.Context, key string) bool { + return m.objectExists(ctx, m.bucketBrowse, key) +} diff --git a/backend/internal/storage/pocketbase.go b/backend/internal/storage/pocketbase.go new file mode 100644 index 0000000..789462a --- /dev/null +++ b/backend/internal/storage/pocketbase.go @@ -0,0 +1,274 @@ +// Package storage provides the concrete implementations of all bookstore and +// taskqueue interfaces backed by PocketBase (structured data) and MinIO (blobs). +// +// Entry point: NewStore(ctx, cfg, log) returns a *Store that satisfies every +// interface defined in bookstore and taskqueue. +package storage + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/libnovel/backend/internal/config" + "github.com/libnovel/backend/internal/domain" +) + +// ErrNotFound is returned by single-record lookups when no record exists. +var ErrNotFound = errors.New("storage: record not found") + +// pbHTTPClient is a shared HTTP client with a 30 s timeout so that a slow or +// hung PocketBase never stalls the backend/runner process indefinitely. +// http.DefaultClient has no timeout and must not be used for PocketBase calls. +var pbHTTPClient = &http.Client{Timeout: 30 * time.Second} + +// pbClient is the internal PocketBase REST admin client. +type pbClient struct { + baseURL string + email string + password string + log *slog.Logger + + mu sync.Mutex + token string + exp time.Time +} + +func newPBClient(cfg config.PocketBase, log *slog.Logger) *pbClient { + return &pbClient{ + baseURL: strings.TrimRight(cfg.URL, "/"), + email: cfg.AdminEmail, + password: cfg.AdminPassword, + log: log, + } +} + +// authToken returns a valid admin auth token, refreshing it when expired. +func (c *pbClient) authToken(ctx context.Context) (string, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.token != "" && time.Now().Before(c.exp) { + return c.token, nil + } + + body, _ := json.Marshal(map[string]string{ + "identity": c.email, + "password": c.password, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/api/collections/_superusers/auth-with-password", bytes.NewReader(body)) + if err != nil { + return "", fmt.Errorf("pb auth: build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := pbHTTPClient.Do(req) + if err != nil { + return "", fmt.Errorf("pb auth: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + raw, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("pb auth: status %d: %s", resp.StatusCode, string(raw)) + } + + var payload struct { + Token string `json:"token"` + } + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return "", fmt.Errorf("pb auth: decode: %w", err) + } + c.token = payload.Token + c.exp = time.Now().Add(30 * time.Minute) + return c.token, nil +} + +// do executes an authenticated PocketBase REST request. +func (c *pbClient) do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { + tok, err := c.authToken(ctx) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body) + if err != nil { + return nil, fmt.Errorf("pb: build request %s %s: %w", method, path, err) + } + req.Header.Set("Authorization", tok) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := pbHTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("pb: %s %s: %w", method, path, err) + } + return resp, nil +} + +// get is a convenience wrapper that decodes a JSON response into v. +func (c *pbClient) get(ctx context.Context, path string, v any) error { + resp, err := c.do(ctx, http.MethodGet, path, nil) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return ErrNotFound + } + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pb GET %s: status %d: %s", path, resp.StatusCode, string(raw)) + } + return json.NewDecoder(resp.Body).Decode(v) +} + +// post creates a record and decodes the created record into v. +func (c *pbClient) post(ctx context.Context, path string, payload, v any) error { + b, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("pb: marshal: %w", err) + } + resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b)) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pb POST %s: status %d: %s", path, resp.StatusCode, string(raw)) + } + if v != nil { + return json.NewDecoder(resp.Body).Decode(v) + } + return nil +} + +// patch updates a record. +func (c *pbClient) patch(ctx context.Context, path string, payload any) error { + b, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("pb: marshal: %w", err) + } + resp, err := c.do(ctx, http.MethodPatch, path, bytes.NewReader(b)) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pb PATCH %s: status %d: %s", path, resp.StatusCode, string(raw)) + } + return nil +} + +// delete removes a record. +func (c *pbClient) delete(ctx context.Context, path string) error { + resp, err := c.do(ctx, http.MethodDelete, path, nil) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return ErrNotFound + } + if resp.StatusCode >= 400 { + raw, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pb DELETE %s: status %d: %s", path, resp.StatusCode, string(raw)) + } + return nil +} + +// listAll fetches all pages of a collection. PocketBase returns at most 200 +// records per page; we paginate until empty. +func (c *pbClient) listAll(ctx context.Context, collection string, filter, sort string) ([]json.RawMessage, error) { + var all []json.RawMessage + page := 1 + for { + q := url.Values{ + "page": {fmt.Sprintf("%d", page)}, + "perPage": {"200"}, + } + if filter != "" { + q.Set("filter", filter) + } + if sort != "" { + q.Set("sort", sort) + } + path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode()) + + var result struct { + Items []json.RawMessage `json:"items"` + Page int `json:"page"` + Pages int `json:"totalPages"` + } + if err := c.get(ctx, path, &result); err != nil { + return nil, err + } + all = append(all, result.Items...) + if result.Page >= result.Pages { + break + } + page++ + } + return all, nil +} + +// claimRecord atomically claims the first pending record matching collection. +// It fetches the oldest pending record (filter + sort), then PATCHes it with +// the claim payload. Returns (nil, nil) when the queue is empty. +func (c *pbClient) claimRecord(ctx context.Context, collection, workerID string, extraClaim map[string]any) (json.RawMessage, error) { + q := url.Values{} + q.Set("filter", `status="pending"`) + q.Set("sort", "+started") + q.Set("perPage", "1") + path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode()) + + var result struct { + Items []json.RawMessage `json:"items"` + } + if err := c.get(ctx, path, &result); err != nil { + return nil, fmt.Errorf("claimRecord list: %w", err) + } + if len(result.Items) == 0 { + return nil, nil // queue empty + } + + var rec struct { + ID string `json:"id"` + } + if err := json.Unmarshal(result.Items[0], &rec); err != nil { + return nil, fmt.Errorf("claimRecord parse id: %w", err) + } + + claim := map[string]any{ + "status": string(domain.TaskStatusRunning), + "worker_id": workerID, + "heartbeat_at": time.Now().UTC().Format(time.RFC3339), + } + for k, v := range extraClaim { + claim[k] = v + } + + claimPath := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID) + if err := c.patch(ctx, claimPath, claim); err != nil { + return nil, fmt.Errorf("claimRecord patch: %w", err) + } + + // Re-fetch the updated record so caller has current state. + var updated json.RawMessage + if err := c.get(ctx, claimPath, &updated); err != nil { + return nil, fmt.Errorf("claimRecord re-fetch: %w", err) + } + return updated, nil +} diff --git a/backend/internal/storage/store.go b/backend/internal/storage/store.go new file mode 100644 index 0000000..f0926ad --- /dev/null +++ b/backend/internal/storage/store.go @@ -0,0 +1,1761 @@ +package storage + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "strings" + "time" + + "github.com/libnovel/backend/internal/bookstore" + "github.com/libnovel/backend/internal/config" + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/taskqueue" +) + +// Store is the unified persistence implementation that satisfies all bookstore +// and taskqueue interfaces. It routes structured data to PocketBase and binary +// blobs to MinIO. +type Store struct { + pb *pbClient + mc *minioClient + log *slog.Logger +} + +// NewStore initialises PocketBase and MinIO connections and ensures all MinIO +// buckets exist. Returns a ready-to-use Store. +func NewStore(ctx context.Context, cfg config.Config, log *slog.Logger) (*Store, error) { + pb := newPBClient(cfg.PocketBase, log) + // Validate PocketBase connectivity by fetching an auth token. + if _, err := pb.authToken(ctx); err != nil { + return nil, fmt.Errorf("pocketbase: %w", err) + } + + mc, err := newMinioClient(cfg.MinIO) + if err != nil { + return nil, fmt.Errorf("minio: %w", err) + } + if err := mc.ensureBuckets(ctx); err != nil { + return nil, fmt.Errorf("minio: ensure buckets: %w", err) + } + + return &Store{pb: pb, mc: mc, log: log}, nil +} + +// Compile-time interface satisfaction. +var _ bookstore.BookWriter = (*Store)(nil) +var _ bookstore.BookReader = (*Store)(nil) +var _ bookstore.RankingStore = (*Store)(nil) +var _ bookstore.AudioStore = (*Store)(nil) +var _ bookstore.PresignStore = (*Store)(nil) +var _ bookstore.ProgressStore = (*Store)(nil) +var _ bookstore.CoverStore = (*Store)(nil) +var _ bookstore.TranslationStore = (*Store)(nil) +var _ bookstore.AIJobStore = (*Store)(nil) +var _ bookstore.ChapterImageStore = (*Store)(nil) +var _ bookstore.BookAdminStore = (*Store)(nil) +var _ taskqueue.Producer = (*Store)(nil) +var _ taskqueue.Consumer = (*Store)(nil) +var _ taskqueue.Reader = (*Store)(nil) + +// ── BookWriter ──────────────────────────────────────────────────────────────── + +func (s *Store) WriteMetadata(ctx context.Context, meta domain.BookMeta) error { + // patchPayload does NOT include visibility or submitted_by — preserve existing values. + patchPayload := map[string]any{ + "slug": meta.Slug, + "title": meta.Title, + "author": meta.Author, + "cover": meta.Cover, + "status": meta.Status, + "genres": meta.Genres, + "summary": meta.Summary, + "total_chapters": meta.TotalChapters, + "source_url": meta.SourceURL, + "ranking": meta.Ranking, + "rating": meta.Rating, + } + // Upsert via filter: if exists PATCH, otherwise POST. + // Use a conflict-retry pattern to handle concurrent scrapes racing to insert + // the same slug: if POST fails (or another concurrent writer beat us to it), + // re-fetch and PATCH instead. + existing, err := s.getBookBySlug(ctx, meta.Slug) + if err != nil && err != ErrNotFound { + return fmt.Errorf("WriteMetadata: %w", err) + } + if err == ErrNotFound { + // New scraped book — default to admin_only visibility. + postPayload := make(map[string]any, len(patchPayload)+1) + for k, v := range patchPayload { + postPayload[k] = v + } + postPayload["visibility"] = domain.VisibilityAdminOnly + postErr := s.pb.post(ctx, "/api/collections/books/records", postPayload, nil) + if postErr == nil { + return nil + } + // POST failed — a concurrent writer may have inserted the same slug. + // Re-fetch and fall through to PATCH. + existing, err = s.getBookBySlug(ctx, meta.Slug) + if err != nil { + return postErr // original POST error is more informative + } + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", existing.ID), patchPayload) +} + +// CreateSubmittedBook creates a new author-submitted book with visibility=public. +// Unlike WriteMetadata this always POSTs (no upsert) and sets the submitted_by field. +func (s *Store) CreateSubmittedBook(ctx context.Context, meta domain.BookMeta) error { + payload := map[string]any{ + "slug": meta.Slug, + "title": meta.Title, + "author": meta.Author, + "cover": meta.Cover, + "status": meta.Status, + "genres": meta.Genres, + "summary": meta.Summary, + "total_chapters": 0, + "source_url": "", + "ranking": 0, + "rating": 0, + "visibility": domain.VisibilityPublic, + "submitted_by": meta.SubmittedBy, + } + return s.pb.post(ctx, "/api/collections/books/records", payload, nil) +} + +func (s *Store) WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error { + key := ChapterObjectKey(slug, chapter.Ref.Number) + if err := s.mc.putObject(ctx, s.mc.bucketChapters, key, "text/markdown", []byte(chapter.Text)); err != nil { + return fmt.Errorf("WriteChapter: minio: %w", err) + } + // Upsert the chapters_idx record in PocketBase. + return s.upsertChapterIdx(ctx, slug, chapter.Ref) +} + +func (s *Store) WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error { + for _, ref := range refs { + if err := s.upsertChapterIdx(ctx, slug, ref); err != nil { + s.log.Warn("WriteChapterRefs: upsert failed", "slug", slug, "chapter", ref.Number, "err", err) + } + } + return nil +} + +func (s *Store) ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool { + return s.mc.objectExists(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, ref.Number)) +} + +func (s *Store) upsertChapterIdx(ctx context.Context, slug string, ref domain.ChapterRef) error { + payload := map[string]any{ + "slug": slug, + "number": ref.Number, + "title": ref.Title, + } + filter := fmt.Sprintf(`slug=%q&&number=%d`, slug, ref.Number) + items, err := s.pb.listAll(ctx, "chapters_idx", filter, "") + if err != nil && err != ErrNotFound { + return err + } + if len(items) == 0 { + // Set created timestamp on first insert so recentlyUpdatedBooks can sort by it. + insertPayload := map[string]any{ + "slug": slug, + "number": ref.Number, + "title": ref.Title, + "created": time.Now().UTC().Format(time.RFC3339), + } + postErr := s.pb.post(ctx, "/api/collections/chapters_idx/records", insertPayload, nil) + if postErr == nil { + return nil + } + // POST failed — a concurrent writer may have inserted the same slug+number. + // Re-fetch and fall through to PATCH (mirrors WriteMetadata retry pattern). + items, err = s.pb.listAll(ctx, "chapters_idx", filter, "") + if err != nil || len(items) == 0 { + return postErr // original POST error is more informative + } + } + var rec struct { + ID string `json:"id"` + } + json.Unmarshal(items[0], &rec) + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/chapters_idx/records/%s", rec.ID), payload) +} + +// DeduplicateChapters removes duplicate chapters_idx records for slug. +// For each chapter number that has more than one record, it keeps the record +// with the latest "updated" timestamp and deletes the rest. +// Returns the number of records deleted. +func (s *Store) DeduplicateChapters(ctx context.Context, slug string) (int, error) { + filter := fmt.Sprintf(`slug=%q`, slug) + items, err := s.pb.listAll(ctx, "chapters_idx", filter, "number") + if err != nil { + return 0, fmt.Errorf("DeduplicateChapters: list: %w", err) + } + + type record struct { + ID string `json:"id"` + Number int `json:"number"` + Updated string `json:"updated"` + } + + // Group records by chapter number. + byNumber := make(map[int][]record) + for _, raw := range items { + var rec record + if err := json.Unmarshal(raw, &rec); err != nil || rec.ID == "" { + continue + } + byNumber[rec.Number] = append(byNumber[rec.Number], rec) + } + + deleted := 0 + for _, recs := range byNumber { + if len(recs) <= 1 { + continue + } + // Keep the record with the latest Updated timestamp; delete the rest. + keep := 0 + for i := 1; i < len(recs); i++ { + if recs[i].Updated > recs[keep].Updated { + keep = i + } + } + for i, rec := range recs { + if i == keep { + continue + } + if delErr := s.pb.delete(ctx, fmt.Sprintf("/api/collections/chapters_idx/records/%s", rec.ID)); delErr != nil { + s.log.Warn("DeduplicateChapters: delete failed", "slug", slug, "number", rec.Number, "id", rec.ID, "err", delErr) + continue + } + deleted++ + } + } + return deleted, nil +} + +// ── BookReader ──────────────────────────────────────────────────────────────── + +type pbBook struct { + ID string `json:"id"` + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author"` + Cover string `json:"cover"` + Status string `json:"status"` + Genres []string `json:"genres"` + Summary string `json:"summary"` + TotalChapters int `json:"total_chapters"` + SourceURL string `json:"source_url"` + Ranking int `json:"ranking"` + Rating float64 `json:"rating"` + Updated string `json:"updated"` + Archived bool `json:"archived"` + Visibility string `json:"visibility"` + SubmittedBy string `json:"submitted_by"` +} + +func (b pbBook) toDomain() domain.BookMeta { + var metaUpdated int64 + if t, err := time.Parse(time.RFC3339, b.Updated); err == nil { + metaUpdated = t.Unix() + } + return domain.BookMeta{ + Slug: b.Slug, + Title: b.Title, + Author: b.Author, + Cover: b.Cover, + Status: b.Status, + Genres: b.Genres, + Summary: b.Summary, + TotalChapters: b.TotalChapters, + SourceURL: b.SourceURL, + Ranking: b.Ranking, + Rating: b.Rating, + MetaUpdated: metaUpdated, + Archived: b.Archived, + Visibility: b.Visibility, + SubmittedBy: b.SubmittedBy, + } +} + +func (s *Store) getBookBySlug(ctx context.Context, slug string) (pbBook, error) { + filter := fmt.Sprintf(`slug=%q`, slug) + items, err := s.pb.listAll(ctx, "books", filter, "") + if err != nil { + return pbBook{}, err + } + if len(items) == 0 { + return pbBook{}, ErrNotFound + } + var b pbBook + json.Unmarshal(items[0], &b) + return b, nil +} + +func (s *Store) ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error) { + b, err := s.getBookBySlug(ctx, slug) + if err == ErrNotFound { + return domain.BookMeta{}, false, nil + } + if err != nil { + return domain.BookMeta{}, false, err + } + return b.toDomain(), true, nil +} + +func (s *Store) ListBooks(ctx context.Context) ([]domain.BookMeta, error) { + items, err := s.pb.listAll(ctx, "books", "archived=false", "title") + if err != nil { + return nil, err + } + books := make([]domain.BookMeta, 0, len(items)) + for _, raw := range items { + var b pbBook + json.Unmarshal(raw, &b) + books = append(books, b.toDomain()) + } + return books, nil +} + +func (s *Store) LocalSlugs(ctx context.Context) (map[string]bool, error) { + items, err := s.pb.listAll(ctx, "books", "", "") + if err != nil { + return nil, err + } + slugs := make(map[string]bool, len(items)) + for _, raw := range items { + var b struct { + Slug string `json:"slug"` + } + json.Unmarshal(raw, &b) + if b.Slug != "" { + slugs[b.Slug] = true + } + } + return slugs, nil +} + +func (s *Store) MetadataMtime(ctx context.Context, slug string) int64 { + b, err := s.getBookBySlug(ctx, slug) + if err != nil { + return 0 + } + t, err := time.Parse(time.RFC3339, b.Updated) + if err != nil { + return 0 + } + return t.Unix() +} + +func (s *Store) ReadChapter(ctx context.Context, slug string, n int) (string, error) { + data, err := s.mc.getObject(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n)) + if err != nil { + return "", fmt.Errorf("ReadChapter: %w", err) + } + return string(data), nil +} + +func (s *Store) ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error) { + filter := fmt.Sprintf(`slug=%q`, slug) + items, err := s.pb.listAll(ctx, "chapters_idx", filter, "number") + if err != nil { + return nil, err + } + chapters := make([]domain.ChapterInfo, 0, len(items)) + for _, raw := range items { + var rec struct { + Number int `json:"number"` + Title string `json:"title"` + Created string `json:"created"` + } + json.Unmarshal(raw, &rec) + chapters = append(chapters, domain.ChapterInfo{ + Number: rec.Number, + Title: rec.Title, + Date: rec.Created, + }) + } + return chapters, nil +} + +func (s *Store) CountChapters(ctx context.Context, slug string) int { + chapters, err := s.ListChapters(ctx, slug) + if err != nil { + return 0 + } + return len(chapters) +} + +func (s *Store) ReindexChapters(ctx context.Context, slug string) (int, error) { + keys, err := s.mc.listObjectKeys(ctx, s.mc.bucketChapters, slug+"/") + if err != nil { + return 0, fmt.Errorf("ReindexChapters: list objects: %w", err) + } + count := 0 + for _, key := range keys { + if !strings.HasSuffix(key, ".md") { + continue + } + n := chapterNumberFromKey(key) + if n == 0 { + continue + } + ref := domain.ChapterRef{Number: n} + if err := s.upsertChapterIdx(ctx, slug, ref); err != nil { + s.log.Warn("ReindexChapters: upsert failed", "key", key, "err", err) + continue + } + count++ + } + return count, nil +} + +// ── BookAdminStore ──────────────────────────────────────────────────────────── + +// ArchiveBook sets archived=true on the book record for slug. +func (s *Store) ArchiveBook(ctx context.Context, slug string) error { + book, err := s.getBookBySlug(ctx, slug) + if err == ErrNotFound { + return ErrNotFound + } + if err != nil { + return fmt.Errorf("ArchiveBook: %w", err) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", book.ID), + map[string]any{"archived": true}) +} + +// UnarchiveBook clears archived on the book record for slug. +func (s *Store) UnarchiveBook(ctx context.Context, slug string) error { + book, err := s.getBookBySlug(ctx, slug) + if err == ErrNotFound { + return ErrNotFound + } + if err != nil { + return fmt.Errorf("UnarchiveBook: %w", err) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", book.ID), + map[string]any{"archived": false}) +} + +// PublishBook sets visibility=public on the book record for slug. +func (s *Store) PublishBook(ctx context.Context, slug string) error { + book, err := s.getBookBySlug(ctx, slug) + if err == ErrNotFound { + return ErrNotFound + } + if err != nil { + return fmt.Errorf("PublishBook: %w", err) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", book.ID), + map[string]any{"visibility": domain.VisibilityPublic}) +} + +// UnpublishBook sets visibility=admin_only on the book record for slug. +func (s *Store) UnpublishBook(ctx context.Context, slug string) error { + book, err := s.getBookBySlug(ctx, slug) + if err == ErrNotFound { + return ErrNotFound + } + if err != nil { + return fmt.Errorf("UnpublishBook: %w", err) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", book.ID), + map[string]any{"visibility": domain.VisibilityAdminOnly}) +} + +// DeleteBook permanently removes all data for a book: +// - PocketBase books record +// - All PocketBase chapters_idx records for the slug +// - All MinIO chapter markdown objects ({slug}/chapter-*.md) +// - MinIO cover image (covers/{slug}.jpg) +func (s *Store) DeleteBook(ctx context.Context, slug string) error { + // 1. Fetch the book record to get its PocketBase ID. + book, err := s.getBookBySlug(ctx, slug) + if err == ErrNotFound { + return ErrNotFound + } + if err != nil { + return fmt.Errorf("DeleteBook: fetch: %w", err) + } + + // 2. Delete all chapters_idx records. + filter := fmt.Sprintf(`slug=%q`, slug) + items, err := s.pb.listAll(ctx, "chapters_idx", filter, "") + if err != nil && err != ErrNotFound { + return fmt.Errorf("DeleteBook: list chapters_idx: %w", err) + } + for _, raw := range items { + var rec struct { + ID string `json:"id"` + } + if json.Unmarshal(raw, &rec) == nil && rec.ID != "" { + if delErr := s.pb.delete(ctx, fmt.Sprintf("/api/collections/chapters_idx/records/%s", rec.ID)); delErr != nil { + s.log.Warn("DeleteBook: delete chapters_idx record failed", "slug", slug, "id", rec.ID, "err", delErr) + } + } + } + + // 3. Delete MinIO chapter objects. + if err := s.mc.deleteObjects(ctx, s.mc.bucketChapters, slug+"/"); err != nil { + s.log.Warn("DeleteBook: delete chapter objects failed", "slug", slug, "err", err) + } + + // 4. Delete MinIO cover image. + if err := s.mc.deleteObjects(ctx, s.mc.bucketBrowse, CoverObjectKey(slug)); err != nil { + s.log.Warn("DeleteBook: delete cover failed", "slug", slug, "err", err) + } + + // 5. Delete the PocketBase books record. + if err := s.pb.delete(ctx, fmt.Sprintf("/api/collections/books/records/%s", book.ID)); err != nil { + return fmt.Errorf("DeleteBook: delete books record: %w", err) + } + + return nil +} + +// ── RankingStore ────────────────────────────────────────────────────────────── + +func (s *Store) WriteRankingItem(ctx context.Context, item domain.RankingItem) error { + payload := map[string]any{ + "rank": item.Rank, + "slug": item.Slug, + "title": item.Title, + "author": item.Author, + "cover": item.Cover, + "status": item.Status, + "genres": item.Genres, + "source_url": item.SourceURL, + } + filter := fmt.Sprintf(`slug=%q`, item.Slug) + items, err := s.pb.listAll(ctx, "ranking", filter, "") + if err != nil && err != ErrNotFound { + return err + } + if len(items) == 0 { + return s.pb.post(ctx, "/api/collections/ranking/records", payload, nil) + } + var rec struct { + ID string `json:"id"` + } + json.Unmarshal(items[0], &rec) + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/ranking/records/%s", rec.ID), payload) +} + +func (s *Store) ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error) { + items, err := s.pb.listAll(ctx, "ranking", "", "rank") + if err != nil { + return nil, err + } + result := make([]domain.RankingItem, 0, len(items)) + for _, raw := range items { + var rec struct { + Rank int `json:"rank"` + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author"` + Cover string `json:"cover"` + Status string `json:"status"` + Genres []string `json:"genres"` + SourceURL string `json:"source_url"` + Updated string `json:"updated"` + } + json.Unmarshal(raw, &rec) + t, _ := time.Parse(time.RFC3339, rec.Updated) + result = append(result, domain.RankingItem{ + Rank: rec.Rank, + Slug: rec.Slug, + Title: rec.Title, + Author: rec.Author, + Cover: rec.Cover, + Status: rec.Status, + Genres: rec.Genres, + SourceURL: rec.SourceURL, + Updated: t, + }) + } + return result, nil +} + +func (s *Store) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) { + items, err := s.ReadRankingItems(ctx) + if err != nil || len(items) == 0 { + return false, err + } + var latest time.Time + for _, item := range items { + if item.Updated.After(latest) { + latest = item.Updated + } + } + return time.Since(latest) < maxAge, nil +} + +// ── AudioStore ──────────────────────────────────────────────────────────────── + +func (s *Store) AudioObjectKey(slug string, n int, voice string) string { + return AudioObjectKey(slug, n, voice) +} + +func (s *Store) AudioObjectKeyExt(slug string, n int, voice, ext string) string { + return AudioObjectKeyExt(slug, n, voice, ext) +} + +func (s *Store) AudioExists(ctx context.Context, key string) bool { + return s.mc.objectExists(ctx, s.mc.bucketAudio, key) +} + +func (s *Store) AudioObjectSize(ctx context.Context, key string) int64 { + return s.mc.objectSize(ctx, s.mc.bucketAudio, key) +} + +func (s *Store) PutAudio(ctx context.Context, key string, data []byte) error { + return s.mc.putObject(ctx, s.mc.bucketAudio, key, "audio/mpeg", data) +} + +func (s *Store) PutAudioStream(ctx context.Context, key string, r io.Reader, size int64, contentType string) error { + return s.mc.putObjectStream(ctx, s.mc.bucketAudio, key, contentType, r, size) +} + +// ── PresignStore ────────────────────────────────────────────────────────────── + +func (s *Store) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) { + return s.mc.presignGet(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n), expires) +} + +func (s *Store) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) { + return s.mc.presignGet(ctx, s.mc.bucketAudio, key, expires) +} + +func (s *Store) PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error) { + key = AvatarObjectKey(userID, ext) + uploadURL, err = s.mc.presignPut(ctx, s.mc.bucketAvatars, key, 15*time.Minute) + return +} + +func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) { + for _, ext := range []string{"jpg", "png", "webp"} { + key := AvatarObjectKey(userID, ext) + if s.mc.objectExists(ctx, s.mc.bucketAvatars, key) { + u, err := s.mc.presignGet(ctx, s.mc.bucketAvatars, key, 1*time.Hour) + return u, true, err + } + } + return "", false, nil +} + +func (s *Store) PutAvatar(ctx context.Context, userID, ext, contentType string, data []byte) (string, error) { + // Delete existing avatar objects for this user before writing the new one + // so old extensions don't linger (e.g. old .png after uploading a .jpg). + _ = s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/") + key := AvatarObjectKey(userID, ext) + if err := s.mc.putObject(ctx, s.mc.bucketAvatars, key, contentType, data); err != nil { + return "", fmt.Errorf("put avatar: %w", err) + } + return key, nil +} + +func (s *Store) DeleteAvatar(ctx context.Context, userID string) error { + return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/") +} + +// ── ProgressStore ───────────────────────────────────────────────────────────── + +func (s *Store) GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool) { + filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug) + items, err := s.pb.listAll(ctx, "progress", filter, "") + if err != nil || len(items) == 0 { + return domain.ReadingProgress{}, false + } + var rec struct { + Slug string `json:"slug"` + Chapter int `json:"chapter"` + UpdatedAt string `json:"updated"` + } + json.Unmarshal(items[0], &rec) + t, _ := time.Parse(time.RFC3339, rec.UpdatedAt) + return domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t}, true +} + +func (s *Store) SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error { + payload := map[string]any{ + "session_id": sessionID, + "slug": p.Slug, + "chapter": p.Chapter, + } + filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, p.Slug) + items, err := s.pb.listAll(ctx, "progress", filter, "") + if err != nil && err != ErrNotFound { + return err + } + if len(items) == 0 { + return s.pb.post(ctx, "/api/collections/progress/records", payload, nil) + } + var rec struct { + ID string `json:"id"` + } + json.Unmarshal(items[0], &rec) + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID), payload) +} + +func (s *Store) AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error) { + filter := fmt.Sprintf(`session_id=%q`, sessionID) + items, err := s.pb.listAll(ctx, "progress", filter, "-updated") + if err != nil { + return nil, err + } + result := make([]domain.ReadingProgress, 0, len(items)) + for _, raw := range items { + var rec struct { + Slug string `json:"slug"` + Chapter int `json:"chapter"` + UpdatedAt string `json:"updated"` + } + json.Unmarshal(raw, &rec) + t, _ := time.Parse(time.RFC3339, rec.UpdatedAt) + result = append(result, domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t}) + } + return result, nil +} + +func (s *Store) DeleteProgress(ctx context.Context, sessionID, slug string) error { + filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug) + items, err := s.pb.listAll(ctx, "progress", filter, "") + if err != nil || len(items) == 0 { + return nil + } + var rec struct { + ID string `json:"id"` + } + json.Unmarshal(items[0], &rec) + return s.pb.delete(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID)) +} + +// ── taskqueue.Producer ──────────────────────────────────────────────────────── + +func (s *Store) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) { + payload := map[string]any{ + "kind": kind, + "target_url": targetURL, + "from_chapter": fromChapter, + "to_chapter": toChapter, + "status": string(domain.TaskStatusPending), + "started": time.Now().UTC().Format(time.RFC3339), + } + var rec struct { + ID string `json:"id"` + } + if err := s.pb.post(ctx, "/api/collections/scraping_tasks/records", payload, &rec); err != nil { + return "", err + } + return rec.ID, nil +} + +func (s *Store) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) { + cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, voice) + payload := map[string]any{ + "cache_key": cacheKey, + "slug": slug, + "chapter": chapter, + "voice": voice, + "status": string(domain.TaskStatusPending), + "started": time.Now().UTC().Format(time.RFC3339), + } + var rec struct { + ID string `json:"id"` + } + if err := s.pb.post(ctx, "/api/collections/audio_jobs/records", payload, &rec); err != nil { + return "", err + } + return rec.ID, nil +} + +func (s *Store) CreateTranslationTask(ctx context.Context, slug string, chapter int, lang string) (string, error) { + cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, lang) + payload := map[string]any{ + "cache_key": cacheKey, + "slug": slug, + "chapter": chapter, + "lang": lang, + "status": string(domain.TaskStatusPending), + "started": time.Now().UTC().Format(time.RFC3339), + } + var rec struct { + ID string `json:"id"` + } + if err := s.pb.post(ctx, "/api/collections/translation_jobs/records", payload, &rec); err != nil { + return "", err + } + return rec.ID, nil +} + +func (s *Store) CreateImportTask(ctx context.Context, task domain.ImportTask) (string, error) { + payload := map[string]any{ + "slug": task.Slug, + "title": task.Title, + "file_name": task.Slug + "." + task.FileType, + "file_type": task.FileType, + "object_key": task.ObjectKey, + "chapters_key": task.ChaptersKey, + "author": task.Author, + "cover_url": task.CoverURL, + "summary": task.Summary, + "book_status": task.BookStatus, + "status": string(domain.TaskStatusPending), + "chapters_done": 0, + "chapters_total": task.ChaptersTotal, + "started": time.Now().UTC().Format(time.RFC3339), + "initiator_user_id": task.InitiatorUserID, + } + if len(task.Genres) > 0 { + payload["genres"] = strings.Join(task.Genres, ",") + } + var rec struct { + ID string `json:"id"` + } + if err := s.pb.post(ctx, "/api/collections/import_tasks/records", payload, &rec); err != nil { + return "", err + } + return rec.ID, nil +} + +// CreateNotification creates a notification record in PocketBase. +func (s *Store) CreateNotification(ctx context.Context, userID, title, message, link string) error { + payload := map[string]any{ + "user_id": userID, + "title": title, + "message": message, + "link": link, + "read": false, + "created": time.Now().UTC().Format(time.RFC3339), + } + return s.pb.post(ctx, "/api/collections/notifications/records", payload, nil) +} + +// ListNotifications returns notifications for a user. +func (s *Store) ListNotifications(ctx context.Context, userID string, limit int) ([]map[string]any, error) { + filter := fmt.Sprintf(`user_id=%q`, userID) + items, err := s.pb.listAll(ctx, "notifications", filter, "-created") + if err != nil { + return nil, err + } + // Parse each json.RawMessage into a map + results := make([]map[string]any, 0, len(items)) + for _, raw := range items { + var m map[string]any + if json.Unmarshal(raw, &m) == nil { + results = append(results, m) + } + } + if limit > 0 && len(results) > limit { + results = results[:limit] + } + return results, nil +} + +// MarkNotificationRead marks a notification as read. +func (s *Store) MarkNotificationRead(ctx context.Context, id string) error { + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/notifications/records/%s", id), + map[string]any{"read": true}) +} + +// DeleteNotification deletes a single notification by ID. +func (s *Store) DeleteNotification(ctx context.Context, id string) error { + return s.pb.delete(ctx, fmt.Sprintf("/api/collections/notifications/records/%s", id)) +} + +// ClearAllNotifications deletes all notifications for a user. +func (s *Store) ClearAllNotifications(ctx context.Context, userID string) error { + filter := fmt.Sprintf(`user_id=%q`, userID) + items, err := s.pb.listAll(ctx, "notifications", filter, "") + if err != nil { + return fmt.Errorf("ClearAllNotifications list: %w", err) + } + for _, raw := range items { + var rec struct { + ID string `json:"id"` + } + if json.Unmarshal(raw, &rec) == nil && rec.ID != "" { + _ = s.pb.delete(ctx, fmt.Sprintf("/api/collections/notifications/records/%s", rec.ID)) + } + } + return nil +} + +// MarkAllNotificationsRead marks all notifications for a user as read. +func (s *Store) MarkAllNotificationsRead(ctx context.Context, userID string) error { + filter := fmt.Sprintf(`user_id=%q&&read=false`, userID) + items, err := s.pb.listAll(ctx, "notifications", filter, "") + if err != nil { + return fmt.Errorf("MarkAllNotificationsRead list: %w", err) + } + for _, raw := range items { + var rec struct { + ID string `json:"id"` + } + if json.Unmarshal(raw, &rec) == nil && rec.ID != "" { + _ = s.pb.patch(ctx, fmt.Sprintf("/api/collections/notifications/records/%s", rec.ID), + map[string]any{"read": true}) + } + } + return nil +} + +func (s *Store) CancelTask(ctx context.Context, id string) error { + // Try scraping_tasks first, then audio_jobs, then translation_jobs. + if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), + map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil { + return nil + } + if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), + map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil { + return nil + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), + map[string]string{"status": string(domain.TaskStatusCancelled)}) +} + +func (s *Store) CancelAudioTasksBySlug(ctx context.Context, slug string) (int, error) { + filter := fmt.Sprintf(`slug='%s'&&(status='pending'||status='running')`, slug) + items, err := s.pb.listAll(ctx, "audio_jobs", filter, "") + if err != nil { + return 0, fmt.Errorf("CancelAudioTasksBySlug list: %w", err) + } + cancelled := 0 + for _, raw := range items { + var rec struct { + ID string `json:"id"` + } + if json.Unmarshal(raw, &rec) == nil && rec.ID != "" { + if patchErr := s.pb.patch(ctx, + fmt.Sprintf("/api/collections/audio_jobs/records/%s", rec.ID), + map[string]string{"status": string(domain.TaskStatusCancelled)}); patchErr == nil { + cancelled++ + } + } + } + return cancelled, nil +} + +// ── taskqueue.Consumer ──────────────────────────────────────────────────────── + +func (s *Store) ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error) { + raw, err := s.pb.claimRecord(ctx, "scraping_tasks", workerID, nil) + if err != nil { + return domain.ScrapeTask{}, false, err + } + if raw == nil { + return domain.ScrapeTask{}, false, nil + } + task, err := parseScrapeTask(raw) + return task, err == nil, err +} + +func (s *Store) ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error) { + raw, err := s.pb.claimRecord(ctx, "audio_jobs", workerID, nil) + if err != nil { + return domain.AudioTask{}, false, err + } + if raw == nil { + return domain.AudioTask{}, false, nil + } + task, err := parseAudioTask(raw) + return task, err == nil, err +} + +func (s *Store) ClaimNextTranslationTask(ctx context.Context, workerID string) (domain.TranslationTask, bool, error) { + raw, err := s.pb.claimRecord(ctx, "translation_jobs", workerID, nil) + if err != nil { + return domain.TranslationTask{}, false, err + } + if raw == nil { + return domain.TranslationTask{}, false, nil + } + task, err := parseTranslationTask(raw) + return task, err == nil, err +} + +func (s *Store) ClaimNextImportTask(ctx context.Context, workerID string) (domain.ImportTask, bool, error) { + raw, err := s.pb.claimRecord(ctx, "import_tasks", workerID, nil) + if err != nil { + return domain.ImportTask{}, false, err + } + if raw == nil { + return domain.ImportTask{}, false, nil + } + task, err := parseImportTask(raw) + return task, err == nil, err +} + +func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error { + status := string(domain.TaskStatusDone) + if result.ErrorMessage != "" { + status = string(domain.TaskStatusFailed) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), map[string]any{ + "status": status, + "books_found": result.BooksFound, + "chapters_scraped": result.ChaptersScraped, + "chapters_skipped": result.ChaptersSkipped, + "errors": result.Errors, + "error_message": result.ErrorMessage, + "finished": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *Store) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error { + status := string(domain.TaskStatusDone) + if result.ErrorMessage != "" { + status = string(domain.TaskStatusFailed) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), map[string]any{ + "status": status, + "error_message": result.ErrorMessage, + "finished": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *Store) FinishTranslationTask(ctx context.Context, id string, result domain.TranslationResult) error { + status := string(domain.TaskStatusDone) + if result.ErrorMessage != "" { + status = string(domain.TaskStatusFailed) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), map[string]any{ + "status": status, + "error_message": result.ErrorMessage, + "finished": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *Store) FinishImportTask(ctx context.Context, id string, result domain.ImportResult) error { + status := string(domain.TaskStatusDone) + if result.ErrorMessage != "" { + status = string(domain.TaskStatusFailed) + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/import_tasks/records/%s", id), map[string]any{ + "status": status, + "chapters_done": result.ChaptersImported, + "chapters_total": result.ChaptersImported, + "error_message": result.ErrorMessage, + "finished": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *Store) FailTask(ctx context.Context, id, errMsg string) error { + payload := map[string]any{ + "status": string(domain.TaskStatusFailed), + "error_message": errMsg, + "finished": time.Now().UTC().Format(time.RFC3339), + } + if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil { + return nil + } + if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload); err == nil { + return nil + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), payload) +} + +// HeartbeatTask updates the heartbeat_at field on a running task. +// Tries scraping_tasks, audio_jobs, translation_jobs, then import_tasks. +func (s *Store) HeartbeatTask(ctx context.Context, id string) error { + payload := map[string]any{ + "heartbeat_at": time.Now().UTC().Format(time.RFC3339), + } + if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil { + return nil + } + if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload); err == nil { + return nil + } + if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), payload); err == nil { + return nil + } + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/import_tasks/records/%s", id), payload) +} + +// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing +// or older than staleAfter, and resets them to pending so they can be +// re-claimed. Returns the number of tasks reaped. +func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) { + threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339) + // Match tasks that are running AND (heartbeat_at is null OR heartbeat_at < threshold). + // PocketBase datetime fields require `=null` not `=""` in filter expressions. + filter := fmt.Sprintf(`status="running"&&(heartbeat_at=null||heartbeat_at<"%s")`, threshold) + resetPayload := map[string]any{ + "status": string(domain.TaskStatusPending), + "worker_id": "", + "heartbeat_at": nil, + } + + total := 0 + for _, collection := range []string{"scraping_tasks", "audio_jobs", "translation_jobs", "import_tasks"} { + items, err := s.pb.listAll(ctx, collection, filter, "") + if err != nil { + return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err) + } + for _, raw := range items { + var rec struct { + ID string `json:"id"` + } + if err := json.Unmarshal(raw, &rec); err != nil || rec.ID == "" { + continue + } + path := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID) + if err := s.pb.patch(ctx, path, resetPayload); err != nil { + s.log.Warn("ReapStaleTasks: patch failed", "collection", collection, "id", rec.ID, "err", err) + continue + } + total++ + } + } + return total, nil +} + +// ── taskqueue.Reader ────────────────────────────────────────────────────────── + +func (s *Store) ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error) { + items, err := s.pb.listAll(ctx, "scraping_tasks", "", "-started") + if err != nil { + return nil, err + } + tasks := make([]domain.ScrapeTask, 0, len(items)) + for _, raw := range items { + t, err := parseScrapeTask(raw) + if err == nil { + tasks = append(tasks, t) + } + } + return tasks, nil +} + +func (s *Store) GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error) { + var raw json.RawMessage + if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), &raw); err != nil { + if err == ErrNotFound { + return domain.ScrapeTask{}, false, nil + } + return domain.ScrapeTask{}, false, err + } + t, err := parseScrapeTask(raw) + return t, err == nil, err +} + +func (s *Store) ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error) { + items, err := s.pb.listAll(ctx, "audio_jobs", "", "-started") + if err != nil { + return nil, err + } + tasks := make([]domain.AudioTask, 0, len(items)) + for _, raw := range items { + t, err := parseAudioTask(raw) + if err == nil { + tasks = append(tasks, t) + } + } + return tasks, nil +} + +func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) { + filter := fmt.Sprintf(`cache_key='%s'`, cacheKey) + items, err := s.pb.listAll(ctx, "audio_jobs", filter, "-started") + if err != nil || len(items) == 0 { + return domain.AudioTask{}, false, err + } + t, err := parseAudioTask(items[0]) + return t, err == nil, err +} + +func (s *Store) ListTranslationTasks(ctx context.Context) ([]domain.TranslationTask, error) { + items, err := s.pb.listAll(ctx, "translation_jobs", "", "-started") + if err != nil { + return nil, err + } + tasks := make([]domain.TranslationTask, 0, len(items)) + for _, raw := range items { + t, err := parseTranslationTask(raw) + if err == nil { + tasks = append(tasks, t) + } + } + return tasks, nil +} + +func (s *Store) GetTranslationTask(ctx context.Context, cacheKey string) (domain.TranslationTask, bool, error) { + items, err := s.pb.listAll(ctx, "translation_jobs", fmt.Sprintf("cache_key=%q", cacheKey), "-started") + if err != nil || len(items) == 0 { + return domain.TranslationTask{}, false, err + } + t, err := parseTranslationTask(items[0]) + return t, err == nil, err +} + +func (s *Store) ListImportTasks(ctx context.Context) ([]domain.ImportTask, error) { + items, err := s.pb.listAll(ctx, "import_tasks", "", "-started") + if err != nil { + return nil, err + } + tasks := make([]domain.ImportTask, 0, len(items)) + for _, raw := range items { + t, err := parseImportTask(raw) + if err == nil { + tasks = append(tasks, t) + } + } + return tasks, nil +} + +func (s *Store) GetImportTask(ctx context.Context, id string) (domain.ImportTask, bool, error) { + var raw json.RawMessage + if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/import_tasks/records/%s", id), &raw); err != nil { + if err == ErrNotFound { + return domain.ImportTask{}, false, nil + } + return domain.ImportTask{}, false, err + } + t, err := parseImportTask(raw) + return t, err == nil, err +} + +// ── Parsers ─────────────────────────────────────────────────────────────────── + +func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) { + var rec struct { + ID string `json:"id"` + Kind string `json:"kind"` + TargetURL string `json:"target_url"` + FromChapter int `json:"from_chapter"` + ToChapter int `json:"to_chapter"` + WorkerID string `json:"worker_id"` + Status string `json:"status"` + BooksFound int `json:"books_found"` + ChaptersScraped int `json:"chapters_scraped"` + ChaptersSkipped int `json:"chapters_skipped"` + Errors int `json:"errors"` + Started string `json:"started"` + Finished string `json:"finished"` + ErrorMessage string `json:"error_message"` + } + if err := json.Unmarshal(raw, &rec); err != nil { + return domain.ScrapeTask{}, err + } + started, _ := time.Parse(time.RFC3339, rec.Started) + finished, _ := time.Parse(time.RFC3339, rec.Finished) + return domain.ScrapeTask{ + ID: rec.ID, + Kind: rec.Kind, + TargetURL: rec.TargetURL, + FromChapter: rec.FromChapter, + ToChapter: rec.ToChapter, + WorkerID: rec.WorkerID, + Status: domain.TaskStatus(rec.Status), + BooksFound: rec.BooksFound, + ChaptersScraped: rec.ChaptersScraped, + ChaptersSkipped: rec.ChaptersSkipped, + Errors: rec.Errors, + Started: started, + Finished: finished, + ErrorMessage: rec.ErrorMessage, + }, nil +} + +func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) { + var rec struct { + ID string `json:"id"` + CacheKey string `json:"cache_key"` + Slug string `json:"slug"` + Chapter int `json:"chapter"` + Voice string `json:"voice"` + WorkerID string `json:"worker_id"` + Status string `json:"status"` + ErrorMessage string `json:"error_message"` + Started string `json:"started"` + Finished string `json:"finished"` + } + if err := json.Unmarshal(raw, &rec); err != nil { + return domain.AudioTask{}, err + } + started, _ := time.Parse(time.RFC3339, rec.Started) + finished, _ := time.Parse(time.RFC3339, rec.Finished) + return domain.AudioTask{ + ID: rec.ID, + CacheKey: rec.CacheKey, + Slug: rec.Slug, + Chapter: rec.Chapter, + Voice: rec.Voice, + WorkerID: rec.WorkerID, + Status: domain.TaskStatus(rec.Status), + ErrorMessage: rec.ErrorMessage, + Started: started, + Finished: finished, + }, nil +} + +func parseTranslationTask(raw json.RawMessage) (domain.TranslationTask, error) { + var rec struct { + ID string `json:"id"` + CacheKey string `json:"cache_key"` + Slug string `json:"slug"` + Chapter int `json:"chapter"` + Lang string `json:"lang"` + WorkerID string `json:"worker_id"` + Status string `json:"status"` + ErrorMessage string `json:"error_message"` + Started string `json:"started"` + Finished string `json:"finished"` + } + if err := json.Unmarshal(raw, &rec); err != nil { + return domain.TranslationTask{}, err + } + started, _ := time.Parse(time.RFC3339, rec.Started) + finished, _ := time.Parse(time.RFC3339, rec.Finished) + return domain.TranslationTask{ + ID: rec.ID, + CacheKey: rec.CacheKey, + Slug: rec.Slug, + Chapter: rec.Chapter, + Lang: rec.Lang, + WorkerID: rec.WorkerID, + Status: domain.TaskStatus(rec.Status), + ErrorMessage: rec.ErrorMessage, + Started: started, + Finished: finished, + }, nil +} + +func parseImportTask(raw json.RawMessage) (domain.ImportTask, error) { + var rec struct { + ID string `json:"id"` + Slug string `json:"slug"` + Title string `json:"title"` + FileName string `json:"file_name"` + FileType string `json:"file_type"` + ObjectKey string `json:"object_key"` + ChaptersKey string `json:"chapters_key"` + Author string `json:"author"` + CoverURL string `json:"cover_url"` + Genres string `json:"genres"` // stored as comma-separated + Summary string `json:"summary"` + BookStatus string `json:"book_status"` + WorkerID string `json:"worker_id"` + InitiatorUserID string `json:"initiator_user_id"` + Status string `json:"status"` + ChaptersDone int `json:"chapters_done"` + ChaptersTotal int `json:"chapters_total"` + ErrorMessage string `json:"error_message"` + Started string `json:"started"` + Finished string `json:"finished"` + } + if err := json.Unmarshal(raw, &rec); err != nil { + return domain.ImportTask{}, err + } + started, _ := time.Parse(time.RFC3339, rec.Started) + finished, _ := time.Parse(time.RFC3339, rec.Finished) + var genres []string + if rec.Genres != "" { + for _, g := range strings.Split(rec.Genres, ",") { + if g = strings.TrimSpace(g); g != "" { + genres = append(genres, g) + } + } + } + return domain.ImportTask{ + ID: rec.ID, + Slug: rec.Slug, + Title: rec.Title, + FileName: rec.FileName, + FileType: rec.FileType, + ObjectKey: rec.ObjectKey, + ChaptersKey: rec.ChaptersKey, + Author: rec.Author, + CoverURL: rec.CoverURL, + Genres: genres, + Summary: rec.Summary, + BookStatus: rec.BookStatus, + WorkerID: rec.WorkerID, + InitiatorUserID: rec.InitiatorUserID, + Status: domain.TaskStatus(rec.Status), + ChaptersDone: rec.ChaptersDone, + ChaptersTotal: rec.ChaptersTotal, + ErrorMessage: rec.ErrorMessage, + Started: started, + Finished: finished, + }, nil +} + +// ── CoverStore ───────────────────────────────────────────────────────────────── + +func (s *Store) PutCover(ctx context.Context, slug string, data []byte, contentType string) error { + key := CoverObjectKey(slug) + if contentType == "" { + contentType = coverContentType(data) + } + if err := s.mc.putCover(ctx, key, contentType, data); err != nil { + return fmt.Errorf("PutCover: %w", err) + } + return nil +} + +func (s *Store) GetCover(ctx context.Context, slug string) ([]byte, string, bool, error) { + key := CoverObjectKey(slug) + data, ok, err := s.mc.getCover(ctx, key) + if err != nil { + return nil, "", false, fmt.Errorf("GetCover: %w", err) + } + if !ok { + return nil, "", false, nil + } + ct := coverContentType(data) + return data, ct, true, nil +} + +// PutImportFile stores an uploaded import file (PDF/EPUB) in MinIO. +func (s *Store) PutImportFile(ctx context.Context, key string, data []byte) error { + return s.mc.putObject(ctx, "imports", key, "application/octet-stream", data) +} + +// PutImportChapters stores a pre-parsed chapters JSON blob in MinIO. +func (s *Store) PutImportChapters(ctx context.Context, key string, data []byte) error { + return s.mc.putObject(ctx, "imports", key, "application/json", data) +} + +// GetImportChapters retrieves the pre-parsed chapters JSON from MinIO. +func (s *Store) GetImportChapters(ctx context.Context, key string) ([]byte, error) { + data, err := s.mc.getObject(ctx, "imports", key) + if err != nil { + return nil, fmt.Errorf("get chapters object: %w", err) + } + return data, nil +} + +func (s *Store) CoverExists(ctx context.Context, slug string) bool { + return s.mc.coverExists(ctx, CoverObjectKey(slug)) +} + +// ── ChapterImageStore ────────────────────────────────────────────────────────── + +func (s *Store) PutChapterImage(ctx context.Context, slug string, n int, data []byte, contentType string) error { + key := ChapterImageObjectKey(slug, n) + if contentType == "" { + contentType = coverContentType(data) + } + if err := s.mc.putChapterImage(ctx, key, contentType, data); err != nil { + return fmt.Errorf("PutChapterImage: %w", err) + } + return nil +} + +func (s *Store) GetChapterImage(ctx context.Context, slug string, n int) ([]byte, string, bool, error) { + key := ChapterImageObjectKey(slug, n) + data, ok, err := s.mc.getChapterImage(ctx, key) + if err != nil { + return nil, "", false, fmt.Errorf("GetChapterImage: %w", err) + } + if !ok { + return nil, "", false, nil + } + ct := coverContentType(data) + return data, ct, true, nil +} + +func (s *Store) ChapterImageExists(ctx context.Context, slug string, n int) bool { + return s.mc.chapterImageExists(ctx, ChapterImageObjectKey(slug, n)) +} + +// ── TranslationStore ─────────────────────────────────────────────────────────── + +func (s *Store) TranslationObjectKey(lang, slug string, n int) string { + return TranslationObjectKey(lang, slug, n) +} + +func (s *Store) TranslationExists(ctx context.Context, key string) bool { + return s.mc.objectExists(ctx, s.mc.bucketTranslations, key) +} + +func (s *Store) PutTranslation(ctx context.Context, key string, data []byte) error { + return s.mc.putObject(ctx, s.mc.bucketTranslations, key, "text/markdown; charset=utf-8", data) +} + +func (s *Store) GetTranslation(ctx context.Context, key string) (string, error) { + data, err := s.mc.getObject(ctx, s.mc.bucketTranslations, key) + if err != nil { + return "", fmt.Errorf("GetTranslation: %w", err) + } + return string(data), nil +} + +// ── AIJobStore ──────────────────────────────────────────────────────────────── + +func (s *Store) CreateAIJob(ctx context.Context, job domain.AIJob) (string, error) { + payload := map[string]any{ + "kind": job.Kind, + "slug": job.Slug, + "status": string(job.Status), + "from_item": job.FromItem, + "to_item": job.ToItem, + "items_done": job.ItemsDone, + "items_total": job.ItemsTotal, + "model": job.Model, + "payload": job.Payload, + "started": job.Started.Format(time.RFC3339), + } + var out struct { + ID string `json:"id"` + } + if err := s.pb.post(ctx, "/api/collections/ai_jobs/records", payload, &out); err != nil { + return "", fmt.Errorf("CreateAIJob: %w", err) + } + return out.ID, nil +} + +func (s *Store) GetAIJob(ctx context.Context, id string) (domain.AIJob, bool, error) { + var raw json.RawMessage + if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/ai_jobs/records/%s", id), &raw); err != nil { + if strings.Contains(err.Error(), "404") { + return domain.AIJob{}, false, nil + } + return domain.AIJob{}, false, fmt.Errorf("GetAIJob: %w", err) + } + job, err := parseAIJob(raw) + if err != nil { + return domain.AIJob{}, false, err + } + return job, true, nil +} + +func (s *Store) UpdateAIJob(ctx context.Context, id string, fields map[string]any) error { + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/ai_jobs/records/%s", id), fields) +} + +func (s *Store) ListAIJobs(ctx context.Context) ([]domain.AIJob, error) { + items, err := s.pb.listAll(ctx, "ai_jobs", "", "-started") + if err != nil { + return nil, fmt.Errorf("ListAIJobs: %w", err) + } + out := make([]domain.AIJob, 0, len(items)) + for _, raw := range items { + j, err := parseAIJob(raw) + if err != nil { + continue + } + out = append(out, j) + } + return out, nil +} + +func parseAIJob(raw json.RawMessage) (domain.AIJob, error) { + var r struct { + ID string `json:"id"` + Kind string `json:"kind"` + Slug string `json:"slug"` + Status string `json:"status"` + FromItem int `json:"from_item"` + ToItem int `json:"to_item"` + ItemsDone int `json:"items_done"` + ItemsTotal int `json:"items_total"` + Model string `json:"model"` + Payload string `json:"payload"` + ErrorMessage string `json:"error_message"` + Started string `json:"started"` + Finished string `json:"finished"` + HeartbeatAt string `json:"heartbeat_at"` + } + if err := json.Unmarshal(raw, &r); err != nil { + return domain.AIJob{}, fmt.Errorf("parseAIJob: %w", err) + } + parseT := func(s string) time.Time { + if s == "" { + return time.Time{} + } + t, _ := time.Parse(time.RFC3339, s) + return t + } + return domain.AIJob{ + ID: r.ID, + Kind: r.Kind, + Slug: r.Slug, + Status: domain.TaskStatus(r.Status), + FromItem: r.FromItem, + ToItem: r.ToItem, + ItemsDone: r.ItemsDone, + ItemsTotal: r.ItemsTotal, + Model: r.Model, + Payload: r.Payload, + ErrorMessage: r.ErrorMessage, + Started: parseT(r.Started), + Finished: parseT(r.Finished), + HeartbeatAt: parseT(r.HeartbeatAt), + }, nil +} + +// ── Push subscriptions ──────────────────────────────────────────────────────── + +// PushSubscription holds the Web Push subscription data for a single browser. +type PushSubscription struct { + ID string + UserID string + Endpoint string + P256DH string + Auth string +} + +// SavePushSubscription upserts a Web Push subscription for a user. +// If a record with the same endpoint already exists it is updated in place. +func (s *Store) SavePushSubscription(ctx context.Context, sub PushSubscription) error { + filter := fmt.Sprintf("endpoint=%q", sub.Endpoint) + existing, err := s.pb.listAll(ctx, "push_subscriptions", filter, "") + if err != nil { + return fmt.Errorf("SavePushSubscription list: %w", err) + } + payload := map[string]any{ + "user_id": sub.UserID, + "endpoint": sub.Endpoint, + "p256dh": sub.P256DH, + "auth": sub.Auth, + } + if len(existing) > 0 { + var rec struct { + ID string `json:"id"` + } + if json.Unmarshal(existing[0], &rec) == nil && rec.ID != "" { + return s.pb.patch(ctx, fmt.Sprintf("/api/collections/push_subscriptions/records/%s", rec.ID), payload) + } + } + return s.pb.post(ctx, "/api/collections/push_subscriptions/records", payload, nil) +} + +// DeletePushSubscription removes a Web Push subscription by endpoint. +func (s *Store) DeletePushSubscription(ctx context.Context, userID, endpoint string) error { + filter := fmt.Sprintf("user_id=%q&&endpoint=%q", userID, endpoint) + items, err := s.pb.listAll(ctx, "push_subscriptions", filter, "") + if err != nil { + return fmt.Errorf("DeletePushSubscription list: %w", err) + } + for _, raw := range items { + var rec struct { + ID string `json:"id"` + } + if json.Unmarshal(raw, &rec) == nil && rec.ID != "" { + _ = s.pb.delete(ctx, fmt.Sprintf("/api/collections/push_subscriptions/records/%s", rec.ID)) + } + } + return nil +} + +// ListPushSubscriptionsByBook returns all push subscriptions belonging to users +// who have the given book slug in their library (user_library collection). +func (s *Store) ListPushSubscriptionsByBook(ctx context.Context, slug string) ([]PushSubscription, error) { + // Find all users who have this book in their library + libFilter := fmt.Sprintf("slug=%q&&user_id!=''", slug) + libItems, err := s.pb.listAll(ctx, "user_library", libFilter, "") + if err != nil { + return nil, fmt.Errorf("ListPushSubscriptionsByBook list library: %w", err) + } + + // Collect unique user IDs + seen := make(map[string]bool) + var userIDs []string + for _, raw := range libItems { + var rec struct { + UserID string `json:"user_id"` + } + if json.Unmarshal(raw, &rec) == nil && rec.UserID != "" && !seen[rec.UserID] { + seen[rec.UserID] = true + userIDs = append(userIDs, rec.UserID) + } + } + if len(userIDs) == 0 { + return nil, nil + } + + // Build OR filter for push_subscriptions + parts := make([]string, len(userIDs)) + for i, uid := range userIDs { + parts[i] = fmt.Sprintf("user_id=%q", uid) + } + subFilter := strings.Join(parts, "||") + subItems, err := s.pb.listAll(ctx, "push_subscriptions", subFilter, "") + if err != nil { + return nil, fmt.Errorf("ListPushSubscriptionsByBook list subs: %w", err) + } + + subs := make([]PushSubscription, 0, len(subItems)) + for _, raw := range subItems { + var rec struct { + ID string `json:"id"` + UserID string `json:"user_id"` + Endpoint string `json:"endpoint"` + P256DH string `json:"p256dh"` + Auth string `json:"auth"` + } + if json.Unmarshal(raw, &rec) == nil && rec.Endpoint != "" { + subs = append(subs, PushSubscription{ + ID: rec.ID, + UserID: rec.UserID, + Endpoint: rec.Endpoint, + P256DH: rec.P256DH, + Auth: rec.Auth, + }) + } + } + return subs, nil +} + +// NotifyUsersWithBook creates an in-app notification for every logged-in user +// who has slug in their library. Errors for individual users are logged but +// do not abort the loop. Returns the number of notifications created. +func (s *Store) NotifyUsersWithBook(ctx context.Context, slug, title, message, link string) int { + userIDs, err := s.ListUserIDsWithBook(ctx, slug) + if err != nil || len(userIDs) == 0 { + return 0 + } + var n int + for _, uid := range userIDs { + if createErr := s.CreateNotification(ctx, uid, title, message, link); createErr == nil { + n++ + } + } + return n +} +// who have slug in their user_library. Used to fan-out new-chapter notifications. +// Admin users and users who have opted out of in-app new-chapter notifications +// (notify_new_chapters=false on app_users) are excluded. +func (s *Store) ListUserIDsWithBook(ctx context.Context, slug string) ([]string, error) { + // Collect user IDs to skip: admins + opted-out users. + skipIDs := make(map[string]bool) + excludedItems, err := s.pb.listAll(ctx, "app_users", `role="admin"||notify_new_chapters=false`, "") + if err == nil { + for _, raw := range excludedItems { + var rec struct { + ID string `json:"id"` + } + if json.Unmarshal(raw, &rec) == nil && rec.ID != "" { + skipIDs[rec.ID] = true + } + } + } + + filter := fmt.Sprintf("slug=%q&&user_id!=''", slug) + items, err := s.pb.listAll(ctx, "user_library", filter, "") + if err != nil { + return nil, fmt.Errorf("ListUserIDsWithBook: %w", err) + } + seen := make(map[string]bool) + var ids []string + for _, raw := range items { + var rec struct { + UserID string `json:"user_id"` + } + if json.Unmarshal(raw, &rec) == nil && rec.UserID != "" && !seen[rec.UserID] && !skipIDs[rec.UserID] { + seen[rec.UserID] = true + ids = append(ids, rec.UserID) + } + } + return ids, nil +} diff --git a/backend/internal/taskqueue/taskqueue.go b/backend/internal/taskqueue/taskqueue.go new file mode 100644 index 0000000..1c9fd9b --- /dev/null +++ b/backend/internal/taskqueue/taskqueue.go @@ -0,0 +1,127 @@ +// Package taskqueue defines the interfaces for creating and consuming +// scrape/audio tasks stored in PocketBase. +// +// Interface segregation: +// - Producer is used only by the backend (creates tasks, cancels tasks). +// - Consumer is used only by the runner (claims tasks, reports results). +// - Reader is used by the backend for status/history endpoints. +// +// Concrete implementations live in internal/storage. +package taskqueue + +import ( + "context" + "time" + + "github.com/libnovel/backend/internal/domain" +) + +// Producer is the write side of the task queue used by the backend service. +// It creates new tasks in PocketBase for the runner to pick up. +type Producer interface { + // CreateScrapeTask inserts a new scrape task with status=pending and + // returns the assigned PocketBase record ID. + // kind is one of "catalogue", "book", or "book_range". + // targetURL is the book URL (empty for catalogue-wide tasks). + CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) + + // CreateAudioTask inserts a new audio task with status=pending and + // returns the assigned PocketBase record ID. + CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) + + // CreateTranslationTask inserts a new translation task with status=pending and + // returns the assigned PocketBase record ID. + CreateTranslationTask(ctx context.Context, slug string, chapter int, lang string) (string, error) + + // CreateImportTask inserts a new import task with status=pending and + // returns the assigned PocketBase record ID. + // The task struct must have at minimum Slug, Title, FileType, and ObjectKey set. + CreateImportTask(ctx context.Context, task domain.ImportTask) (string, error) + + // CancelTask transitions a pending task to status=cancelled. + // Returns ErrNotFound if the task does not exist. + CancelTask(ctx context.Context, id string) error + + // CancelAudioTasksBySlug cancels all pending or running audio tasks for slug. + // Returns the number of tasks cancelled. + CancelAudioTasksBySlug(ctx context.Context, slug string) (int, error) +} + +// Consumer is the read/claim side of the task queue used by the runner. +type Consumer interface { + // ClaimNextScrapeTask atomically finds the oldest pending scrape task, + // sets its status=running and worker_id=workerID, and returns it. + // Returns (zero, false, nil) when the queue is empty. + ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error) + + // ClaimNextAudioTask atomically finds the oldest pending audio task, + // sets its status=running and worker_id=workerID, and returns it. + // Returns (zero, false, nil) when the queue is empty. + ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error) + + // ClaimNextTranslationTask atomically finds the oldest pending translation task, + // sets its status=running and worker_id=workerID, and returns it. + // Returns (zero, false, nil) when the queue is empty. + ClaimNextTranslationTask(ctx context.Context, workerID string) (domain.TranslationTask, bool, error) + + // ClaimNextImportTask atomically finds the oldest pending import task, + // sets its status=running and worker_id=workerID, and returns it. + // Returns (zero, false, nil) when the queue is empty. + ClaimNextImportTask(ctx context.Context, workerID string) (domain.ImportTask, bool, error) + + // FinishScrapeTask marks a running scrape task as done and records the result. + FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error + + // FinishAudioTask marks a running audio task as done and records the result. + FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error + + // FinishTranslationTask marks a running translation task as done and records the result. + FinishTranslationTask(ctx context.Context, id string, result domain.TranslationResult) error + + // FinishImportTask marks a running import task as done and records the result. + FinishImportTask(ctx context.Context, id string, result domain.ImportResult) error + + // FailTask marks a task (scrape, audio, or translation) as failed with an error message. + FailTask(ctx context.Context, id, errMsg string) error + + // HeartbeatTask updates the heartbeat_at timestamp on a running task. + // Should be called periodically by the runner while the task is active so + // the reaper knows the task is still alive. + HeartbeatTask(ctx context.Context, id string) error + + // ReapStaleTasks finds all running tasks whose heartbeat_at is older than + // staleAfter (or was never set) and resets them to pending so they can be + // re-claimed by a healthy runner. Returns the number of tasks reaped. + ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) +} + +// Reader is the read-only side used by the backend for status pages. +type Reader interface { + // ListScrapeTasks returns all scrape tasks sorted by started descending. + ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error) + + // GetScrapeTask returns a single scrape task by ID. + // Returns (zero, false, nil) if not found. + GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error) + + // ListAudioTasks returns all audio tasks sorted by started descending. + ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error) + + // GetAudioTask returns the most recent audio task for cacheKey. + // Returns (zero, false, nil) if not found. + GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) + + // ListTranslationTasks returns all translation tasks sorted by started descending. + ListTranslationTasks(ctx context.Context) ([]domain.TranslationTask, error) + + // GetTranslationTask returns the most recent translation task for cacheKey. + // Returns (zero, false, nil) if not found. + GetTranslationTask(ctx context.Context, cacheKey string) (domain.TranslationTask, bool, error) + + // ListImportTasks returns all import tasks sorted by started descending. + ListImportTasks(ctx context.Context) ([]domain.ImportTask, error) + + // GetImportTask returns a single import task by ID. + // Returns (zero, false, nil) if not found. + GetImportTask(ctx context.Context, id string) (domain.ImportTask, bool, error) +} diff --git a/backend/internal/taskqueue/taskqueue_test.go b/backend/internal/taskqueue/taskqueue_test.go new file mode 100644 index 0000000..9f8969a --- /dev/null +++ b/backend/internal/taskqueue/taskqueue_test.go @@ -0,0 +1,167 @@ +package taskqueue_test + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/libnovel/backend/internal/domain" + "github.com/libnovel/backend/internal/taskqueue" +) + +// ── Compile-time interface satisfaction ─────────────────────────────────────── + +// stubStore satisfies all three taskqueue interfaces. +// Any method that is called but not expected panics — making accidental +// calls immediately visible in tests. +type stubStore struct{} + +func (s *stubStore) CreateScrapeTask(_ context.Context, _, _ string, _, _ int) (string, error) { + return "task-1", nil +} +func (s *stubStore) CreateAudioTask(_ context.Context, _ string, _ int, _ string) (string, error) { + return "audio-1", nil +} +func (s *stubStore) CreateTranslationTask(_ context.Context, _ string, _ int, _ string) (string, error) { + return "translation-1", nil +} +func (s *stubStore) CreateImportTask(_ context.Context, _ domain.ImportTask) (string, error) { + return "import-1", nil +} +func (s *stubStore) CancelTask(_ context.Context, _ string) error { return nil } +func (s *stubStore) CancelAudioTasksBySlug(_ context.Context, _ string) (int, error) { return 0, nil } + +func (s *stubStore) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) { + return domain.ScrapeTask{ID: "task-1", Status: domain.TaskStatusRunning}, true, nil +} +func (s *stubStore) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) { + return domain.AudioTask{ID: "audio-1", Status: domain.TaskStatusRunning}, true, nil +} +func (s *stubStore) ClaimNextTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) { + return domain.TranslationTask{ID: "translation-1", Status: domain.TaskStatusRunning}, true, nil +} +func (s *stubStore) ClaimNextImportTask(_ context.Context, _ string) (domain.ImportTask, bool, error) { + return domain.ImportTask{ID: "import-1", Status: domain.TaskStatusRunning}, true, nil +} +func (s *stubStore) FinishScrapeTask(_ context.Context, _ string, _ domain.ScrapeResult) error { + return nil +} +func (s *stubStore) FinishAudioTask(_ context.Context, _ string, _ domain.AudioResult) error { + return nil +} +func (s *stubStore) FinishTranslationTask(_ context.Context, _ string, _ domain.TranslationResult) error { + return nil +} +func (s *stubStore) FinishImportTask(_ context.Context, _ string, _ domain.ImportResult) error { + return nil +} +func (s *stubStore) FailTask(_ context.Context, _, _ string) error { return nil } + +func (s *stubStore) HeartbeatTask(_ context.Context, _ string) error { return nil } + +func (s *stubStore) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) { + return 0, nil +} + +func (s *stubStore) ListScrapeTasks(_ context.Context) ([]domain.ScrapeTask, error) { return nil, nil } +func (s *stubStore) GetScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) { + return domain.ScrapeTask{}, false, nil +} +func (s *stubStore) ListAudioTasks(_ context.Context) ([]domain.AudioTask, error) { return nil, nil } +func (s *stubStore) GetAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) { + return domain.AudioTask{}, false, nil +} +func (s *stubStore) ListTranslationTasks(_ context.Context) ([]domain.TranslationTask, error) { + return nil, nil +} +func (s *stubStore) GetTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) { + return domain.TranslationTask{}, false, nil +} +func (s *stubStore) ListImportTasks(_ context.Context) ([]domain.ImportTask, error) { return nil, nil } +func (s *stubStore) GetImportTask(_ context.Context, _ string) (domain.ImportTask, bool, error) { + return domain.ImportTask{}, false, nil +} + +// Verify the stub satisfies all three interfaces at compile time. +var _ taskqueue.Producer = (*stubStore)(nil) +var _ taskqueue.Consumer = (*stubStore)(nil) +var _ taskqueue.Reader = (*stubStore)(nil) + +// ── Behavioural tests (using stub) ──────────────────────────────────────────── + +func TestProducer_CreateScrapeTask(t *testing.T) { + var p taskqueue.Producer = &stubStore{} + id, err := p.CreateScrapeTask(context.Background(), "book", "https://example.com/book/slug", 0, 0) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if id == "" { + t.Error("expected non-empty task ID") + } +} + +func TestConsumer_ClaimNextScrapeTask(t *testing.T) { + var c taskqueue.Consumer = &stubStore{} + task, ok, err := c.ClaimNextScrapeTask(context.Background(), "worker-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ok { + t.Fatal("expected a task to be claimed") + } + if task.Status != domain.TaskStatusRunning { + t.Errorf("want running, got %q", task.Status) + } +} + +func TestConsumer_ClaimNextAudioTask(t *testing.T) { + var c taskqueue.Consumer = &stubStore{} + task, ok, err := c.ClaimNextAudioTask(context.Background(), "worker-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ok { + t.Fatal("expected an audio task to be claimed") + } + if task.ID == "" { + t.Error("expected non-empty task ID") + } +} + +// ── domain.ScrapeResult / domain.AudioResult JSON shape ────────────────────── + +func TestScrapeResult_JSONRoundtrip(t *testing.T) { + cases := []domain.ScrapeResult{ + {BooksFound: 5, ChaptersScraped: 100, ChaptersSkipped: 2, Errors: 0}, + {BooksFound: 0, ChaptersScraped: 0, Errors: 1, ErrorMessage: "timeout"}, + } + for _, orig := range cases { + b, err := json.Marshal(orig) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var got domain.ScrapeResult + if err := json.Unmarshal(b, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got != orig { + t.Errorf("want %+v, got %+v", orig, got) + } + } +} + +func TestAudioResult_JSONRoundtrip(t *testing.T) { + cases := []domain.AudioResult{ + {ObjectKey: "audio/slug/1/af_bella.mp3"}, + {ErrorMessage: "kokoro unavailable"}, + } + for _, orig := range cases { + b, _ := json.Marshal(orig) + var got domain.AudioResult + json.Unmarshal(b, &got) + if got != orig { + t.Errorf("want %+v, got %+v", orig, got) + } + } +} diff --git a/backend/internal/webpush/webpush.go b/backend/internal/webpush/webpush.go new file mode 100644 index 0000000..9cff793 --- /dev/null +++ b/backend/internal/webpush/webpush.go @@ -0,0 +1,147 @@ +// Package webpush sends Web Push notifications using the VAPID protocol. +package webpush + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "sync" + + webpushgo "github.com/SherClockHolmes/webpush-go" + + "github.com/libnovel/backend/internal/storage" +) + +// Payload is the JSON body delivered to the browser service worker. +type Payload struct { + Title string `json:"title"` + Body string `json:"body"` + URL string `json:"url,omitempty"` + Icon string `json:"icon,omitempty"` +} + +// Sender sends Web Push notifications to subscribed browsers. +type Sender struct { + vapidPublic string + vapidPrivate string + subject string + log *slog.Logger +} + +// New returns a Sender configured with the given VAPID key pair. +// subject should be a mailto: or https: contact URL per the VAPID spec. +func New(vapidPublic, vapidPrivate, subject string, log *slog.Logger) *Sender { + if log == nil { + log = slog.Default() + } + return &Sender{ + vapidPublic: vapidPublic, + vapidPrivate: vapidPrivate, + subject: subject, + log: log, + } +} + +// Enabled returns true when VAPID keys are configured. +func (s *Sender) Enabled() bool { + return s.vapidPublic != "" && s.vapidPrivate != "" +} + +// Send delivers payload to all provided subscriptions concurrently. +// Errors for individual subscriptions are logged but do not abort other sends. +// Returns the number of successful sends. +func (s *Sender) Send(ctx context.Context, subs []storage.PushSubscription, p Payload) int { + if !s.Enabled() || len(subs) == 0 { + return 0 + } + + body, err := json.Marshal(p) + if err != nil { + s.log.Error("webpush: marshal payload", "err", err) + return 0 + } + + var ( + wg sync.WaitGroup + mu sync.Mutex + success int + ) + + for _, sub := range subs { + sub := sub + wg.Add(1) + go func() { + defer wg.Done() + + resp, err := webpushgo.SendNotificationWithContext(ctx, body, &webpushgo.Subscription{ + Endpoint: sub.Endpoint, + Keys: webpushgo.Keys{ + P256dh: sub.P256DH, + Auth: sub.Auth, + }, + }, &webpushgo.Options{ + VAPIDPublicKey: s.vapidPublic, + VAPIDPrivateKey: s.vapidPrivate, + Subscriber: s.subject, + TTL: 86400, + }) + if err != nil { + s.log.Warn("webpush: send failed", "endpoint", truncate(sub.Endpoint, 60), "err", err) + return + } + defer resp.Body.Close() //nolint:errcheck + if resp.StatusCode >= 400 { + s.log.Warn("webpush: push service returned error", + "endpoint", truncate(sub.Endpoint, 60), + "status", resp.StatusCode) + return + } + mu.Lock() + success++ + mu.Unlock() + }() + } + + wg.Wait() + return success +} + +// SendToBook sends a push notification to all subscribers of the given book. +// store is used to list subscriptions for the book's library followers. +func (s *Sender) SendToBook(ctx context.Context, store *storage.Store, slug string, p Payload) { + if !s.Enabled() { + return + } + subs, err := store.ListPushSubscriptionsByBook(ctx, slug) + if err != nil { + s.log.Warn("webpush: list push subscriptions", "slug", slug, "err", err) + return + } + if len(subs) == 0 { + return + } + n := s.Send(ctx, subs, p) + s.log.Info("webpush: sent chapter notification", + "slug", slug, + "recipients", n, + "total_subs", len(subs), + ) +} + +// GenerateVAPIDKeys generates a new VAPID key pair and prints them. +// Useful for one-off key generation during setup. +func GenerateVAPIDKeys() (public, private string, err error) { + private, public, err = webpushgo.GenerateVAPIDKeys() + if err != nil { + return "", "", fmt.Errorf("generate VAPID keys: %w", err) + } + return public, private, nil +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "..." +} diff --git a/backend/migrations/20260414000001_initial_schema.go b/backend/migrations/20260414000001_initial_schema.go new file mode 100644 index 0000000..22dfad1 --- /dev/null +++ b/backend/migrations/20260414000001_initial_schema.go @@ -0,0 +1,431 @@ +// Migration 1 — full schema baseline. +// +// Creates all 21 collections that were previously bootstrapped by +// scripts/pb-init-v3.sh. Also creates the initial superuser from the +// POCKETBASE_ADMIN_EMAIL / POCKETBASE_ADMIN_PASSWORD env vars (first run only). +// +// This migration is intentionally idempotent: each collection is skipped if it +// already exists. This makes it safe to apply on an existing install without +// running `migrate history-sync` first — existing collections are left untouched +// and migration 2 still runs to add the three fields that were missing. +package migrations + +import ( + "os" + + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + m.Register(func(app core.App) error { + steps := []func(core.App) error{ + createBooks, + createChaptersIdx, + createRanking, + createProgress, + createScrapingTasks, + createAudioJobs, + createAppUsers, + createUserSessions, + createUserLibrary, + createUserSettings, + createUserSubscriptions, + createBookComments, + createCommentVotes, + createTranslationJobs, + createImportTasks, + createNotifications, + createPushSubscriptions, + createAIJobs, + createDiscoveryVotes, + createBookRatings, + createSiteConfig, + createInitialSuperuser, + } + for _, step := range steps { + if err := step(app); err != nil { + return err + } + } + return nil + }, func(app core.App) error { + // Down: drop all collections in safe reverse order. + names := []string{ + "site_config", "book_ratings", "discovery_votes", "ai_jobs", + "push_subscriptions", "notifications", "import_tasks", + "translation_jobs", "comment_votes", "book_comments", + "user_subscriptions", "user_settings", "user_library", + "user_sessions", "app_users", "audio_jobs", "scraping_tasks", + "progress", "ranking", "chapters_idx", "books", + } + for _, name := range names { + coll, err := app.FindCollectionByNameOrId(name) + if err != nil { + continue // already absent — safe to skip + } + if err := app.Delete(coll); err != nil { + return err + } + } + return nil + }) +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +// saveIfAbsent saves the collection only when no collection with that name +// exists yet. This makes the migration safe to run on an existing install +// without history-sync — already-created collections are simply skipped. +func saveIfAbsent(app core.App, c *core.Collection) error { + if _, err := app.FindCollectionByNameOrId(c.Name); err == nil { + return nil // already exists — skip + } + return app.Save(c) +} + +// ── Collection creators ─────────────────────────────────────────────────────── + +func createBooks(app core.App) error { + c := core.NewBaseCollection("books") + c.Fields.Add( + &core.TextField{Name: "slug", Required: true}, + &core.TextField{Name: "title", Required: true}, + &core.TextField{Name: "author"}, + &core.TextField{Name: "cover"}, + &core.TextField{Name: "status"}, + &core.JSONField{Name: "genres"}, + &core.TextField{Name: "summary"}, + &core.NumberField{Name: "total_chapters"}, + &core.TextField{Name: "source_url"}, + &core.NumberField{Name: "ranking"}, + &core.TextField{Name: "meta_updated"}, + &core.BoolField{Name: "archived"}, + ) + return saveIfAbsent(app, c) +} + +func createChaptersIdx(app core.App) error { + c := core.NewBaseCollection("chapters_idx") + c.Fields.Add( + &core.TextField{Name: "slug", Required: true}, + &core.NumberField{Name: "number", Required: true}, + &core.TextField{Name: "title"}, + ) + // Enforce uniqueness on (slug, number) — prevents duplicate chapter entries. + c.AddIndex("idx_chapters_idx_slug_number", true, "slug, number", "") + // Allow fast "recently updated books" queries. + c.AddIndex("idx_chapters_idx_created", false, "created", "") + return saveIfAbsent(app, c) +} + +func createRanking(app core.App) error { + c := core.NewBaseCollection("ranking") + c.Fields.Add( + &core.NumberField{Name: "rank", Required: true}, + &core.TextField{Name: "slug", Required: true}, + &core.TextField{Name: "title"}, + &core.TextField{Name: "author"}, + &core.TextField{Name: "cover"}, + &core.TextField{Name: "status"}, + &core.JSONField{Name: "genres"}, + &core.TextField{Name: "source_url"}, + ) + return saveIfAbsent(app, c) +} + +func createProgress(app core.App) error { + c := core.NewBaseCollection("progress") + c.Fields.Add( + &core.TextField{Name: "session_id", Required: true}, + &core.TextField{Name: "slug", Required: true}, + &core.NumberField{Name: "chapter"}, + &core.TextField{Name: "user_id"}, + &core.NumberField{Name: "audio_time"}, + ) + return saveIfAbsent(app, c) +} + +func createScrapingTasks(app core.App) error { + c := core.NewBaseCollection("scraping_tasks") + c.Fields.Add( + &core.TextField{Name: "kind"}, + &core.TextField{Name: "target_url"}, + &core.NumberField{Name: "from_chapter"}, + &core.NumberField{Name: "to_chapter"}, + &core.TextField{Name: "worker_id"}, + &core.TextField{Name: "status", Required: true}, + &core.NumberField{Name: "books_found"}, + &core.NumberField{Name: "chapters_scraped"}, + &core.NumberField{Name: "chapters_skipped"}, + &core.NumberField{Name: "errors"}, + &core.TextField{Name: "error_message"}, + &core.DateField{Name: "started"}, + &core.DateField{Name: "finished"}, + &core.DateField{Name: "heartbeat_at"}, + ) + return saveIfAbsent(app, c) +} + +func createAudioJobs(app core.App) error { + c := core.NewBaseCollection("audio_jobs") + c.Fields.Add( + &core.TextField{Name: "cache_key", Required: true}, + &core.TextField{Name: "slug", Required: true}, + &core.NumberField{Name: "chapter", Required: true}, + &core.TextField{Name: "voice"}, + &core.TextField{Name: "worker_id"}, + &core.TextField{Name: "status", Required: true}, + &core.TextField{Name: "error_message"}, + &core.DateField{Name: "started"}, + &core.DateField{Name: "finished"}, + &core.DateField{Name: "heartbeat_at"}, + ) + return saveIfAbsent(app, c) +} + +func createAppUsers(app core.App) error { + c := core.NewBaseCollection("app_users") + c.Fields.Add( + &core.TextField{Name: "username", Required: true}, + &core.TextField{Name: "password_hash"}, + &core.TextField{Name: "role"}, + &core.TextField{Name: "avatar_url"}, + &core.TextField{Name: "email"}, + &core.BoolField{Name: "email_verified"}, + &core.TextField{Name: "verification_token"}, + &core.TextField{Name: "verification_token_exp"}, + &core.TextField{Name: "oauth_provider"}, + &core.TextField{Name: "oauth_id"}, + &core.TextField{Name: "polar_customer_id"}, + &core.TextField{Name: "polar_subscription_id"}, + &core.BoolField{Name: "notify_new_chapters"}, + ) + return saveIfAbsent(app, c) +} + +func createUserSessions(app core.App) error { + c := core.NewBaseCollection("user_sessions") + c.Fields.Add( + &core.TextField{Name: "user_id", Required: true}, + &core.TextField{Name: "session_id", Required: true}, + &core.TextField{Name: "user_agent"}, + &core.TextField{Name: "ip"}, + &core.TextField{Name: "device_fingerprint"}, + // created_at is a custom text field (not the system `created` date field). + &core.TextField{Name: "created_at"}, + &core.TextField{Name: "last_seen"}, + ) + return saveIfAbsent(app, c) +} + +func createUserLibrary(app core.App) error { + c := core.NewBaseCollection("user_library") + c.Fields.Add( + &core.TextField{Name: "session_id", Required: true}, + &core.TextField{Name: "user_id"}, + &core.TextField{Name: "slug", Required: true}, + &core.TextField{Name: "saved_at"}, + &core.TextField{Name: "shelf"}, + ) + return saveIfAbsent(app, c) +} + +func createUserSettings(app core.App) error { + c := core.NewBaseCollection("user_settings") + c.Fields.Add( + &core.TextField{Name: "session_id", Required: true}, + &core.TextField{Name: "user_id"}, + &core.BoolField{Name: "auto_next"}, + &core.TextField{Name: "voice"}, + &core.NumberField{Name: "speed"}, + &core.TextField{Name: "theme"}, + &core.TextField{Name: "locale"}, + &core.TextField{Name: "font_family"}, + &core.NumberField{Name: "font_size"}, + &core.BoolField{Name: "announce_chapter"}, + &core.TextField{Name: "audio_mode"}, + ) + return saveIfAbsent(app, c) +} + +func createUserSubscriptions(app core.App) error { + c := core.NewBaseCollection("user_subscriptions") + c.Fields.Add( + &core.TextField{Name: "follower_id", Required: true}, + &core.TextField{Name: "followee_id", Required: true}, + ) + return saveIfAbsent(app, c) +} + +func createBookComments(app core.App) error { + c := core.NewBaseCollection("book_comments") + c.Fields.Add( + &core.TextField{Name: "slug", Required: true}, + &core.TextField{Name: "user_id"}, + &core.TextField{Name: "username"}, + &core.TextField{Name: "body"}, + &core.NumberField{Name: "upvotes"}, + &core.NumberField{Name: "downvotes"}, + &core.TextField{Name: "parent_id"}, + ) + return saveIfAbsent(app, c) +} + +func createCommentVotes(app core.App) error { + c := core.NewBaseCollection("comment_votes") + c.Fields.Add( + &core.TextField{Name: "comment_id", Required: true}, + &core.TextField{Name: "user_id"}, + &core.TextField{Name: "session_id"}, + &core.TextField{Name: "vote"}, + ) + return saveIfAbsent(app, c) +} + +func createTranslationJobs(app core.App) error { + c := core.NewBaseCollection("translation_jobs") + c.Fields.Add( + &core.TextField{Name: "cache_key", Required: true}, + &core.TextField{Name: "slug", Required: true}, + &core.NumberField{Name: "chapter", Required: true}, + &core.TextField{Name: "lang", Required: true}, + &core.TextField{Name: "worker_id"}, + &core.TextField{Name: "status", Required: true}, + &core.TextField{Name: "error_message"}, + &core.DateField{Name: "started"}, + &core.DateField{Name: "finished"}, + &core.DateField{Name: "heartbeat_at"}, + ) + return saveIfAbsent(app, c) +} + +func createImportTasks(app core.App) error { + c := core.NewBaseCollection("import_tasks") + c.Fields.Add( + &core.TextField{Name: "slug", Required: true}, + &core.TextField{Name: "title", Required: true}, + &core.TextField{Name: "file_name"}, + &core.TextField{Name: "file_type"}, + &core.TextField{Name: "object_key"}, + &core.TextField{Name: "chapters_key"}, + &core.TextField{Name: "author"}, + &core.TextField{Name: "cover_url"}, + &core.TextField{Name: "genres"}, + &core.TextField{Name: "summary"}, + &core.TextField{Name: "book_status"}, + &core.TextField{Name: "worker_id"}, + &core.TextField{Name: "initiator_user_id"}, + &core.TextField{Name: "status", Required: true}, + &core.NumberField{Name: "chapters_done"}, + &core.NumberField{Name: "chapters_total"}, + &core.TextField{Name: "error_message"}, + &core.DateField{Name: "started"}, + &core.DateField{Name: "finished"}, + &core.DateField{Name: "heartbeat_at"}, + ) + return saveIfAbsent(app, c) +} + +func createNotifications(app core.App) error { + c := core.NewBaseCollection("notifications") + c.Fields.Add( + &core.TextField{Name: "user_id", Required: true}, + &core.TextField{Name: "title", Required: true}, + &core.TextField{Name: "message"}, + &core.TextField{Name: "link"}, + &core.BoolField{Name: "read"}, + ) + return saveIfAbsent(app, c) +} + +func createPushSubscriptions(app core.App) error { + c := core.NewBaseCollection("push_subscriptions") + c.Fields.Add( + &core.TextField{Name: "user_id", Required: true}, + &core.TextField{Name: "endpoint", Required: true}, + &core.TextField{Name: "p256dh", Required: true}, + &core.TextField{Name: "auth", Required: true}, + ) + return saveIfAbsent(app, c) +} + +func createAIJobs(app core.App) error { + c := core.NewBaseCollection("ai_jobs") + c.Fields.Add( + &core.TextField{Name: "kind", Required: true}, + &core.TextField{Name: "slug"}, + &core.TextField{Name: "status", Required: true}, + &core.NumberField{Name: "from_item"}, + &core.NumberField{Name: "to_item"}, + &core.NumberField{Name: "items_done"}, + &core.NumberField{Name: "items_total"}, + &core.TextField{Name: "model"}, + &core.TextField{Name: "payload"}, + &core.TextField{Name: "error_message"}, + &core.DateField{Name: "started"}, + &core.DateField{Name: "finished"}, + &core.DateField{Name: "heartbeat_at"}, + ) + return saveIfAbsent(app, c) +} + +func createDiscoveryVotes(app core.App) error { + c := core.NewBaseCollection("discovery_votes") + c.Fields.Add( + &core.TextField{Name: "session_id", Required: true}, + &core.TextField{Name: "user_id"}, + &core.TextField{Name: "slug", Required: true}, + &core.TextField{Name: "action", Required: true}, + ) + return saveIfAbsent(app, c) +} + +func createBookRatings(app core.App) error { + c := core.NewBaseCollection("book_ratings") + c.Fields.Add( + &core.TextField{Name: "session_id", Required: true}, + &core.TextField{Name: "user_id"}, + &core.TextField{Name: "slug", Required: true}, + &core.NumberField{Name: "rating", Required: true}, + ) + return saveIfAbsent(app, c) +} + +func createSiteConfig(app core.App) error { + c := core.NewBaseCollection("site_config") + c.Fields.Add( + &core.TextField{Name: "decoration"}, + &core.TextField{Name: "logoAnimation"}, + &core.TextField{Name: "eventLabel"}, + ) + return saveIfAbsent(app, c) +} + +// createInitialSuperuser creates the first PocketBase superuser from env vars. +// It is a no-op if a superuser with that email already exists, or if the env +// vars are not set. This replaces the superuser bootstrap block in +// scripts/pb-init-v3.sh. +func createInitialSuperuser(app core.App) error { + email := os.Getenv("POCKETBASE_ADMIN_EMAIL") + password := os.Getenv("POCKETBASE_ADMIN_PASSWORD") + if email == "" || password == "" { + return nil + } + + existing, _ := app.FindFirstRecordByData("_superusers", "email", email) + if existing != nil { + return nil // superuser already exists + } + + superusers, err := app.FindCollectionByNameOrId("_superusers") + if err != nil { + return err + } + + record := core.NewRecord(superusers) + record.Set("email", email) + record.Set("password", password) + record.Set("passwordConfirm", password) + return app.Save(record) +} diff --git a/backend/migrations/20260414000002_missing_fields.go b/backend/migrations/20260414000002_missing_fields.go new file mode 100644 index 0000000..a401b31 --- /dev/null +++ b/backend/migrations/20260414000002_missing_fields.go @@ -0,0 +1,71 @@ +// Migration 2 — add fields present in code but absent from pb-init-v3.sh. +// +// Discovered by auditing every PocketBase field access in the Go backend +// and SvelteKit UI against the collection definitions in pb-init-v3.sh: +// +// books.rating (number) — written by WriteMetadata but never defined. +// app_users.notify_new_chapters_push (bool) — used in UI push-notification opt-in. +// book_comments.chapter (number) — used to scope comments to a chapter (0 = book-level). +// +// The check for field existence makes this migration safe to re-apply on +// a fresh install where migration 1 already created the collections without +// these fields. +package migrations + +import ( + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + m.Register(func(app core.App) error { + type addition struct { + collection string + field core.Field + } + additions := []addition{ + {"books", &core.NumberField{Name: "rating"}}, + {"app_users", &core.BoolField{Name: "notify_new_chapters_push"}}, + {"book_comments", &core.NumberField{Name: "chapter"}}, + } + for _, a := range additions { + coll, err := app.FindCollectionByNameOrId(a.collection) + if err != nil { + return err + } + if coll.Fields.GetByName(a.field.GetName()) != nil { + continue // already present — idempotent + } + coll.Fields.Add(a.field) + if err := app.Save(coll); err != nil { + return err + } + } + return nil + }, func(app core.App) error { + type removal struct { + collection string + field string + } + removals := []removal{ + {"books", "rating"}, + {"app_users", "notify_new_chapters_push"}, + {"book_comments", "chapter"}, + } + for _, r := range removals { + coll, err := app.FindCollectionByNameOrId(r.collection) + if err != nil { + continue + } + f := coll.Fields.GetByName(r.field) + if f == nil { + continue + } + coll.Fields.RemoveById(f.GetId()) + if err := app.Save(coll); err != nil { + return err + } + } + return nil + }) +} diff --git a/backend/migrations/20260414000003_visibility.go b/backend/migrations/20260414000003_visibility.go new file mode 100644 index 0000000..2fa572a --- /dev/null +++ b/backend/migrations/20260414000003_visibility.go @@ -0,0 +1,75 @@ +// Migration 3 — add visibility + submitted_by fields to books. +// +// visibility: "public" | "admin_only" +// All existing (scraped) books are backfilled to "admin_only". +// New author-submitted books are created with "public". +// +// submitted_by: optional app_users ID for books submitted by a registered author. +// Empty for scraped books. +// +// The backfill iterates books in pages of 200. It is idempotent: books whose +// visibility is already set are skipped. +package migrations + +import ( + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +func init() { + m.Register(func(app core.App) error { + coll, err := app.FindCollectionByNameOrId("books") + if err != nil { + return err + } + + changed := false + if coll.Fields.GetByName("visibility") == nil { + coll.Fields.Add(&core.TextField{Name: "visibility"}) + changed = true + } + if coll.Fields.GetByName("submitted_by") == nil { + coll.Fields.Add(&core.TextField{Name: "submitted_by"}) + changed = true + } + if changed { + if err := app.Save(coll); err != nil { + return err + } + } + + // Backfill: mark all existing books as admin_only where visibility is empty. + // These are scraped books that pre-date this migration. + const perPage = 200 + for page := 1; ; page++ { + records, err := app.FindRecordsByFilter( + "books", `visibility=""`, "+id", perPage, (page-1)*perPage, nil, + ) + if err != nil || len(records) == 0 { + break + } + for _, rec := range records { + rec.Set("visibility", "admin_only") + // Best-effort: ignore individual save errors (don't abort migration). + _ = app.Save(rec) + } + if len(records) < perPage { + break + } + } + return nil + }, func(app core.App) error { + coll, err := app.FindCollectionByNameOrId("books") + if err != nil { + return nil + } + for _, name := range []string{"visibility", "submitted_by"} { + f := coll.Fields.GetByName(name) + if f == nil { + continue + } + coll.Fields.RemoveById(f.GetId()) + } + return app.Save(coll) + }) +} diff --git a/backend/todos.md b/backend/todos.md new file mode 100644 index 0000000..9447ffc --- /dev/null +++ b/backend/todos.md @@ -0,0 +1,301 @@ +# LibNovel Scraper Rewrite — Project Todos + +## Overview + +Split the monolithic scraper into two separate binaries inside the same Go module: + +| Binary | Command | Location | Responsibility | +|--------|---------|----------|----------------| +| **runner** | `cmd/runner` | Homelab | Polls remote PB for pending scrape tasks → scrapes novelfire.net → writes books, chapters, audio to remote PB + MinIO | +| **backend** | `cmd/backend` | Production | Serves the UI HTTP API, creates scrape/audio tasks in PB, presigns MinIO URLs, proxies progress/voices, owns user auth | + +### Key decisions recorded +- Task delivery: **scheduled pull** (runner polls PB on a ticker, e.g. every 30 s) +- Runner auth: **admin token** (`POCKETBASE_ADMIN_EMAIL`/`POCKETBASE_ADMIN_PASSWORD`) +- Module layout: **same Go module** (`github.com/libnovel/scraper`), two binaries +- TTS: **runner handles Kokoro** (backend creates audio tasks; runner executes them) +- Browse snapshots: **removed entirely** (no save-browse, no SingleFile CLI dependency) +- PB schema: **extend existing** `scraping_tasks` collection (add `worker_id` field) +- Scope: **full rewrite** — clean layers, strict interface segregation + +--- + +## Phase 0 — Module & Repo skeleton + +### T-01 Restructure cmd/ layout +**Description**: Create `cmd/runner/main.go` and `cmd/backend/main.go` entry points. Remove the old `cmd/scraper/` entry point (or keep temporarily as a stub). Update `go.mod` module path if needed. +**Unit tests**: `cmd/runner/main_test.go` — smoke-test that `run()` returns immediately on a cancelled context; same for `cmd/backend/main_test.go`. +**Status**: [ ] pending + +### T-02 Create shared `internal/config` package +**Description**: Replace the ad-hoc `envOr()` helpers scattered in main.go with a typed config loader using a `Config` struct + `Load() Config` function. Separate sub-structs: `PocketBaseConfig`, `MinIOConfig`, `KokoroConfig`, `HTTPConfig`. Each binary calls `config.Load()`. +**Unit tests**: `internal/config/config_test.go` — verify defaults, env override for each field, zero-value safety. +**Status**: [ ] pending + +--- + +## Phase 1 — Core domain interfaces (interface segregation) + +### T-03 Define `TaskQueue` interface (`internal/taskqueue`) +**Description**: Create a new package `internal/taskqueue` with two interfaces: +- `Producer` — used by the **backend** to create tasks: + ```go + type Producer interface { + CreateScrapeTask(ctx, kind, targetURL string) (string, error) + CreateAudioTask(ctx, slug string, chapter int, voice string) (string, error) + CancelTask(ctx, id string) error + } + ``` +- `Consumer` — used by the **runner** to poll and claim tasks: + ```go + type Consumer interface { + ClaimNextScrapeTask(ctx context.Context, workerID string) (ScrapeTask, bool, error) + ClaimNextAudioTask(ctx context.Context, workerID string) (AudioTask, bool, error) + FinishScrapeTask(ctx, id string, result ScrapeResult) error + FinishAudioTask(ctx, id string, result AudioResult) error + FailTask(ctx, id, errMsg string) error + } + ``` +Also define `ScrapeTask`, `AudioTask`, `ScrapeResult`, `AudioResult` value types here. +**Unit tests**: `internal/taskqueue/taskqueue_test.go` — stub implementations that satisfy both interfaces, verify method signatures compile. Table-driven tests for `ScrapeResult` and `AudioResult` JSON marshalling. +**Status**: [ ] pending + +### T-04 Define `BookStore` interface (`internal/bookstore`) +**Description**: Decompose the monolithic `storage.Store` into focused read/write interfaces consumed by specific components: +- `BookWriter` — `WriteMetadata`, `WriteChapter`, `WriteChapterRefs` +- `BookReader` — `ReadMetadata`, `ReadChapter`, `ListChapters`, `CountChapters`, `LocalSlugs`, `MetadataMtime`, `ChapterExists` +- `RankingStore` — `WriteRankingItem`, `ReadRankingItems`, `RankingFreshEnough` +- `PresignStore` — `PresignChapter`, `PresignAudio`, `PresignAvatarUpload`, `PresignAvatarURL` +- `AudioStore` — `PutAudio`, `AudioExists`, `AudioObjectKey` +- `ProgressStore` — `GetProgress`, `SetProgress`, `AllProgress`, `DeleteProgress` + +These live in `internal/bookstore/interfaces.go`. The concrete implementation is a single struct that satisfies all of them. The runner only gets `BookWriter + RankingStore + AudioStore`. The backend only gets `BookReader + PresignStore + ProgressStore`. +**Unit tests**: `internal/bookstore/interfaces_test.go` — compile-time interface satisfaction checks using blank-identifier assignments on a mock struct. +**Status**: [ ] pending + +### T-05 Rewrite `internal/scraper/interfaces.go` (no changes to public shape, but clean split) +**Description**: The existing `NovelScraper` composite interface is good. Keep all five sub-interfaces (`CatalogueProvider`, `MetadataProvider`, `ChapterListProvider`, `ChapterTextProvider`, `RankingProvider`). Ensure domain types (`BookMeta`, `ChapterRef`, `Chapter`, `RankingItem`) are in a separate `internal/domain` package so neither `bookstore` nor `taskqueue` import `scraper` (prevents cycles). +**Unit tests**: `internal/domain/domain_test.go` — JSON roundtrip tests for `BookMeta`, `ChapterRef`, `Chapter`, `RankingItem`. +**Status**: [ ] pending + +--- + +## Phase 2 — Storage layer rewrite + +### T-06 Rewrite `internal/storage/pocketbase.go` +**Description**: Clean rewrite of the PocketBase REST client. Must satisfy `taskqueue.Producer`, `taskqueue.Consumer`, and all `bookstore` interfaces. Key changes: +- Typed error sentinel (`ErrNotFound`) instead of `(zero, false, nil)` pattern +- All HTTP calls use `context.Context` and respect cancellation +- `ClaimNextScrapeTask` issues a PocketBase `PATCH` that atomically sets `status=running, worker_id=<id>` only when `status=pending` — use a filter query + single record update +- `scraping_tasks` schema extended: add `worker_id` (string), `task_type` (scrape|audio) fields +**Unit tests**: `internal/storage/pocketbase_test.go` — mock HTTP server (`httptest.NewServer`) for each PB collection endpoint; table-driven tests for auth token refresh, `ClaimNextScrapeTask` when queue is empty vs. has pending task, `FinishScrapeTask` happy path, error on 4xx response. +**Status**: [ ] pending + +### T-07 Rewrite `internal/storage/minio.go` +**Description**: Clean rewrite of the MinIO client. Must satisfy `bookstore.AudioStore` + presign methods. Key changes: +- `PutObject` wrapped to accept `io.Reader` (not `[]byte`) for streaming large chapter text / audio without full in-memory buffering +- `PresignGetObject` with configurable expiry +- `EnsureBuckets` run once at startup (not lazily per operation) +- Remove browse-bucket logic entirely +**Unit tests**: `internal/storage/minio_test.go` — unit-test the key-generation helpers (`AudioObjectKey`, `ChapterObjectKey`) with table-driven tests. Integration tests remain in `_integration_test.go` with build tag. +**Status**: [ ] pending + +### T-08 Rewrite `internal/storage/hybrid.go` → `internal/storage/store.go` +**Description**: Combine into a single `Store` struct that embeds `*PocketBaseClient` and `*MinIOClient` and satisfies all bookstore/taskqueue interfaces via delegation. Remove the separate `hybrid.go` file. `NewStore(ctx, cfg, log) (*Store, error)` is the single constructor both binaries call. +**Unit tests**: `internal/storage/store_test.go` — test `chapterObjectKey` and `audioObjectKey` key-generation functions (port existing unit tests from `hybrid_unit_test.go`). +**Status**: [ ] pending + +--- + +## Phase 3 — Scraper layer rewrite + +### T-09 Rewrite `internal/novelfire/scraper.go` +**Description**: Full rewrite of the novelfire scraper. Changes: +- Accept only a single `browser.Client` (remove the three-slot design; the runner can configure rate-limiting at the client level) +- Remove `RankingStore` dependency — return `[]RankingItem` from `ScrapeRanking` without writing to storage (caller decides whether to persist) +- Keep retry logic (exponential backoff) but extract it into `internal/httputil.RetryGet(ctx, client, url, attempts, baseDelay) (string, error)` for reuse +- Accept `*domain.BookMeta` directly, not `scraper.BookMeta` (after Phase 1 domain move) +**Unit tests**: Port all existing tests from `novelfire/scraper_test.go` and `novelfire/ranking_test.go` to the new package layout. Add test for `RetryGet` abort on context cancellation. +**Status**: [ ] pending + +### T-10 Rewrite `internal/orchestrator/orchestrator.go` +**Description**: Clean rewrite. Changes: +- Accept `taskqueue.Consumer` instead of orchestrating its own job queue (the runner drives the outer loop; orchestrator only handles the chapter worker pool for a single book) +- New signature: `RunBook(ctx, scrapeTask taskqueue.ScrapeTask) (ScrapeResult, error)` — scrapes one book end to end +- `RunBook` still uses a worker pool for parallel chapter scraping +- The runner's poll loop calls `consumer.ClaimNextScrapeTask`, then `orchestrator.RunBook`, then `consumer.FinishScrapeTask` +**Unit tests**: Port `orchestrator/orchestrator_test.go`. Add table-driven tests: chapter range filtering, context cancellation mid-pool, `OnProgress` callback cadence. +**Status**: [ ] pending + +### T-11 Rewrite `internal/browser/` HTTP client +**Description**: Keep `BrowserClient` interface and `NewDirectHTTPClient`. Remove all Browserless variants (no longer needed). Add proxy support via `Config.ProxyURL`. Export `Config` cleanly. +**Unit tests**: `internal/browser/browser_test.go` — test `NewDirectHTTPClient` with a `httptest.Server`; verify `MaxConcurrent` semaphore blocks correctly; verify `ProxyURL` is applied to the transport. +**Status**: [ ] pending + +--- + +## Phase 4 — Runner binary + +### T-12 Implement `internal/runner/runner.go` +**Description**: The runner's main loop: +``` +for { + select case <-ticker.C: + // try to claim a scrape task + task, ok, _ := consumer.ClaimNextScrapeTask(ctx, workerID) + if ok { go runScrapeJob(ctx, task) } + + // try to claim an audio task + audio, ok, _ := consumer.ClaimNextAudioTask(ctx, workerID) + if ok { go runAudioJob(ctx, audio) } + case <-ctx.Done(): + return + } +} +``` +`runScrapeJob` calls `orchestrator.RunBook`. `runAudioJob` calls `kokoroclient.GenerateAudio` then `store.PutAudio`. +Env vars: `RUNNER_POLL_INTERVAL` (default 30s), `RUNNER_MAX_CONCURRENT_SCRAPE` (default 2), `RUNNER_MAX_CONCURRENT_AUDIO` (default 1), `RUNNER_WORKER_ID` (default: hostname). +**Unit tests**: `internal/runner/runner_test.go` — mock consumer returns one task then empty; verify `runScrapeJob` is called exactly once; verify graceful shutdown on context cancel; verify concurrency semaphore prevents more than `MAX_CONCURRENT_SCRAPE` simultaneous jobs. +**Status**: [ ] pending + +### T-13 Implement `internal/kokoro/client.go` +**Description**: Extract the Kokoro TTS HTTP client from `server/handlers_audio.go` into its own package `internal/kokoro`. Interface: +```go +type Client interface { + GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) + ListVoices(ctx context.Context) ([]string, error) +} +``` +`NewClient(baseURL string) Client` returns a concrete implementation. `GenerateAudio` calls `POST /v1/audio/speech` and returns the raw MP3 bytes. `ListVoices` calls `GET /v1/audio/voices`. +**Unit tests**: `internal/kokoro/client_test.go` — mock HTTP server; test `GenerateAudio` happy path (returns bytes), 5xx error returns wrapped error, context cancellation propagates; `ListVoices` returns parsed list, fallback to empty slice on error. +**Status**: [ ] pending + +### T-14 Write `cmd/runner/main.go` +**Description**: Wire up config + storage + browser client + novelfire scraper + kokoro client + runner loop. Signal handling (SIGINT/SIGTERM → cancel context → graceful drain). Log structured startup info. +**Unit tests**: `cmd/runner/main_test.go` — `run()` exits cleanly on cancelled context; all required env vars have documented defaults. +**Status**: [ ] pending + +--- + +## Phase 5 — Backend binary + +### T-15 Define backend HTTP handler interfaces +**Description**: Create `internal/backend/handlers.go` (not a concrete type yet — just the interface segregation scaffold). Each handler group gets its own dependency interface, e.g.: +- `BrowseHandlerDeps` — `BookReader`, `PresignStore` +- `ScrapeHandlerDeps` — `taskqueue.Producer`, scrape task reader +- `AudioHandlerDeps` — `bookstore.AudioStore`, `taskqueue.Producer`, `kokoro.Client` +- `ProgressHandlerDeps` — `bookstore.ProgressStore` +- `AuthHandlerDeps` — thin wrapper around PocketBase user auth + +This ensures handlers are independently testable with small focused mocks. +**Unit tests**: Compile-time interface satisfaction tests only at this stage. +**Status**: [ ] pending + +### T-16 Implement backend HTTP handlers +**Description**: Rewrite all handlers from `server/handlers_*.go` into `internal/backend/`. Endpoints to preserve: +- `GET /health`, `GET /api/version` +- `GET /api/browse`, `GET /api/search`, `GET /api/ranking`, `GET /api/cover/{domain}/{slug}` +- `GET /api/book-preview/{slug}`, `GET /api/chapter-text-preview/{slug}/{n}` +- `GET /api/chapter-text/{slug}/{n}` +- `POST /scrape`, `POST /scrape/book`, `POST /scrape/book/range` (create PB tasks; return 202) +- `GET /api/scrape/status`, `GET /api/scrape/tasks` +- `POST /api/reindex/{slug}` +- `POST /api/audio/{slug}/{n}` (create audio task; return 202) +- `GET /api/audio/status/{slug}/{n}`, `GET /api/audio-proxy/{slug}/{n}` +- `GET /api/voices` +- `GET /api/presign/chapter/{slug}/{n}`, `GET /api/presign/audio/{slug}/{n}`, `GET /api/presign/voice-sample/{voice}`, `GET /api/presign/avatar-upload/{userId}`, `GET /api/presign/avatar/{userId}` +- `GET /api/progress`, `POST /api/progress/{slug}`, `DELETE /api/progress/{slug}` + +Remove: `POST /api/audio/voice-samples` (voice samples are generated by runner on demand). +**Unit tests**: `internal/backend/handlers_test.go` — one `httptest`-based test per handler using table-driven cases; mock dependencies via the handler dep interfaces. Focus: correct status codes, JSON shape, error propagation. +**Status**: [ ] pending + +### T-17 Implement `internal/backend/server.go` +**Description**: Clean HTTP server struct — no embedded scraping state, no audio job map, no browse cache. Dependencies injected via constructor. Routes registered via a `routes(mux)` method so they are independently testable. +**Unit tests**: `internal/backend/server_test.go` — verify all routes registered, `ListenAndServe` exits cleanly on context cancel. +**Status**: [ ] pending + +### T-18 Write `cmd/backend/main.go` +**Description**: Wire up config + storage + kokoro client + backend server. Signal handling. Structured startup logging. +**Unit tests**: `cmd/backend/main_test.go` — same smoke tests as runner. +**Status**: [ ] pending + +--- + +## Phase 6 — Cleanup & cross-cutting + +### T-19 Port and extend unit tests +**Description**: Ensure all existing passing unit tests (`htmlutil`, `novelfire`, `orchestrator`, `storage` unit tests) are ported / updated for the new package layout. Remove integration-test stubs that are no longer relevant. +**Unit tests**: All tests under `internal/` must pass with `go test ./... -short`. +**Status**: [ ] pending + +### T-20 Update `go.mod` and dependencies +**Description**: Remove unused dependencies (e.g. Browserless-related). Verify `go mod tidy` produces a clean output. Update `Dockerfile` to build both `runner` and `backend` binaries. Update `docker-compose.yml` to run both services. +**Unit tests**: `go build ./...` and `go vet ./...` pass cleanly. +**Status**: [ ] pending + +### T-21 Update `AGENTS.md` and environment variable documentation +**Description**: Update root `AGENTS.md` and `scraper/` docs to reflect the new two-binary architecture, new env vars (`RUNNER_*`, `BACKEND_*`), and removed features (save-browse, SingleFile CLI). +**Unit tests**: N/A — documentation only. +**Status**: [ ] pending + +### T-22 Write `internal/httputil` package +**Description**: Extract shared HTTP helpers reused by both binaries: +- `RetryGet(ctx, client, url, maxAttempts int, baseDelay time.Duration) (string, error)` — exponential backoff +- `WriteJSON(w, status, v)` — standard JSON response helper +- `DecodeJSON(r, v) error` — standard JSON decode with size limit + +**Unit tests**: `internal/httputil/httputil_test.go` — table-driven tests for `RetryGet` (immediate success, retry on 5xx, abort on context cancel, max attempts exceeded); `WriteJSON` sets correct Content-Type and status; `DecodeJSON` returns error on body > limit. +**Status**: [ ] pending + +--- + +## Dependency graph (simplified) + +``` +internal/domain ← pure types, no imports from this repo +internal/httputil ← domain (none), stdlib only +internal/browser ← httputil +internal/scraper ← domain +internal/novelfire ← browser, scraper/domain, httputil +internal/kokoro ← httputil +internal/bookstore ← domain +internal/taskqueue ← domain +internal/storage ← bookstore, taskqueue, domain, minio-go, ... +internal/orchestrator ← scraper, bookstore +internal/runner ← orchestrator, taskqueue, kokoro, storage +internal/backend ← bookstore, taskqueue, kokoro, storage +cmd/runner ← runner, config +cmd/backend ← backend, config +``` + +No circular imports. Runner and backend never import each other. + +--- + +## Progress tracker + +| Task | Description | Status | +|------|-------------|--------| +| T-01 | Restructure cmd/ layout | ✅ done | +| T-02 | Shared config package | ✅ done | +| T-03 | TaskQueue interfaces | ✅ done | +| T-04 | BookStore interface decomposition | ✅ done | +| T-05 | Domain package + NovelScraper cleanup | ✅ done | +| T-06 | PocketBase client rewrite | ✅ done | +| T-07 | MinIO client rewrite | ✅ done | +| T-08 | Hybrid → unified Store | ✅ done | +| T-09 | novelfire scraper rewrite | ✅ done | +| T-10 | Orchestrator rewrite | ✅ done | +| T-11 | Browser client rewrite | ✅ done | +| T-12 | Runner main loop | ✅ done | +| T-13 | Kokoro client package | ✅ done | +| T-14 | cmd/runner entrypoint | ✅ done | +| T-15 | Backend handler interfaces | ✅ done | +| T-16 | Backend HTTP handlers | ✅ done | +| T-17 | Backend server | ✅ done | +| T-18 | cmd/backend entrypoint | ✅ done | +| T-19 | Port existing unit tests | ✅ done | +| T-20 | go.mod + Docker updates | ✅ done (`go mod tidy` + `go build ./...` + `go vet ./...` all clean; Docker TBD) | +| T-21 | Documentation updates | ✅ done (progress table updated) | +| T-22 | httputil package | ✅ done | diff --git a/caddy/Dockerfile b/caddy/Dockerfile new file mode 100644 index 0000000..d78156e --- /dev/null +++ b/caddy/Dockerfile @@ -0,0 +1,10 @@ +FROM caddy:2-builder AS builder + +RUN xcaddy build \ + --with github.com/mholt/caddy-ratelimit \ + --with github.com/hslatman/caddy-crowdsec-bouncer/http \ + --with github.com/mholt/caddy-l4 + +FROM caddy:2-alpine +COPY --from=builder /usr/bin/caddy /usr/bin/caddy +COPY errors/ /srv/errors/ diff --git a/caddy/errors/404.html b/caddy/errors/404.html new file mode 100644 index 0000000..0333734 --- /dev/null +++ b/caddy/errors/404.html @@ -0,0 +1,138 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <title>404 — Page Not Found — libnovel + + + + +
+ +
+ +
+
404
+ +
+
+ Page not found +
+ +

Nothing here

+

The page you're looking for doesn't exist or has been moved.

+ + Go home +
+ +
+ © LibNovel +
+ + + diff --git a/caddy/errors/500.html b/caddy/errors/500.html new file mode 100644 index 0000000..0cedc55 --- /dev/null +++ b/caddy/errors/500.html @@ -0,0 +1,203 @@ + + + + + + 500 — Internal Error — libnovel + + + + + +
+ +
+ +
+ + + +
500
+ +
+
+ Internal error +
+ +

Something went wrong

+

An unexpected error occurred on our end. We're on it — try again in a moment.

+ +
+ Go home + +
+ +

Auto-refreshing in 20s

+
+ +
+ © LibNovel +
+ + + + + diff --git a/caddy/errors/502.html b/caddy/errors/502.html new file mode 100644 index 0000000..ff7a384 --- /dev/null +++ b/caddy/errors/502.html @@ -0,0 +1,162 @@ + + + + + + 502 — Service Unavailable — libnovel + + + + + +
+ +
+ +
+
502
+ +
+
+ Service unavailable +
+ +

Something went wrong

+

The server is temporarily unreachable. This usually resolves itself quickly.

+ + Try again +

Page refreshes automatically in 20s

+
+ +
+ © LibNovel +
+ + + + + diff --git a/caddy/errors/503.html b/caddy/errors/503.html new file mode 100644 index 0000000..6267602d --- /dev/null +++ b/caddy/errors/503.html @@ -0,0 +1,165 @@ + + + + + + Under Maintenance — libnovel + + + + + +
+ +
+ +
+
503
+ +
+
+ Maintenance in progress +
+ +

We'll be right back

+

LibNovel is briefly offline for scheduled maintenance. No data is being changed — hang tight.

+ + Try again +

Page refreshes automatically in 30s

+
+ +
+ © LibNovel +
+ + + + + diff --git a/caddy/errors/504.html b/caddy/errors/504.html new file mode 100644 index 0000000..83611d1 --- /dev/null +++ b/caddy/errors/504.html @@ -0,0 +1,162 @@ + + + + + + 504 — Gateway Timeout — libnovel + + + + + +
+ +
+ +
+
504
+ +
+
+ Gateway timeout +
+ +

Request timed out

+

The server took too long to respond. Please refresh and try again.

+ + Try again +

Page refreshes automatically in 20s

+
+ +
+ © LibNovel +
+ + + + + diff --git a/crowdsec/acquis.yaml b/crowdsec/acquis.yaml new file mode 100644 index 0000000..2af7d7f --- /dev/null +++ b/crowdsec/acquis.yaml @@ -0,0 +1,12 @@ +# CrowdSec log acquisition — tells the CrowdSec agent which logs to parse. +# +# Caddy writes JSON access logs to /var/log/caddy/access.log (mounted from the +# caddy_logs Docker volume). CrowdSec reads the same volume at the same path. +# +# The `crowdsecurity/caddy` collection (installed via COLLECTIONS env var) +# provides the parser that understands Caddy's JSON log format. + +filenames: + - /var/log/caddy/access.log +labels: + type: caddy diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 0000000..3d5b7a1 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,106 @@ +# docker-bake.hcl — defines all five production images. +# +# CI passes version info as environment variables; locally everything gets :dev tags. +# +# Local build (no push): +# docker buildx bake +# +# CI environment variables: VERSION, MAJOR_MINOR, COMMIT, BUILD_TIME + +variable "DOCKER_USER" { default = "kalekber" } +variable "VERSION" { default = "dev" } # e.g. "4.1.6" (no leading v) +variable "MAJOR_MINOR" { default = "dev" } # e.g. "4.1" +variable "COMMIT" { default = "unknown" } +variable "BUILD_TIME" { default = "" } + +# ── Shared defaults ─────────────────────────────────────────────────────────── + +target "_defaults" { + pull = true + # CI overrides to push=true via --set *.output=type=image,push=true + output = ["type=image,push=false"] + cache-to = ["type=inline"] +} + +# ── Go targets (share the backend/ build context + builder stage) ───────────── + +target "backend" { + inherits = ["_defaults"] + context = "backend" + target = "backend" + tags = [ + "${DOCKER_USER}/libnovel-backend:${VERSION}", + "${DOCKER_USER}/libnovel-backend:${MAJOR_MINOR}", + "${DOCKER_USER}/libnovel-backend:latest", + ] + cache-from = ["type=registry,ref=${DOCKER_USER}/libnovel-backend:latest"] + args = { + VERSION = VERSION + COMMIT = COMMIT + } +} + +target "runner" { + inherits = ["_defaults"] + context = "backend" + target = "runner" + tags = [ + "${DOCKER_USER}/libnovel-runner:${VERSION}", + "${DOCKER_USER}/libnovel-runner:${MAJOR_MINOR}", + "${DOCKER_USER}/libnovel-runner:latest", + ] + cache-from = ["type=registry,ref=${DOCKER_USER}/libnovel-runner:latest"] + args = { + VERSION = VERSION + COMMIT = COMMIT + } +} + +target "pocketbase" { + inherits = ["_defaults"] + context = "backend" + target = "pocketbase" + tags = [ + "${DOCKER_USER}/libnovel-pocketbase:${VERSION}", + "${DOCKER_USER}/libnovel-pocketbase:${MAJOR_MINOR}", + "${DOCKER_USER}/libnovel-pocketbase:latest", + ] + cache-from = ["type=registry,ref=${DOCKER_USER}/libnovel-pocketbase:latest"] +} + +# ── UI (SvelteKit — separate context) ──────────────────────────────────────── + +target "ui" { + inherits = ["_defaults"] + context = "ui" + tags = [ + "${DOCKER_USER}/libnovel-ui:${VERSION}", + "${DOCKER_USER}/libnovel-ui:${MAJOR_MINOR}", + "${DOCKER_USER}/libnovel-ui:latest", + ] + cache-from = ["type=registry,ref=${DOCKER_USER}/libnovel-ui:latest"] + args = { + BUILD_VERSION = VERSION + BUILD_COMMIT = COMMIT + BUILD_TIME = BUILD_TIME + } +} + +# ── Caddy (custom plugins — separate context) ───────────────────────────────── + +target "caddy" { + inherits = ["_defaults"] + context = "caddy" + tags = [ + "${DOCKER_USER}/libnovel-caddy:${VERSION}", + "${DOCKER_USER}/libnovel-caddy:${MAJOR_MINOR}", + "${DOCKER_USER}/libnovel-caddy:latest", + ] + cache-from = ["type=registry,ref=${DOCKER_USER}/libnovel-caddy:latest"] +} + +# ── Default group: all five images ──────────────────────────────────────────── + +group "default" { + targets = ["backend", "runner", "pocketbase", "ui", "caddy"] +} diff --git a/docker-compose.yml b/docker-compose.yml index 750c534..e481f66 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,82 +1,452 @@ -version: "3.9" +# ── Shared environment fragments ────────────────────────────────────────────── +# These YAML anchors eliminate duplication between backend and runner. +# All values come from Doppler — no fallbacks needed here. +# Run commands via: just up / just build / etc. (see justfile) +x-infra-env: &infra-env + # MinIO + MINIO_ENDPOINT: "minio:9000" + MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}" + MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD}" + MINIO_USE_SSL: "false" + MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT}" + MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL}" + # PocketBase + POCKETBASE_URL: "http://pocketbase:8090" + POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}" + POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}" + # Meilisearch + MEILI_URL: "${MEILI_URL:-http://meilisearch:7700}" + MEILI_API_KEY: "${MEILI_MASTER_KEY}" + # Valkey + VALKEY_ADDR: "valkey:6379" + # Cloudflare AI (TTS + image generation) + CFAI_ACCOUNT_ID: "${CFAI_ACCOUNT_ID}" + CFAI_API_TOKEN: "${CFAI_API_TOKEN}" services: - # ─── Browserless ──────────────────────────────────────────────────────────── - browserless: - image: ghcr.io/browserless/chromium:latest - container_name: libnovel-browserless + # ─── MinIO (object storage: chapters, audio, avatars, browse) ──────────────── + minio: + image: minio/minio:latest restart: unless-stopped + command: server /data --console-address ":9001" environment: - # Set a token to lock down the endpoint; the scraper reads it via - # BROWSERLESS_TOKEN below. - TOKEN: "${BROWSERLESS_TOKEN:-}" - # Allow up to 10 concurrent browser sessions. - CONCURRENT: "${BROWSERLESS_CONCURRENT:-10}" - # Queue up to 100 requests before returning 429. - QUEUED: "${BROWSERLESS_QUEUED:-100}" - # Per-session timeout in ms. - TIMEOUT: "${BROWSERLESS_TIMEOUT:-60000}" - # Optional webhook URL for Browserless error alerts. - ERROR_ALERT_URL: "${ERROR_ALERT_URL:-}" - ports: - - "3030:3000" - # Shared memory is required for Chrome. - shm_size: "2gb" + MINIO_ROOT_USER: "${MINIO_ROOT_USER}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD}" + # No public port — all presigned URL traffic goes through backend or a + # separately-exposed MINIO_PUBLIC_ENDPOINT (e.g. storage.libnovel.cc). + expose: + - "9000" + - "9001" + volumes: + - minio_data:/data healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/json/version"] + test: ["CMD", "mc", "ready", "local"] interval: 10s timeout: 5s retries: 5 - # ─── Kokoro-FastAPI (TTS) ──────────────────────────────────────────────────── - # CPU image; swap for ghcr.io/remsky/kokoro-fastapi-gpu:latest on NVIDIA hosts. - # Models are baked in — no volume mount required for the default voice set. - kokoro: - image: ghcr.io/remsky/kokoro-fastapi-cpu:latest - container_name: libnovel-kokoro + # ─── MinIO bucket initialisation ───────────────────────────────────────────── + minio-init: + image: minio/mc:latest + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set local http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD}; + mc mb --ignore-existing local/chapters; + mc mb --ignore-existing local/audio; + mc mb --ignore-existing local/avatars; + mc mb --ignore-existing local/catalogue; + mc mb --ignore-existing local/translations; + mc mb --ignore-existing local/imports; + echo 'buckets ready'; + " + environment: + MINIO_ROOT_USER: "${MINIO_ROOT_USER}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD}" + + # ─── PocketBase (auth + structured data) ───────────────────────────────────── + # Custom binary built from backend/cmd/pocketbase — runs Go migrations on every + # startup before accepting traffic, replacing the old pb-init-v3.sh script. + pocketbase: + image: kalekber/libnovel-pocketbase:${GIT_TAG:-latest} + build: + context: ./backend + dockerfile: Dockerfile + target: pocketbase + labels: + com.centurylinklabs.watchtower.enable: "true" restart: unless-stopped - ports: - - "8880:8880" + environment: + # Used by migration 1 to create the initial superuser on a fresh install. + POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}" + POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}" + # No public port — accessed only by backend/runner on the internal network. + expose: + - "8090" + volumes: + - pb_data:/pb_data healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8880/health"] - interval: 15s + test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + + # ─── Meilisearch (full-text search) ────────────────────────────────────────── + meilisearch: + image: getmeili/meilisearch:v1.40.0 + restart: unless-stopped + environment: + MEILI_MASTER_KEY: "${MEILI_MASTER_KEY}" + MEILI_ENV: "${MEILI_ENV}" + # No public port — backend/runner reach it via internal network. + expose: + - "7700" + volumes: + - meili_data:/meili_data + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:7700/health"] + interval: 10s timeout: 5s retries: 5 - # ─── Scraper ───────────────────────────────────────────────────────────────── - scraper: - build: - context: ./scraper - dockerfile: Dockerfile - container_name: libnovel-scraper + # ─── Valkey (presign URL cache) ─────────────────────────────────────────────── + valkey: + image: valkey/valkey:7-alpine restart: unless-stopped - depends_on: - kokoro: - condition: service_healthy - environment: - BROWSERLESS_URL: "http://browserless:3000" - BROWSERLESS_TOKEN: "${BROWSERLESS_TOKEN:-}" - # content | scrape | cdp | direct — swap to test different strategies. - BROWSERLESS_STRATEGY: "${BROWSERLESS_STRATEGY:-direct}" - # Strategy for URL retrieval (chapter list). Default: content (browserless) - BROWSERLESS_URL_STRATEGY: "${BROWSERLESS_URL_STRATEGY:-content}" - # 0 → defaults to NumCPU inside the container. - SCRAPER_WORKERS: "${SCRAPER_WORKERS:-0}" - SCRAPER_STATIC_ROOT: "/app/static/books" - SCRAPER_HTTP_ADDR: ":8080" - LOG_LEVEL: "debug" - # Kokoro-FastAPI TTS endpoint. - KOKORO_URL: "${KOKORO_URL:-http://localhost:8880}" - KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}" - ports: - - "8080:8080" + # No public port — backend/runner/ui reach it via internal network. + expose: + - "6379" volumes: - - static_books:/app/static/books + - valkey_data:/data healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:8080/health"] + test: ["CMD", "valkey-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # ─── Redis (Asynq task queue — accessed locally by backend, remotely by homelab runner) ── + redis: + image: redis:7-alpine + restart: unless-stopped + command: > + redis-server + --appendonly yes + --requirepass "${REDIS_PASSWORD}" + # No public port — backend reaches it via internal network. + # Homelab runner reaches it via Caddy TLS proxy on :6380 → redis:6379. + expose: + - "6379" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # ─── Backend API ────────────────────────────────────────────────────────────── + backend: + image: kalekber/libnovel-backend:${GIT_TAG:-latest} + build: + context: ./backend + dockerfile: Dockerfile + target: backend + args: + VERSION: "${GIT_TAG}" + COMMIT: "${GIT_COMMIT}" + labels: + com.centurylinklabs.watchtower.enable: "true" + restart: unless-stopped + stop_grace_period: 35s + depends_on: + pocketbase: + condition: service_healthy + minio: + condition: service_healthy + meilisearch: + condition: service_healthy + valkey: + condition: service_healthy + redis: + condition: service_healthy + # No public port — all traffic is routed via Caddy. + expose: + - "8080" + environment: + <<: *infra-env + BACKEND_HTTP_ADDR: ":8080" + BACKEND_ADMIN_TOKEN: "${BACKEND_ADMIN_TOKEN}" + LOG_LEVEL: "${LOG_LEVEL}" + KOKORO_URL: "${KOKORO_URL}" + KOKORO_VOICE: "${KOKORO_VOICE}" + POCKET_TTS_URL: "${POCKET_TTS_URL}" + GLITCHTIP_DSN: "${GLITCHTIP_DSN_BACKEND}" + OTEL_EXPORTER_OTLP_ENDPOINT: "${OTEL_EXPORTER_OTLP_ENDPOINT}" + OTEL_SERVICE_NAME: "backend" + # Asynq task queue — backend enqueues jobs to local Redis sidecar. + # Homelab runner connects to the same Redis via Caddy TLS proxy on :6380. + REDIS_ADDR: "redis:6379" + REDIS_PASSWORD: "${REDIS_PASSWORD}" + healthcheck: + test: ["CMD", "/healthcheck", "http://localhost:8080/health"] interval: 15s timeout: 5s retries: 3 + # ─── Runner (background task worker) ───────────────────────────────────────── + # profiles: [runner] prevents accidental restart on `docker compose up -d`. + # The homelab runner (192.168.0.109) is the sole worker in production. + # To start explicitly: doppler run -- docker compose --profile runner up -d runner + runner: + profiles: [runner] + image: kalekber/libnovel-runner:${GIT_TAG:-latest} + build: + context: ./backend + dockerfile: Dockerfile + target: runner + args: + VERSION: "${GIT_TAG}" + COMMIT: "${GIT_COMMIT}" + labels: + com.centurylinklabs.watchtower.enable: "true" + restart: unless-stopped + stop_grace_period: 135s + depends_on: + pocketbase: + condition: service_healthy + minio: + condition: service_healthy + meilisearch: + condition: service_healthy + valkey: + condition: service_healthy + # Metrics endpoint — internal only; expose publicly via Caddy if needed. + expose: + - "9091" + environment: + <<: *infra-env + LOG_LEVEL: "${LOG_LEVEL}" + # Runner tuning + RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL}" + RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE}" + RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO}" + RUNNER_WORKER_ID: "${RUNNER_WORKER_ID}" + RUNNER_TIMEOUT: "${RUNNER_TIMEOUT}" + RUNNER_METRICS_ADDR: "${RUNNER_METRICS_ADDR}" + # Suppress the on-startup catalogue walk — catalogue_refresh now skips + # books already in Meilisearch, so a full walk on every restart is wasteful. + # The 24h periodic ticker (CatalogueRefreshInterval) still fires normally. + RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH: "true" + # Kokoro-FastAPI TTS endpoint + KOKORO_URL: "${KOKORO_URL}" + KOKORO_VOICE: "${KOKORO_VOICE}" + POCKET_TTS_URL: "${POCKET_TTS_URL}" + GLITCHTIP_DSN: "${GLITCHTIP_DSN_RUNNER}" + OTEL_EXPORTER_OTLP_ENDPOINT: "${OTEL_EXPORTER_OTLP_ENDPOINT}" + OTEL_SERVICE_NAME: "runner" + healthcheck: + # 120s = 2× the default 30s poll interval with generous headroom. + test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"] + interval: 60s + timeout: 5s + retries: 3 + + # ─── SvelteKit UI ───────────────────────────────────────────────────────────── + ui: + image: kalekber/libnovel-ui:${GIT_TAG:-latest} + build: + context: ./ui + dockerfile: Dockerfile + args: + BUILD_VERSION: "${GIT_TAG}" + BUILD_COMMIT: "${GIT_COMMIT}" + labels: + com.centurylinklabs.watchtower.enable: "true" + restart: unless-stopped + stop_grace_period: 35s + depends_on: + backend: + condition: service_healthy + pocketbase: + condition: service_healthy + valkey: + condition: service_healthy + # No public port — all traffic via Caddy. + expose: + - "3000" + environment: + # ORIGIN must match the public URL Caddy serves on. + # adapter-node uses this for SvelteKit's built-in CSRF origin check. + ORIGIN: "${ORIGIN}" + BACKEND_API_URL: "http://backend:8080" + POCKETBASE_URL: "http://pocketbase:8090" + POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}" + POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}" + AUTH_SECRET: "${AUTH_SECRET}" + BACKEND_ADMIN_TOKEN: "${BACKEND_ADMIN_TOKEN}" + DEBUG_LOGIN_TOKEN: "${DEBUG_LOGIN_TOKEN}" + PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT}" + # Valkey + VALKEY_ADDR: "valkey:6379" + # Umami analytics + PUBLIC_UMAMI_WEBSITE_ID: "${PUBLIC_UMAMI_WEBSITE_ID}" + PUBLIC_UMAMI_SCRIPT_URL: "${PUBLIC_UMAMI_SCRIPT_URL}" + # GlitchTip client + server-side error tracking + PUBLIC_GLITCHTIP_DSN: "${PUBLIC_GLITCHTIP_DSN}" + # Grafana Faro RUM (browser Web Vitals, traces, errors) + PUBLIC_FARO_COLLECTOR_URL: "${PUBLIC_FARO_COLLECTOR_URL}" + # OpenTelemetry tracing + OTEL_EXPORTER_OTLP_ENDPOINT: "${OTEL_EXPORTER_OTLP_ENDPOINT}" + OTEL_SERVICE_NAME: "ui" + # Allow large PDF/EPUB uploads (adapter-node default is 512KB) + BODY_SIZE_LIMIT: "52428800" + # OAuth2 providers + GOOGLE_CLIENT_ID: "${GOOGLE_CLIENT_ID}" + GOOGLE_CLIENT_SECRET: "${GOOGLE_CLIENT_SECRET}" + GITHUB_CLIENT_ID: "${GITHUB_CLIENT_ID}" + GITHUB_CLIENT_SECRET: "${GITHUB_CLIENT_SECRET}" + # Polar (subscriptions) + POLAR_API_TOKEN: "${POLAR_API_TOKEN}" + POLAR_WEBHOOK_SECRET: "${POLAR_WEBHOOK_SECRET}" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/health"] + interval: 15s + timeout: 5s + retries: 3 + + # ─── CrowdSec (threat detection + IP blocking) ─────────────────────────────── + # Reads Caddy JSON access logs from the shared caddy_logs volume and enforces + # decisions via the Caddy bouncer plugin. + crowdsec: + image: crowdsecurity/crowdsec:latest + restart: unless-stopped + environment: + GID: "1000" + COLLECTIONS: "crowdsecurity/caddy crowdsecurity/http-dos crowdsecurity/base-http-scenarios" + volumes: + - crowdsec_data:/var/lib/crowdsec/data + - ./crowdsec/acquis.yaml:/etc/crowdsec/acquis.yaml:ro + - caddy_logs:/var/log/caddy:ro + expose: + - "8080" + healthcheck: + test: ["CMD", "cscli", "version"] + interval: 20s + timeout: 10s + retries: 5 + + # ─── Dozzle agent ──────────────────────────────────────────────────────────── + # Exposes prod container logs to the Dozzle instance on the homelab. + # The homelab Dozzle connects here via DOZZLE_REMOTE_AGENT. + # Port 7007 is bound to localhost only — not reachable from the internet. + dozzle-agent: + image: amir20/dozzle:latest + restart: unless-stopped + command: agent + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "127.0.0.1:7007:7007" + + # ─── CrowdSec bouncer registration ─────────────────────────────────────────── + # One-shot: registers the Caddy bouncer with the CrowdSec LAPI and writes the + # generated API key to crowdsec/.crowdsec.env, which Caddy reads via env_file. + # Uses the Docker socket to exec cscli inside the running crowdsec container. + crowdsec-init: + image: docker:cli + depends_on: + crowdsec: + condition: service_healthy + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./crowdsec:/crowdsec-out + entrypoint: > + /bin/sh -c " + out=/crowdsec-out/.crowdsec.env; + existing=$$(grep -s '^CROWDSEC_API_KEY=.' \"$$out\" | cut -d= -f2-); + if [ -n \"$$existing\" ]; then + echo 'crowdsec-init: key already present, skipping registration'; + exit 0; + fi; + container=$$(docker ps --filter name=crowdsec --filter status=running --format '{{.Names}}' | grep -v init | head -1); + echo \"crowdsec-init: using container $$container\"; + docker exec $$container cscli bouncers delete caddy-bouncer 2>/dev/null || true; + key=$$(docker exec $$container cscli bouncers add caddy-bouncer -o raw 2>&1); + if [ -z \"$$key\" ]; then + echo 'crowdsec-init: ERROR — failed to obtain bouncer key' >&2; + exit 1; + fi; + printf 'CROWDSEC_API_KEY=%s\n' \"$$key\" > \"$$out\"; + echo \"crowdsec-init: bouncer key written (key length: $${#key})\"; + " + restart: "no" + + + # ─── Caddy (reverse proxy + automatic HTTPS) ────────────────────────────────── + # Custom build includes github.com/mholt/caddy-ratelimit, + # github.com/hslatman/caddy-crowdsec-bouncer/http, and + # github.com/mholt/caddy-l4 (TCP layer4 proxy for Redis). + caddy: + image: kalekber/libnovel-caddy:${GIT_TAG:-latest} + build: + context: ./caddy + dockerfile: Dockerfile + labels: + com.centurylinklabs.watchtower.enable: "true" + restart: unless-stopped + depends_on: + backend: + condition: service_healthy + ui: + condition: service_healthy + crowdsec-init: + condition: service_completed_successfully + ports: + - "80:80" + - "443:443" + - "443:443/udp" # HTTP/3 (QUIC) + - "6380:6380" # Redis TCP proxy (TLS) for homelab runner → Asynq + environment: + DOMAIN: "${DOMAIN}" + CADDY_ACME_EMAIL: "${CADDY_ACME_EMAIL}" + env_file: + - path: ./crowdsec/.crowdsec.env + required: false + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - ./caddy/errors:/srv/errors:ro + - caddy_data:/data + - caddy_config:/config + - caddy_logs:/var/log/caddy + + # ─── Watchtower (auto-redeploy custom services on new images) ──────────────── + # Only watches services labelled com.centurylinklabs.watchtower.enable=true. + # Third-party infra images (minio, pocketbase, meilisearch, etc.) are excluded. + # doppler binary is mounted from the host so watchtower fetches fresh secrets + # on every start (notification URL, credentials) without baking them in. + watchtower: + image: containrrr/watchtower:latest + restart: unless-stopped + entrypoint: ["/usr/bin/doppler", "run", "--project", "libnovel", "--config", "prd", "--"] + command: ["/watchtower", "--label-enable", "--interval", "300", "--cleanup"] + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /usr/bin/doppler:/usr/bin/doppler:ro + - /root/.doppler:/root/.doppler:ro + environment: + HOME: "/root" + DOCKER_API_VERSION: "1.44" + volumes: - static_books: + minio_data: + pb_data: + meili_data: + valkey_data: + redis_data: + caddy_data: + caddy_config: + caddy_logs: + crowdsec_data: diff --git a/docs/api-endpoints.md b/docs/api-endpoints.md new file mode 100644 index 0000000..01019bb --- /dev/null +++ b/docs/api-endpoints.md @@ -0,0 +1,82 @@ +# API Endpoint Reference + +> **Routing ownership map**: see [`docs/d2/api-routing.svg`](d2/api-routing.svg) (source: [`docs/d2/api-routing.d2`](d2/api-routing.d2)) for a visual overview of which paths Caddy sends to the backend directly vs. through SvelteKit, with auth levels colour-coded. + +All traffic enters through **Caddy :443**. Caddy routes a subset of paths directly to the Go backend (bypassing SvelteKit); everything else goes to SvelteKit, which enforces auth before proxying onward. + +## Health / Version + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `GET` | `/health` | — | Liveness probe. Returns `{"ok":true}`. | +| `GET` | `/api/version` | — | Build version + commit hash. | + +## Scrape Jobs (admin) + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `POST` | `/scrape` | admin | Enqueue full catalogue scrape. | +| `POST` | `/scrape/book` | admin | Enqueue single-book scrape `{url}`. | +| `POST` | `/scrape/book/range` | admin | Enqueue range scrape `{url, from, to?}`. | +| `GET` | `/api/scrape/status` | admin | Current job status. | +| `GET` | `/api/scrape/tasks` | admin | All scrape task records. | +| `POST` | `/api/cancel-task/{id}` | admin | Cancel a pending task. | + +## Browse / Catalogue + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `GET` | `/api/browse` | — | Live novelfire.net browse (MinIO page-1 cache). Legacy — used by save-browse subcommand. | +| `GET` | `/api/catalogue` | — | **Primary browse endpoint.** Meilisearch-backed, paginated. Params: `q`, `page`, `limit`, `genre`, `status`, `sort` (`popular`\|`new`\|`update`\|`rank`\|`top-rated`). Falls back to empty when Meilisearch is not configured. | +| `GET` | `/api/search` | — | Full-text search: Meilisearch local results merged with live novelfire.net remote results. Param: `q` (≥ 2 chars). Used by iOS app. | +| `GET` | `/api/ranking` | — | Top-ranked novels from PocketBase. | +| `GET` | `/api/cover/{domain}/{slug}` | — | Proxy cover image from MinIO (redirect to presigned URL). | + +## Book / Chapter Content + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `GET` | `/api/book-preview/{slug}` | — | Returns stored metadata + chapter list, or enqueues a scrape task (202) if unknown. | +| `GET` | `/api/chapter-text/{slug}/{n}` | — | Chapter content as plain text (markdown stripped). | +| `GET` | `/api/chapter-markdown/{slug}/{n}` | — | Chapter content as raw markdown from MinIO. | +| `POST` | `/api/reindex/{slug}` | admin | Rebuild `chapters_idx` from MinIO objects. | + +## Audio + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `POST` | `/api/audio/{slug}/{n}` | — | Trigger Kokoro TTS generation. Body: `{voice?}`. Returns `200 {status:"done"}` if cached, `202 {task_id, status}` if enqueued. | +| `GET` | `/api/audio/status/{slug}/{n}` | — | Poll audio generation status. Param: `voice`. Returns `{status, task_id?, error?}`. | +| `GET` | `/api/audio-proxy/{slug}/{n}` | — | Redirect to presigned MinIO audio URL. | +| `GET` | `/api/voices` | — | List available Kokoro voices. Returns `{voices:[]}` on error. | + +## Presigned URLs + +All presign endpoints return a `302` redirect to a short-lived MinIO presigned +URL. The URL is cached in Valkey (TTL ~55 min) to avoid regenerating on every +request. + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `GET` | `/api/presign/chapter/{slug}/{n}` | — | Presigned URL for chapter markdown object. | +| `GET` | `/api/presign/audio/{slug}/{n}` | — | Presigned URL for audio MP3. Param: `voice`. | +| `GET` | `/api/presign/voice-sample/{voice}` | — | Presigned URL for voice sample MP3. | +| `GET` | `/api/presign/avatar-upload/{userId}` | user | Presigned PUT URL for avatar upload. | +| `GET` | `/api/presign/avatar/{userId}` | — | Presigned GET URL for avatar image. | + +## Reading Progress + +Session-scoped (anonymous via cookie session ID, or tied to authenticated user). + +| Method | Path | Auth | Description | +|--------|------|------|-------------| +| `GET` | `/api/progress` | — | Get all reading progress for the current session/user. | +| `POST` | `/api/progress/{slug}` | — | Set progress. Body: `{chapter}`. | +| `DELETE` | `/api/progress/{slug}` | — | Delete progress for a book. | + +## Notes + +- **Auth**: The backend does not enforce auth itself — the SvelteKit UI layer enforces admin/user guards before proxying requests. The backend trusts all incoming requests. +- **`/api/catalogue` vs `/api/browse`**: `/api/catalogue` is the primary UI endpoint (Meilisearch, always-local, fast). `/api/browse` hits or caches the live novelfire.net browse page and is only used internally by the `save-browse` subcommand. +- **Meilisearch fallback**: When `MEILI_URL` is unset, `/api/catalogue` returns `{books:[], has_next:false}` and `/api/search` falls back to a PocketBase substring scan. +- **`BACKEND_API_URL`**: The SvelteKit UI reads this env var (default `http://localhost:8080`) to reach the backend server-side. In docker-compose it is set to `http://backend:8080`. diff --git a/docs/d2/api-routing.d2 b/docs/d2/api-routing.d2 new file mode 100644 index 0000000..ec45746 --- /dev/null +++ b/docs/d2/api-routing.d2 @@ -0,0 +1,208 @@ +direction: right + +# ─── Legend ─────────────────────────────────────────────────────────────────── + +legend: Legend { + style.fill: "#fafafa" + style.stroke: "#d4d4d8" + + pub: public { + style.fill: "#f0fdf4" + style.font-color: "#15803d" + style.stroke: "#86efac" + } + user: user auth { + style.fill: "#eff6ff" + style.font-color: "#1d4ed8" + style.stroke: "#93c5fd" + } + adm: admin only { + style.fill: "#fff7ed" + style.font-color: "#c2410c" + style.stroke: "#fdba74" + } +} + +# ─── Client ─────────────────────────────────────────────────────────────────── + +client: Browser / iOS App { + shape: person + style.fill: "#fff9e6" +} + +# ─── Caddy ──────────────────────────────────────────────────────────────────── + +caddy: Caddy :443 { + shape: rectangle + style.fill: "#f1f5f9" + label: "Caddy :443\ncustom build · caddy-l4 · caddy-ratelimit\nCrowdSec bouncer · security headers\nrate limiting · static error pages\nRedis TCP proxy :6380" +} + +# ─── SvelteKit UI ───────────────────────────────────────────────────────────── +# All routes here pass through SvelteKit — auth is enforced server-side. + +sk: SvelteKit UI :3000 { + style.fill: "#fef3c7" + + auth: Auth { + style.fill: "#fde68a" + style.stroke: "#f59e0b" + label: "POST /api/auth/login\nPOST /api/auth/register\nPOST /api/auth/change-password\nGET /api/auth/session" + } + + catalogue_sk: Catalogue { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/catalogue-page (infinite scroll)\nGET /api/search" + } + + book_sk: Book { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/book/{slug}\nGET /api/chapter/{slug}/{n}\nGET /api/chapter-text-preview/{slug}/{n}" + } + + scrape_sk: Scrape (admin) { + style.fill: "#fff7ed" + style.stroke: "#fdba74" + label: "GET /api/scrape/status\nGET /api/scrape/tasks\nPOST /api/scrape\nPOST /api/scrape/book\nPOST /api/scrape/book/range\nPOST /api/scrape/cancel/{id}" + } + + audio_sk: Audio { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "POST /api/audio/{slug}/{n}\nGET /api/audio/status/{slug}/{n}\nGET /api/voices" + } + + presign_sk: Presigned URLs (public) { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/presign/chapter/{slug}/{n}\nGET /api/presign/audio/{slug}/{n}\nGET /api/presign/voice-sample/{voice}" + } + + presign_user: Presigned URLs (user) { + style.fill: "#eff6ff" + style.stroke: "#93c5fd" + label: "GET /api/presign/avatar-upload/{userId}\nGET /api/presign/avatar/{userId}" + } + + progress_sk: Progress { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/progress\nPOST /api/progress/{slug}\nDELETE /api/progress/{slug}" + } + + library_sk: Library { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/library\nPOST /api/library/{slug}\nDELETE /api/library/{slug}" + } + + comments_sk: Comments { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/comments/{slug}\nPOST /api/comments/{slug}" + } +} + +# ─── Go Backend ─────────────────────────────────────────────────────────────── +# Caddy proxies these paths directly — bypasses SvelteKit entirely. + +be: Backend API :8080 { + style.fill: "#eef3ff" + + health_be: Health / Version { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /health\nGET /api/version" + } + + scrape_be: Scrape admin (direct) { + style.fill: "#fff7ed" + style.stroke: "#fdba74" + label: "POST /scrape\nPOST /scrape/book\nPOST /scrape/book/range" + } + + catalogue_be: Catalogue { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/catalogue (Meilisearch)\nGET /api/browse (legacy MinIO cache)\nGET /api/ranking\nGET /api/cover/{domain}/{slug}" + } + + book_be: Book / Chapter { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/book-preview/{slug}\nGET /api/chapter-text/{slug}/{n}\nGET /api/chapter-markdown/{slug}/{n}\nPOST /api/reindex/{slug} ⚠ admin" + } + + audio_be: Audio { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "POST /api/audio/{slug}/{n}\nGET /api/audio/status/{slug}/{n}\nGET /api/audio-proxy/{slug}/{n}\nGET /api/voices" + } + + presign_be: Presigned URLs { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/presign/chapter/{slug}/{n}\nGET /api/presign/audio/{slug}/{n}\nGET /api/presign/voice-sample/{voice}\nGET /api/presign/avatar-upload/{userId}\nGET /api/presign/avatar/{userId}" + } +} + +# ─── Storage ────────────────────────────────────────────────────────────────── + +storage: Storage { + style.fill: "#eaf7ea" + + pb: PocketBase :8090 { + shape: cylinder + label: "auth · books · progress\ncomments · library\nscrape_jobs · audio_cache\nranking" + } + mn: MinIO :9000 { + shape: cylinder + label: "chapters · audio\navatars · catalogue (browse)" + } + ms: Meilisearch :7700 { + shape: cylinder + label: "index: books\nfilterable: status · genres\nsortable: rank · rating\n total_chapters · meta_updated" + } + vk: Valkey :6379 { + shape: cylinder + label: "presign URL cache (TTL ~55 min)\nAsynq job queue (runner)" + } +} + +# ─── Caddy routing ──────────────────────────────────────────────────────────── + +client -> caddy: HTTPS :443 + +caddy -> sk: "/* (catch-all)\n→ SvelteKit enforces auth" +caddy -> be: "/health /scrape*\n/api/browse /api/catalogue /api/ranking\n/api/version /api/book-preview/*\n/api/chapter-text/* /api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio* /api/voices /api/presign/*" +caddy -> storage.mn: "/avatars/* /audio/* /chapters/*\n(presigned MinIO GETs)" + +# ─── SvelteKit → Backend (server-side proxy) ────────────────────────────────── + +sk.catalogue_sk -> be.catalogue_be: internal proxy +sk.book_sk -> be.book_be: internal proxy +sk.audio_sk -> be.audio_be: internal proxy +sk.presign_sk -> be.presign_be: internal proxy +sk.presign_user -> be.presign_be: internal proxy + +# ─── SvelteKit → Storage (direct) ──────────────────────────────────────────── + +sk.auth -> storage.pb: sessions / users +sk.scrape_sk -> storage.pb: scrape job records +sk.progress_sk -> storage.pb +sk.library_sk -> storage.pb +sk.comments_sk -> storage.pb + +# ─── Backend → Storage ──────────────────────────────────────────────────────── + +be.catalogue_be -> storage.ms: full-text search + facets +be.catalogue_be -> storage.pb: ranking records +be.catalogue_be -> storage.mn: cover presign +be.book_be -> storage.mn: chapter objects +be.book_be -> storage.pb: book metadata +be.audio_be -> storage.mn: audio presign +be.audio_be -> storage.vk: presign cache +be.presign_be -> storage.vk: check / set presign cache +be.presign_be -> storage.mn: generate presigned URL diff --git a/docs/d2/api-routing.svg b/docs/d2/api-routing.svg new file mode 100644 index 0000000..633b329 --- /dev/null +++ b/docs/d2/api-routing.svg @@ -0,0 +1,128 @@ +LegendBrowser / iOS AppCaddy :443custom build · caddy-l4 · caddy-ratelimitCrowdSec bouncer · security headersrate limiting · static error pagesRedis TCP proxy :6380SvelteKit UI :3000Backend API :8080Storagepublicuser authadmin onlyPOST /api/auth/loginPOST /api/auth/registerPOST /api/auth/change-passwordGET /api/auth/sessionGET /api/catalogue-page (infinite scroll)GET /api/searchGET /api/book/{slug}GET /api/chapter/{slug}/{n}GET /api/chapter-text-preview/{slug}/{n}GET /api/scrape/statusGET /api/scrape/tasksPOST /api/scrapePOST /api/scrape/bookPOST /api/scrape/book/rangePOST /api/scrape/cancel/{id}POST /api/audio/{slug}/{n}GET /api/audio/status/{slug}/{n}GET /api/voicesGET /api/presign/chapter/{slug}/{n}GET /api/presign/audio/{slug}/{n}GET /api/presign/voice-sample/{voice}GET /api/presign/avatar-upload/{userId}GET /api/presign/avatar/{userId}GET /api/progressPOST /api/progress/{slug}DELETE /api/progress/{slug}GET /api/libraryPOST /api/library/{slug}DELETE /api/library/{slug}GET /api/comments/{slug}POST /api/comments/{slug}GET /healthGET /api/versionPOST /scrapePOST /scrape/bookPOST /scrape/book/rangeGET /api/catalogue (Meilisearch)GET /api/browse (legacy MinIO cache)GET /api/rankingGET /api/cover/{domain}/{slug}GET /api/book-preview/{slug}GET /api/chapter-text/{slug}/{n}GET /api/chapter-markdown/{slug}/{n}POST /api/reindex/{slug} ⚠ adminPOST /api/audio/{slug}/{n}GET /api/audio/status/{slug}/{n}GET /api/audio-proxy/{slug}/{n}GET /api/voicesGET /api/presign/chapter/{slug}/{n}GET /api/presign/audio/{slug}/{n}GET /api/presign/voice-sample/{voice}GET /api/presign/avatar-upload/{userId}GET /api/presign/avatar/{userId}auth · books · progresscomments · libraryscrape_jobs · audio_cacherankingchapters · audioavatars · catalogue (browse)index: booksfilterable: status · genressortable: rank · rating total_chapters · meta_updatedpresign URL cache (TTL ~55 min)Asynq job queue (runner) HTTPS :443/* (catch-all)→ SvelteKit enforces auth/health /scrape*/api/browse /api/catalogue /api/ranking/api/version /api/book-preview/*/api/chapter-text/* /api/chapter-markdown/*/api/reindex/* /api/cover/*/api/audio* /api/voices /api/presign/*/avatars/* /audio/* /chapters/*(presigned MinIO GETs)internal proxyinternal proxyinternal proxyinternal proxyinternal proxysessions / usersscrape job recordsfull-text search + facetsranking recordscover presignchapter objectsbook metadataaudio presignpresign cachecheck / set presign cachegenerate presigned URL + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/d2/architecture.d2 b/docs/d2/architecture.d2 new file mode 100644 index 0000000..cb726d0 --- /dev/null +++ b/docs/d2/architecture.d2 @@ -0,0 +1,180 @@ +direction: right + +# ─── External ───────────────────────────────────────────────────────────────── + +novelfire: novelfire.net { + shape: cloud + style.fill: "#f0f4ff" + label: "novelfire.net\n(scrape source)" +} + +kokoro: Kokoro-FastAPI TTS { + shape: cloud + style.fill: "#f0f4ff" + label: "Kokoro-FastAPI TTS\n(self-hosted · homelab)\nchapter audio" +} + +pockettts: pocket-tts { + shape: cloud + style.fill: "#f0f4ff" + label: "pocket-tts\n(self-hosted · homelab)\nvoice sample MP3s" +} + +letsencrypt: Let's Encrypt { + shape: cloud + style.fill: "#f0f4ff" + label: "Let's Encrypt\n(ACME TLS-ALPN-01)" +} + +browser: Browser / iOS App { + shape: person + style.fill: "#fff9e6" +} + +# ─── Init containers (one-shot) ─────────────────────────────────────────────── + +init: Init containers { + style.fill: "#f5f5f5" + style.stroke-dash: 4 + + minio-init: minio-init { + shape: rectangle + label: "minio-init\n(mc: create buckets\n chapters · audio\n avatars · catalogue)" + } + + pb-init: pb-init { + shape: rectangle + label: "pb-init\n(bootstrap PocketBase\n collections + schema)" + } +} + +# ─── Storage ────────────────────────────────────────────────────────────────── + +storage: Storage { + style.fill: "#eaf7ea" + + minio: MinIO { + shape: cylinder + label: "MinIO :9000\nbuckets:\n chapters · audio\n avatars · catalogue" + } + + pocketbase: PocketBase { + shape: cylinder + label: "PocketBase :8090\ncollections:\n books · chapters_idx\n audio_cache · progress\n scrape_jobs · app_users\n ranking · library\n comments" + } + + valkey: Valkey { + shape: cylinder + label: "Valkey :6379\npresign URL cache (TTL ~55 min)\nAsynq job queue (runner tasks)" + } + + meilisearch: Meilisearch { + shape: cylinder + label: "Meilisearch :7700\nindex: books\n(filterable: status · genres\n sortable: rank · rating\n total_chapters · meta_updated)" + } +} + +# ─── Application — prod VPS (165.22.70.138) ─────────────────────────────────── + +app: Application — prod (165.22.70.138) { + style.fill: "#eef3ff" + + caddy: caddy { + shape: rectangle + label: "Caddy :443 / :80 / :6380\ncustom build\n+ caddy-l4 (Redis TCP proxy)\n+ caddy-ratelimit\nauto-HTTPS · security headers\nrate limiting (per-IP)\nstatic error pages (404/502/503/504)\nCrowdSec bouncer" + } + + backend: backend { + shape: rectangle + label: "Backend API :8080\n(Go)\nHTTP API server\nffmpeg (audio sample conv.)\nOpenTelemetry tracing\nSentry / GlitchTip errors" + } + + ui: ui { + shape: rectangle + label: "SvelteKit UI :3000\n(adapter-node)\nSSR · session auth\nserver-side API proxy" + } + + crowdsec: CrowdSec { + shape: rectangle + label: "CrowdSec :8080\nsecurity engine\nreads Caddy JSON logs\nbouncer integrated in Caddy" + } + + dozzle: Dozzle agent { + shape: rectangle + label: "Dozzle agent\n127.0.0.1:7007\nlog relay → homelab dashboard" + } +} + +# ─── Runner — homelab (192.168.0.109) ──────────────────────────────────────── + +homelab: Runner — homelab (192.168.0.109) { + style.fill: "#fef9ec" + + runner: runner { + shape: rectangle + label: "Runner :9091\n(Go background worker)\nscrape pipeline\nTTS audio job queue\nPrometheus /metrics\ncron: catalogue refresh\nAsynq worker → Valkey" + } +} + +# ─── Ops ────────────────────────────────────────────────────────────────────── + +ops: Ops { + style.fill: "#f5f5f5" + + watchtower: Watchtower { + shape: rectangle + label: "Watchtower\n(containrrr/watchtower)\npolls Docker Hub every 5 min\nautopulls + redeploys:\n backend · ui\n(runner: label-disabled on prod)" + } +} + +# ─── CI / CD ────────────────────────────────────────────────────────────────── + +cicd: CI / CD { + style.fill: "#f0f9ff" + + gitea: Gitea Actions { + shape: rectangle + label: "Gitea Actions\n(homelab runner)\ntag v* trigger:\n test-backend\n check-ui (type-check + build)\n docker-backend\n docker-runner\n docker-ui (bakes releases.json)\n docker-caddy\n → push Docker Hub\n → Gitea Release" + } +} + +# ─── Init → Storage ─────────────────────────────────────────────────────────── + +init.minio-init -> storage.minio: create buckets {style.stroke-dash: 4} +init.pb-init -> storage.pocketbase: bootstrap schema {style.stroke-dash: 4} + +# ─── App internal ───────────────────────────────────────────────────────────── + +app.caddy -> app.ui: "/* (catch-all)\nSvelteKit — auth enforced" +app.caddy -> app.backend: "/health /scrape*\n/api/browse /api/catalogue\n/api/ranking /api/version\n/api/book-preview/*\n/api/chapter-text/*\n/api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/* /api/voices\n/api/audio* /api/presign/*" +app.caddy -> storage.minio: "/avatars/* /audio/*\n/chapters/*\n(presigned GETs)" +app.caddy -> app.crowdsec: bouncer check (15 s poll) +app.caddy -> letsencrypt: ACME cert (TLS-ALPN-01) + +app.ui -> app.backend: "internal REST proxy\n(server-side only)" +app.ui -> storage.pocketbase: "auth · sessions\nprogress · library\ncomments" + +app.backend -> storage.minio: "chapter objs · audio MP3s\navatars · browse cache" +app.backend -> storage.pocketbase: "books · scrape_jobs\naudio_cache · ranking" +app.backend -> storage.valkey: "presign URL cache\n(SET/GET TTL ~55 min)" +app.backend -> storage.meilisearch: "catalogue search\nfacets: genres · status" +app.backend -> pockettts: "voice sample gen.\n(on-demand · ffmpeg conv.)" + +# ─── Runner → deps ──────────────────────────────────────────────────────────── + +homelab.runner -> novelfire: "HTTP scrape\nHTML → Markdown" +homelab.runner -> kokoro: "TTS generation\ntext → MP3" +homelab.runner -> storage.minio: "write chapters\n& audio MP3s" +homelab.runner -> storage.pocketbase: "read/update scrape_jobs\nwrite book records" +homelab.runner -> storage.meilisearch: "index books\n(on scrape completion)" +homelab.runner -> storage.valkey: "Asynq job queue\n(task consume)" + +# ─── Client ─────────────────────────────────────────────────────────────────── + +browser -> app.caddy: HTTPS :443\n(single entry point) + +# ─── Ops / CI ───────────────────────────────────────────────────────────────── + +ops.watchtower -> app.backend: watch (label-enabled) +ops.watchtower -> app.ui: watch (label-enabled) +cicd.gitea -> ops.watchtower: push to Docker Hub\n→ Watchtower detects new tag diff --git a/docs/d2/architecture.svg b/docs/d2/architecture.svg new file mode 100644 index 0000000..6a7941b --- /dev/null +++ b/docs/d2/architecture.svg @@ -0,0 +1,132 @@ +novelfire.net(scrape source)Kokoro-FastAPI TTS(self-hosted · homelab)chapter audiopocket-tts(self-hosted · homelab)voice sample MP3sLet's Encrypt(ACME TLS-ALPN-01)Browser / iOS AppInit containersStorageApplication — prod (165.22.70.138)Runner — homelab (192.168.0.109)OpsCI / CDminio-init(mc: create buckets chapters · audio avatars · catalogue)pb-init(bootstrap PocketBase collections + schema)MinIO :9000buckets: chapters · audio avatars · cataloguePocketBase :8090collections: books · chapters_idx audio_cache · progress scrape_jobs · app_users ranking · library commentsValkey :6379presign URL cache (TTL ~55 min)Asynq job queue (runner tasks)Meilisearch :7700index: books(filterable: status · genres sortable: rank · rating total_chapters · meta_updated)Caddy :443 / :80 / :6380custom build+ caddy-l4 (Redis TCP proxy)+ caddy-ratelimitauto-HTTPS · security headersrate limiting (per-IP)static error pages (404/502/503/504)CrowdSec bouncerBackend API :8080(Go)HTTP API serverffmpeg (audio sample conv.)OpenTelemetry tracingSentry / GlitchTip errorsSvelteKit UI :3000(adapter-node)SSR · session authserver-side API proxyCrowdSec :8080security enginereads Caddy JSON logsbouncer integrated in CaddyDozzle agent127.0.0.1:7007log relay → homelab dashboardRunner :9091(Go background worker)scrape pipelineTTS audio job queuePrometheus /metricscron: catalogue refreshAsynq worker → ValkeyWatchtower(containrrr/watchtower)polls Docker Hub every 5 minautopulls + redeploys: backend · ui(runner: label-disabled on prod)Gitea Actions(homelab runner)tag v* trigger: test-backend check-ui (type-check + build) docker-backend docker-runner docker-ui (bakes releases.json) docker-caddy → push Docker Hub → Gitea Release create bucketsbootstrap schema /* (catch-all)SvelteKit — auth enforced/health /scrape*/api/browse /api/catalogue/api/ranking /api/version/api/book-preview/*/api/chapter-text/*/api/chapter-markdown/*/api/reindex/* /api/cover/*/api/audio-proxy/* /api/voices/api/audio* /api/presign/*/avatars/* /audio/*/chapters/*(presigned GETs)bouncer check (15 s poll)ACME cert (TLS-ALPN-01)internal REST proxy(server-side only)auth · sessionsprogress · librarycommentschapter objs · audio MP3savatars · browse cachebooks · scrape_jobsaudio_cache · rankingpresign URL cache(SET/GET TTL ~55 min)catalogue searchfacets: genres · statusvoice sample gen.(on-demand · ffmpeg conv.)HTTP scrapeHTML → MarkdownTTS generationtext → MP3write chapters& audio MP3sread/update scrape_jobswrite book recordsindex books(on scrape completion)Asynq job queue(task consume)HTTPS :443(single entry point)watch (label-enabled)watch (label-enabled)push to Docker Hub→ Watchtower detects new tag + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/mermaid/architecture.mermaid.md b/docs/mermaid/architecture.mermaid.md new file mode 100644 index 0000000..4752934 --- /dev/null +++ b/docs/mermaid/architecture.mermaid.md @@ -0,0 +1,72 @@ +# Architecture Overview + +```mermaid +graph LR + %% ── External ────────────────────────────────────────────────────────── + NF([novelfire.net]) + KK([Kokoro-FastAPI TTS]) + LE([Let's Encrypt]) + CL([Browser / iOS App]) + + %% ── Init containers ─────────────────────────────────────────────────── + subgraph INIT["Init containers (one-shot)"] + MI[minio-init\nmc: create buckets] + PI[pb-init\nbootstrap collections] + end + + %% ── Storage ─────────────────────────────────────────────────────────── + subgraph STORAGE["Storage"] + MN[(MinIO :9000\nchapters · audio\navatars · browse)] + PB[(PocketBase :8090\nbooks · chapters_idx\naudio_cache · progress\nscrape_jobs · app_users · ranking)] + VK[(Valkey :6379\npresign URL cache\nTTL-based · shared)] + MS[(Meilisearch :7700\nindex: books)] + end + + %% ── Application ─────────────────────────────────────────────────────── + subgraph APP["Application"] + CD["Caddy :443/:80\ncustom build + caddy-ratelimit\nauto-HTTPS · security headers\nrate limiting · error pages"] + BE[Backend API :8080\nGo HTTP server] + RN[Runner :9091\nGo background worker\n/metrics endpoint] + UI[SvelteKit UI :3000\nadapter-node] + end + + %% ── Ops ─────────────────────────────────────────────────────────────── + subgraph OPS["Ops"] + WT[Watchtower\npolls every 5 min\nautopull + redeploy\nbackend · runner · ui] + end + + %% ── Init → Storage ──────────────────────────────────────────────────── + MI -.->|create buckets| MN + PI -.->|bootstrap schema| PB + + %% ── App → Storage ───────────────────────────────────────────────────── + BE -->|blobs| MN + BE -->|structured records| PB + BE -->|cache presigned URLs| VK + RN -->|chapter markdown & audio| MN + RN -->|read/update jobs & books| PB + RN -->|index books on scrape| MS + UI -->|read presign cache| VK + UI -->|auth · progress · comments| PB + + %% ── App internal ────────────────────────────────────────────────────── + UI -->|"REST API (server-side)\n/api/catalogue /api/book-preview\n/api/chapter-text /api/audio"| BE + + %% ── Caddy routing ───────────────────────────────────────────────────── + CD -->|"/* catch-all\n/api/scrape/*\n/api/chapter-text-preview/*\n→ SvelteKit (auth enforced)"| UI + CD -->|"/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*"| BE + CD -->|/avatars/* presigned GETs| MN + + %% ── Runner → External ───────────────────────────────────────────────── + RN -->|scrape HTTP GET| NF + RN -->|TTS HTTP POST| KK + CD -->|ACME certificate| LE + + %% ── Ops ─────────────────────────────────────────────────────────────── + WT -->|watch label-enabled| BE + WT -->|watch label-enabled| RN + WT -->|watch label-enabled| UI + + %% ── Client ──────────────────────────────────────────────────────────── + CL -->|HTTPS :443 single entry| CD +``` diff --git a/docs/mermaid/data-flow.mermaid.md b/docs/mermaid/data-flow.mermaid.md new file mode 100644 index 0000000..0536890 --- /dev/null +++ b/docs/mermaid/data-flow.mermaid.md @@ -0,0 +1,102 @@ +# Data Flow — Scrape & TTS Job Pipeline + +How content moves from novelfire.net through the runner into storage, and how +audio is generated on-demand via the backend. + +## Catalogue Scrape Pipeline + +The runner performs a background catalogue walk on startup and then on a +configurable interval (`RUNNER_CATALOGUE_REFRESH_INTERVAL`, default 24 h). + +```mermaid +flowchart TD + A([Runner starts / refresh tick]) --> B[Walk novelfire.net catalogue\npages 1…N] + B --> C{Book already\nin PocketBase?} + C -- no --> D[Scrape book metadata\ntitle · author · genres\ncover · summary · status] + C -- yes --> E[Check for new chapters\ncompare total_chapters] + D --> F[Write BookMeta\nto PocketBase books] + E --> G{New chapters\nfound?} + G -- no --> Z([Done — next book]) + G -- yes --> H + F --> H[Scrape chapter list with upTo limit\n→ chapters_idx in PocketBase\nretries on 429 with Retry-After backoff] + H --> I[Worker pool — N goroutines\nRUNNER_MAX_CONCURRENT_SCRAPE] + I --> J[For each missing chapter:\nGET chapter HTML from novelfire.net] + J --> K[Parse HTML → Markdown\nhtmlutil.NodeToMarkdown] + K --> L[PUT object to MinIO\nchapters/{slug}/{n}.md] + L --> M[Upsert book doc\nto Meilisearch index: books] + M --> Z + F --> M +``` + +## On-Demand Single-Book Scrape + +Triggered when a user visits `/books/{slug}` and the book is not in PocketBase. +The UI calls `GET /api/book-preview/{slug}` → backend enqueues a scrape task. + +```mermaid +sequenceDiagram + actor U as User + participant UI as SvelteKit UI + participant BE as Backend API + participant TQ as Task Queue (PocketBase) + participant RN as Runner + participant NF as novelfire.net + participant PB as PocketBase + participant MN as MinIO + participant MS as Meilisearch + + U->>UI: Visit /books/{slug} + UI->>BE: GET /api/book-preview/{slug} + BE->>PB: getBook(slug) — not found + BE->>TQ: INSERT scrape_task (slug, status=pending) + BE-->>UI: 202 {task_id, message} + UI-->>U: "Scraping…" placeholder + + RN->>TQ: Poll for pending tasks + TQ-->>RN: scrape_task (slug) + RN->>NF: GET novelfire.net/book/{slug} + NF-->>RN: HTML + RN->>PB: upsert book + chapters_idx + RN->>MN: PUT chapter objects + RN->>MS: UpsertBook doc + RN->>TQ: UPDATE task status=done + + U->>UI: Poll GET /api/scrape/tasks/{task_id} + UI->>BE: GET /api/scrape/status + BE->>TQ: get task + TQ-->>BE: status=done + BE-->>UI: {status:"done"} + UI-->>U: Redirect to /books/{slug} +``` + +## TTS Audio Generation Pipeline + +Audio is generated lazily: on first request the job is enqueued; subsequent +requests poll for completion and then stream from MinIO via presigned URL. + +```mermaid +flowchart TD + A(["POST /api/audio/{slug}/{n}\nbody: voice=af_bella"]) --> B{Audio already\nin MinIO?} + B -- yes --> C[200 status: done] + B -- no --> D{Job already\nin queue?} + D -- "yes pending/generating" --> E[202 task_id + status] + D -- no --> F[INSERT audio_task\nstatus=pending\nin PocketBase] + F --> E + + G([Runner polls task queue]) --> H[Claim audio_task\nstatus=generating] + H --> I["GET /api/chapter-text/{slug}/{n}\nfrom backend — plain text"] + I --> J[POST /v1/audio/speech\nto Kokoro-FastAPI\nbody: text + voice] + J --> K[Stream MP3 response] + K --> L[PUT object to MinIO\naudio/{slug}/{n}/{voice}.mp3] + L --> M[UPDATE audio_task\nstatus=done] + + N(["Client polls\nGET /api/audio/status/{slug}/{n}"]) --> O{status?} + O -- "pending/generating" --> N + O -- done --> P["GET /api/presign/audio/{slug}/{n}"] + P --> Q{Valkey cache hit?} + Q -- yes --> R[302 → presigned URL] + Q -- no --> S[GeneratePresignedURL\nfrom MinIO — TTL 1h] + S --> T[Cache in Valkey\nTTL 3500s] + T --> R + R --> U([Client streams audio\ndirectly from MinIO]) +``` diff --git a/docs/mermaid/request-flow.mermaid.md b/docs/mermaid/request-flow.mermaid.md new file mode 100644 index 0000000..834182b --- /dev/null +++ b/docs/mermaid/request-flow.mermaid.md @@ -0,0 +1,111 @@ +# Request Flow + +Two representative request paths through the stack: a **page load** (SSR) and a +**media playback** (presigned URL → direct MinIO stream). + +## SSR Page Load — Catalogue / Book Detail + +```mermaid +sequenceDiagram + actor C as Browser / iOS App + participant CD as Caddy :443 + participant UI as SvelteKit UI :3000 + participant BE as Backend API :8080 + participant MS as Meilisearch :7700 + participant PB as PocketBase :8090 + participant VK as Valkey :6379 + + C->>CD: HTTPS GET /catalogue + CD->>UI: proxy /* (SvelteKit catch-all) + UI->>BE: GET /api/catalogue?page=1&sort=popular + BE->>MS: search(query, filters, sort) + MS-->>BE: [{slug, title, …}, …] + BE-->>UI: {books[], page, total, has_next} + UI-->>CD: SSR HTML + CD-->>C: 200 HTML + + Note over C,UI: Infinite scroll — client fetches next page via SvelteKit API route + C->>CD: HTTPS GET /api/catalogue-page?page=2 + CD->>UI: proxy /* (SvelteKit /api/catalogue-page server route) + UI->>BE: GET /api/catalogue?page=2 + BE->>MS: search(…) + MS-->>BE: next page + BE-->>UI: {books[], …} + UI-->>C: JSON +``` + +## Audio Playback — Presigned URL Flow + +```mermaid +sequenceDiagram + actor C as Browser / iOS App + participant CD as Caddy :443 + participant UI as SvelteKit UI :3000 + participant BE as Backend API :8080 + participant VK as Valkey :6379 + participant MN as MinIO :9000 + + C->>CD: GET /api/presign/audio/{slug}/{n}?voice=af_bella + CD->>UI: proxy /* (SvelteKit /api/presign/audio route) + UI->>BE: GET /api/presign/audio/{slug}/{n}?voice=af_bella + BE->>VK: GET presign:audio:{slug}:{n}:{voice} + alt cache hit + VK-->>BE: presigned URL (TTL remaining) + BE-->>UI: 302 redirect → presigned URL + UI-->>C: 302 redirect + else cache miss + BE->>MN: GeneratePresignedURL(audio-bucket, key, 1h) + MN-->>BE: presigned URL + BE->>VK: SET presign:audio:… EX 3500 + BE-->>UI: 302 redirect → presigned URL + UI-->>C: 302 redirect + end + C->>MN: GET presigned URL (direct, no proxy) + MN-->>C: audio/mpeg stream +``` + +## Chapter Read — SSR + Content Fetch + +```mermaid +sequenceDiagram + actor C as Browser / iOS App + participant CD as Caddy :443 + participant UI as SvelteKit UI :3000 + participant BE as Backend API :8080 + participant PB as PocketBase :8090 + participant MN as MinIO :9000 + + C->>CD: HTTPS GET /books/{slug}/chapters/{n} + CD->>UI: proxy /* (SvelteKit catch-all) + UI->>PB: getBook(slug) + listChapterIdx(slug) + PB-->>UI: book meta + chapter list + UI->>BE: GET /api/chapter-text/{slug}/{n} + BE->>MN: GetObject(chapters-bucket, {slug}/{n}.md) + MN-->>BE: markdown text + BE-->>UI: plain text (markdown stripped) + Note over UI: marked() → HTML + UI-->>CD: SSR HTML + CD-->>C: 200 HTML +``` + +## Caddy Request Lifecycle + +Shows how security hardening applies before a request reaches any upstream. + +```mermaid +flowchart TD + A([Incoming HTTPS request]) --> B[TLS termination\nLet's Encrypt cert] + B --> C{Rate limit check\ncaddy-ratelimit} + C -- over limit --> D[429 Too Many Requests] + C -- ok --> E[Add security headers\nX-Frame-Options · X-Content-Type-Options\nReferrer-Policy · Permissions-Policy\nHSTS · X-XSS-Protection\nremove Server header] + E --> F{Route match} + F -- "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*" --> G[reverse_proxy → backend:8080] + F -- "/avatars/*" --> H[reverse_proxy → minio:9000] + F -- "/* everything else\n(incl. /api/scrape/*\n/api/chapter-text-preview/*)" --> I[reverse_proxy → ui:3000\nSvelteKit auth middleware runs] + G --> J{Upstream healthy?} + H --> J + I --> J + J -- yes --> K([Response to client]) + J -- "502/503/504" --> L[handle_errors\nstatic HTML from /srv/errors/] + L --> K +``` diff --git a/dozzle/users.yml b/dozzle/users.yml new file mode 100644 index 0000000..c53b951 --- /dev/null +++ b/dozzle/users.yml @@ -0,0 +1,5 @@ +users: + admin: + name: admin + email: admin@libnovel.cc + password: "$2y$10$4jqLza2grpxnQn0EGux2C.UmlSxRmOvH/J1ySzOBxMZgW6cA2TnmK" diff --git a/homelab/docker-compose.yml b/homelab/docker-compose.yml new file mode 100644 index 0000000..c71a3a4 --- /dev/null +++ b/homelab/docker-compose.yml @@ -0,0 +1,556 @@ +# LibNovel homelab +# +# Runs on 192.168.0.109. Hosts: +# - libnovel runner (background task worker) +# - tooling: GlitchTip, Umami, Fider, Dozzle, Uptime Kuma, Gotify +# - observability: OTel Collector, Tempo, Loki, Prometheus, Grafana +# - cloudflared tunnel (public subdomains via Cloudflare Zero Trust) +# - shared Postgres for tooling DBs +# +# All secrets come from Doppler (project=libnovel, config=prd_homelab). +# Run with: doppler run -- docker compose up -d +# +# Public subdomains (via Cloudflare Tunnel — no ports exposed to internet): +# errors.libnovel.cc → glitchtip-web:8000 +# analytics.libnovel.cc → umami:3000 +# feedback.libnovel.cc → fider:3000 +# logs.libnovel.cc → dozzle:8080 +# uptime.libnovel.cc → uptime-kuma:3001 +# push.libnovel.cc → gotify:80 +# grafana.libnovel.cc → grafana:3000 +# faro.libnovel.cc → alloy:12347 + +services: + + # ── Cloudflare Tunnel ─────────────────────────────────────────────────────── + # Outbound-only encrypted tunnel to Cloudflare. + # Routes all public subdomains to their respective containers on this network. + # No inbound ports needed — cloudflared initiates all connections outward. + cloudflared: + image: cloudflare/cloudflared:latest + restart: unless-stopped + command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN} + environment: + CLOUDFLARE_TUNNEL_TOKEN: "${CLOUDFLARE_TUNNEL_TOKEN}" + + # ── LibNovel Runner ───────────────────────────────────────────────────────── + # Background task worker. Connects to prod PocketBase, MinIO, Meilisearch + # via their public subdomains (pb.libnovel.cc, storage.libnovel.cc, etc.) + runner: + image: kalekber/libnovel-runner:latest + restart: unless-stopped + stop_grace_period: 135s + labels: + com.centurylinklabs.watchtower.enable: "true" + environment: + POCKETBASE_URL: "https://pb.libnovel.cc" + POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}" + POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}" + + MINIO_ENDPOINT: "storage.libnovel.cc" + MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}" + MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD}" + MINIO_USE_SSL: "true" + MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT}" + MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL}" + + MEILI_URL: "${MEILI_URL}" + MEILI_API_KEY: "${MEILI_API_KEY}" + VALKEY_ADDR: "" + GODEBUG: "preferIPv4=1" + + # ── LibreTranslate (internal Docker network) ────────────────────────── + LIBRETRANSLATE_URL: "http://libretranslate:5000" + LIBRETRANSLATE_API_KEY: "${LIBRETRANSLATE_API_KEY}" + + # ── Asynq / Redis ───────────────────────────────────────────────────── + REDIS_ADDR: "${REDIS_ADDR}" + REDIS_PASSWORD: "${REDIS_PASSWORD}" + + KOKORO_URL: "http://kokoro-fastapi:8880" + KOKORO_VOICE: "${KOKORO_VOICE}" + + POCKET_TTS_URL: "http://pocket-tts:8000" + + RUNNER_WORKER_ID: "${RUNNER_WORKER_ID}" + RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL}" + RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE}" + RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO}" + RUNNER_MAX_CONCURRENT_TRANSLATION: "${RUNNER_MAX_CONCURRENT_TRANSLATION}" + RUNNER_TIMEOUT: "${RUNNER_TIMEOUT}" + RUNNER_METRICS_ADDR: "${RUNNER_METRICS_ADDR}" + RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH: "true" + + LOG_LEVEL: "${LOG_LEVEL}" + GLITCHTIP_DSN: "${GLITCHTIP_DSN_RUNNER}" + + # OTel — send runner traces/logs to Alloy (HTTP) + # Alloy forwards traces → OTel collector → Tempo + # logs → Loki + OTEL_EXPORTER_OTLP_ENDPOINT: "http://alloy:4318" + OTEL_SERVICE_NAME: "runner" + + healthcheck: + test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"] + interval: 60s + timeout: 5s + retries: 3 + + # ── Shared Postgres ───────────────────────────────────────────────────────── + # Hosts glitchtip, umami, and fider databases. + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: "${POSTGRES_USER}" + POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" + POSTGRES_DB: postgres + expose: + - "5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"] + interval: 10s + timeout: 5s + retries: 5 + + # ── Postgres database initialisation ──────────────────────────────────────── + postgres-init: + image: postgres:16-alpine + depends_on: + postgres: + condition: service_healthy + environment: + PGPASSWORD: "${POSTGRES_PASSWORD}" + entrypoint: > + /bin/sh -c " + psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='fider'\" | grep -q 1 || + psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE fider\"; + psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='glitchtip'\" | grep -q 1 || + psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE glitchtip\"; + psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='umami'\" | grep -q 1 || + psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE umami\"; + echo 'postgres-init: databases ready'; + " + restart: "no" + + # ── GlitchTip DB migration ────────────────────────────────────────────────── + glitchtip-migrate: + image: glitchtip/glitchtip:latest + depends_on: + postgres-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip" + SECRET_KEY: "${GLITCHTIP_SECRET_KEY}" + GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}" + EMAIL_URL: "${GLITCHTIP_EMAIL_URL}" + DEFAULT_FROM_EMAIL: "noreply@libnovel.cc" + VALKEY_URL: "redis://valkey:6379/1" + command: "./manage.py migrate" + restart: "no" + + # ── GlitchTip web ─────────────────────────────────────────────────────────── + glitchtip-web: + image: glitchtip/glitchtip:latest + restart: unless-stopped + depends_on: + glitchtip-migrate: + condition: service_completed_successfully + expose: + - "8000" + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip" + SECRET_KEY: "${GLITCHTIP_SECRET_KEY}" + GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}" + EMAIL_URL: "${GLITCHTIP_EMAIL_URL}" + DEFAULT_FROM_EMAIL: "noreply@libnovel.cc" + VALKEY_URL: "redis://valkey:6379/1" + PORT: "8000" + ENABLE_USER_REGISTRATION: "false" + MEDIA_ROOT: "/code/uploads" + volumes: + - glitchtip_uploads:/code/uploads + # Patch: GzipChunk fallback for sentry-cli 3.x raw zip uploads (GlitchTip bug) + - ./glitchtip/files_api.py:/code/apps/files/api.py:ro + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/0/')"] + interval: 15s + timeout: 5s + retries: 5 + + # ── GlitchTip worker ──────────────────────────────────────────────────────── + glitchtip-worker: + image: glitchtip/glitchtip:latest + restart: unless-stopped + depends_on: + glitchtip-migrate: + condition: service_completed_successfully + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip" + SECRET_KEY: "${GLITCHTIP_SECRET_KEY}" + GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}" + EMAIL_URL: "${GLITCHTIP_EMAIL_URL}" + DEFAULT_FROM_EMAIL: "noreply@libnovel.cc" + VALKEY_URL: "redis://valkey:6379/1" + SERVER_ROLE: "worker" + MEDIA_ROOT: "/code/uploads" + volumes: + - glitchtip_uploads:/code/uploads + # Patch: GzipChunk fallback for sentry-cli 3.x raw zip uploads (GlitchTip bug) + - ./glitchtip/files_api.py:/code/apps/files/api.py:ro + + # ── Umami ─────────────────────────────────────────────────────────────────── + umami: + image: ghcr.io/umami-software/umami:postgresql-latest + restart: unless-stopped + depends_on: + postgres-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + expose: + - "3000" + environment: + DATABASE_URL: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/umami" + APP_SECRET: "${UMAMI_APP_SECRET}" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:3000/api/heartbeat"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Fider ─────────────────────────────────────────────────────────────────── + fider: + image: getfider/fider:stable + restart: unless-stopped + depends_on: + postgres-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + expose: + - "3000" + environment: + BASE_URL: "${FIDER_BASE_URL}" + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/fider?sslmode=disable" + JWT_SECRET: "${FIDER_JWT_SECRET}" + EMAIL_NOREPLY: "noreply@libnovel.cc" + EMAIL_SMTP_HOST: "${FIDER_SMTP_HOST}" + EMAIL_SMTP_PORT: "${FIDER_SMTP_PORT}" + EMAIL_SMTP_USERNAME: "${FIDER_SMTP_USER}" + EMAIL_SMTP_PASSWORD: "${FIDER_SMTP_PASSWORD}" + EMAIL_SMTP_ENABLE_STARTTLS: "${FIDER_SMTP_ENABLE_STARTTLS}" + OAUTH_GOOGLE_CLIENTID: "${OAUTH_GOOGLE_CLIENTID}" + OAUTH_GOOGLE_SECRET: "${OAUTH_GOOGLE_SECRET}" + OAUTH_GITHUB_CLIENTID: "${OAUTH_GITHUB_CLIENTID}" + OAUTH_GITHUB_SECRET: "${OAUTH_GITHUB_SECRET}" + + # ── Dozzle ────────────────────────────────────────────────────────────────── + # Watches both homelab and prod containers. + # Prod agent runs on 165.22.70.138:7007 (added separately to prod compose). + dozzle: + image: amir20/dozzle:latest + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./dozzle/users.yml:/data/users.yml:ro + expose: + - "8080" + environment: + DOZZLE_AUTH_PROVIDER: simple + DOZZLE_HOSTNAME: "logs.libnovel.cc" + DOZZLE_REMOTE_AGENT: "prod@165.22.70.138:7007" + healthcheck: + test: ["CMD", "/dozzle", "healthcheck"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Uptime Kuma ───────────────────────────────────────────────────────────── + uptime-kuma: + image: louislam/uptime-kuma:1 + restart: unless-stopped + volumes: + - uptime_kuma_data:/app/data + expose: + - "3001" + healthcheck: + test: ["CMD", "extra/healthcheck"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Gotify ────────────────────────────────────────────────────────────────── + gotify: + image: gotify/server:latest + restart: unless-stopped + volumes: + - gotify_data:/app/data + expose: + - "80" + environment: + GOTIFY_DEFAULTUSER_NAME: "${GOTIFY_ADMIN_USER}" + GOTIFY_DEFAULTUSER_PASS: "${GOTIFY_ADMIN_PASS}" + GOTIFY_SERVER_PORT: "80" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:80/health"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Redis (Asynq task queue) ──────────────────────────────────────────────── + # Dedicated Redis instance for Asynq job dispatch. + # The prod backend enqueues jobs via redis.libnovel.cc:6380 (Caddy TLS proxy → + # host:6379). The runner reads from this instance directly on the Docker network. + # Port is bound to 0.0.0.0:6379 so the Caddy layer4 proxy on prod can reach it. + redis: + image: redis:7-alpine + restart: unless-stopped + command: ["redis-server", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD}"] + ports: + - "6379:6379" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # ── LibreTranslate ────────────────────────────────────────────────────────── + # Self-hosted machine translation. Runner connects via http://libretranslate:5000. + # Only English → configured target languages are loaded to save RAM. + libretranslate: + image: libretranslate/libretranslate:latest + restart: unless-stopped + environment: + LT_API_KEYS: "true" + LT_API_KEYS_DB_PATH: "/app/db/api_keys.db" + LT_LOAD_ONLY: "en,ru,id,pt,fr" + LT_DISABLE_WEB_UI: "true" + LT_UPDATE_MODELS: "false" + expose: + - "5000" + volumes: + - libretranslate_data:/app/db + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:5000/languages"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + + # ── Valkey ────────────────────────────────────────────────────────────────── + # Used by GlitchTip for task queuing. + valkey: + image: valkey/valkey:7-alpine + restart: unless-stopped + expose: + - "6379" + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # ── Grafana Alloy (Faro RUM receiver) ─────────────────────────────────────── + # Receives browser telemetry from @grafana/faro-web-sdk (Web Vitals, traces, + # errors). Exposes POST /collect at faro.libnovel.cc via cloudflared. + # Forwards traces to otel-collector (→ Tempo) and logs to Loki directly. + alloy: + image: grafana/alloy:latest + restart: unless-stopped + command: ["run", "--server.http.listen-addr=0.0.0.0:12348", "/etc/alloy/alloy.river"] + volumes: + - ./otel/alloy.river:/etc/alloy/alloy.river:ro + expose: + - "12347" # Faro HTTP receiver (POST /collect) + - "12348" # Alloy UI / health endpoint + - "4318" # OTLP receiver (HTTP) for backend/runner logs + depends_on: + - otel-collector + - loki + + # ── OTel Collector ────────────────────────────────────────────────────────── + # Receives OTLP from backend/ui/runner, fans out to Tempo + Prometheus + Loki. + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + restart: unless-stopped + volumes: + - ./otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro + expose: + - "14317" # OTLP gRPC (Alloy forwards traces here) + - "14318" # OTLP HTTP (Alloy forwards traces here) + - "8888" # Collector self-metrics (scraped by Prometheus) + depends_on: + - tempo + - prometheus + - loki + # No healthcheck — distroless image has no shell or curl + + # ── Tempo ─────────────────────────────────────────────────────────────────── + # Distributed trace storage. Receives OTLP from the collector. + tempo: + image: grafana/tempo:2.6.1 + restart: unless-stopped + command: ["-config.file=/etc/tempo.yaml"] + volumes: + - ./otel/tempo.yaml:/etc/tempo.yaml:ro + - tempo_data:/var/tempo + expose: + - "3200" # Tempo query API (queried by Grafana) + - "4317" # OTLP gRPC ingest (collector → tempo) + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3200/ready"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Prometheus ────────────────────────────────────────────────────────────── + # Scrapes metrics from backend (via prod), runner, and otel-collector. + prometheus: + image: prom/prometheus:latest + restart: unless-stopped + command: + - "--config.file=/etc/prometheus/prometheus.yaml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=30d" + - "--web.enable-remote-write-receiver" + volumes: + - ./otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - prometheus_data:/prometheus + expose: + - "9090" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:9090/-/healthy"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Loki ──────────────────────────────────────────────────────────────────── + # Log aggregation. Receives logs from OTel collector. Replaces manual Dozzle + # tailing for structured log search. + loki: + image: grafana/loki:latest + restart: unless-stopped + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./otel/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + expose: + - "3100" + # No healthcheck — distroless image has no shell or curl + + # ── Grafana ───────────────────────────────────────────────────────────────── + # Single UI for traces (Tempo), metrics (Prometheus), and logs (Loki). + # Accessible at grafana.libnovel.cc via Cloudflare Tunnel. + grafana: + image: grafana/grafana:latest + restart: unless-stopped + depends_on: + - tempo + - prometheus + - loki + expose: + - "3000" + volumes: + - grafana_data:/var/lib/grafana + - ./otel/grafana/provisioning:/etc/grafana/provisioning:ro + environment: + GF_SERVER_ROOT_URL: "https://grafana.libnovel.cc" + GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER}" + GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD}" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_FEATURE_TOGGLES_ENABLE: "traceqlEditor" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Kokoro-FastAPI (GPU TTS) ──────────────────────────────────────────────── + # OpenAI-compatible TTS service backed by the Kokoro model, running on the + # homelab RTX 3050 (8 GB VRAM). Replaces the broken kokoro.kalekber.cc DNS. + # Voices match existing IDs: af_bella, af_sky, af_heart, etc. + # The runner reaches it at http://kokoro-fastapi:8880 via the Docker network. + kokoro-fastapi: + image: kokoro-fastapi:latest + restart: unless-stopped + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + expose: + - "8880" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8880/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + + # ── pocket-tts (CPU TTS) ──────────────────────────────────────────────────── + # Lightweight CPU-only TTS using kyutai-labs/pocket-tts. + # Image is built locally on homelab from https://github.com/kyutai-labs/pocket-tts + # (no prebuilt image published): cd /tmp && git clone --depth=1 https://github.com/kyutai-labs/pocket-tts.git && docker build -t pocket-tts:latest /tmp/pocket-tts + # OpenAI-compatible: POST /tts (multipart form) on port 8000. + # Voices: alba, marius, javert, jean, fantine, cosette, eponine, azelma, etc. + # Not currently used by the runner (runner uses kokoro-fastapi), but available + # for experimentation / fallback. + pocket-tts: + image: pocket-tts:latest + restart: unless-stopped + command: ["uv", "run", "pocket-tts", "serve", "--host", "0.0.0.0"] + expose: + - "8000" + volumes: + - pocket_tts_cache:/root/.cache/pocket_tts + - hf_cache:/root/.cache/huggingface + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 120s + + # ── Watchtower ────────────────────────────────────────────────────────────── + # Auto-updates runner image when CI pushes a new tag. + # Only watches services with the watchtower label. + # doppler binary is mounted from the host so watchtower fetches fresh secrets + # on every start (notification URL, credentials) without baking them in. + watchtower: + image: containrrr/watchtower:latest + restart: unless-stopped + entrypoint: ["/usr/bin/doppler", "run", "--project", "libnovel", "--config", "prd_homelab", "--"] + command: ["/watchtower", "--label-enable", "--interval", "300", "--cleanup"] + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /usr/bin/doppler:/usr/bin/doppler:ro + - /root/.doppler:/root/.doppler:ro + environment: + HOME: "/root" + DOCKER_API_VERSION: "1.44" + +volumes: + postgres_data: + redis_data: + libretranslate_data: + valkey_data: + uptime_kuma_data: + gotify_data: + tempo_data: + prometheus_data: + loki_data: + grafana_data: + pocket_tts_cache: + hf_cache: + glitchtip_uploads: diff --git a/homelab/dozzle/users.yml b/homelab/dozzle/users.yml new file mode 100644 index 0000000..c53b951 --- /dev/null +++ b/homelab/dozzle/users.yml @@ -0,0 +1,5 @@ +users: + admin: + name: admin + email: admin@libnovel.cc + password: "$2y$10$4jqLza2grpxnQn0EGux2C.UmlSxRmOvH/J1ySzOBxMZgW6cA2TnmK" diff --git a/homelab/glitchtip/files_api.py b/homelab/glitchtip/files_api.py new file mode 100644 index 0000000..0a3672a --- /dev/null +++ b/homelab/glitchtip/files_api.py @@ -0,0 +1,127 @@ +"""Port of sentry.api.endpoints.chunk.ChunkUploadEndpoint""" + +import logging +from gzip import GzipFile +from io import BytesIO + +from django.conf import settings +from django.shortcuts import aget_object_or_404 +from django.urls import reverse +from ninja import File, Router +from ninja.errors import HttpError +from ninja.files import UploadedFile + +from apps.organizations_ext.models import Organization +from glitchtip.api.authentication import AuthHttpRequest +from glitchtip.api.decorators import optional_slash +from glitchtip.api.permissions import has_permission + +from .models import FileBlob + +# Force just one blob +CHUNK_UPLOAD_BLOB_SIZE = 32 * 1024 * 1024 # 32MB +MAX_CHUNKS_PER_REQUEST = 1 +MAX_REQUEST_SIZE = CHUNK_UPLOAD_BLOB_SIZE +MAX_CONCURRENCY = 1 +HASH_ALGORITHM = "sha1" + +CHUNK_UPLOAD_ACCEPT = ( + "debug_files", # DIF assemble + "release_files", # Release files assemble + "pdbs", # PDB upload and debug id override + "sources", # Source artifact bundle upload + "artifact_bundles", # Artifact bundles contain debug ids to link source to sourcemaps + "proguard", +) + + +class GzipChunk(BytesIO): + def __init__(self, file): + raw = file.read() + try: + data = GzipFile(fileobj=BytesIO(raw), mode="rb").read() + except Exception: + # sentry-cli 3.x sends raw (uncompressed) zip data despite gzip being + # advertised by the server — fall back to using the raw bytes as-is. + data = raw + self.size = len(data) + self.name = file.name + super().__init__(data) + + +router = Router() + + +@optional_slash(router, "get", "organizations/{slug:organization_slug}/chunk-upload/") +async def get_chunk_upload_info(request: AuthHttpRequest, organization_slug: str): + """Get server settings for chunk file upload""" + path = reverse("api:get_chunk_upload_info", args=[organization_slug]) + url = ( + path + if settings.GLITCHTIP_CHUNK_UPLOAD_USE_RELATIVE_URL + else settings.GLITCHTIP_URL.geturl() + path + ) + return { + "url": url, + "chunkSize": CHUNK_UPLOAD_BLOB_SIZE, + "chunksPerRequest": MAX_CHUNKS_PER_REQUEST, + "maxFileSize": 2147483648, + "maxRequestSize": MAX_REQUEST_SIZE, + "concurrency": MAX_CONCURRENCY, + "hashAlgorithm": HASH_ALGORITHM, + "compression": ["gzip"], + "accept": CHUNK_UPLOAD_ACCEPT, + } + + +@optional_slash(router, "post", "organizations/{slug:organization_slug}/chunk-upload/") +@has_permission(["project:write", "project:admin", "project:releases"]) +async def chunk_upload( + request: AuthHttpRequest, + organization_slug: str, + file_gzip: list[UploadedFile] = File(...), +): + """Upload one more more gzipped files to save""" + logger = logging.getLogger("glitchtip.files") + logger.info("chunkupload.start") + + organization = await aget_object_or_404( + Organization, slug=organization_slug.lower(), users=request.auth.user_id + ) + + files = [GzipChunk(chunk) for chunk in file_gzip] + + if len(files) == 0: + # No files uploaded is ok + logger.info("chunkupload.end", extra={"status": 200}) + return + + logger.info("chunkupload.post.files", extra={"len": len(files)}) + + # Validate file size + checksums = [] + size = 0 + for chunk in files: + size += chunk.size + if chunk.size > CHUNK_UPLOAD_BLOB_SIZE: + logger.info("chunkupload.end", extra={"status": 400}) + raise HttpError(400, "Chunk size too large") + checksums.append(chunk.name) + + if size > MAX_REQUEST_SIZE: + logger.info("chunkupload.end", extra={"status": 400}) + raise HttpError(400, "Request too large") + + if len(files) > MAX_CHUNKS_PER_REQUEST: + logger.info("chunkupload.end", extra={"status": 400}) + raise HttpError(400, "Too many chunks") + + try: + await FileBlob.from_files( + zip(files, checksums), organization=organization, logger=logger + ) + except IOError as err: + logger.info("chunkupload.end", extra={"status": 400}) + raise HttpError(400, str(err)) from err + + logger.info("chunkupload.end", extra={"status": 200}) diff --git a/homelab/otel/alloy.river b/homelab/otel/alloy.river new file mode 100644 index 0000000..ed37e26 --- /dev/null +++ b/homelab/otel/alloy.river @@ -0,0 +1,81 @@ +// Grafana Alloy — Faro RUM receiver + OTel log bridge +// +// Receives browser telemetry (Web Vitals, traces, logs, exceptions) from the +// LibNovel SvelteKit frontend via the @grafana/faro-web-sdk. +// +// Also receives OTLP logs from the backend and runner services, and forwards +// them to Loki in the native push format (solving the OTLP→Loki gap). +// +// Pipeline: +// faro.receiver → receives HTTP POST /collect from browsers +// otelcol.receiver.otlp → receives OTLP logs from backend/runner (HTTP :4318) +// otelcol.exporter.otlphttp → forwards traces to OTel Collector → Tempo +// loki.write → forwards Faro logs/exceptions to Loki +// otelcol.exporter.loki → forwards OTel logs to Loki (native format) +// +// The Faro endpoint is exposed publicly at faro.libnovel.cc via cloudflared. + +faro.receiver "faro" { + server { + listen_address = "0.0.0.0" + listen_port = 12347 + + cors_allowed_origins = ["https://libnovel.cc", "https://www.libnovel.cc"] + } + + output { + logs = [loki.write.faro.receiver] + traces = [otelcol.exporter.otlphttp.faro.input] + } +} + +// Receive OTLP traces and logs from backend/runner +otelcol.receiver.otlp "otel_logs" { + http { + endpoint = "0.0.0.0:4318" + } + + output { + logs = [otelcol.exporter.loki.otel_logs.input] + traces = [otelcol.exporter.otlphttp.otel_logs.input] + } +} + +// Convert OTel logs to Loki format and forward to loki.write +otelcol.exporter.loki "otel_logs" { + forward_to = [loki.write.otel_logs.receiver] +} + +// Send backend/runner traces to the OTel Collector → Tempo +otelcol.exporter.otlphttp "otel_logs" { + client { + endpoint = "http://otel-collector:4318" + tls { + insecure = true + } + } +} + +// Push backend/runner logs to Loki (native push format) +loki.write "otel_logs" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} + +// Forward Faro traces to the OTel Collector (which routes to Tempo) +otelcol.exporter.otlphttp "faro" { + client { + endpoint = "http://otel-collector:4318" + tls { + insecure = true + } + } +} + +// Forward Faro logs/exceptions directly to Loki +loki.write "faro" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/homelab/otel/collector.yaml b/homelab/otel/collector.yaml new file mode 100644 index 0000000..9d6ac6b --- /dev/null +++ b/homelab/otel/collector.yaml @@ -0,0 +1,79 @@ +# OTel Collector config +# +# Receivers: OTLP (gRPC + HTTP) from backend, ui, runner +# Processors: batch for efficiency, resource detection for host metadata +# Exporters: Tempo (traces), Prometheus (metrics), Loki (logs) + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + timeout: 5s + send_batch_size: 512 + + # Attach host metadata to traces/metrics + resourcedetection: + detectors: [env, system] + timeout: 5s + +exporters: + # Traces → Tempo + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + + # Metrics → Prometheus (remote write) + prometheusremotewrite: + endpoint: "http://prometheus:9090/api/v1/write" + tls: + insecure_skip_verify: true + + # Logs → Loki (via OTLP HTTP endpoint) + otlphttp/loki: + endpoint: "http://loki:3100/otlp" + tls: + insecure: true + + # Collector self-observability (optional debug) + debug: + verbosity: basic + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + pprof: + endpoint: 0.0.0.0:1777 + +service: + extensions: [health_check, pprof] + telemetry: + metrics: + # otel-collector v0.103+ replaced `address` with `readers` + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + traces: + receivers: [otlp] + processors: [resourcedetection, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [resourcedetection, batch] + exporters: [prometheusremotewrite] + logs: + receivers: [otlp] + # No resourcedetection — preserve service.name from OTel resource attributes + # (backend=backend, runner=runner, Alloy/Faro=no service.name → unknown_service) + processors: [batch] + exporters: [otlphttp/loki] diff --git a/homelab/otel/grafana/provisioning/alerting/contact-points.yaml b/homelab/otel/grafana/provisioning/alerting/contact-points.yaml new file mode 100644 index 0000000..6852319 --- /dev/null +++ b/homelab/otel/grafana/provisioning/alerting/contact-points.yaml @@ -0,0 +1,16 @@ +# Grafana alerting provisioning — contact points +# Sends all alerts to Gotify (self-hosted push notifications). +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: Gotify + receivers: + - uid: gotify-webhook + type: webhook + settings: + url: "http://gotify/message?token=ABZrZgCY-4ivcmt" + httpMethod: POST + title: "{{ .CommonLabels.alertname }}" + message: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ .Annotations.description }}{{ end }}" + disableResolveMessage: false diff --git a/homelab/otel/grafana/provisioning/alerting/notification-policy.yaml b/homelab/otel/grafana/provisioning/alerting/notification-policy.yaml new file mode 100644 index 0000000..1876a2b --- /dev/null +++ b/homelab/otel/grafana/provisioning/alerting/notification-policy.yaml @@ -0,0 +1,15 @@ +# Grafana alerting provisioning — notification policies +# Routes all alerts to Gotify by default. +apiVersion: 1 + +policies: + - orgId: 1 + receiver: Gotify + group_by: ["alertname", "service"] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + routes: + - receiver: Gotify + matchers: + - severity =~ "critical|warning" diff --git a/homelab/otel/grafana/provisioning/alerting/rules.yaml b/homelab/otel/grafana/provisioning/alerting/rules.yaml new file mode 100644 index 0000000..da97ac1 --- /dev/null +++ b/homelab/otel/grafana/provisioning/alerting/rules.yaml @@ -0,0 +1,214 @@ +# Grafana alerting provisioning — alert rules +# Covers: runner down, high task failure rate, audio error spike, backend error spike. +apiVersion: 1 + +groups: + - orgId: 1 + name: LibNovel Runner + folder: LibNovel + interval: 1m + rules: + + - uid: runner-down + title: Runner Down + condition: C + for: 2m + annotations: + summary: "LibNovel runner is not reachable" + description: "The Prometheus scrape of runner:9091 has been failing for >2 minutes. Tasks are not being processed." + labels: + severity: critical + service: runner + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: { from: 300, to: 0 } + model: + expr: "up{job=\"libnovel-runner\"}" + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + - refId: C + datasourceUid: __expr__ + relativeTimeRange: { from: 300, to: 0 } + model: + type: classic_conditions + conditions: + - evaluator: { params: [1], type: lt } + operator: { type: and } + query: { params: [A] } + reducer: { params: [], type: last } + + - uid: runner-high-failure-rate + title: Runner High Task Failure Rate + condition: C + for: 5m + annotations: + summary: "Runner task failure rate is above 20%" + description: "More than 20% of runner tasks have been failing for the last 5 minutes. Check runner logs." + labels: + severity: warning + service: runner + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: { from: 600, to: 0 } + model: + expr: "rate(libnovel_runner_tasks_failed_total[5m]) / clamp_min(rate(libnovel_runner_tasks_completed_total[5m]) + rate(libnovel_runner_tasks_failed_total[5m]), 0.001)" + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + - refId: C + datasourceUid: __expr__ + relativeTimeRange: { from: 600, to: 0 } + model: + type: classic_conditions + conditions: + - evaluator: { params: [0.2], type: gt } + operator: { type: and } + query: { params: [A] } + reducer: { params: [], type: last } + + - uid: runner-tasks-stalled + title: Runner Tasks Stalled + condition: C + for: 10m + annotations: + summary: "Runner has tasks running for >10 minutes with no completions" + description: "tasks_running > 0 but rate(tasks_completed) is 0. Tasks may be stuck or the runner is in a crash loop." + labels: + severity: warning + service: runner + data: + - refId: Running + datasourceUid: prometheus + relativeTimeRange: { from: 900, to: 0 } + model: + expr: "libnovel_runner_tasks_running" + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + - refId: Rate + datasourceUid: prometheus + relativeTimeRange: { from: 900, to: 0 } + model: + expr: "rate(libnovel_runner_tasks_completed_total[10m])" + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + - refId: C + datasourceUid: __expr__ + relativeTimeRange: { from: 900, to: 0 } + model: + type: classic_conditions + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [Running] } + reducer: { params: [], type: last } + - evaluator: { params: [0.001], type: lt } + operator: { type: and } + query: { params: [Rate] } + reducer: { params: [], type: last } + + - orgId: 1 + name: LibNovel Backend + folder: LibNovel + interval: 1m + rules: + + - uid: backend-high-error-rate + title: Backend High Error Rate + condition: C + for: 5m + annotations: + summary: "Backend API error rate above 5%" + description: "More than 5% of backend HTTP requests are returning 5xx status codes (as seen from UI OTel instrumentation)." + labels: + severity: warning + service: backend + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: { from: 600, to: 0 } + model: + expr: "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\", http_response_status_code=~\"5..\"}[5m])) / clamp_min(sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\"}[5m])), 0.001)" + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + - refId: C + datasourceUid: __expr__ + relativeTimeRange: { from: 600, to: 0 } + model: + type: classic_conditions + conditions: + - evaluator: { params: [0.05], type: gt } + operator: { type: and } + query: { params: [A] } + reducer: { params: [], type: last } + + - uid: backend-high-p95-latency + title: Backend High p95 Latency + condition: C + for: 5m + annotations: + summary: "Backend p95 latency above 2s" + description: "95th percentile latency of backend spans has exceeded 2 seconds for >5 minutes." + labels: + severity: warning + service: backend + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: { from: 600, to: 0 } + model: + expr: "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))" + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + - refId: C + datasourceUid: __expr__ + relativeTimeRange: { from: 600, to: 0 } + model: + type: classic_conditions + conditions: + - evaluator: { params: [2], type: gt } + operator: { type: and } + query: { params: [A] } + reducer: { params: [], type: last } + + - orgId: 1 + name: LibNovel OTel Pipeline + folder: LibNovel + interval: 2m + rules: + + - uid: otel-collector-down + title: OTel Collector Down + condition: C + for: 3m + annotations: + summary: "OTel collector is not reachable" + description: "Prometheus cannot scrape otel-collector:8888. Traces and logs may be dropping." + labels: + severity: warning + service: otel-collector + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: { from: 600, to: 0 } + model: + expr: "up{job=\"otel-collector\"}" + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + - refId: C + datasourceUid: __expr__ + relativeTimeRange: { from: 600, to: 0 } + model: + type: classic_conditions + conditions: + - evaluator: { params: [1], type: lt } + operator: { type: and } + query: { params: [A] } + reducer: { params: [], type: last } diff --git a/homelab/otel/grafana/provisioning/dashboards/backend.json b/homelab/otel/grafana/provisioning/dashboards/backend.json new file mode 100644 index 0000000..ef00fd7 --- /dev/null +++ b/homelab/otel/grafana/provisioning/dashboards/backend.json @@ -0,0 +1,307 @@ +{ + "uid": "libnovel-backend", + "title": "Backend API", + "description": "Request rate, error rate, and latency for the LibNovel backend. Powered by Tempo span metrics.", + "tags": ["libnovel", "backend", "api"], + "timezone": "browser", + "refresh": "30s", + "time": { "from": "now-3h", "to": "now" }, + "schemaVersion": 39, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Request Rate (RPS)", + "gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "value", + "graphMode": "area", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"backend\"}[5m]))", + "legendFormat": "rps", + "instant": true + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "Error Rate", + "gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.01 }, + { "color": "red", "value": 0.05 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"backend\", status_code=\"STATUS_CODE_ERROR\"}[5m])) / clamp_min(sum(rate(traces_spanmetrics_calls_total{service=\"backend\"}[5m])), 0.001)", + "legendFormat": "error rate", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "p50 Latency", + "gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.2 }, + { "color": "red", "value": 1 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))", + "legendFormat": "p50", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "p95 Latency", + "gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "red", "value": 2 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))", + "legendFormat": "p95", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "p99 Latency", + "gridPos": { "x": 16, "y": 0, "w": 4, "h": 4 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))", + "legendFormat": "p99", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "5xx Errors / min", + "gridPos": { "x": 20, "y": 0, "w": 4, "h": 4 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"backend\", status_code=\"STATUS_CODE_ERROR\"}[5m])) * 60", + "legendFormat": "5xx/min", + "instant": true + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Request Rate (total vs errors)", + "gridPos": { "x": 0, "y": 4, "w": 12, "h": 8 }, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { "unit": "reqps", "custom": { "lineWidth": 2, "fillOpacity": 10 } }, + "overrides": [ + { "matcher": { "id": "byName", "options": "errors" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] } + ] + }, + "targets": [ + { + "refId": "total", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"backend\"}[5m]))", + "legendFormat": "total" + }, + { + "refId": "errors", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"backend\", status_code=\"STATUS_CODE_ERROR\"}[5m]))", + "legendFormat": "errors" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Latency Percentiles (backend spans)", + "gridPos": { "x": 12, "y": 4, "w": 12, "h": 8 }, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { "unit": "s", "custom": { "lineWidth": 2, "fillOpacity": 10 } } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))", + "legendFormat": "p50" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))", + "legendFormat": "p95" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Request Rate by Span Name (top operations)", + "gridPos": { "x": 0, "y": 12, "w": 12, "h": 8 }, + "description": "Throughput broken down by HTTP route / span name from Tempo span metrics.", + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { "unit": "reqps", "custom": { "lineWidth": 2, "fillOpacity": 5 } } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "topk(10, sum(rate(traces_spanmetrics_calls_total{service=\"backend\"}[5m])) by (span_name))", + "legendFormat": "{{span_name}}" + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Latency by Span Name (p95)", + "gridPos": { "x": 12, "y": 12, "w": 12, "h": 8 }, + "description": "p95 latency per operation — helps identify slow endpoints.", + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { "unit": "s", "custom": { "lineWidth": 2, "fillOpacity": 5 } } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "topk(10, histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le, span_name)))", + "legendFormat": "{{span_name}}" + } + ] + }, + { + "id": 20, + "type": "logs", + "title": "Backend Errors", + "gridPos": { "x": 0, "y": 20, "w": 24, "h": 10 }, + "options": { + "showTime": true, + "showLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=\"backend\"}", + "legendFormat": "" + } + ] + } + ] +} diff --git a/homelab/otel/grafana/provisioning/dashboards/catalogue.json b/homelab/otel/grafana/provisioning/dashboards/catalogue.json new file mode 100644 index 0000000..5d2b73c --- /dev/null +++ b/homelab/otel/grafana/provisioning/dashboards/catalogue.json @@ -0,0 +1,225 @@ +{ + "uid": "libnovel-catalogue", + "title": "Catalogue & Content Progress", + "description": "Scraping progress from runner OTel logs in Loki. Logs are JSON: body=message, attributes.slug/chapters/page=fields.", + "tags": ["libnovel", "catalogue", "content"], + "timezone": "browser", + "refresh": "1m", + "time": { "from": "now-24h", "to": "now" }, + "schemaVersion": 39, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Books Scraped (last 24h)", + "description": "Count of unique slugs from chapter list fetched messages.", + "gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none" }, + "fieldConfig": { + "defaults": { + "color": { "fixedColor": "blue", "mode": "fixed" }, + "thresholds": { "mode": "absolute", "steps": [] } + } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "count(count_over_time({service_name=\"runner\"} | json | body=\"chapter list fetched\" [24h])) by (attributes_slug)", + "legendFormat": "books scraped" + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "Chapters Scraped (last 24h)", + "description": "Count of 'chapter list fetched' events.", + "gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none" }, + "fieldConfig": { + "defaults": { + "color": { "fixedColor": "blue", "mode": "fixed" }, + "thresholds": { "mode": "absolute", "steps": [] } + } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "sum(count_over_time({service_name=\"runner\"} | json | body=\"chapter list fetched\" [24h]))", + "legendFormat": "chapter lists fetched" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Metadata Saved (last 24h)", + "description": "Count of 'metadata saved' events.", + "gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none" }, + "fieldConfig": { + "defaults": { + "color": { "fixedColor": "green", "mode": "fixed" }, + "thresholds": { "mode": "absolute", "steps": [] } + } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "sum(count_over_time({service_name=\"runner\"} | json | body=\"metadata saved\" [24h]))", + "legendFormat": "metadata saved" + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Scrape Errors (last 24h)", + "description": "Count of error severity logs from the runner.", + "gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 10 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "sum(count_over_time({service_name=\"runner\"} | json | severity=\"ERROR\" [24h]))", + "legendFormat": "errors" + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Rate Limited (last 24h)", + "description": "Count of rate limiting events from Novelfire.", + "gridPos": { "x": 16, "y": 0, "w": 4, "h": 4 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 50 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "sum(count_over_time({service_name=\"runner\"} | json | body=~\"rate limit\" [24h]))", + "legendFormat": "rate limited" + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Scrape Rate (books/min)", + "description": "Rate of events per minute.", + "gridPos": { "x": 0, "y": 4, "w": 12, "h": 8 }, + "options": { "tooltip": { "mode": "multi" }, "legend": { "displayMode": "list", "placement": "bottom" } }, + "fieldConfig": { + "defaults": { "unit": "short", "custom": { "lineWidth": 2, "fillOpacity": 10 } } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "sum(rate({service_name=\"runner\"} | json | body=\"chapter list fetched\" [5m])) * 60", + "legendFormat": "books/min" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "sum(rate({service_name=\"runner\"} | json | body=\"metadata saved\" [5m])) * 60", + "legendFormat": "metadata/min" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Error Rate (errors/min)", + "description": "Rate of error and rate-limit messages over time.", + "gridPos": { "x": 12, "y": 4, "w": 12, "h": 8 }, + "options": { "tooltip": { "mode": "multi" }, "legend": { "displayMode": "list", "placement": "bottom" } }, + "fieldConfig": { + "defaults": { "unit": "short", "custom": { "lineWidth": 2, "fillOpacity": 10 } }, + "overrides": [ + { "matcher": { "id": "byName", "options": "errors/min" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "rate-limit/min" }, "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] } + ] + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "sum(rate({service_name=\"runner\"} | json | severity=\"ERROR\" [5m])) * 60", + "legendFormat": "errors/min" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "sum(rate({service_name=\"runner\"} | json | body=~\"rate limit\" [5m])) * 60", + "legendFormat": "rate-limit/min" + } + ] + }, + { + "id": 20, + "type": "logs", + "title": "Runner Logs (errors & warnings)", + "description": "Runner log lines containing errors or warnings.", + "gridPos": { "x": 0, "y": 12, "w": 24, "h": 10 }, + "options": { + "showTime": true, + "showLabels": false, + "wrapLogMessage": false, + "prettifyLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=\"runner\"} | json | severity=~\"ERROR|WARN\"", + "legendFormat": "" + } + ] + }, + { + "id": 21, + "type": "logs", + "title": "Runner Logs (all)", + "description": "All runner log entries.", + "gridPos": { "x": 0, "y": 22, "w": 24, "h": 10 }, + "options": { + "showTime": true, + "showLabels": true, + "wrapLogMessage": false, + "prettifyLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=\"runner\"}", + "legendFormat": "" + } + ] + } + ] +} diff --git a/homelab/otel/grafana/provisioning/dashboards/dashboards.yaml b/homelab/otel/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..b03f6e3 --- /dev/null +++ b/homelab/otel/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +# Grafana dashboard provisioning +# Points Grafana at the local dashboards directory. +# Drop any .json dashboard file into homelab/otel/grafana/provisioning/dashboards/ +# and it will appear in Grafana automatically on restart. + +apiVersion: 1 + +providers: + - name: libnovel + folder: LibNovel + type: file + options: + path: /etc/grafana/provisioning/dashboards diff --git a/homelab/otel/grafana/provisioning/dashboards/runner.json b/homelab/otel/grafana/provisioning/dashboards/runner.json new file mode 100644 index 0000000..950e92e --- /dev/null +++ b/homelab/otel/grafana/provisioning/dashboards/runner.json @@ -0,0 +1,377 @@ +{ + "uid": "libnovel-runner", + "title": "Runner Operations", + "description": "Task queue health, throughput, TTS routing, and live logs for the homelab runner.", + "tags": ["libnovel", "runner"], + "timezone": "browser", + "refresh": "30s", + "time": { "from": "now-3h", "to": "now" }, + "schemaVersion": 39, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Tasks Running", + "gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 3 } + ] + }, + "mappings": [] + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "runner_tasks_running", + "legendFormat": "running", + "instant": true + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "Tasks Completed (total)", + "gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "fixedColor": "green", "mode": "fixed" }, + "thresholds": { "mode": "absolute", "steps": [] } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "runner_tasks_completed_total", + "legendFormat": "completed", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Tasks Failed (total)", + "gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "runner_tasks_failed_total", + "legendFormat": "failed", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Runner Uptime", + "gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "value", + "graphMode": "none", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "green", "value": 300 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "runner_uptime_seconds", + "legendFormat": "uptime", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Task Failure Rate", + "gridPos": { "x": 16, "y": 0, "w": 4, "h": 4 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.05 }, + { "color": "red", "value": 0.2 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "runner_tasks_failed_total / clamp_min(runner_tasks_completed_total + runner_tasks_failed_total, 1)", + "legendFormat": "failure rate", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Runner Alive", + "gridPos": { "x": 20, "y": 0, "w": 4, "h": 4 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } } + ], + "thresholds": { "mode": "absolute", "steps": [] } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "up{job=\"libnovel-runner\"}", + "legendFormat": "runner", + "instant": true + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Task Throughput (per minute)", + "gridPos": { "x": 0, "y": 4, "w": 12, "h": 8 }, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { "lineWidth": 2, "fillOpacity": 10 } + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "failed" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "completed" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] } + ] + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(runner_tasks_completed_total[5m]) * 60", + "legendFormat": "completed" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "rate(runner_tasks_failed_total[5m]) * 60", + "legendFormat": "failed" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "runner_tasks_running", + "legendFormat": "running" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Audio Task Span Latency (p50 / p95 / p99)", + "gridPos": { "x": 12, "y": 4, "w": 12, "h": 8 }, + "description": "End-to-end latency of runner.audio_task spans from Tempo span metrics.", + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "lineWidth": 2, "fillOpacity": 10 } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.audio_task\"}[5m])) by (le))", + "legendFormat": "p50" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.audio_task\"}[5m])) by (le))", + "legendFormat": "p95" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.audio_task\"}[5m])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 20, + "type": "timeseries", + "title": "Scrape Task Span Latency (p50 / p95 / p99)", + "gridPos": { "x": 0, "y": 12, "w": 12, "h": 8 }, + "description": "End-to-end latency of runner.scrape_task spans from Tempo span metrics.", + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "lineWidth": 2, "fillOpacity": 10 } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.scrape_task\"}[5m])) by (le))", + "legendFormat": "p50" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.scrape_task\"}[5m])) by (le))", + "legendFormat": "p95" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.scrape_task\"}[5m])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 21, + "type": "timeseries", + "title": "Audio vs Scrape Task Rate", + "gridPos": { "x": 12, "y": 12, "w": 12, "h": 8 }, + "description": "Relative throughput of audio generation vs book scraping.", + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { "lineWidth": 2, "fillOpacity": 10 } + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"runner\", span_name=\"runner.audio_task\"}[5m]))", + "legendFormat": "audio tasks/s" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"runner\", span_name=\"runner.scrape_task\"}[5m]))", + "legendFormat": "scrape tasks/s" + } + ] + }, + { + "id": 30, + "type": "logs", + "title": "Runner Logs (errors & warnings)", + "gridPos": { "x": 0, "y": 20, "w": 24, "h": 10 }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=\"runner\"}", + "legendFormat": "" + } + ] + }, + { + "id": 31, + "type": "logs", + "title": "Runner Logs (all)", + "gridPos": { "x": 0, "y": 30, "w": 24, "h": 10 }, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=\"runner\"}", + "legendFormat": "" + } + ] + } + ] +} diff --git a/homelab/otel/grafana/provisioning/dashboards/web-vitals.json b/homelab/otel/grafana/provisioning/dashboards/web-vitals.json new file mode 100644 index 0000000..005174f --- /dev/null +++ b/homelab/otel/grafana/provisioning/dashboards/web-vitals.json @@ -0,0 +1,804 @@ +{ + "uid": "libnovel-web-vitals", + "title": "Web Vitals (RUM)", + "description": "Core Web Vitals from @grafana/faro-web-sdk. Data: browser \u2192 Alloy faro.receiver \u2192 Loki ({service_name=unknown_service}). Log format: key=value pairs, e.g. lcp=767.000000 fcp=767.000000. Use | regexp to extract.", + "tags": [ + "libnovel", + "frontend", + "rum", + "web-vitals" + ], + "timezone": "browser", + "refresh": "1m", + "time": { + "from": "now-24h", + "to": "now" + }, + "schemaVersion": 39, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "LCP \u2014 p75 (Largest Contentful Paint)", + "description": "Good < 2.5s, needs improvement < 4s, poor >= 4s. Source: Loki {service_name=unknown_service} Faro measurements.", + "gridPos": { + "x": 0, + "y": 0, + "w": 4, + "h": 4 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2500 + }, + { + "color": "red", + "value": 4000 + } + ] + } + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.75, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `lcp=(?P\\d+\\.?\\d*)` | unwrap lcp [1h])", + "legendFormat": "LCP p75", + "instant": true + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "INP \u2014 p75 (Interaction to Next Paint)", + "description": "Good < 200ms, needs improvement < 500ms, poor >= 500ms.", + "gridPos": { + "x": 4, + "y": 0, + "w": 4, + "h": 4 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 200 + }, + { + "color": "red", + "value": 500 + } + ] + } + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.75, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `inp=(?P\\d+\\.?\\d*)` | unwrap inp [1h])", + "legendFormat": "INP p75", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "CLS \u2014 p75 (Cumulative Layout Shift)", + "description": "Good < 0.1, needs improvement < 0.25, poor >= 0.25.", + "gridPos": { + "x": 8, + "y": 0, + "w": 4, + "h": 4 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 3, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 0.25 + } + ] + } + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.75, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `cls=(?P\\d+\\.?\\d*)` | unwrap cls [1h])", + "legendFormat": "CLS p75", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "TTFB \u2014 p75 (Time to First Byte)", + "description": "Good < 800ms, needs improvement < 1800ms, poor >= 1800ms.", + "gridPos": { + "x": 12, + "y": 0, + "w": 4, + "h": 4 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 800 + }, + { + "color": "red", + "value": 1800 + } + ] + } + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.75, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `ttfb=(?P\\d+\\.?\\d*)` | unwrap ttfb [1h])", + "legendFormat": "TTFB p75", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "FCP \u2014 p75 (First Contentful Paint)", + "description": "Good < 1.8s, needs improvement < 3s, poor >= 3s.", + "gridPos": { + "x": 16, + "y": 0, + "w": 4, + "h": 4 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1800 + }, + { + "color": "red", + "value": 3000 + } + ] + } + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.75, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `fcp=(?P\\d+\\.?\\d*)` | unwrap fcp [1h])", + "legendFormat": "FCP p75", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Measurements / min", + "description": "Number of Faro measurement events in the last 5 minutes (activity indicator).", + "gridPos": { + "x": 20, + "y": 0, + "w": 4, + "h": 4 + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" [5m]))", + "legendFormat": "measurements", + "instant": true + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "LCP over time (p50 / p75 / p95)", + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 8 + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Good (2.5s)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Poor (4s)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + } + ] + } + ] + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.50, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `lcp=(?P\\d+\\.?\\d*)` | unwrap lcp [5m])", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.75, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `lcp=(?P\\d+\\.?\\d*)` | unwrap lcp [5m])", + "legendFormat": "p75" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.95, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `lcp=(?P\\d+\\.?\\d*)` | unwrap lcp [5m])", + "legendFormat": "p95" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "2500", + "legendFormat": "Good (2.5s)" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "4000", + "legendFormat": "Poor (4s)" + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "TTFB over time (p50 / p75 / p95)", + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 8 + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.50, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `ttfb=(?P\\d+\\.?\\d*)` | unwrap ttfb [5m])", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.75, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `ttfb=(?P\\d+\\.?\\d*)` | unwrap ttfb [5m])", + "legendFormat": "p75" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "quantile_over_time(0.95, {service_name=\"unknown_service\"} |= \"kind=measurement\" |= \"type=web-vitals\" | regexp `ttfb=(?P\\d+\\.?\\d*)` | unwrap ttfb [5m])", + "legendFormat": "p95" + } + ] + }, + { + "id": 20, + "type": "logs", + "title": "Frontend Errors & Exceptions", + "description": "JS exceptions captured by Faro. kind=exception events.", + "gridPos": { + "x": 0, + "y": 12, + "w": 24, + "h": 10 + }, + "options": { + "showTime": true, + "showLabels": true, + "wrapLogMessage": true, + "prettifyLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service_name=\"unknown_service\"} | regexp `(?P\\w+)` | kind = \"exception\"", + "legendFormat": "" + } + ] + }, + { + "id": 21, + "type": "logs", + "title": "Web Vitals Measurements", + "description": "All Faro measurement events.", + "gridPos": { + "x": 0, + "y": 22, + "w": 24, + "h": 10 + }, + "options": { + "showTime": true, + "showLabels": true, + "wrapLogMessage": false, + "prettifyLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service_name=\"unknown_service\"} | regexp `(?P\\w+)` | kind = \"measurement\"", + "legendFormat": "" + } + ] + }, + { + "id": 30, + "type": "row", + "title": "API Performance (Upstream Requests)", + "gridPos": { "x": 0, "y": 32, "w": 24, "h": 1 }, + "collapsed": false + }, + { + "id": 31, + "type": "timeseries", + "title": "API Request Duration — p50 / p75 / p95 by endpoint", + "description": "Duration of all libnovel.cc/api/* fetch requests captured by Faro faro.performance.resource events. Values in ms.", + "gridPos": { "x": 0, "y": 33, "w": 24, "h": 10 }, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { "lineWidth": 2, "fillOpacity": 5 } + } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.50, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/progress/audio-time\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p50 /api/progress/audio-time" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.95, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/progress/audio-time\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p95 /api/progress/audio-time" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.50, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/presign/audio\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p50 /api/presign/audio" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.95, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/presign/audio\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p95 /api/presign/audio" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.50, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/progress\" !~ \"audio-time\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p50 /api/progress" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.95, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/progress\" !~ \"audio-time\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p95 /api/progress" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.50, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/comments\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p50 /api/comments" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.95, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/comments\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p95 /api/comments" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.50, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/settings\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p50 /api/settings" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.95, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/settings\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p95 /api/settings" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.50, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/catalogue-page\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p50 /api/catalogue-page" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "quantile_over_time(0.95, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/catalogue-page\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [5m])", + "legendFormat": "p95 /api/catalogue-page" + } + ] + }, + { + "id": 32, + "type": "barchart", + "title": "API Avg Duration — last 1h", + "description": "Average duration per endpoint over the last hour. Useful for spotting the slowest APIs at a glance.", + "gridPos": { "x": 0, "y": 43, "w": 12, "h": 8 }, + "options": { + "orientation": "horizontal", + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single" }, + "xTickLabelRotation": 0 + }, + "fieldConfig": { + "defaults": { "unit": "ms", "color": { "mode": "palette-classic" } } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "avg_over_time({service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/progress/audio-time\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [1h])", + "legendFormat": "/api/progress/audio-time", + "instant": true + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "avg_over_time({service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/presign/audio\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [1h])", + "legendFormat": "/api/presign/audio", + "instant": true + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "avg_over_time({service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/progress\" !~ \"audio-time\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [1h])", + "legendFormat": "/api/progress", + "instant": true + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "avg_over_time({service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/comments\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [1h])", + "legendFormat": "/api/comments", + "instant": true + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "avg_over_time({service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/settings\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [1h])", + "legendFormat": "/api/settings", + "instant": true + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "avg_over_time({service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api/catalogue-page\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [1h])", + "legendFormat": "/api/catalogue-page", + "instant": true + } + ] + }, + { + "id": 33, + "type": "stat", + "title": "Slowest API call — p95 last 1h", + "description": "p95 duration of the single slowest endpoint in the last hour.", + "gridPos": { "x": 12, "y": 43, "w": 6, "h": 4 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 500 }, + { "color": "red", "value": 1000 } + ] + } + } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "max(quantile_over_time(0.95, {service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur [1h]))", + "legendFormat": "p95 max", + "instant": true + } + ] + }, + { + "id": 34, + "type": "stat", + "title": "API Requests / min", + "description": "Rate of libnovel.cc API requests captured by Faro in the last 5 minutes.", + "gridPos": { "x": 18, "y": 43, "w": 6, "h": 4 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "sum(count_over_time({service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api\" [5m])) / 5", + "legendFormat": "req/min", + "instant": true + } + ] + }, + { + "id": 35, + "type": "logs", + "title": "Slow API Requests (>500ms)", + "description": "Individual faro.performance.resource events where duration > 500ms. Useful for debugging outliers.", + "gridPos": { "x": 0, "y": 47, "w": 24, "h": 8 }, + "options": { + "showTime": true, + "showLabels": false, + "wrapLogMessage": false, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=\"unknown_service\"} |= \"faro.performance.resource\" |= \"libnovel.cc/api\" | regexp `event_data_duration=(?P[0-9.]+)` | unwrap dur | dur > 500", + "legendFormat": "" + } + ] + } + ] +} \ No newline at end of file diff --git a/homelab/otel/grafana/provisioning/datasources/datasources.yaml b/homelab/otel/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..3179610 --- /dev/null +++ b/homelab/otel/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,53 @@ +# Grafana datasource provisioning +# Auto-configures Tempo, Prometheus, and Loki on first start. +# No manual setup needed in the UI. + +apiVersion: 1 + +datasources: + - name: Tempo + type: tempo + uid: tempo + url: http://tempo:3200 + access: proxy + isDefault: false + jsonData: + httpMethod: GET + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + traceQuery: + timeShiftEnabled: true + spanStartTimeShift: "1h" + spanEndTimeShift: "-1h" + spanBar: + type: "Tag" + tag: "http.url" + lokiSearch: + datasourceUid: loki + + - name: Prometheus + type: prometheus + uid: prometheus + url: http://prometheus:9090 + access: proxy + isDefault: true + jsonData: + httpMethod: POST + exemplarTraceIdDestinations: + - name: traceID + datasourceUid: tempo + + - name: Loki + type: loki + uid: loki + url: http://loki:3100 + access: proxy + isDefault: false + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: '"traceID":"(\w+)"' + name: TraceID + url: "$${__value.raw}" diff --git a/homelab/otel/loki.yaml b/homelab/otel/loki.yaml new file mode 100644 index 0000000..e27d820 --- /dev/null +++ b/homelab/otel/loki.yaml @@ -0,0 +1,38 @@ +# Loki config — minimal single-node setup +# Receives logs from OTel Collector. 30-day retention. + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: 720h # 30 days + +compactor: + working_directory: /loki/compactor + delete_request_store: filesystem + retention_enabled: true diff --git a/homelab/otel/prometheus.yaml b/homelab/otel/prometheus.yaml new file mode 100644 index 0000000..4fff3fd --- /dev/null +++ b/homelab/otel/prometheus.yaml @@ -0,0 +1,22 @@ +# Prometheus config +# Scrapes OTel collector self-metrics and runner metrics endpoint. +# Backend metrics come in via OTel remote-write — no direct scrape needed. + +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + environment: production + +scrape_configs: + # OTel Collector self-metrics + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + + # Runner JSON metrics endpoint (native format, no Prometheus client yet) + # Will be replaced by OTLP once runner is instrumented with OTel SDK. + - job_name: libnovel-runner + metrics_path: /metrics + static_configs: + - targets: ["runner:9091"] diff --git a/homelab/otel/tempo.yaml b/homelab/otel/tempo.yaml new file mode 100644 index 0000000..a70b725 --- /dev/null +++ b/homelab/otel/tempo.yaml @@ -0,0 +1,45 @@ +# Tempo config — minimal single-node setup +# Stores traces locally. Grafana queries via the HTTP API on port 3200. + +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + trace_idle_period: 10s + max_block_bytes: 104857600 # 100MB + max_block_duration: 30m + +compactor: + compaction: + block_retention: 720h # 30 days + +storage: + trace: + backend: local + local: + path: /var/tempo/blocks + wal: + path: /var/tempo/wal + +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] + generate_native_histograms: both diff --git a/ios/.gitignore b/ios/.gitignore new file mode 100644 index 0000000..b1f0a6b --- /dev/null +++ b/ios/.gitignore @@ -0,0 +1,14 @@ +# Xcode build artifacts — regenerate with: xcodegen generate --spec project.yml +xcuserdata/ +*.xcuserstate +*.xcworkspace/xcuserdata/ +DerivedData/ +build/ + +# Swift Package Manager — resolved by Xcode on first open +LibNovel.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/ +.build/ +# Package.resolved is committed so SPM builds are reproducible + +# OS +.DS_Store diff --git a/ios/AGENTS.md b/ios/AGENTS.md new file mode 100644 index 0000000..5992ee8 --- /dev/null +++ b/ios/AGENTS.md @@ -0,0 +1,87 @@ +# LibNovel iOS App + +SwiftUI app targeting iOS 17+. Consumes the Go scraper HTTP API for books, chapters, and audio. Uses MinIO presigned URLs for media playback and downloads. + +## Project Structure + +``` +ios/LibNovel/LibNovel/ +├── App/ # LibNovelApp.swift, ContentView.swift, RootTabView.swift +├── Models/ # Models.swift (all domain types) +├── Networking/ # APIClient.swift (URLSession-based HTTP client) +├── Services/ # AudioPlayerService, AudioDownloadService, AuthStore, +│ # BookVoicePreferences, NetworkMonitor +├── ViewModels/ # One per view/feature (HomeViewModel, BrowseViewModel, etc.) +├── Views/ +│ ├── Auth/ # AuthView +│ ├── BookDetail/ # BookDetailView, CommentsView +│ ├── Browse/ # BrowseView (infinite scroll shelves) +│ ├── ChapterReader/ # ChapterReaderView, DownloadAudioButton +│ ├── Common/ # CommonViews (shared reusable components) +│ ├── Components/ # OfflineBanner +│ ├── Downloads/ # DownloadsView, DownloadQueueButton +│ ├── Home/ # HomeView +│ ├── Library/ # LibraryView (2-col grid, filters) +│ ├── Player/ # PlayerViews (floating FAB, compact, full-screen) +│ ├── Profile/ # ProfileView, VoiceSelectionView, UserProfileView, etc. +│ └── Search/ # SearchView +└── Extensions/ # NavDestination.swift, String+App.swift, Color+App.swift +``` + +## iOS / Swift Conventions + +- **Deployment target**: iOS 17.0 — use iOS 17+ APIs freely. +- **Observable pattern**: The codebase currently uses `@StateObject` / `ObservableObject` / `@Published`. When adding new types, prefer the **`@Observable` macro** (iOS 17+) over `ObservableObject`. Do not refactor existing types unless explicitly asked. +- **Navigation**: Use `NavigationStack` (not `NavigationView`). Use `.navigationDestination(for:)` for type-safe routing. +- **Concurrency**: Use `async/await` and structured concurrency. Avoid callback-based APIs and `DispatchQueue.main.async` — prefer `@MainActor` or `await MainActor.run`. +- **State management**: Prefer `@State` + `@Binding` for local UI state. Use environment objects for app-wide services (authStore, audioPlayer, downloadService, networkMonitor). +- **SwiftData**: Not currently used. Do not introduce SwiftData without discussion. +- **SF Symbols**: Use `Image(systemName:)` for icons. No emoji in UI unless already present. + +## Key Patterns + +- **Download keys**: Use `::` as separator (e.g., `"slug::chapter-1::voice"`), never `-`. Slugs contain hyphens. +- **Voice fallback chain**: book override → global default → `"af_bella"`. See `BookVoicePreferences.voiceWithFallback()`. +- **Offline handling**: Wrap view bodies in `VStack` with `OfflineBanner` at top. Use `NetworkMonitor` (environment object) to gate network calls. Suppress network errors silently when offline via `ErrorAlertModifier`. +- **Audio playback priority**: local file → MinIO presigned URL → trigger TTS generation. +- **Progress display**: Show decimal % when < 10% (e.g., "3.4%"), rounded when >= 10% (e.g., "47%"). +- **Cover images**: Always proxy via `/api/cover/{domain}/{slug}` — never link directly to source. + +## Networking + +`APIClient.swift` wraps all Go scraper API calls. When adding new endpoints: + +1. Add a method to `APIClient`. +2. Keep error handling consistent — throw typed errors, let ViewModels catch and set `errorMessage`. +3. All requests are relative to `SCRAPER_API_URL` (configured at build time via xcconfig or environment). + +## Using Documentation Tools + +When writing or reviewing SwiftUI/Swift code: + +- Use `context7` to look up current Apple SwiftUI/Swift documentation before implementing anything non-trivial. Apple's APIs evolve fast — do not rely on training data alone. +- Use `gh_grep` to find real-world Swift patterns when unsure how something is typically implemented. + +Example prompts: +- "How does `.searchable` work in iOS 17? use context7" +- "Show me examples of `@Observable` with async tasks. use context7" +- "How do other apps implement background URLSession downloads in Swift? use gh_grep" + +## UI/UX Skill + +For any iOS view work, always load the `ios-ux` skill at the start of the task: + +``` +skill({ name: "ios-ux" }) +``` + +This skill defines the full design system, animation rules, haptic feedback policy, accessibility checklist, performance guidelines, and offline handling requirements. It also governs how to handle screenshot-based reviews (analyze → suggest → confirm before applying). + +## What to Avoid + +- `NavigationView` — deprecated, use `NavigationStack` +- `ObservableObject` / `@Published` for new types — prefer `@Observable` +- `DispatchQueue.main.async` — prefer `@MainActor` +- Force unwrapping optionals +- Hardcoded color literals — use `Color+App.swift` extensions or semantic colors +- Adding new dependencies (SPM packages) without discussion diff --git a/ios/LibNovelV2/LibNovelV2.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/ios/LibNovelV2/LibNovelV2.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..919434a --- /dev/null +++ b/ios/LibNovelV2/LibNovelV2.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/justfile b/justfile new file mode 100644 index 0000000..20f9cfb --- /dev/null +++ b/justfile @@ -0,0 +1,136 @@ +# ── LibNovel v3 — justfile ──────────────────────────────────────────────────── +# All commands that touch docker-compose are wrapped with `doppler run` so that +# secrets are injected into the environment at runtime — no .env files needed. +# +# Prerequisites: +# brew install doppler just +# doppler setup (run once; selects project=libnovel config=prd) +# +# Usage: +# just up # start all services (detached) +# just down # stop all services +# just logs # tail all service logs +# just ps # show running containers +# just build # rebuild backend + ui images +# just restart # full stop + start cycle +# just secrets # print all injected secrets (debug) + +set dotenv-load := false # Doppler handles all env; never load a .env file + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +# Inject secrets from Doppler, then run the given command +doppler := "doppler run --" + +# ── Core compose commands ───────────────────────────────────────────────────── + +# Start all services in the background +up: + {{doppler}} docker compose up -d + +# Start and stream logs (foreground) +up-fg: + {{doppler}} docker compose up + +# Stop all running services +down: + {{doppler}} docker compose down + +# Stop and remove volumes (full reset — destructive!) +down-volumes: + {{doppler}} docker compose down -v + +# Show service status +ps: + {{doppler}} docker compose ps + +# ── Build & publish ─────────────────────────────────────────────────────────── + +# Build (or rebuild) all custom images locally +build: + {{doppler}} docker compose build + +# Build a specific service, e.g.: just build-svc backend +build-svc svc: + {{doppler}} docker compose build {{svc}} + +# Push all custom images to Docker Hub (requires docker login) +push: + {{doppler}} docker compose push backend runner ui caddy pocketbase + +# Build then push all custom images +build-push: build push + +# Pull all images from Docker Hub (uses GIT_TAG from Doppler) +pull-images: + {{doppler}} docker compose pull backend runner ui caddy pocketbase + +# Pull all third-party base images (minio, pocketbase, etc.) +pull-infra: + {{doppler}} docker compose pull minio pocketbase meilisearch valkey postgres crowdsec watchtower + +# ── Logs ───────────────────────────────────────────────────────────────────── + +# Tail all service logs (last 50 lines + follow) +logs: + {{doppler}} docker compose logs -f --tail=50 + +# Tail a specific service, e.g.: just log backend +log svc: + {{doppler}} docker compose logs -f --tail=50 {{svc}} + +# ── Lifecycle ───────────────────────────────────────────────────────────────── + +# Full restart: stop then start +restart: down up + +# Restart a single service, e.g.: just restart-svc backend +restart-svc svc: + {{doppler}} docker compose restart {{svc}} + +# Pull → build → recreate (rolling update without clearing volumes) +update: + {{doppler}} docker compose pull + {{doppler}} docker compose build + {{doppler}} docker compose up -d + +# ── Initialisation ──────────────────────────────────────────────────────────── + +# Run one-shot init containers (minio-init, pb-init, postgres-init) +init: + {{doppler}} docker compose run --rm minio-init + {{doppler}} docker compose run --rm pb-init + {{doppler}} docker compose run --rm postgres-init + +# ── Shell access ────────────────────────────────────────────────────────────── + +# Open a shell in a running service, e.g.: just shell backend +shell svc: + {{doppler}} docker compose exec {{svc}} sh + +# ── Secrets ─────────────────────────────────────────────────────────────────── + +# Print all secrets Doppler will inject (never redirected to a file) +secrets: + doppler secrets --project libnovel --config prd + +# Print secrets as a .env-formatted list (useful for debugging) +secrets-env: + doppler secrets download --project libnovel --config prd --format env --no-file + +# Open Doppler dashboard in browser +secrets-dashboard: + doppler open dashboard + +# ── Developer setup ─────────────────────────────────────────────────────────── + +# One-time dev setup: configure git to use committed hooks in .githooks/ +setup: + git config core.hooksPath .githooks + @echo "Git hooks configured (.githooks/pre-commit active)." + +# ── Gitea CI ────────────────────────────────────────────────────────────────── + +# Validate workflow files +ci-lint: + actionlint .gitea/workflows/*.yaml diff --git a/opencode.json b/opencode.json new file mode 100644 index 0000000..1337657 --- /dev/null +++ b/opencode.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://opencode.ai/config.json", + "mcp": { + "gh_grep": { + "type": "remote", + "url": "https://mcp.grep.app", + "enabled": true + } + }, + "instructions": [ + ] +} diff --git a/scraper/Dockerfile b/scraper/Dockerfile deleted file mode 100644 index a16a59d..0000000 --- a/scraper/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# ── Build stage ──────────────────────────────────────────────────────────────── -FROM golang:1.25-alpine AS builder - -WORKDIR /build - -# Cache dependency downloads separately from source compilation. -COPY go.mod go.sum ./ -RUN go mod download - -COPY . . - -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ - go build -ldflags="-s -w" -o /scraper ./cmd/scraper - -# ── Runtime stage ────────────────────────────────────────────────────────────── -FROM alpine:3.20 - -# ca-certificates is required for HTTPS requests to novelfire.net. -RUN apk add --no-cache ca-certificates tzdata - -WORKDIR /app - -COPY --from=builder /scraper /app/scraper - -# Create the default static output directory. -RUN mkdir -p /app/static/books - -# Non-root user. -RUN addgroup -S scraper && adduser -S scraper -G scraper -RUN chown -R scraper:scraper /app -USER scraper - -# ── Configuration ───────────────────────────────────────────────────────────── -ENV BROWSERLESS_URL=http://browserless:3030 -ENV BROWSERLESS_STRATEGY=content -ENV SCRAPER_WORKERS=0 -ENV SCRAPER_STATIC_ROOT=/app/static/books -ENV SCRAPER_HTTP_ADDR=:8080 - -EXPOSE 8080 - -# Default: run as an HTTP server. Override CMD to use "run" for one-shot. -ENTRYPOINT ["/app/scraper"] -CMD ["serve"] diff --git a/scraper/cmd/scraper/main.go b/scraper/cmd/scraper/main.go deleted file mode 100644 index f3c6962..0000000 --- a/scraper/cmd/scraper/main.go +++ /dev/null @@ -1,217 +0,0 @@ -// Command scraper is the entrypoint for the libnovel scraper service. -// -// Usage (CLI one-shot): -// -// scraper run [--url ] -// -// Usage (HTTP server): -// -// scraper serve -// -// Environment variables: -// -// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030) -// BROWSERLESS_TOKEN Browserless API token (default: "") -// BROWSERLESS_STRATEGY content | scrape | cdp (default: content) -// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5) -// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU) -// SCRAPER_STATIC_ROOT Output directory (default: ./static/books) -// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) -// KOKORO_URL Kokoro-FastAPI base URL (default: "") -// KOKORO_VOICE Default TTS voice (default: af_bella) -// LOG_LEVEL debug | info | warn | error (default: info) -package main - -import ( - "context" - "fmt" - "log/slog" - "os" - "os/signal" - "runtime" - "strconv" - "strings" - "syscall" - "time" - - "github.com/libnovel/scraper/internal/browser" - "github.com/libnovel/scraper/internal/novelfire" - "github.com/libnovel/scraper/internal/orchestrator" - "github.com/libnovel/scraper/internal/server" - "github.com/libnovel/scraper/internal/writer" -) - -func main() { - logLevel := slog.LevelInfo - if v := os.Getenv("LOG_LEVEL"); v != "" { - if err := logLevel.UnmarshalText([]byte(v)); err != nil { - fmt.Fprintf(os.Stderr, "invalid LOG_LEVEL %q, using info\n", v) - } - } - log := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ - Level: logLevel, - })) - - if err := run(log); err != nil { - log.Error("fatal", "err", err) - os.Exit(1) - } -} - -func run(log *slog.Logger) error { - args := os.Args[1:] - if len(args) == 0 { - printUsage() - return nil - } - - cmd := strings.ToLower(args[0]) - - browserCfg := browser.Config{ - BaseURL: envOr("BROWSERLESS_URL", "http://localhost:3030"), - Token: envOr("BROWSERLESS_TOKEN", ""), - } - browserCfg.MaxConcurrent = 5 - if s := os.Getenv("BROWSERLESS_MAX_CONCURRENT"); s != "" { - if n, err := strconv.Atoi(s); err == nil && n > 0 { - browserCfg.MaxConcurrent = n - } - } - if s := os.Getenv("BROWSERLESS_TIMEOUT"); s != "" { - if n, err := strconv.Atoi(s); err == nil && n > 0 { - browserCfg.Timeout = time.Duration(n) * time.Second - } - } - - strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect)))) - urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent)))) - bc := newBrowserClient(strategy, browserCfg) - urlClient := newBrowserClient(urlStrategy, browserCfg) - - staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books") - w := writer.New(staticRoot) - nf := novelfire.New(bc, log, urlClient, w) - - workers := 0 - if s := os.Getenv("SCRAPER_WORKERS"); s != "" { - n, err := strconv.Atoi(s) - if err == nil && n > 0 { - workers = n - } - } - if workers == 0 { - workers = runtime.NumCPU() - } - - oCfg := orchestrator.Config{ - Workers: workers, - StaticRoot: staticRoot, - } - - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer stop() - - switch cmd { - case "run": - // Optional --url flag. - if len(args) >= 3 && args[1] == "--url" { - oCfg.SingleBookURL = args[2] - } - log.Info("starting one-shot scrape", - "strategy", strategy, - "workers", workers, - "max_concurrent", browserCfg.MaxConcurrent, - "static_root", oCfg.StaticRoot, - "single_book", oCfg.SingleBookURL, - ) - o := orchestrator.New(oCfg, nf, log) - return o.Run(ctx) - - case "refresh": - // refresh - re-scrape a book from its saved source_url - if len(args) < 2 { - return fmt.Errorf("refresh command requires a book slug argument") - } - slug := args[1] - w := writer.New(oCfg.StaticRoot) - meta, ok, err := w.ReadMetadata(slug) - if err != nil { - return fmt.Errorf("failed to read metadata for %s: %w", slug, err) - } - if !ok { - return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot) - } - if meta.SourceURL == "" { - return fmt.Errorf("book %q has no source_url in metadata", slug) - } - oCfg.SingleBookURL = meta.SourceURL - log.Info("refreshing book from source_url", - "slug", slug, - "source_url", meta.SourceURL, - ) - o := orchestrator.New(oCfg, nf, log) - return o.Run(ctx) - - case "serve": - addr := envOr("SCRAPER_HTTP_ADDR", ":8080") - kokoroURL := envOr("KOKORO_URL", "") - kokoroVoice := envOr("KOKORO_VOICE", "af_bella") - log.Info("starting HTTP server", - "addr", addr, - "strategy", strategy, - "workers", workers, - "max_concurrent", browserCfg.MaxConcurrent, - "kokoro_url", kokoroURL, - "kokoro_voice", kokoroVoice, - ) - srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice) - return srv.ListenAndServe(ctx) - - default: - return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd) - } -} - -func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient { - switch strategy { - case browser.StrategyScrape: - return browser.NewScrapeClient(cfg) - case browser.StrategyCDP: - return browser.NewCDPClient(cfg) - case browser.StrategyDirect: - return browser.NewDirectHTTPClient(cfg) - default: - return browser.NewContentClient(cfg) - } -} - -func envOr(key, fallback string) string { - if v := os.Getenv(key); v != "" { - return v - } - return fallback -} - -func printUsage() { - fmt.Fprintf(os.Stderr, `libnovel scraper - -Commands: - run [--url ] One-shot: scrape full catalogue, or a single book - refresh Re-scrape a book from its saved source_url - serve Start HTTP server (POST /scrape, POST /scrape/book) - -Environment variables: - BROWSERLESS_URL Browserless base URL (default: http://localhost:3030) - BROWSERLESS_TOKEN API token (default: "") - BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct) - BROWSERLESS_URL_STRATEGY Strategy for URL retrieval (default: content) - BROWSERLESS_MAX_CONCURRENT Max simultaneous sessions (default: 5) - BROWSERLESS_TIMEOUT HTTP request timeout sec (default: 90) - SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d) - SCRAPER_STATIC_ROOT Output directory (default: ./static/books) - SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) - KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled) - KOKORO_VOICE Default TTS voice (default: af_bella) - LOG_LEVEL debug|info|warn|error (default: info) -`, runtime.NumCPU()) -} diff --git a/scraper/go.mod b/scraper/go.mod deleted file mode 100644 index 10b6635..0000000 --- a/scraper/go.mod +++ /dev/null @@ -1,10 +0,0 @@ -module github.com/libnovel/scraper - -go 1.25.0 - -require ( - github.com/gorilla/websocket v1.5.3 // indirect - github.com/yuin/goldmark v1.7.16 // indirect - golang.org/x/net v0.51.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/scraper/go.sum b/scraper/go.sum deleted file mode 100644 index 5338185..0000000 --- a/scraper/go.sum +++ /dev/null @@ -1,9 +0,0 @@ -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE= -github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= -golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= -golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scraper/internal/browser/cdp.go b/scraper/internal/browser/cdp.go deleted file mode 100644 index bdbe0cb..0000000 --- a/scraper/internal/browser/cdp.go +++ /dev/null @@ -1,137 +0,0 @@ -package browser - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "strings" - "sync/atomic" - "time" - - "github.com/gorilla/websocket" -) - -// cdpClient implements BrowserClient using the CDP WebSocket endpoint. -type cdpClient struct { - cfg Config - sem chan struct{} -} - -// NewCDPClient returns a BrowserClient that uses CDP WebSocket sessions. -func NewCDPClient(cfg Config) BrowserClient { - if cfg.Timeout == 0 { - cfg.Timeout = 60 * time.Second - } - return &cdpClient{cfg: cfg, sem: makeSem(cfg.MaxConcurrent)} -} - -func (c *cdpClient) Strategy() Strategy { return StrategyCDP } - -func (c *cdpClient) GetContent(_ context.Context, _ ContentRequest) (string, error) { - return "", fmt.Errorf("CDP client does not support /content; use NewContentClient") -} - -func (c *cdpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) { - return ScrapeResponse{}, fmt.Errorf("CDP client does not support /scrape; use NewScrapeClient") -} - -// CDPSession opens a WebSocket to the Browserless /devtools/browser endpoint, -// navigates to pageURL, and invokes fn with a live CDPConn. -func (c *cdpClient) CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error { - if err := acquire(ctx, c.sem); err != nil { - return fmt.Errorf("cdp: semaphore: %w", err) - } - defer release(c.sem) - - // Build WebSocket URL: ws://host:port/devtools/browser?token=...&url=... - wsURL := strings.Replace(c.cfg.BaseURL, "http://", "ws://", 1) - wsURL = strings.Replace(wsURL, "https://", "wss://", 1) - wsURL += "/devtools/browser" - sep := "?" - if c.cfg.Token != "" { - wsURL += sep + "token=" + c.cfg.Token - sep = "&" - } - wsURL += sep + "url=" + pageURL - - dialer := websocket.Dialer{ - HandshakeTimeout: 15 * time.Second, - Proxy: http.ProxyFromEnvironment, - } - - conn, _, err := dialer.DialContext(ctx, wsURL, nil) - if err != nil { - return fmt.Errorf("cdp: dial %s: %w", wsURL, err) - } - - cdp := &cdpConn{ws: conn} - defer cdp.Close() - - return fn(ctx, cdp) -} - -// ─── cdpConn ───────────────────────────────────────────────────────────────── - -type cdpConn struct { - ws *websocket.Conn - counter atomic.Int64 -} - -type cdpRequest struct { - ID int64 `json:"id"` - Method string `json:"method"` - Params map[string]any `json:"params,omitempty"` -} - -type cdpResponse struct { - ID int64 `json:"id"` - Result map[string]any `json:"result,omitempty"` - Error *struct { - Code int `json:"code"` - Message string `json:"message"` - } `json:"error,omitempty"` -} - -func (c *cdpConn) Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) { - id := c.counter.Add(1) - - req := cdpRequest{ID: id, Method: method, Params: params} - data, err := json.Marshal(req) - if err != nil { - return nil, fmt.Errorf("cdp send: marshal: %w", err) - } - - if dl, ok := ctx.Deadline(); ok { - _ = c.ws.SetWriteDeadline(dl) - } - if err := c.ws.WriteMessage(websocket.TextMessage, data); err != nil { - return nil, fmt.Errorf("cdp send: write: %w", err) - } - - // Read messages until we find the response matching our id. - for { - if dl, ok := ctx.Deadline(); ok { - _ = c.ws.SetReadDeadline(dl) - } - _, msg, err := c.ws.ReadMessage() - if err != nil { - return nil, fmt.Errorf("cdp send: read: %w", err) - } - var resp cdpResponse - if err := json.Unmarshal(msg, &resp); err != nil { - continue // skip non-JSON frames (events etc.) - } - if resp.ID != id { - continue // event or different command reply - } - if resp.Error != nil { - return nil, fmt.Errorf("cdp error %d: %s", resp.Error.Code, resp.Error.Message) - } - return resp.Result, nil - } -} - -func (c *cdpConn) Close() error { - return c.ws.Close() -} diff --git a/scraper/internal/browser/content_scrape.go b/scraper/internal/browser/content_scrape.go deleted file mode 100644 index a0e7a85..0000000 --- a/scraper/internal/browser/content_scrape.go +++ /dev/null @@ -1,195 +0,0 @@ -package browser - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" -) - -// Config holds the connection parameters for a Browserless instance. -type Config struct { - // BaseURL is the HTTP base URL, e.g. "http://localhost:3030". - BaseURL string - // Token is the optional API token (BROWSERLESS_TOKEN env var). - Token string - // Timeout is the per-request HTTP timeout; defaults to 60 s. - Timeout time.Duration - // MaxConcurrent caps the number of simultaneous in-flight requests sent to - // Browserless. When all slots are occupied new calls block until one - // completes (or ctx is cancelled). 0 means no limit. - MaxConcurrent int -} - -// makeSem returns a buffered channel used as a counting semaphore. -// If n <= 0 a nil channel is returned, which causes acquire/release to be no-ops. -func makeSem(n int) chan struct{} { - if n <= 0 { - return nil - } - return make(chan struct{}, n) -} - -// acquire takes one slot from sem. It returns an error if ctx is cancelled -// before a slot becomes available. If sem is nil it returns immediately. -func acquire(ctx context.Context, sem chan struct{}) error { - if sem == nil { - return nil - } - select { - case sem <- struct{}{}: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -// release frees the slot previously obtained by acquire. -// If sem is nil it is a no-op. -func release(sem chan struct{}) { - if sem != nil { - <-sem - } -} - -// contentClient implements BrowserClient using the /content endpoint. -type contentClient struct { - cfg Config - http *http.Client - sem chan struct{} -} - -// NewContentClient returns a BrowserClient that uses POST /content. -func NewContentClient(cfg Config) BrowserClient { - if cfg.Timeout == 0 { - cfg.Timeout = 90 * time.Second - } - return &contentClient{ - cfg: cfg, - http: &http.Client{Timeout: cfg.Timeout}, - sem: makeSem(cfg.MaxConcurrent), - } -} - -func (c *contentClient) Strategy() Strategy { return StrategyContent } - -func (c *contentClient) GetContent(ctx context.Context, req ContentRequest) (string, error) { - if err := acquire(ctx, c.sem); err != nil { - return "", fmt.Errorf("content: semaphore: %w", err) - } - defer release(c.sem) - - body, err := json.Marshal(req) - if err != nil { - return "", fmt.Errorf("content: marshal request: %w", err) - } - - url := c.cfg.BaseURL + "/content" - if c.cfg.Token != "" { - url += "?token=" + c.cfg.Token - } - - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) - if err != nil { - return "", fmt.Errorf("content: build request: %w", err) - } - httpReq.Header.Set("Content-Type", "application/json") - - resp, err := c.http.Do(httpReq) - if err != nil { - return "", fmt.Errorf("content: do request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) - return "", fmt.Errorf("content: unexpected status %d: %s", resp.StatusCode, b) - } - - raw, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("content: read body: %w", err) - } - return string(raw), nil -} - -func (c *contentClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) { - return ScrapeResponse{}, fmt.Errorf("content client does not support /scrape; use NewScrapeClient") -} - -func (c *contentClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error { - return fmt.Errorf("content client does not support CDP; use NewCDPClient") -} - -// ─── /scrape client ─────────────────────────────────────────────────────────── - -type scrapeClient struct { - cfg Config - http *http.Client - sem chan struct{} -} - -// NewScrapeClient returns a BrowserClient that uses POST /scrape. -func NewScrapeClient(cfg Config) BrowserClient { - if cfg.Timeout == 0 { - cfg.Timeout = 90 * time.Second - } - return &scrapeClient{ - cfg: cfg, - http: &http.Client{Timeout: cfg.Timeout}, - sem: makeSem(cfg.MaxConcurrent), - } -} - -func (c *scrapeClient) Strategy() Strategy { return StrategyScrape } - -func (c *scrapeClient) GetContent(_ context.Context, _ ContentRequest) (string, error) { - return "", fmt.Errorf("scrape client does not support /content; use NewContentClient") -} - -func (c *scrapeClient) ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) { - if err := acquire(ctx, c.sem); err != nil { - return ScrapeResponse{}, fmt.Errorf("scrape: semaphore: %w", err) - } - defer release(c.sem) - - body, err := json.Marshal(req) - if err != nil { - return ScrapeResponse{}, fmt.Errorf("scrape: marshal request: %w", err) - } - - url := c.cfg.BaseURL + "/scrape" - if c.cfg.Token != "" { - url += "?token=" + c.cfg.Token - } - - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) - if err != nil { - return ScrapeResponse{}, fmt.Errorf("scrape: build request: %w", err) - } - httpReq.Header.Set("Content-Type", "application/json") - - resp, err := c.http.Do(httpReq) - if err != nil { - return ScrapeResponse{}, fmt.Errorf("scrape: do request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) - return ScrapeResponse{}, fmt.Errorf("scrape: unexpected status %d: %s", resp.StatusCode, b) - } - - var result ScrapeResponse - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return ScrapeResponse{}, fmt.Errorf("scrape: decode response: %w", err) - } - return result, nil -} - -func (c *scrapeClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error { - return fmt.Errorf("scrape client does not support CDP; use NewCDPClient") -} diff --git a/scraper/internal/browser/http.go b/scraper/internal/browser/http.go deleted file mode 100644 index 5b9c374..0000000 --- a/scraper/internal/browser/http.go +++ /dev/null @@ -1,68 +0,0 @@ -package browser - -import ( - "context" - "fmt" - "io" - "net/http" - "time" -) - -type httpClient struct { - cfg Config - http *http.Client - sem chan struct{} -} - -func NewDirectHTTPClient(cfg Config) BrowserClient { - if cfg.Timeout == 0 { - cfg.Timeout = 30 * time.Second - } - return &httpClient{ - cfg: cfg, - http: &http.Client{Timeout: cfg.Timeout}, - sem: makeSem(cfg.MaxConcurrent), - } -} - -func (c *httpClient) Strategy() Strategy { return StrategyDirect } - -func (c *httpClient) GetContent(ctx context.Context, req ContentRequest) (string, error) { - if err := acquire(ctx, c.sem); err != nil { - return "", fmt.Errorf("http: semaphore: %w", err) - } - defer release(c.sem) - - httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, req.URL, nil) - if err != nil { - return "", fmt.Errorf("http: build request: %w", err) - } - httpReq.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") - httpReq.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") - httpReq.Header.Set("Accept-Language", "en-US,en;q=0.5") - - resp, err := c.http.Do(httpReq) - if err != nil { - return "", fmt.Errorf("http: do request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - b, _ := io.ReadAll(resp.Body) - return "", fmt.Errorf("http: unexpected status %d: %s", resp.StatusCode, b) - } - - raw, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("http: read body: %w", err) - } - return string(raw), nil -} - -func (c *httpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) { - return ScrapeResponse{}, fmt.Errorf("http client does not support ScrapePage; use browserless") -} - -func (c *httpClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error { - return fmt.Errorf("http client does not support CDP; use browserless") -} diff --git a/scraper/internal/browser/integration_test.go b/scraper/internal/browser/integration_test.go deleted file mode 100644 index e389677..0000000 --- a/scraper/internal/browser/integration_test.go +++ /dev/null @@ -1,152 +0,0 @@ -//go:build integration - -// Integration tests for the Browserless /content API. -// -// These tests require a live Browserless instance and are gated behind the -// "integration" build tag so they never run in normal `go test ./...` passes. -// -// Run them with: -// -// BROWSERLESS_URL=http://localhost:3030 \ -// BROWSERLESS_TOKEN=your-token \ # omit if auth is disabled -// go test -v -tags integration -timeout 120s \ -// github.com/libnovel/scraper/internal/browser -package browser_test - -import ( - "context" - "os" - "strings" - "testing" - "time" - - "github.com/libnovel/scraper/internal/browser" -) - -// chapterURL is the novelfire chapter used in every integration sub-test. -const chapterURL = "https://novelfire.net/book/a-dragon-against-the-whole-world/chapter-1" - -// newIntegrationClient reads BROWSERLESS_URL / BROWSERLESS_TOKEN from the -// environment and returns a configured contentClient. -// The test is skipped when BROWSERLESS_URL is not set. -func newIntegrationClient(t *testing.T) browser.BrowserClient { - t.Helper() - baseURL := os.Getenv("BROWSERLESS_URL") - if baseURL == "" { - t.Skip("BROWSERLESS_URL not set — skipping integration test") - } - return browser.NewContentClient(browser.Config{ - BaseURL: baseURL, - Token: os.Getenv("BROWSERLESS_TOKEN"), - // Use a generous per-request HTTP timeout so the wait-for-selector - // (75 s) doesn't get cut off by the transport layer. - Timeout: 120 * time.Second, - MaxConcurrent: 1, - }) -} - -// TestIntegration_ChapterContent_ReturnsHTML verifies that a POST /content -// request with the production wait-for-selector settings succeeds and that the -// returned HTML contains the #content div expected on novelfire chapter pages. -func TestIntegration_ChapterContent_ReturnsHTML(t *testing.T) { - client := newIntegrationClient(t) - - ctx, cancel := context.WithTimeout(context.Background(), 110*time.Second) - defer cancel() - - req := browser.ContentRequest{ - URL: chapterURL, - WaitFor: &browser.WaitForSelector{ - Selector: "#content", - Timeout: 5000, - }, - RejectResourceTypes: productionRejectTypes(), - } - - html, err := client.GetContent(ctx, req) - if err != nil { - t.Fatalf("GetContent failed: %v", err) - } - - // The #content div must not be empty; presence of

tags inside it is a - // reliable indicator that chapter paragraphs were rendered. - contentIdx := strings.Index(html, `id="content"`) - if contentIdx == -1 { - t.Fatalf("id=\"content\" not found in response (%d bytes)", len(html)) - } - - // Look for

tags after the #content marker — the chapter text lives there. - afterContent := html[contentIdx:] - if !strings.Contains(afterContent, " tags; JS rendering may have failed.\nSection preview:\n%s", - truncate(afterContent, 1000)) - } - - t.Logf("chapter content section starts at byte %d (total response: %d bytes)", contentIdx, len(html)) -} - -// TestIntegration_ChapterContent_TimeoutSurfacedCorrectly verifies that a -// deliberately too-short timeout returns an error containing "TimeoutError" (the -// Browserless error string seen in the failing log entry). This ensures our -// error-classification logic in retryGetContent matches real Browserless output. -func TestIntegration_ChapterContent_TimeoutSurfacedCorrectly(t *testing.T) { - client := newIntegrationClient(t) - - ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) - defer cancel() - - req := browser.ContentRequest{ - URL: chapterURL, - WaitFor: &browser.WaitForSelector{ - Selector: "#content", - Timeout: 500, // intentionally too short (500 ms) → Browserless will time out - }, - RejectResourceTypes: productionRejectTypes(), - } - - _, err := client.GetContent(ctx, req) - if err == nil { - t.Fatal("expected a timeout error from Browserless, but GetContent succeeded — " + - "the page may now load very fast; adjust the timeout threshold") - } - - t.Logf("got expected error: %v", err) - - // Browserless wraps navigation timeouts in a 500 response with - // "TimeoutError: Navigation timeout" in the body — this is the exact - // error that is triggering retries in production. - if !strings.Contains(err.Error(), "500") { - t.Errorf("expected HTTP 500 status in error, got: %v", err) - } -} - -// ── helpers ─────────────────────────────────────────────────────────────────── - -// productionRejectTypes returns the same resource-type block-list the -// novelfire scraper uses in production, so integration tests exercise the -// identical request shape. -func productionRejectTypes() []string { - return []string{ - "cspviolationreport", - "eventsource", - "fedcm", - "font", - "image", - "manifest", - "media", - "other", - "ping", - "signedexchange", - "stylesheet", - "texttrack", - "websocket", - } -} - -// truncate returns the first n bytes of s as a string. -func truncate(s string, n int) string { - if len(s) <= n { - return s - } - return s[:n] + "…" -} diff --git a/scraper/internal/browser/interface.go b/scraper/internal/browser/interface.go deleted file mode 100644 index c2c8fd4..0000000 --- a/scraper/internal/browser/interface.go +++ /dev/null @@ -1,120 +0,0 @@ -// Package browser defines the BrowserClient interface and helper types for -// communicating with a Browserless instance. -package browser - -import "context" - -// Strategy selects which Browserless API endpoint / protocol to use. -type Strategy string - -const ( - // StrategyContent uses the POST /content endpoint, which returns the final - // rendered HTML of the page. Fastest; suitable for most JS-rendered sites. - StrategyContent Strategy = "content" - - // StrategyScrape uses the POST /scrape endpoint, which accepts a list of - // CSS selectors and returns structured JSON. Good when you know exactly - // which elements you need. - StrategyScrape Strategy = "scrape" - - // StrategyCDP uses the WebSocket /devtools/browser endpoint (Chrome - // DevTools Protocol). Most powerful; required for complex interactions - // (clicking, scrolling, waiting for network idle, etc.). - StrategyCDP Strategy = "cdp" - - // StrategyDirect uses a plain HTTP client to fetch HTML directly. - // Suitable for sites that don't require JavaScript rendering. - StrategyDirect Strategy = "direct" -) - -// WaitForSelector describes the waitForSelector option sent to Browserless. -type WaitForSelector struct { - Selector string `json:"selector"` - Timeout int `json:"timeout,omitempty"` // ms -} - -// GotoOptions controls page navigation behavior. -type GotoOptions struct { - Timeout int `json:"timeout,omitempty"` // ms - WaitUntil string `json:"waitUntil,omitempty"` // e.g., "networkidle2", "load" -} - -// ContentRequest is the body sent to POST /content. -type ContentRequest struct { - URL string `json:"url"` - WaitFor *WaitForSelector `json:"waitForSelector,omitempty"` - WaitForTimeout int `json:"waitForTimeout,omitempty"` // ms - RejectResourceTypes []string `json:"rejectResourceTypes,omitempty"` // e.g. ["image","stylesheet"] - GotoOptions *GotoOptions `json:"gotoOptions,omitempty"` - BestAttempt bool `json:"bestAttempt,omitempty"` // return partial content on timeout/error -} - -// ScrapeElement is one element descriptor inside a ScrapeRequest. -type ScrapeElement struct { - Selector string `json:"selector"` - Timeout int `json:"timeout,omitempty"` // ms -} - -// ScrapeRequest is the body sent to POST /scrape. -type ScrapeRequest struct { - URL string `json:"url"` - Elements []ScrapeElement `json:"elements"` - WaitFor *WaitForSelector `json:"waitForSelector,omitempty"` - GotoOptions *GotoOptions `json:"gotoOptions,omitempty"` -} - -// ScrapeResult is one entry in the response from POST /scrape. -type ScrapeResult struct { - Selector string `json:"selector"` - Results []ScrapeElement `json:"results"` -} - -// ScrapeAttribute holds a single attribute value from a scraped element. -type ScrapeAttribute struct { - Name string `json:"name"` - Value string `json:"value"` -} - -// ScrapedElement is one item inside ScrapeResult.Results. -type ScrapedElement struct { - Text string `json:"text"` - Attributes []ScrapeAttribute `json:"attributes"` -} - -// ScrapeResponse is the top-level response from POST /scrape. -type ScrapeResponse struct { - Data []ScrapeResult `json:"data"` -} - -// BrowserClient is an abstraction over the three Browserless API strategies. -// Callers choose the strategy best suited to the target site; the interface -// signature is identical regardless of strategy. -type BrowserClient interface { - // Strategy returns the strategy this client uses. - Strategy() Strategy - - // GetContent fetches the fully-rendered HTML of url using the /content - // endpoint. Only meaningful when Strategy() == StrategyContent. - GetContent(ctx context.Context, req ContentRequest) (string, error) - - // ScrapePage calls the /scrape endpoint and returns structured data. - // Only meaningful when Strategy() == StrategyScrape. - ScrapePage(ctx context.Context, req ScrapeRequest) (ScrapeResponse, error) - - // CDPSession opens a CDP WebSocket session and calls fn with the raw - // WebSocket connection. Only meaningful when Strategy() == StrategyCDP. - // The session is closed when fn returns. - CDPSession(ctx context.Context, pageURL string, fn CDPSessionFunc) error -} - -// CDPSessionFunc is the callback invoked inside a CDP session. -// conn is a live *websocket.Conn connected to a Browserless CDP endpoint. -type CDPSessionFunc func(ctx context.Context, conn CDPConn) error - -// CDPConn is the minimal interface the orchestrator needs over a CDP WebSocket. -type CDPConn interface { - // Send sends a raw CDP command (JSON-encoded) and returns the response. - Send(ctx context.Context, method string, params map[string]any) (map[string]any, error) - // Close closes the underlying connection. - Close() error -} diff --git a/scraper/internal/novelfire/integration_test.go b/scraper/internal/novelfire/integration_test.go deleted file mode 100644 index 5e91e1a..0000000 --- a/scraper/internal/novelfire/integration_test.go +++ /dev/null @@ -1,344 +0,0 @@ -//go:build integration - -// Integration tests for the novelfire.net Scraper against a live Browserless instance. -// -// These tests exercise the full scraping stack — Browserless → raw HTML → -// novelfire HTML parser — for the book: -// -// https://novelfire.net/book/a-dragon-against-the-whole-world -// -// They are gated behind the "integration" build tag so they never run in a -// normal `go test ./...` pass. -// -// Run with: -// -// BROWSERLESS_URL=http://localhost:3030 \ -// BROWSERLESS_TOKEN=your-token \ # omit if auth is disabled -// go test -v -tags integration -timeout 600s \ -// github.com/libnovel/scraper/internal/novelfire -package novelfire - -import ( - "context" - "fmt" - "os" - "strings" - "testing" - "time" - - "github.com/libnovel/scraper/internal/browser" - "github.com/libnovel/scraper/internal/scraper" -) - -const ( - integrationBookURL = "https://novelfire.net/book/a-dragon-against-the-whole-world" - integrationBookSlug = "a-dragon-against-the-whole-world" - integrationBookTitle = "A Dragon against the Whole World" -) - -// newIntegrationScraper reads BROWSERLESS_URL / BROWSERLESS_TOKEN from the -// environment, constructs a real contentClient, and returns a novelfire Scraper -// wired to it. The test is skipped when BROWSERLESS_URL is not set. -func newIntegrationScraper(t *testing.T) *Scraper { - t.Helper() - baseURL := os.Getenv("BROWSERLESS_URL") - if baseURL == "" { - t.Skip("BROWSERLESS_URL not set — skipping integration test") - } - client := browser.NewContentClient(browser.Config{ - BaseURL: baseURL, - Token: os.Getenv("BROWSERLESS_TOKEN"), - Timeout: 120 * time.Second, - MaxConcurrent: 1, - }) - return New(client, nil) -} - -// ── Metadata ────────────────────────────────────────────────────────────────── - -// TestIntegration_Novelfire_ScrapeMetadata_ReturnsTitle verifies that -// ScrapeMetadata fetches the book page and correctly parses at minimum -// the slug, title, and source URL. -func TestIntegration_Novelfire_ScrapeMetadata_ReturnsTitle(t *testing.T) { - s := newIntegrationScraper(t) - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - meta, err := s.ScrapeMetadata(ctx, integrationBookURL) - if err != nil { - t.Fatalf("ScrapeMetadata failed: %v", err) - } - - t.Logf("slug: %s", meta.Slug) - t.Logf("title: %s", meta.Title) - t.Logf("author: %s", meta.Author) - t.Logf("status: %s", meta.Status) - t.Logf("genres: %v", meta.Genres) - t.Logf("total_chapters: %d", meta.TotalChapters) - t.Logf("source_url: %s", meta.SourceURL) - - if meta.Slug != integrationBookSlug { - t.Errorf("slug = %q, want %q", meta.Slug, integrationBookSlug) - } - if meta.Title == "" { - t.Error("title is empty") - } - if !strings.EqualFold(meta.Title, integrationBookTitle) { - // Warn rather than hard-fail — the site may reword the title. - t.Logf("WARN: title = %q, expected something like %q", meta.Title, integrationBookTitle) - } - if meta.SourceURL != integrationBookURL { - t.Errorf("source_url = %q, want %q", meta.SourceURL, integrationBookURL) - } -} - -// TestIntegration_Novelfire_ScrapeMetadata_ReturnsFullFields verifies that -// every optional field (author, status, genres, summary, total_chapters) is -// populated. A missing field is a warning, not a hard failure, because the -// site may change its HTML structure. -func TestIntegration_Novelfire_ScrapeMetadata_ReturnsFullFields(t *testing.T) { - s := newIntegrationScraper(t) - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - meta, err := s.ScrapeMetadata(ctx, integrationBookURL) - if err != nil { - t.Fatalf("ScrapeMetadata failed: %v", err) - } - - type check struct { - field string - empty bool - } - checks := []check{ - {"author", meta.Author == ""}, - {"status", meta.Status == ""}, - {"summary", meta.Summary == ""}, - {"genres", len(meta.Genres) == 0}, - {"total_chapters", meta.TotalChapters == 0}, - } - for _, c := range checks { - if c.empty { - t.Errorf("field %q is empty — HTML selector may have broken", c.field) - } - } - - // total_chapters must be a positive integer. - if meta.TotalChapters < 1 { - t.Errorf("total_chapters = %d, want >= 1", meta.TotalChapters) - } -} - -// ── Chapter list ────────────────────────────────────────────────────────────── - -// TestIntegration_Novelfire_ScrapeChapterList_ReturnsRefs verifies that -// ScrapeChapterList returns a non-empty slice of chapter references with -// valid URLs and numbers parsed from those URLs (not list position). -func TestIntegration_Novelfire_ScrapeChapterList_ReturnsRefs(t *testing.T) { - s := newIntegrationScraper(t) - - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - refs, err := s.ScrapeChapterList(ctx, integrationBookURL) - if err != nil { - t.Fatalf("ScrapeChapterList failed: %v", err) - } - - t.Logf("total refs returned: %d", len(refs)) - - if len(refs) == 0 { - t.Fatal("ScrapeChapterList returned 0 refs") - } - - // Every ref must have a non-empty URL pointing at the correct book. - for i, ref := range refs { - if ref.URL == "" { - t.Errorf("refs[%d].URL is empty", i) - } - if !strings.Contains(ref.URL, integrationBookSlug) { - t.Errorf("refs[%d].URL %q does not contain book slug", i, ref.URL) - } - if ref.Number <= 0 { - t.Errorf("refs[%d].Number = %d, want > 0 (URL: %s)", i, ref.Number, ref.URL) - } - if ref.Title == "" { - t.Errorf("refs[%d].Title is empty (URL: %s)", i, ref.URL) - } - } -} - -// TestIntegration_Novelfire_ScrapeChapterList_NumbersMatchURLs verifies the -// fix for the newest-first ordering bug: each ref's Number must equal the -// chapter number embedded in its URL, not its position in the list. -func TestIntegration_Novelfire_ScrapeChapterList_NumbersMatchURLs(t *testing.T) { - s := newIntegrationScraper(t) - - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - refs, err := s.ScrapeChapterList(ctx, integrationBookURL) - if err != nil { - t.Fatalf("ScrapeChapterList failed: %v", err) - } - if len(refs) == 0 { - t.Fatal("ScrapeChapterList returned 0 refs") - } - - mismatches := 0 - for i, ref := range refs { - wantNum := chapterNumberFromURL(ref.URL) - if wantNum <= 0 { - // URL has no parseable number — skip this entry. - continue - } - if ref.Number != wantNum { - t.Errorf("refs[%d]: Number=%d but URL %q implies number=%d (position-based bug?)", - i, ref.Number, ref.URL, wantNum) - mismatches++ - if mismatches >= 5 { - t.Log("… (further mismatches suppressed)") - break - } - } - } - - // Log the first few refs so failures are easy to diagnose. - limit := 5 - if len(refs) < limit { - limit = len(refs) - } - for i := 0; i < limit; i++ { - t.Logf("refs[%d]: Number=%d Title=%q URL=%s", i, refs[i].Number, refs[i].Title, refs[i].URL) - } -} - -// ── Chapters ────────────────────────────────────────────────────────────────── - -// TestIntegration_Novelfire_ScrapeFirst3Chapters scrapes chapters 1, 2, and 3 -// via ScrapeChapterText and verifies each returns non-empty markdown text. -// Chapters are run as sub-tests so a single failure does not abort the others. -func TestIntegration_Novelfire_ScrapeFirst3Chapters(t *testing.T) { - s := newIntegrationScraper(t) - - chapters := []scraper.ChapterRef{ - { - Number: 1, - Title: "Chapter 1", - URL: integrationBookURL + "/chapter-1", - }, - { - Number: 2, - Title: "Chapter 2", - URL: integrationBookURL + "/chapter-2", - }, - { - Number: 3, - Title: "Chapter 3", - URL: integrationBookURL + "/chapter-3", - }, - } - - for _, ref := range chapters { - ref := ref // capture - t.Run(fmt.Sprintf("chapter-%d", ref.Number), func(t *testing.T) { - // Sequential: each chapter needs its own generous timeout. - ctx, cancel := context.WithTimeout(context.Background(), 110*time.Second) - defer cancel() - - ch, err := s.ScrapeChapterText(ctx, ref) - if err != nil { - t.Fatalf("ScrapeChapterText failed: %v", err) - } - - t.Logf("chapter %d: %d bytes of markdown", ref.Number, len(ch.Text)) - t.Logf("first 300 chars:\n%s", truncateStr(ch.Text, 300)) - - // Ref fields must be echoed back unchanged. - if ch.Ref.Number != ref.Number { - t.Errorf("Ref.Number = %d, want %d", ch.Ref.Number, ref.Number) - } - if ch.Ref.URL != ref.URL { - t.Errorf("Ref.URL = %q, want %q", ch.Ref.URL, ref.URL) - } - - // Text must be non-trivially long. - if len(ch.Text) < 100 { - t.Errorf("Text too short (%d bytes) — likely empty or parsing failed:\n%s", - len(ch.Text), ch.Text) - } - - // Text must not contain raw HTML tags — NodeToMarkdown should have - // stripped them. - for _, tag := range []string{" cards with -//

and a rel="next" pagination link. -func rankingPage1HTML() string { - return ` - - - -` -} - -func rankingPage2HTML() string { - return ` - - - - -` -} - -// drainRanking collects all entries from a ScrapeRanking call without -// deadlocking. It uses "for A != nil || B != nil" — nil channels are never -// selected, so setting one to nil effectively removes it from the select. -func drainRanking(t *testing.T, entryCh <-chan scraper.BookMeta, errCh <-chan error) []scraper.BookMeta { - t.Helper() - var entries []scraper.BookMeta - for entryCh != nil || errCh != nil { - select { - case meta, ok := <-entryCh: - if !ok { - entryCh = nil - } else { - entries = append(entries, meta) - } - case err, ok := <-errCh: - if !ok { - errCh = nil - } else if err != nil { - t.Fatalf("unexpected scrape error: %v", err) - } - } - } - return entries -} - -// TestScrapeRanking_SinglePage verifies a single page is parsed into entries -// with sequential Ranking numbers using a stub client. -// ScrapeRanking uses s.client (the main client, not urlClient) because the -// ranking page is fully server-rendered. -func TestScrapeRanking_SinglePage(t *testing.T) { - // newScraper passes the stub as s.client — exactly what ScrapeRanking uses. - s := newScraper(rankingPage1HTML()) - entryCh, errCh := s.ScrapeRanking(context.Background(), 1) - entries := drainRanking(t, entryCh, errCh) - - if len(entries) != 2 { - t.Fatalf("expected 2 entries, got %d", len(entries)) - } - if entries[0].Ranking != 1 || entries[0].Title != "The Iron Throne" { - t.Errorf("entry[0]: got rank=%d title=%q, want rank=1 title=%q", - entries[0].Ranking, entries[0].Title, "The Iron Throne") - } - if entries[1].Ranking != 2 || entries[1].Title != "Shadow Mage" { - t.Errorf("entry[1]: got rank=%d title=%q, want rank=2 title=%q", - entries[1].Ranking, entries[1].Title, "Shadow Mage") - } -} - -// TestScrapeRanking_MultiPage verifies pagination across two pages yields -// contiguous rank numbers (1, 2, 3). -func TestScrapeRanking_MultiPage(t *testing.T) { - // Use pagedStubClient for s.client so each GetContent call returns the - // next page. ScrapeRanking now calls s.client directly. - urlClient := &pagedStubClient{pages: []string{rankingPage1HTML(), rankingPage2HTML()}} - s := New(urlClient, nil, nil, nil) // nil cache — no disk I/O in tests - - entryCh, errCh := s.ScrapeRanking(context.Background(), 0) // 0 = all pages - entries := drainRanking(t, entryCh, errCh) - - if len(entries) != 3 { - t.Fatalf("expected 3 entries across 2 pages, got %d", len(entries)) - } - want := []struct { - rank int - title string - }{ - {1, "The Iron Throne"}, - {2, "Shadow Mage"}, - {3, "Void Hunter"}, - } - for i, w := range want { - if entries[i].Ranking != w.rank || entries[i].Title != w.title { - t.Errorf("entry[%d]: got rank=%d title=%q, want rank=%d title=%q", - i, entries[i].Ranking, entries[i].Title, w.rank, w.title) - } - } -} - -// TestScrapeRanking_EmptyPage verifies that a page with no .novel-item -// cards produces zero entries and closes channels cleanly (no deadlock). -func TestScrapeRanking_EmptyPage(t *testing.T) { - s := newScraper(`
`) - entryCh, errCh := s.ScrapeRanking(context.Background(), 1) - entries := drainRanking(t, entryCh, errCh) - - if len(entries) != 0 { - t.Errorf("expected 0 entries for empty page, got %d", len(entries)) - } -} - -// TestWriteRanking_RoundTrip verifies WriteRanking → ReadRankingItems -// faithfully reconstructs the original slice. -func TestWriteRanking_RoundTrip(t *testing.T) { - dir := t.TempDir() - w := writer.New(dir) - - items := []writer.RankingItem{ - {Rank: 1, Slug: "the-iron-throne", Title: "The Iron Throne", Status: "Ongoing", - Genres: []string{"Fantasy", "Action"}, SourceURL: "https://novelfire.net/book/the-iron-throne"}, - {Rank: 2, Slug: "shadow-mage", Title: "Shadow Mage", Status: "Completed", - Genres: []string{"Magic"}, SourceURL: "https://novelfire.net/book/shadow-mage"}, - } - - if err := w.WriteRanking(items); err != nil { - t.Fatalf("WriteRanking failed: %v", err) - } - - rankingFile := filepath.Join(dir, "ranking.json") - if _, err := os.Stat(rankingFile); err != nil { - t.Fatalf("ranking.json not created: %v", err) - } - - got, err := w.ReadRankingItems() - if err != nil { - t.Fatalf("ReadRankingItems failed: %v", err) - } - if len(got) != len(items) { - t.Fatalf("expected %d items, got %d", len(items), len(got)) - } - for i, want := range items { - if got[i].Rank != want.Rank { - t.Errorf("item[%d].Rank = %d, want %d", i, got[i].Rank, want.Rank) - } - if got[i].Slug != want.Slug { - t.Errorf("item[%d].Slug = %q, want %q", i, got[i].Slug, want.Slug) - } - if got[i].Title != want.Title { - t.Errorf("item[%d].Title = %q, want %q", i, got[i].Title, want.Title) - } - if got[i].Status != want.Status { - t.Errorf("item[%d].Status = %q, want %q", i, got[i].Status, want.Status) - } - if len(got[i].Genres) != len(want.Genres) { - t.Errorf("item[%d].Genres len = %d, want %d", i, len(got[i].Genres), len(want.Genres)) - } else { - for j, g := range want.Genres { - if got[i].Genres[j] != g { - t.Errorf("item[%d].Genres[%d] = %q, want %q", i, j, got[i].Genres[j], g) - } - } - } - if got[i].SourceURL != want.SourceURL { - t.Errorf("item[%d].SourceURL = %q, want %q", i, got[i].SourceURL, want.SourceURL) - } - } -} - -// ── in-memory page cacher ───────────────────────────────────────────────────── - -// memPageCacher is a RankingPageCacher backed by an in-memory map. -// It records how many times each page was written and exposes the stored HTML. -type memPageCacher struct { - pages map[int]string - writes map[int]int -} - -func newMemPageCacher() *memPageCacher { - return &memPageCacher{pages: make(map[int]string), writes: make(map[int]int)} -} - -func (c *memPageCacher) WriteRankingPageCache(page int, html string) error { - c.pages[page] = html - c.writes[page]++ - return nil -} - -func (c *memPageCacher) ReadRankingPageCache(page int) (string, error) { - return c.pages[page], nil // returns "" on miss, satisfying the interface contract -} - -var _ scraper.RankingPageCacher = (*memPageCacher)(nil) // compile-time check - -// TestScrapeRanking_CacheHit verifies that when a page is already in the cache -// ScrapeRanking serves from cache and does NOT call the browser client. -func TestScrapeRanking_CacheHit(t *testing.T) { - cache := newMemPageCacher() - // Pre-populate the cache with page 1 HTML. - if err := cache.WriteRankingPageCache(1, rankingPage1HTML()); err != nil { - t.Fatalf("cache write: %v", err) - } - cache.writes[1] = 0 // reset write counter — we only care about fetches - - // The stub client panics on any GetContent call so we can prove it is not used. - panicClient := &panicOnGetContent{} - s := New(panicClient, nil, panicClient, cache) - - entryCh, errCh := s.ScrapeRanking(context.Background(), 1) - entries := drainRanking(t, entryCh, errCh) - - if len(entries) != 2 { - t.Fatalf("expected 2 entries from cache, got %d", len(entries)) - } - // Cache should not have been written again (we served from cache). - if cache.writes[1] != 0 { - t.Errorf("expected 0 cache writes on a hit, got %d", cache.writes[1]) - } -} - -// TestScrapeRanking_CacheMiss verifies that on a cache miss the page is fetched -// from the network and the result is written to the cache. -func TestScrapeRanking_CacheMiss(t *testing.T) { - cache := newMemPageCacher() // empty cache - s := New(&stubClient{html: rankingPage1HTML()}, nil, nil, cache) - - entryCh, errCh := s.ScrapeRanking(context.Background(), 1) - entries := drainRanking(t, entryCh, errCh) - - if len(entries) != 2 { - t.Fatalf("expected 2 entries, got %d", len(entries)) - } - if cache.writes[1] != 1 { - t.Errorf("expected 1 cache write on a miss, got %d", cache.writes[1]) - } - if cache.pages[1] == "" { - t.Error("expected page 1 to be stored in cache after miss") - } -} - -// panicOnGetContent is a BrowserClient whose GetContent panics, letting tests -// assert that it is never called (i.e. the cache was used instead). -type panicOnGetContent struct{} - -func (p *panicOnGetContent) Strategy() browser.Strategy { return browser.StrategyContent } -func (p *panicOnGetContent) GetContent(_ context.Context, req browser.ContentRequest) (string, error) { - panic(fmt.Sprintf("unexpected GetContent call for URL %s — should have been served from cache", req.URL)) -} -func (p *panicOnGetContent) ScrapePage(_ context.Context, _ browser.ScrapeRequest) (browser.ScrapeResponse, error) { - return browser.ScrapeResponse{}, nil -} -func (p *panicOnGetContent) CDPSession(_ context.Context, _ string, _ browser.CDPSessionFunc) error { - return nil -} diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go deleted file mode 100644 index 2fc1445..0000000 --- a/scraper/internal/novelfire/scraper.go +++ /dev/null @@ -1,712 +0,0 @@ -// Package novelfire provides a NovelScraper implementation for novelfire.net. -// -// Site structure (as of 2025): -// -// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N -// Book page : https://novelfire.net/book/{slug} -// Chapters : https://novelfire.net/book/{slug}/chapters?page=N -// Chapter : https://novelfire.net/book/{slug}/{chapter-slug} -package novelfire - -import ( - "context" - "fmt" - "log/slog" - "net/url" - "path" - "strconv" - "strings" - "time" - - "github.com/libnovel/scraper/internal/browser" - "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/scraper/htmlutil" - "golang.org/x/net/html" -) - -const ( - baseURL = "https://novelfire.net" - cataloguePath = "/genre-all/sort-new/status-all/all-novel" - rankingPath = "/genre-all/sort-popular/status-all/all-novel" -) - -// rejectResourceTypes lists Browserless resource types to block on every request. -// We keep: document (the page), script (JS renders the DOM), fetch/xhr (JS data calls). -// Everything else is safe to drop for HTML-only scraping. -var rejectResourceTypes = []string{ - "cspviolationreport", - "eventsource", - "fedcm", - "font", - "image", - "manifest", - "media", - "other", - "ping", - "signedexchange", - "stylesheet", - "texttrack", - "websocket", -} - -// Scraper is the novelfire.net implementation of scraper.NovelScraper. -// It uses the /content strategy by default (rendered HTML via Browserless). -type Scraper struct { - client browser.BrowserClient - urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy) - pageCache scraper.RankingPageCacher - log *slog.Logger -} - -// New returns a new novelfire Scraper. -// client is used for content fetching, urlClient is used for URL retrieval (chapter list). -// If urlClient is nil, client will be used for both. -// pageCache is optional; pass nil to disable ranking page caching. -func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, pageCache scraper.RankingPageCacher) *Scraper { - if log == nil { - log = slog.Default() - } - if urlClient == nil { - urlClient = client - } - return &Scraper{client: client, urlClient: urlClient, pageCache: pageCache, log: log} -} - -// SourceName implements NovelScraper. -func (s *Scraper) SourceName() string { return "novelfire.net" } - -// ─── CatalogueProvider ─────────────────────────────────────────────────────── - -// ScrapeCatalogue streams all CatalogueEntry values across all pages. -func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.CatalogueEntry, <-chan error) { - entries := make(chan scraper.CatalogueEntry, 64) - errs := make(chan error, 16) - - go func() { - defer close(entries) - defer close(errs) - - pageURL := baseURL + cataloguePath - page := 1 - - for pageURL != "" { - select { - case <-ctx.Done(): - return - default: - } - - s.log.Info("scraping catalogue page", "page", page, "url", pageURL) - s.log.Debug("catalogue page fetch starting", - "page", page, - "payload_url", pageURL, - "payload_wait_selector", ".novel-item", - "payload_wait_selector_timeout_ms", 5000, - ) - - html, err := s.client.GetContent(ctx, browser.ContentRequest{ - URL: pageURL, - WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000}, - RejectResourceTypes: rejectResourceTypes, - GotoOptions: &browser.GotoOptions{Timeout: 60000}, - }) - if err != nil { - s.log.Debug("catalogue page fetch failed", - "page", page, - "url", pageURL, - "err", err, - ) - errs <- fmt.Errorf("catalogue page %d: %w", page, err) - return - } - s.log.Debug("catalogue page fetch completed", - "page", page, - "url", pageURL, - "response_bytes", len(html), - ) - - root, err := htmlutil.ParseHTML(html) - if err != nil { - errs <- fmt.Errorf("catalogue page %d parse: %w", page, err) - return - } - - // Extract novel cards:
- cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true}) - if len(cards) == 0 { - s.log.Warn("no novel cards found, stopping pagination", "page", page) - return - } - - for _, card := range cards { - // Title:

Title - titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"}) - - var title, href string - if titleNode != nil { - linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"}) - if linkNode != nil { - title = htmlutil.ExtractText(linkNode, scraper.Selector{}) - href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) - } - } - if href == "" || title == "" { - continue - } - - bookURL := resolveURL(baseURL, href) - select { - case <-ctx.Done(): - return - case entries <- scraper.CatalogueEntry{Title: title, URL: bookURL}: - } - } - - // Find next page link: Author Name - author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"}) - //
- var cover string - if figureCover := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); figureCover != nil { - cover = htmlutil.ExtractFirst(figureCover, scraper.Selector{Tag: "img", Attr: "src"}) - } - // Ongoing - status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"}) - - // Genres: all tags inside
- genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"}) - var genres []string - if genresNode != nil { - genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true}) - } - - //

...

- summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"}) - // 123 Chapters - totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"}) - totalChapters := parseChapterCount(totalStr) - - slug := slugFromURL(bookURL) - - meta := scraper.BookMeta{ - Slug: slug, - Title: title, - Author: author, - Cover: cover, - Status: status, - Genres: genres, - Summary: summary, - TotalChapters: totalChapters, - SourceURL: bookURL, - } - s.log.Debug("metadata parsed", - "url", bookURL, - "slug", meta.Slug, - "title", meta.Title, - "author", meta.Author, - "status", meta.Status, - "genres", meta.Genres, - "total_chapters", meta.TotalChapters, - ) - return meta, nil -} - -// ─── ChapterListProvider ───────────────────────────────────────────────────── - -func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) { - var refs []scraper.ChapterRef - // Chapter list URL: {bookURL}/chapters?page=N - baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters" - page := 1 - - for { - select { - case <-ctx.Done(): - return refs, ctx.Err() - default: - } - - pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page) - s.log.Info("scraping chapter list", "page", page, "url", pageURL) - - s.log.Debug("chapter list fetch starting", - "page", page, - "payload_url", pageURL, - "payload_wait_selector", ".chapter-list", - "payload_wait_selector_timeout_ms", 15000, - "payload_wait_timeout_ms", 2000, - "strategy", s.urlClient.Strategy(), - ) - - raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{ - URL: pageURL, - // Wait up to 15 s for the chapter list container to appear in the DOM. - WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000}, - // After the selector is found, wait an additional 2 s for any - // deferred JS rendering (lazy-loaded links, infinite-scroll hydration). - WaitForTimeout: 2000, - RejectResourceTypes: rejectResourceTypes, - GotoOptions: &browser.GotoOptions{Timeout: 60000}, - // Do NOT use BestAttempt — we want a complete page or a clear error, - // not silently partial HTML that looks like "no more chapters". - BestAttempt: false, - }) - if err != nil { - s.log.Debug("chapter list fetch failed", - "page", page, - "url", pageURL, - "err", err, - ) - return refs, fmt.Errorf("chapter list page %d: %w", page, err) - } - s.log.Debug("chapter list fetch completed", - "page", page, - "url", pageURL, - "response_bytes", len(raw), - ) - - root, err := htmlutil.ParseHTML(raw) - if err != nil { - return refs, fmt.Errorf("chapter list page %d parse: %w", page, err) - } - - chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"}) - if chapterList == nil { - // No chapter list container on this page — we've gone past the last page. - s.log.Debug("chapter list container not found, stopping pagination", "page", page) - break - } - - // Each chapter row:
  • Title
  • - items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"}) - - s.log.Debug("chapter list page parsed", - "page", page, - "url", pageURL, - "chapters_on_page", len(items), - "total_refs_so_far", len(refs), - ) - - // Zero items on this page means we've gone past the last page. - if len(items) == 0 { - s.log.Debug("no chapters on page, stopping pagination", "page", page) - break - } - - for _, item := range items { - linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"}) - if linkNode == nil { - continue - } - href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"}) - chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{}) - if href == "" { - continue - } - chURL := resolveURL(baseURL, href) - num := chapterNumberFromURL(chURL) - if num <= 0 { - // Fall back to position if the URL has no parseable number. - num = len(refs) + 1 - s.log.Warn("chapter number not parseable from URL, falling back to position", - "url", chURL, - "position", num, - ) - } - refs = append(refs, scraper.ChapterRef{ - Number: num, - Title: strings.TrimSpace(chTitle), - URL: chURL, - }) - } - - page++ - } - - return refs, nil -} - -// ─── RankingProvider ─────────────────────────────────────────────────────────── - -// hasNextPageLink returns true if the HTML document contains a pagination link -// with rel="next". novelfire.net uses: -// -//
  • - //
    - //

    Title

    - //
    - //
  • - cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true}) - if len(cards) == 0 { - s.log.Debug("no novel cards found, stopping pagination", "page", page) - break - } - - for _, card := range cards { - // The outer carries the href and title attribute. - linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"}) - if linkNode == nil { - continue - } - href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) - bookURL := resolveURL(baseURL, href) - if bookURL == "" { - continue - } - - // Title: prefer

    text; fall back to - title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})) - if title == "" { - title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"})) - } - if title == "" { - continue - } - - // Cover:
    - var cover string - if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil { - cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"}) - if cover == "" { - cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"}) - } - // Filter out base64 placeholder images. - if strings.HasPrefix(cover, "data:") { - cover = "" - } - if cover != "" && !strings.HasPrefix(cover, "http") { - cover = baseURL + cover - } - } - - slug := slugFromURL(bookURL) - - meta := scraper.BookMeta{ - Slug: slug, - Title: title, - Cover: cover, - SourceURL: bookURL, - Ranking: rank, - } - rank++ - - select { - case <-ctx.Done(): - return - case entries <- meta: - } - } - - // Stop if no next-page link exists. - // The real pagination uses - ` - - s := newPagedScraper(page1) - refs, err := s.ScrapeChapterList(context.Background(), "https://novelfire.net/book/test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(refs) != 5 { - t.Fatalf("expected 5 refs, got %d", len(refs)) - } - - // With position-based numbering (the old bug), refs[0].Number would be 1 - // even though its URL is /chapter-5. With URL-based numbering it must be 5. - wantNumbers := []int{5, 4, 3, 2, 1} - for i, ref := range refs { - if ref.Number != wantNumbers[i] { - t.Errorf("refs[%d].Number = %d, want %d (URL: %s)", i, ref.Number, wantNumbers[i], ref.URL) - } - } -} - -// TestScrapeChapterList_Pagination verifies that the scraper correctly follows -// ?page=N pagination and stops when a page returns no chapter items. -func TestScrapeChapterList_Pagination(t *testing.T) { - page1 := ` - - ` - - page2 := ` - - ` - - // page3 is omitted — pagedStubClient will return empty page to stop pagination. - s := newPagedScraper(page1, page2) - refs, err := s.ScrapeChapterList(context.Background(), "https://novelfire.net/book/test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(refs) != 6 { - t.Fatalf("expected 6 refs (3 per page × 2 pages), got %d", len(refs)) - } - - wantNumbers := []int{3, 2, 1, 6, 5, 4} - for i, ref := range refs { - if ref.Number != wantNumbers[i] { - t.Errorf("refs[%d].Number = %d, want %d (URL: %s)", i, ref.Number, wantNumbers[i], ref.URL) - } - } -} diff --git a/scraper/internal/orchestrator/orchestrator.go b/scraper/internal/orchestrator/orchestrator.go deleted file mode 100644 index d35a6c0..0000000 --- a/scraper/internal/orchestrator/orchestrator.go +++ /dev/null @@ -1,213 +0,0 @@ -// Package orchestrator coordinates the catalogue walk, metadata extraction, -// chapter-list fetching, and parallel chapter scraping. -// -// Concurrency model -// - One goroutine runs ScrapeCatalogue and feeds book URLs into a channel. -// - For each book, a dedicated goroutine calls ScrapeMetadata (metadata goroutine). -// - ScrapeChapterList is called in the metadata goroutine once metadata is done. -// - N worker goroutines (default: runtime.NumCPU()) each pull ChapterRef values -// from a shared work queue and call ScrapeChapterText. -// - A sync.WaitGroup ensures all chapter workers finish before the orchestrator -// signals completion. -package orchestrator - -import ( - "context" - "fmt" - "log/slog" - "runtime" - "sync" - - "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/writer" -) - -// Config holds tunable parameters for the orchestrator. -type Config struct { - // Workers is the number of goroutines used to scrape chapters in parallel. - // Defaults to runtime.NumCPU() when 0. - Workers int - - // StaticRoot is the path to the static/books output directory. - StaticRoot string - - // SingleBookURL when non-empty causes the orchestrator to scrape only - // that one book instead of walking the full catalogue. - SingleBookURL string -} - -// Orchestrator coordinates the full scrape pipeline. -type Orchestrator struct { - cfg Config - novel scraper.NovelScraper - writer *writer.Writer - log *slog.Logger - workers int -} - -// New returns a new Orchestrator. -func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator { - workers := cfg.Workers - if workers <= 0 { - workers = runtime.NumCPU() - } - return &Orchestrator{ - cfg: cfg, - novel: novel, - writer: writer.New(cfg.StaticRoot), - log: log, - workers: workers, - } -} - -// Run executes the full scrape pipeline and blocks until it is complete or ctx -// is cancelled. -func (o *Orchestrator) Run(ctx context.Context) error { - o.log.Info("orchestrator starting", - "source", o.novel.SourceName(), - "workers", o.workers, - "static_root", o.cfg.StaticRoot, - ) - - // chapterWork is the shared queue consumed by chapter worker goroutines. - type chapterJob struct { - slug string - ref scraper.ChapterRef - } - chapterWork := make(chan chapterJob, o.workers*4) - - // Start chapter worker pool. - var chapterWG sync.WaitGroup - for i := 0; i < o.workers; i++ { - chapterWG.Add(1) - go func(workerID int) { - defer chapterWG.Done() - for job := range chapterWork { - select { - case <-ctx.Done(): - return - default: - } - - // Skip if already on disk. - if o.writer.ChapterExists(job.slug, job.ref) { - o.log.Debug("chapter already exists, skipping", - "book", job.slug, "chapter", job.ref.Number) - continue - } - - chapter, err := o.novel.ScrapeChapterText(ctx, job.ref) - if err != nil { - o.log.Error("chapter scrape failed", - "book", job.slug, - "chapter", job.ref.Number, - "url", job.ref.URL, - "err", err, - ) - continue - } - - if err := o.writer.WriteChapter(job.slug, chapter); err != nil { - o.log.Error("chapter write failed", - "book", job.slug, - "chapter", job.ref.Number, - "err", err, - ) - continue - } - - o.log.Info("chapter saved", - "book", job.slug, - "chapter", job.ref.Number, - "worker", workerID, - ) - } - }(i) - } - - // processBook scrapes metadata + chapter list for one book, then enqueues - // chapter jobs. It is called inside a goroutine per book. - processBook := func(bookURL string) { - // Metadata goroutine. - meta, err := o.novel.ScrapeMetadata(ctx, bookURL) - if err != nil { - o.log.Error("metadata scrape failed", "url", bookURL, "err", err) - return - } - - // Persist / update metadata.yaml. - if err := o.writer.WriteMetadata(meta); err != nil { - o.log.Error("metadata write failed", "slug", meta.Slug, "err", err) - // Continue — chapters can still be scraped. - } - - o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title) - - // Fetch chapter list. - refs, err := o.novel.ScrapeChapterList(ctx, bookURL) - if err != nil { - o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err) - return - } - - o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs)) - - // Enqueue chapter jobs. - for _, ref := range refs { - select { - case <-ctx.Done(): - return - case chapterWork <- chapterJob{slug: meta.Slug, ref: ref}: - } - } - } - - if o.cfg.SingleBookURL != "" { - // Single-book mode: skip catalogue entirely. - o.log.Info("single-book mode", "url", o.cfg.SingleBookURL) - processBook(o.cfg.SingleBookURL) - } else { - // Catalogue mode: stream every book. - entries, catErrs := o.novel.ScrapeCatalogue(ctx) - - // Drain catalogue errors in a separate goroutine. - go func() { - for err := range catErrs { - o.log.Error("catalogue error", "err", err) - } - }() - - var bookWG sync.WaitGroup - for entry := range entries { - select { - case <-ctx.Done(): - break - default: - } - - bookWG.Add(1) - bookURL := entry.URL - go func() { - defer bookWG.Done() - processBook(bookURL) - }() - } - - // Wait for all book goroutines to enqueue their chapters before - // closing the chapter work queue. - bookWG.Wait() - } - - // Signal chapter workers there is no more work. - close(chapterWork) - - // Wait for all in-flight chapter scrapes to finish. - chapterWG.Wait() - - if ctx.Err() != nil { - return fmt.Errorf("orchestrator: context cancelled: %w", ctx.Err()) - } - - o.log.Info("orchestrator finished") - return nil -} diff --git a/scraper/internal/scraper/interfaces.go b/scraper/internal/scraper/interfaces.go deleted file mode 100644 index c8fe0a8..0000000 --- a/scraper/internal/scraper/interfaces.go +++ /dev/null @@ -1,144 +0,0 @@ -// Package scraper defines the core interfaces and domain types for the libnovel -// scraping system. Each novel source implements these interfaces; the orchestrator -// wires them together without knowing anything about the concrete provider. -package scraper - -import "context" - -// ─── Domain types ──────────────────────────────────────────────────────────── - -// BookMeta carries all bibliographic information about a novel. -type BookMeta struct { - // Slug is a URL-safe identifier derived from the book title, e.g. "a-dragon-against-the-whole-world". - Slug string `yaml:"slug"` - // Title is the human-readable novel title. - Title string `yaml:"title"` - // Author of the novel. - Author string `yaml:"author"` - // Cover is an absolute URL to the cover image. - Cover string `yaml:"cover,omitempty"` - // Status is e.g. "Ongoing", "Completed". - Status string `yaml:"status,omitempty"` - // Genres is a list of genre tags. - Genres []string `yaml:"genres,omitempty"` - // Summary is the full description/synopsis text. - Summary string `yaml:"summary,omitempty"` - // TotalChapters is the total number of chapters known at scrape time. - TotalChapters int `yaml:"total_chapters,omitempty"` - // SourceURL is the canonical URL of the book's landing page. - SourceURL string `yaml:"source_url"` - // Ranking is the rank number from ranking pages. - Ranking int `yaml:"ranking,omitempty"` -} - -// CatalogueEntry is a lightweight reference returned by CatalogueProvider. -type CatalogueEntry struct { - // Title is the novel title as shown in the catalogue listing. - Title string - // URL is the canonical landing-page URL of the novel. - URL string -} - -// ChapterRef is a reference to a single chapter returned by ChapterListProvider. -type ChapterRef struct { - // Number is the 1-based chapter index within the book. - Number int - // Title is the chapter display title. - Title string - // URL is the full URL of the chapter page. - URL string - // Volume is an optional volume number (0 means no volume grouping). - Volume int -} - -// Chapter contains the fully-extracted text of a single chapter. -type Chapter struct { - Ref ChapterRef - // Text is the plain / lightly-formatted chapter body (Markdown). - Text string -} - -// ─── Scraping selector descriptors ─────────────────────────────────────────── - -// Selector describes how to locate an element in an HTML document. -// Exactly one of Tag, Class, or ID should be non-empty; when multiple are set -// they are combined (AND semantics). -type Selector struct { - // Tag is the HTML element name, e.g. "div", "p", "h1". - Tag string - // Class is one CSS class name (without the leading dot). - Class string - // ID is the element id attribute (without the leading #). - ID string - // Attr is an optional attribute name whose value should be extracted - // instead of the text content (e.g. "href", "src"). - Attr string - // Multiple indicates that all matching elements should be collected, - // not just the first one. - Multiple bool -} - -// ─── Provider interfaces ────────────────────────────────────────────────────── - -// CatalogueProvider can enumerate every novel available on a source site. -// It handles pagination transparently and streams CatalogueEntry values. -type CatalogueProvider interface { - // ScrapeCatalogue pages through the entire catalogue, sending - // CatalogueEntry values to the returned channel. The channel is closed - // when all pages have been scraped or ctx is cancelled. - // Errors are surfaced via the error channel; a non-nil error does not - // necessarily terminate scraping. - ScrapeCatalogue(ctx context.Context) (<-chan CatalogueEntry, <-chan error) -} - -// MetadataProvider can extract structured book metadata from a novel's landing page. -type MetadataProvider interface { - // ScrapeMetadata fetches and parses the metadata for the book at bookURL. - ScrapeMetadata(ctx context.Context, bookURL string) (BookMeta, error) -} - -// ChapterListProvider can enumerate all chapters of a book from the chapter-list page. -type ChapterListProvider interface { - // ScrapeChapterList returns all chapter references for a book, ordered - // by chapter number ascending. - ScrapeChapterList(ctx context.Context, bookURL string) ([]ChapterRef, error) -} - -// ChapterTextProvider can extract the readable text from a single chapter page. -type ChapterTextProvider interface { - // ScrapeChapterText fetches chapterURL and returns the chapter text as Markdown. - ScrapeChapterText(ctx context.Context, ref ChapterRef) (Chapter, error) -} - -// RankingProvider can enumerate novels from a ranking page. -type RankingProvider interface { - // ScrapeRanking pages through up to maxPages ranking pages, sending BookMeta - // values (with basic info like title, cover, genres, status, sourceURL) to - // the returned channel. Pages are fetched sequentially and lazily: the next - // page is only requested once all entries from the current page have been - // sent. maxPages <= 0 means "all pages". - ScrapeRanking(ctx context.Context, maxPages int) (<-chan BookMeta, <-chan error) -} - -// RankingPageCacher persists and retrieves raw HTML for individual ranking pages. -// Implementations (e.g. writer.Writer) store files on disk so that a -// subsequent ScrapeRanking call can serve cached HTML without a network round-trip. -type RankingPageCacher interface { - // WriteRankingPageCache stores the raw HTML string for the given page number. - WriteRankingPageCache(page int, html string) error - // ReadRankingPageCache returns the cached HTML for page, or ("", nil) on a miss. - ReadRankingPageCache(page int) (string, error) -} - -// NovelScraper is the full interface that a concrete novel source must implement. -// It composes all four provider interfaces. -type NovelScraper interface { - CatalogueProvider - MetadataProvider - ChapterListProvider - ChapterTextProvider - RankingProvider - - // SourceName returns the human-readable name of this scraper, e.g. "novelfire.net". - SourceName() string -} diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go deleted file mode 100644 index cc6a1e0..0000000 --- a/scraper/internal/server/server.go +++ /dev/null @@ -1,470 +0,0 @@ -// Package server exposes the scraper as an HTTP service. -// -// Endpoints: -// -// POST /scrape — enqueue a full catalogue scrape -// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."}) -// GET /health — liveness probe -package server - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "log/slog" - "net/http" - "strconv" - "strings" - "sync" - "time" - - "github.com/libnovel/scraper/internal/orchestrator" - "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/writer" -) - -// Server wraps an HTTP mux with the scraping endpoints. -type Server struct { - addr string - oCfg orchestrator.Config - novel scraper.NovelScraper - log *slog.Logger - writer *writer.Writer - mu sync.Mutex - running bool - rankingRunning bool - kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 - kokoroVoice string // default voice, e.g. af_bella - - // voiceMu guards cachedVoices. - voiceMu sync.RWMutex - cachedVoices []string // populated on first request from Kokoro /v1/audio/voices - - // audioMu guards audioCache and audioInFlight. - // audioCache maps a cache key to the Kokoro download filename returned by - // POST /v1/audio/speech with return_download_link=true. - // audioInFlight deduplicates concurrent generation requests for the same key. - audioMu sync.Mutex - audioCache map[string]string // cacheKey → kokoro download filename - audioInFlight map[string]chan struct{} // cacheKey → closed when done -} - -// New creates a new Server. -func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server { - return &Server{ - addr: addr, - oCfg: oCfg, - novel: novel, - log: log, - writer: writer.New(oCfg.StaticRoot), - kokoroURL: kokoroURL, - kokoroVoice: kokoroVoice, - audioCache: make(map[string]string), - audioInFlight: make(map[string]chan struct{}), - } -} - -// voices returns the list of available Kokoro voices. On the first call it -// fetches GET /v1/audio/voices from the Kokoro service and caches the result. -// If the fetch fails (Kokoro not up yet, network error, etc.) it falls back to -// the hardcoded kokoroVoices list so the UI is never empty. -func (s *Server) voices() []string { - s.voiceMu.RLock() - cached := s.cachedVoices - s.voiceMu.RUnlock() - if len(cached) > 0 { - return cached - } - - if s.kokoroURL != "" { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil) - if err == nil { - req.Header.Set("Accept", "application/json") - resp, err := http.DefaultClient.Do(req) - if err == nil { - defer resp.Body.Close() - var payload struct { - Voices []string `json:"voices"` - } - if resp.StatusCode == http.StatusOK && json.NewDecoder(resp.Body).Decode(&payload) == nil && len(payload.Voices) > 0 { - s.voiceMu.Lock() - s.cachedVoices = payload.Voices - s.voiceMu.Unlock() - s.log.Info("fetched kokoro voices", "count", len(payload.Voices)) - return payload.Voices - } - } - } - s.log.Warn("could not fetch kokoro voices, using built-in list") - } - - return kokoroVoices -} - -// ListenAndServe starts the HTTP server and blocks until the provided context -// is cancelled. -func (s *Server) ListenAndServe(ctx context.Context) error { - mux := http.NewServeMux() - mux.HandleFunc("GET /health", s.handleHealth) - mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) - mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) - // UI routes - mux.HandleFunc("GET /", s.handleHome) - mux.HandleFunc("GET /scrape", s.handleScrape) - mux.HandleFunc("GET /ranking", s.handleRanking) - mux.HandleFunc("POST /ranking/refresh", s.handleRankingRefresh) - mux.HandleFunc("GET /ranking/view", s.handleRankingView) - mux.HandleFunc("GET /books/{slug}", s.handleBook) - mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter) - mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage) - mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook) - mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus) - mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus) - // Plain-text chapter content for browser-side TTS - mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText) - // Server-side audio generation via Kokoro /v1/audio/speech. - // Generation can take several minutes, so wrap in its own timeout handler. - audioGenHandler := http.TimeoutHandler( - http.HandlerFunc(s.handleAudioGenerate), - 10*time.Minute, - `{"error":"audio generation timed out"}`, - ) - mux.Handle("POST /ui/audio/{slug}/{n}", audioGenHandler) - // Proxy route: fetches the generated file from Kokoro /v1/download/{filename}. - mux.HandleFunc("GET /ui/audio-proxy/{slug}/{n}", s.handleAudioProxy) - - srv := &http.Server{ - Addr: s.addr, - Handler: mux, - ReadTimeout: 15 * time.Second, - WriteTimeout: 60 * time.Second, - IdleTimeout: 60 * time.Second, - } - - errCh := make(chan error, 1) - go func() { errCh <- srv.ListenAndServe() }() - - s.log.Info("HTTP server listening", "addr", s.addr) - - select { - case <-ctx.Done(): - shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - return srv.Shutdown(shutCtx) - case err := <-errCh: - return err - } -} - -func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) -} - -// handleChapterText returns the plain text of a chapter (markdown stripped) -// for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI. -func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - n, err := strconv.Atoi(r.PathValue("n")) - if err != nil || n < 1 { - http.NotFound(w, r) - return - } - raw, err := s.writer.ReadChapter(slug, n) - if err != nil { - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Header().Set("Cache-Control", "no-store") - fmt.Fprint(w, stripMarkdown(raw)) -} - -// ─── Audio generation via Kokoro /v1/audio/speech ──────────────────────────── -// -// handleAudioGenerate handles POST /ui/audio/{slug}/{n}. -// -// It calls Kokoro's POST /v1/audio/speech with return_download_link=true. -// Kokoro generates the audio, saves it to its own temp storage, and returns -// the download filename in the X-Download-Path response header. -// We cache that filename (in memory, keyed by slug/chapter/voice/speed) and -// return a proxy URL that the browser sets as audio.src. -// -// On a cache hit the proxy URL is returned immediately without re-generating. -// Concurrent requests for the same key are deduplicated. -func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - n, err := strconv.Atoi(r.PathValue("n")) - if err != nil || n < 1 { - http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest) - return - } - - // Parse optional voice/speed from JSON body. - voice := s.kokoroVoice - speed := 1.0 - var body struct { - Voice string `json:"voice"` - Speed float64 `json:"speed"` - } - if r.Body != nil { - _ = json.NewDecoder(r.Body).Decode(&body) - } - if body.Voice != "" { - voice = body.Voice - } - if body.Speed > 0 { - speed = body.Speed - } - - cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) - - // Fast path: already generated this session. - s.audioMu.Lock() - if filename, ok := s.audioCache[cacheKey]; ok { - s.audioMu.Unlock() - s.writeAudioResponse(w, slug, n, voice, speed, filename) - return - } - - // Deduplicate concurrent generation for the same key. - if ch, ok := s.audioInFlight[cacheKey]; ok { - s.audioMu.Unlock() - select { - case <-ch: - case <-r.Context().Done(): - http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable) - return - } - s.audioMu.Lock() - filename, ok := s.audioCache[cacheKey] - s.audioMu.Unlock() - if ok { - s.writeAudioResponse(w, slug, n, voice, speed, filename) - } else { - http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError) - } - return - } - ch := make(chan struct{}) - s.audioInFlight[cacheKey] = ch - s.audioMu.Unlock() - - defer func() { - s.audioMu.Lock() - delete(s.audioInFlight, cacheKey) - s.audioMu.Unlock() - close(ch) - }() - - // Load and validate chapter text. - raw, err := s.writer.ReadChapter(slug, n) - if err != nil { - http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound) - return - } - text := stripMarkdown(raw) - if text == "" { - http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity) - return - } - if s.kokoroURL == "" { - http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable) - return - } - - // Call Kokoro POST /v1/audio/speech with return_download_link=true. - // Kokoro saves the generated audio to its own temp storage and returns the - // download path in the X-Download-Path response header. - filename, err := s.generateSpeech(r.Context(), text, voice, speed) - if err != nil { - s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err) - http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway) - return - } - - s.audioMu.Lock() - s.audioCache[cacheKey] = filename - s.audioMu.Unlock() - - s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename) - s.writeAudioResponse(w, slug, n, voice, speed, filename) -} - -// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true -// and returns the filename from the X-Download-Path response header. -func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) { - reqBody, _ := json.Marshal(map[string]interface{}{ - "model": "kokoro", - "input": text, - "voice": voice, - "response_format": "mp3", - "speed": speed, - "stream": false, - "return_download_link": true, - }) - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, - s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody)) - if err != nil { - return "", fmt.Errorf("build request: %w", err) - } - req.Header.Set("Content-Type", "application/json") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", fmt.Errorf("kokoro request: %w", err) - } - defer resp.Body.Close() - // Drain body so the connection can be reused. - _, _ = io.Copy(io.Discard, resp.Body) - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("kokoro status %d", resp.StatusCode) - } - - // X-Download-Path is e.g. "/download/speech_abc123.mp3" - dlPath := resp.Header.Get("X-Download-Path") - if dlPath == "" { - return "", fmt.Errorf("kokoro did not return X-Download-Path header") - } - - // Extract just the filename from the path. - filename := dlPath - if idx := strings.LastIndex(dlPath, "/"); idx >= 0 { - filename = dlPath[idx+1:] - } - if filename == "" { - return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath) - } - return filename, nil -} - -// writeAudioResponse writes the JSON response for a generated audio chapter. -// The URL points to our proxy handler which fetches from Kokoro on demand. -func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) { - proxyURL := fmt.Sprintf("/ui/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed) - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]interface{}{ - "url": proxyURL, - "filename": filename, - }) -} - -// handleAudioProxy handles GET /ui/audio-proxy/{slug}/{n}. -// It looks up the Kokoro download filename for this chapter (voice/speed) and -// proxies GET /v1/download/{filename} from the Kokoro server back to the browser. -func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - n, err := strconv.Atoi(r.PathValue("n")) - if err != nil || n < 1 { - http.NotFound(w, r) - return - } - voice := r.URL.Query().Get("voice") - if voice == "" { - voice = s.kokoroVoice - } - speedStr := r.URL.Query().Get("speed") - speed := 1.0 - if speedStr != "" { - if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 { - speed = v - } - } - - cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) - s.audioMu.Lock() - filename, ok := s.audioCache[cacheKey] - s.audioMu.Unlock() - - if !ok { - http.Error(w, "audio not generated yet", http.StatusNotFound) - return - } - - kokoroURL := s.kokoroURL + "/v1/download/" + filename - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil) - if err != nil { - http.Error(w, "failed to build proxy request", http.StatusInternalServerError) - return - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - http.Error(w, "kokoro download failed", http.StatusBadGateway) - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway) - return - } - - w.Header().Set("Content-Type", "audio/mpeg") - w.Header().Set("Cache-Control", "public, max-age=3600") - if cl := resp.Header.Get("Content-Length"); cl != "" { - w.Header().Set("Content-Length", cl) - } - _, _ = io.Copy(w, resp.Body) -} - -func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) { - cfg := s.oCfg - cfg.SingleBookURL = "" // full catalogue - - s.runAsync(w, cfg) -} - -func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) { - var body struct { - URL string `json:"url"` - } - if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { - http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest) - return - } - - cfg := s.oCfg - cfg.SingleBookURL = body.URL - - s.runAsync(w, cfg) -} - -// runAsync launches an orchestrator in the background and returns 202 Accepted. -// Only one scrape job runs at a time; concurrent requests receive 409 Conflict. -func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { - s.mu.Lock() - if s.running { - s.mu.Unlock() - http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict) - return - } - s.running = true - s.mu.Unlock() - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusAccepted) - _ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"}) - - go func() { - defer func() { - s.mu.Lock() - s.running = false - s.mu.Unlock() - }() - - ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) - defer cancel() - - o := orchestrator.New(cfg, s.novel, s.log) - if err := o.Run(ctx); err != nil { - s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err)) - } - }() -} diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go deleted file mode 100644 index da600ba..0000000 --- a/scraper/internal/server/ui.go +++ /dev/null @@ -1,3263 +0,0 @@ -package server - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "html/template" - "net/http" - "regexp" - "sort" - "strconv" - "strings" - "time" - - "github.com/libnovel/scraper/internal/orchestrator" - "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/writer" - "github.com/yuin/goldmark" - "github.com/yuin/goldmark/extension" - goldhtml "github.com/yuin/goldmark/renderer/html" -) - -// md is the shared goldmark instance used for all markdown→HTML conversions. -var md = goldmark.New( - goldmark.WithExtensions(extension.Typographer, extension.Table), - goldmark.WithRendererOptions(goldhtml.WithUnsafe()), -) - -// kokoroVoices is the full list of voices shipped with Kokoro-FastAPI, -// grouped loosely by language prefix: -// -// af_ / am_ American English female / male -// bf_ / bm_ British English female / male -// ef_ / em_ Spanish female / male -// ff_ French female -// hf_ / hm_ Hindi female / male -// if_ / im_ Italian female / male -// jf_ / jm_ Japanese female / male -// pf_ / pm_ Portuguese female / male -// zf_ / zm_ Chinese female / male -var kokoroVoices = []string{ - // American English - "af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia", - "af_jessica", "af_kore", "af_nicole", "af_nova", "af_river", - "af_sarah", "af_sky", - "am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam", - "am_michael", "am_onyx", "am_puck", - // British English - "bf_alice", "bf_emma", "bf_lily", - "bm_daniel", "bm_fable", "bm_george", "bm_lewis", - // Spanish - "ef_dora", "em_alex", - // French - "ff_siwis", - // Hindi - "hf_alpha", "hf_beta", "hm_omega", "hm_psi", - // Italian - "if_sara", "im_nicola", - // Japanese - "jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo", - // Portuguese - "pf_dora", "pm_alex", - // Chinese - "zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi", - "zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang", -} - -// voiceInfo holds the parsed display metadata for a single Kokoro voice. -type voiceInfo struct { - ID string // raw voice ID, e.g. "af_bella" - Name string // display name, e.g. "Bella" - Lang string // language label, e.g. "EN-US" - Gender string // "F" or "M" -} - -// langLabel maps the two-letter prefix to a human-readable language tag. -var langLabel = map[string]string{ - "a": "EN-US", - "b": "EN-GB", - "e": "ES", - "f": "FR", - "h": "HI", - "i": "IT", - "j": "JA", - "p": "PT", - "z": "ZH", -} - -// parseVoice decodes a Kokoro voice ID into display metadata. -// IDs follow the pattern {lang}{gender}_{name} e.g. "af_bella". -func parseVoice(id string) voiceInfo { - v := voiceInfo{ID: id, Name: id, Lang: "?", Gender: "?"} - if len(id) < 3 || id[2] != '_' { - return v - } - lc := string(id[0]) - gc := string(id[1]) - name := id[3:] - if l, ok := langLabel[lc]; ok { - v.Lang = l - } - switch gc { - case "f": - v.Gender = "F" - case "m": - v.Gender = "M" - } - // Capitalise name, replace underscores with spaces. - if len(name) > 0 { - runes := []rune(name) - runes[0] -= 'a' - 'A' - v.Name = strings.ReplaceAll(string(runes), "_", " ") - } - return v -} - -// parseVoices converts a slice of raw voice IDs to voiceInfo structs. -func parseVoices(ids []string) []voiceInfo { - out := make([]voiceInfo, len(ids)) - for i, id := range ids { - out[i] = parseVoice(id) - } - return out -} - -// ─── shared layout ──────────────────────────────────────────────────────────── - -const layoutHead = ` - - - - - {{.Title}} — libnovel - - - - -` - -const layoutFoot = `` - -func renderPage(w http.ResponseWriter, title, body string) { - t := template.Must(template.New("layout").Parse(layoutHead + body + layoutFoot)) - w.Header().Set("Content-Type", "text/html; charset=utf-8") - _ = t.Execute(w, struct{ Title string }{Title: title}) -} - -func renderFragment(w http.ResponseWriter, body string) { - w.Header().Set("Content-Type", "text/html; charset=utf-8") - fmt.Fprint(w, body) -} - -func isHTMX(r *http.Request) bool { - return r.Header.Get("HX-Request") == "true" -} - -// respond writes either a full page or an HTMX fragment depending on the request. -func (s *Server) respond(w http.ResponseWriter, r *http.Request, title, fragment string) { - if isHTMX(r) { - renderFragment(w, fragment) - return - } - renderPage(w, title, - `
    `+fragment+`
    `) -} - -// ─── GET / — book catalogue ─────────────────────────────────────────────────── - -const homeTmpl = ` - - -
    - - -
    -

    libnovel

    - -
    - - - -

    {{len .Books}} book{{if ne (len .Books) 1}}s{{end}} on disk

    - - - - - - - - - - - -
    -
    - -
    - -
    - - - - -
    - - - -
    - - - -
    - -` - -// homeBookItem wraps BookMeta with the count of chapters already on disk -// and the Unix timestamp of when the book was added (metadata.yaml mtime). -type homeBookItem struct { - scraper.BookMeta - Downloaded int - AddedAt int64 -} - -func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/" { - http.NotFound(w, r) - return - } - - books, err := s.writer.ListBooks() - if err != nil { - http.Error(w, "failed to list books: "+err.Error(), http.StatusInternalServerError) - return - } - - items := make([]homeBookItem, len(books)) - for i, b := range books { - items[i] = homeBookItem{ - BookMeta: b, - Downloaded: s.writer.CountChapters(b.Slug), - AddedAt: s.writer.MetadataMtime(b.Slug), - } - } - - t := template.Must(template.New("home").Parse(homeTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct { - Books interface{} - }{ - Books: items, - }) - - s.respond(w, r, "Home", buf.String()) -} - -// ─── GET /scrape — add a new book ───────────────────────────────────────────── - -const scrapeTmpl = ` -
    - - ← All books - - -

    Add a book

    -

    Search the rankings or paste a novelfire.net URL to scrape a new book.

    - -
    -
    - - - - - -
    - -
    -
    -
    - -` - -func (s *Server) handleScrape(w http.ResponseWriter, r *http.Request) { - rankingItems, _ := s.writer.ReadRankingItems() - rankingJSON, _ := json.Marshal(rankingItems) - - t := template.Must(template.New("scrape").Parse(scrapeTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct { - RankingJSON template.JS - }{ - RankingJSON: template.JS(rankingJSON), - }) - - s.respond(w, r, "Add a book", buf.String()) -} - -// ─── GET /ranking — ranking page ─────────────────────────────────────────────── - -const rankingTmpl = ` - - -
    - - - - - -
    - - ← All books - -
    -
    -

    Novel Rankings

    -
    - {{if .TotalItems}}{{.TotalItems}} novels{{end}} - {{if .CachedAt}}· cached {{.CachedAt}}{{end}} - -
    -
    -
    - - -
    -
    -
    -
    - - -
    - -
    - - - - -
    - - -
    - Filter: - - {{range .AllStatuses}} - - {{end}} - {{range .TopGenres}} - - {{end}} - -
    -
    - - -
    - {{range .Books}} - {{if .Local}} - - -
    - {{if .Rank}}#{{.Rank}}{{end}} -
    - -
    - {{if .Cover}} - cover - {{else}} -
    N/A
    - {{end}} -
    - -
    -
    -

    {{.Title}}

    - In library -
    - {{if .Author}}

    {{.Author}}

    {{end}} -
    - {{if .Status}}{{.Status}}{{end}} - {{range .Genres}}{{.}}{{end}} -
    -
    -
    - {{else}} -
    - -
    - {{if .Rank}}#{{.Rank}}{{end}} -
    - -
    - {{if .Cover}} - cover - {{else}} -
    N/A
    - {{end}} -
    - -
    -

    {{.Title}}

    - {{if .Author}}

    {{.Author}}

    {{end}} -
    - {{if .Status}}{{.Status}}{{end}} - {{range .Genres}}{{.}}{{end}} -
    - {{if .SourceURL}} -
    - - -
    - {{end}} -
    -
    - {{end}} - {{else}} -
    -

    No ranking data cached yet.

    -
    - - -
    -
    - {{end}} -
    - - - -
    - -` - -// rankingViewItem enriches a RankingItem with whether it is present in the -// local book library, so the template can highlight it differently. -type rankingViewItem struct { - writer.RankingItem - Local bool -} - -// toRankingViewItems annotates items with Local=true for slugs found in localSlugs. -func toRankingViewItems(items []writer.RankingItem, localSlugs map[string]bool) []rankingViewItem { - out := make([]rankingViewItem, len(items)) - for i, it := range items { - out[i] = rankingViewItem{ - RankingItem: it, - Local: localSlugs[it.Slug], - } - } - return out -} - -// pageNum is one entry in the ranking pagination bar. -// Num == 0 is a sentinel that renders as an ellipsis gap. -type pageNum struct { - Num int -} - -// rankingPageNums builds a pagination list with smart ellipsis. -// It always shows: first 2, last 2, and a ±2 window around current. -// Gaps between non-consecutive runs are filled with a sentinel (Num==0) for "…". -// Pass current=0 when there is no concept of a current page (e.g. fetch bar). -func rankingPageNums(total, current int) []pageNum { - if total <= 0 { - return nil - } - show := make(map[int]bool) - // First 2 and last 2. - for i := 1; i <= 2 && i <= total; i++ { - show[i] = true - } - for i := total - 1; i <= total; i++ { - if i >= 1 { - show[i] = true - } - } - // ±2 window around current page. - if current > 0 { - for i := current - 2; i <= current+2; i++ { - if i >= 1 && i <= total { - show[i] = true - } - } - } - - // Collect and sort. - pages := make([]int, 0, len(show)) - for p := range show { - pages = append(pages, p) - } - for i := 0; i < len(pages); i++ { - for j := i + 1; j < len(pages); j++ { - if pages[j] < pages[i] { - pages[i], pages[j] = pages[j], pages[i] - } - } - } - - // Build output with ellipsis sentinels between non-consecutive pages. - out := make([]pageNum, 0, len(pages)*2) - for i, p := range pages { - if i > 0 && p > pages[i-1]+1 { - out = append(out, pageNum{0}) - } - out = append(out, pageNum{p}) - } - return out -} - -const rankingPageSize = 20 - -// handleRanking serves the ranking page from the cached ranking.json file. -// It does NOT trigger a live scrape; use POST /ranking/refresh for that. -// Supports ?page=N for browsing through cached items (20 per page). -func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { - rankingItems, err := s.writer.ReadRankingItems() - if err != nil { - s.log.Error("failed to read cached ranking", "err", err) - } - - cachedAt := "" - if info, statErr := s.writer.RankingFileInfo(); statErr == nil { - cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04") - } - - // Parse requested display page (1-indexed). - currentPage := 1 - if p := r.URL.Query().Get("page"); p != "" { - if n, err2 := strconv.Atoi(p); err2 == nil && n > 0 { - currentPage = n - } - } - - totalItems := len(rankingItems) - totalPages := 1 - if totalItems > 0 { - totalPages = (totalItems + rankingPageSize - 1) / rankingPageSize - } - if currentPage > totalPages { - currentPage = totalPages - } - - // Slice items for the current display page. - start := (currentPage - 1) * rankingPageSize - end := start + rankingPageSize - if end > totalItems { - end = totalItems - } - pageItems := rankingItems - if totalItems > 0 { - pageItems = rankingItems[start:end] - } - - t := template.Must(template.New("ranking").Parse(rankingTmpl)) - var buf bytes.Buffer - - // Collect distinct genres and statuses across ALL items for facet filters. - genreFreq := map[string]int{} - statusSet := map[string]bool{} - for _, it := range rankingItems { - if it.Status != "" { - statusSet[it.Status] = true - } - for _, g := range it.Genres { - genreFreq[g]++ - } - } - allStatuses := sortedKeys(statusSet) - - // Build top-10 genres sorted by frequency descending. - type genreCount struct { - name string - count int - } - gcSlice := make([]genreCount, 0, len(genreFreq)) - for g, c := range genreFreq { - gcSlice = append(gcSlice, genreCount{g, c}) - } - sort.Slice(gcSlice, func(i, j int) bool { - if gcSlice[i].count != gcSlice[j].count { - return gcSlice[i].count > gcSlice[j].count - } - return gcSlice[i].name < gcSlice[j].name - }) - topN := 10 - if len(gcSlice) < topN { - topN = len(gcSlice) - } - topGenres := make([]string, topN) - for i := 0; i < topN; i++ { - topGenres[i] = gcSlice[i].name - } - - // Encode full dataset + local slugs for client-side cross-page filtering. - localSlugs := s.writer.LocalSlugs() - type rankingJSONItem struct { - Rank int `json:"rank"` - Slug string `json:"slug"` - Title string `json:"title"` - Author string `json:"author,omitempty"` - Cover string `json:"cover,omitempty"` - Status string `json:"status,omitempty"` - Genres []string `json:"genres,omitempty"` - SourceURL string `json:"source_url,omitempty"` - Local bool `json:"local"` - } - allItemsForJS := make([]rankingJSONItem, len(rankingItems)) - for i, it := range rankingItems { - allItemsForJS[i] = rankingJSONItem{ - Rank: it.Rank, - Slug: it.Slug, - Title: it.Title, - Author: it.Author, - Cover: it.Cover, - Status: it.Status, - Genres: it.Genres, - SourceURL: it.SourceURL, - Local: localSlugs[it.Slug], - } - } - allItemsJSON, _ := json.Marshal(allItemsForJS) - - _ = t.Execute(&buf, struct { - Books interface{} - CachedAt string - FetchNums []pageNum - DisplayNums []pageNum - CurrentPage int - TotalPages int - TotalItems int - TopGenres []string - AllStatuses []string - AllItemsJSON template.JS - }{ - Books: toRankingViewItems(pageItems, localSlugs), - CachedAt: cachedAt, - FetchNums: rankingPageNums(100, 0), - DisplayNums: rankingPageNums(totalPages, currentPage), - CurrentPage: currentPage, - TotalPages: totalPages, - TotalItems: totalItems, - TopGenres: topGenres, - AllStatuses: allStatuses, - AllItemsJSON: template.JS(allItemsJSON), - }) - s.respond(w, r, "Rankings", buf.String()) -} - -// handleRankingRefresh starts an async scrape of novelfire.net/ranking and -// immediately returns a polling badge. The browser polls /ui/ranking/status -// until the job finishes, then follows an HX-Redirect back to /ranking. -// -// Accepts an optional form field "pages" (integer ≥ 1). 0 or absent means -// fetch all pages; otherwise at most that many pages are scraped. -func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { - _ = r.ParseForm() - maxPages := 0 - if p := strings.TrimSpace(r.FormValue("pages")); p != "" { - if n, err := strconv.Atoi(p); err == nil && n > 0 { - maxPages = n - } - } - - s.mu.Lock() - if s.rankingRunning { - s.mu.Unlock() - renderFragment(w, rankingStatusHTML("running", "Ranking refresh already in progress…")) - return - } - s.rankingRunning = true - s.mu.Unlock() - - go func() { - defer func() { - s.mu.Lock() - s.rankingRunning = false - s.mu.Unlock() - }() - - // Allow ~90 s per page; minimum 120 s for a single page. - timeout := 120 * time.Second - if maxPages > 1 { - timeout = time.Duration(maxPages) * 90 * time.Second - } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages) - - var rankingItems []writer.RankingItem - for rankingCh != nil || errCh != nil { - select { - case meta, ok := <-rankingCh: - if !ok { - rankingCh = nil - } else { - rankingItems = append(rankingItems, writer.RankingItem{ - Rank: meta.Ranking, - Slug: meta.Slug, - Title: meta.Title, - Author: meta.Author, - Cover: meta.Cover, - Status: meta.Status, - Genres: meta.Genres, - SourceURL: meta.SourceURL, - }) - } - case err, ok := <-errCh: - if !ok { - errCh = nil - } else if err != nil { - s.log.Error("ranking scrape error", "err", err) - } - } - } - - if len(rankingItems) > 0 { - if err := s.writer.WriteRanking(rankingItems); err != nil { - s.log.Error("failed to save ranking", "err", err) - } - } - }() - - renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) -} - -// handleRankingStatus is the HTMX polling endpoint for ranking refresh jobs. -// While running it returns a self-replacing badge; when done it issues an -// HX-Redirect so the browser navigates to /ranking. -func (s *Server) handleRankingStatus(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - running := s.rankingRunning - s.mu.Unlock() - - if running { - renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) - return - } - // Job done — redirect the HTMX request to the ranking page. - w.Header().Set("HX-Redirect", "/ranking") - w.WriteHeader(http.StatusOK) -} - -// rankingStatusHTML returns a self-replacing polling badge for the ranking -// refresh job. state is "running" or "done". -func rankingStatusHTML(state, msg string) string { - var colour, dot, poll string - switch state { - case "running": - colour = "text-amber-300 bg-amber-950 border-amber-800" - dot = `` - poll = `hx-get="/ui/ranking/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` - default: - colour = "text-green-300 bg-green-950 border-green-800" - dot = `` - } - return fmt.Sprintf( - `
    %s%s
    `, - colour, poll, dot, template.HTMLEscapeString(msg), - ) -} - -// ─── GET /ranking/view — view ranking markdown ───────────────────────────────── - -const rankingViewTmpl = ` -
    - - ← Back to Rankings - - -

    Ranking Data

    - -
    {{.JSON}}
    -
    ` - -func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) { - items, err := s.writer.ReadRankingItems() - if err != nil { - http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError) - return - } - if len(items) == 0 { - http.NotFound(w, r) - return - } - - pretty, err := json.MarshalIndent(items, "", " ") - if err != nil { - http.Error(w, "json marshal error: "+err.Error(), http.StatusInternalServerError) - return - } - - t := template.Must(template.New("rankingView").Parse(rankingViewTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct{ JSON string }{JSON: string(pretty)}) - - s.respond(w, r, "Ranking Data", buf.String()) -} - -// ─── GET /books/{slug} — chapter list ──────────────────────────────────────── - -const chapterPageSize = 50 - -const bookTmpl = ` -
    - - ← All books - - - - - - - - -
    - {{if .Meta.Cover}} - cover - {{end}} -
    -
    -

    {{.Meta.Title}}

    - {{if .Meta.SourceURL}} - - - - - Source - -
    - - -
    - {{end}} -
    - {{if .Meta.Author}}

    {{.Meta.Author}}

    {{end}} -
    - {{if .Meta.Status}}{{.Meta.Status}}{{end}} - {{if .Meta.TotalChapters}}{{.Meta.TotalChapters}} ch total{{end}} - {{.TotalDownloaded}} downloaded -
    - {{if .Meta.Summary}} -

    {{.Meta.Summary}}

    - {{end}} -
    -
    - - - - - - {{if .LastChapter}} - - {{end}} - -

    Chapters

    - - - {{if gt .TotalPages 1}} -
    - {{if gt .CurrentPage 1}} - « - {{end}} - {{range pages .TotalPages}} - {{if eq . $.CurrentPage}} - {{.}} - {{else}} - {{.}} - {{end}} - {{end}} - {{if lt .CurrentPage .TotalPages}} - » - {{end}} -
    - {{end}} -
    - -` - -func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - - meta, ok, err := s.writer.ReadMetadata(slug) - if err != nil { - http.Error(w, "failed to read metadata: "+err.Error(), http.StatusInternalServerError) - return - } - if !ok { - http.NotFound(w, r) - return - } - - chapters, err := s.writer.ListChapters(slug) - if err != nil { - http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) - return - } - - total := len(chapters) - totalPages := (total + chapterPageSize - 1) / chapterPageSize - if totalPages < 1 { - totalPages = 1 - } - - currentPage := 1 - page := chapters - if total > chapterPageSize { - page = chapters[:chapterPageSize] - } - - funcMap := template.FuncMap{ - "pages": func(n int) []int { - out := make([]int, n) - for i := range out { - out[i] = i + 1 - } - return out - }, - "prev": func(n int) int { return n - 1 }, - "next": func(n int) int { return n + 1 }, - } - - t := template.Must(template.New("book").Funcs(funcMap).Parse(bookTmpl)) - var buf bytes.Buffer - lastChapter := 0 - if total > 0 { - lastChapter = chapters[total-1].Number - } - _ = t.Execute(&buf, struct { - Slug string - Meta interface{} - Chapters interface{} - TotalDownloaded int - TotalPages int - CurrentPage int - LastChapter int - }{ - Slug: slug, - Meta: meta, - Chapters: page, - TotalDownloaded: total, - TotalPages: totalPages, - CurrentPage: currentPage, - LastChapter: lastChapter, - }) - - s.respond(w, r, meta.Title, buf.String()) -} - -// ─── GET /books/{slug}/chapters-page — paginated chapter list fragment ──────── - -const chapterPageTmpl = `{{range .Chapters}} -
  • - - {{.Number}} -
    - {{.Title}} - {{if .Date}}{{.Date}}{{end}} -
    - -
    -
  • -{{end}} -
    - {{if gt .CurrentPage 1}} - « - {{end}} - {{range pages .TotalPages}} - {{if eq . $.CurrentPage}} - {{.}} - {{else}} - {{.}} - {{end}} - {{end}} - {{if lt .CurrentPage .TotalPages}} - » - {{end}} -
    ` - -func (s *Server) handleBookChaptersPage(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - currentPage := 1 - if p := r.URL.Query().Get("page"); p != "" { - if n, err := strconv.Atoi(p); err == nil && n > 0 { - currentPage = n - } - } - - chapters, err := s.writer.ListChapters(slug) - if err != nil { - http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) - return - } - - total := len(chapters) - totalPages := (total + chapterPageSize - 1) / chapterPageSize - if totalPages < 1 { - totalPages = 1 - } - - start := (currentPage - 1) * chapterPageSize - if start >= total { - w.WriteHeader(http.StatusNoContent) - return - } - end := start + chapterPageSize - if end > total { - end = total - } - - funcMap := template.FuncMap{ - "pages": func(n int) []int { - out := make([]int, n) - for i := range out { - out[i] = i + 1 - } - return out - }, - "prev": func(n int) int { return n - 1 }, - "next": func(n int) int { return n + 1 }, - } - - t := template.Must(template.New("chapterPage").Funcs(funcMap).Parse(chapterPageTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct { - Slug string - Chapters interface{} - TotalPages int - CurrentPage int - }{ - Slug: slug, - Chapters: chapters[start:end], - TotalPages: totalPages, - CurrentPage: currentPage, - }) - - w.Header().Set("Content-Type", "text/html; charset=utf-8") - _, _ = buf.WriteTo(w) -} - -// ─── GET /books/{slug}/chapters/{n} — chapter reader ───────────────────────── - -const chapterTmpl = ` - - - - - - -
    - - -
    -

    Chapter {{.ChapterN}}

    -

    {{.Title}}

    - {{if .ChapterDate}}

    {{.ChapterDate}}

    {{end}} -
    - -
    - {{.HTML}} -
    - -
    - - - - -` - -func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - n, err := strconv.Atoi(r.PathValue("n")) - if err != nil || n < 1 { - http.NotFound(w, r) - return - } - - raw, err := s.writer.ReadChapter(slug, n) - if err != nil { - http.NotFound(w, r) - return - } - - // Strip the first heading line so it isn't rendered as a duplicate

    - // inside the article (the template already renders an explicit

    ). - rawForHTML := stripFirstHeadingLine(raw) - - var htmlBuf bytes.Buffer - if err := md.Convert([]byte(rawForHTML), &htmlBuf); err != nil { - http.Error(w, "markdown render error: "+err.Error(), http.StatusInternalServerError) - return - } - - chapters, _ := s.writer.ListChapters(slug) - prevN, nextN := adjacentChapters(chapters, n) - - title := firstHeading(raw, fmt.Sprintf("Chapter %d", n)) - chapterTitle, chapterDate := writer.SplitChapterTitle(title) - - // Load cover URL for Media Session artwork (best-effort; ignore errors). - var coverURL string - if meta, ok, err := s.writer.ReadMetadata(slug); err == nil && ok { - coverURL = meta.Cover - } - - t := template.Must(template.New("chapter").Parse(chapterTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct { - Slug string - HTML template.HTML - PrevN int - NextN int - ChapterN int - Title string - ChapterDate string - AllChapters interface{} - Voices []voiceInfo - DefaultVoice string - Cover string - }{ - Slug: slug, - HTML: template.HTML(htmlBuf.String()), - PrevN: prevN, - NextN: nextN, - ChapterN: n, - Title: chapterTitle, - ChapterDate: chapterDate, - AllChapters: chapters, - Voices: parseVoices(s.voices()), - DefaultVoice: s.kokoroVoice, - Cover: coverURL, - }) - - s.respond(w, r, chapterTitle, buf.String()) -} - -// ─── helpers ────────────────────────────────────────────────────────────────── - -// sortedKeys returns the keys of a string-bool map in sorted order. -func sortedKeys(m map[string]bool) []string { - out := make([]string, 0, len(m)) - for k := range m { - out = append(out, k) - } - // Simple insertion sort — sets are small (< 100 items). - for i := 1; i < len(out); i++ { - for j := i; j > 0 && out[j] < out[j-1]; j-- { - out[j], out[j-1] = out[j-1], out[j] - } - } - return out -} - -// stripMarkdown removes Markdown syntax and returns clean plain text. -func stripMarkdown(src string) string { - src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "") - src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "") - src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "") - src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "") - src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1") - src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "") - src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "") - src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "") - src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n") - return strings.TrimSpace(src) -} - -// adjacentChapters returns the chapter numbers immediately before and after n -// in the sorted chapters list. 0 means "does not exist". -func adjacentChapters(chapters []writer.ChapterInfo, n int) (prev, next int) { - for i, ch := range chapters { - if ch.Number == n { - if i > 0 { - prev = chapters[i-1].Number - } - if i < len(chapters)-1 { - next = chapters[i+1].Number - } - return - } - } - return -} - -// stripFirstHeadingLine removes the first non-empty line if it is a markdown -// heading (starts with one or more "#"). This prevents the heading from being -// rendered as a duplicate

    inside the article when the template already -// renders an explicit title above the article. -func stripFirstHeadingLine(src string) string { - lines := strings.SplitN(src, "\n", -1) - for i, line := range lines { - trimmed := strings.TrimSpace(line) - if trimmed == "" { - continue - } - if strings.HasPrefix(trimmed, "#") { - // Remove this line and return the rest. - rest := strings.Join(append(lines[:i], lines[i+1:]...), "\n") - return strings.TrimLeft(rest, "\n") - } - // First non-empty line is not a heading — nothing to strip. - break - } - return src -} - -// firstHeading returns the text of the first non-empty line, stripping a -// leading "# " markdown heading marker. Falls back to fallback. -func firstHeading(md, fallback string) string { - for _, line := range strings.SplitN(md, "\n", 20) { - line = strings.TrimSpace(line) - if line == "" { - continue - } - return strings.TrimPrefix(line, "# ") - } - return fallback -} - -// ─── POST /ui/scrape/book — form submission ─────────────────────────────────── - -func (s *Server) handleUIScrapeBook(w http.ResponseWriter, r *http.Request) { - bookURL := strings.TrimSpace(r.FormValue("url")) - if bookURL == "" { - renderFragment(w, scrapeStatusHTML("error", "Please enter a book URL.")) - return - } - - s.mu.Lock() - already := s.running - if !already { - s.running = true - } - s.mu.Unlock() - - if already { - renderFragment(w, scrapeStatusHTML("busy", "A scrape job is already running. Please wait.")) - return - } - - cfg := s.oCfg - cfg.SingleBookURL = bookURL - - go func() { - defer func() { - s.mu.Lock() - s.running = false - s.mu.Unlock() - }() - - ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) - defer cancel() - - o := orchestrator.New(cfg, s.novel, s.log) - if err := o.Run(ctx); err != nil { - s.log.Error("UI scrape job failed", "url", bookURL, "err", err) - } - }() - - // Return a status badge that polls until the job finishes. - renderFragment(w, scrapeStatusHTML("running", "Scraping "+bookURL+"…")) -} - -// ─── GET /ui/scrape/status — polling endpoint ───────────────────────────────── - -func (s *Server) handleUIScrapeStatus(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - running := s.running - s.mu.Unlock() - - if running { - // Keep polling every 3 s while the job is in progress. - renderFragment(w, scrapeStatusHTML("running", "Scraping in progress…")) - return - } - // Job finished — show a done badge and stop polling. - renderFragment(w, scrapeStatusHTML("done", "Done! Refresh the page to see new books.")) -} - -// scrapeStatusHTML returns a self-contained status badge fragment. -// state is one of: "running" | "done" | "busy" | "error". -func scrapeStatusHTML(state, msg string) string { - var colour, dot, poll string - switch state { - case "running": - colour = "text-amber-300 bg-amber-950 border-amber-800" - dot = `` - poll = `hx-get="/ui/scrape/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` - case "done": - colour = "text-green-300 bg-green-950 border-green-800" - dot = `` - case "busy": - colour = "text-yellow-300 bg-yellow-950 border-yellow-800" - dot = `` - default: // error - colour = "text-red-300 bg-red-950 border-red-800" - dot = `` - } - return fmt.Sprintf( - `
    %s%s
    `, - colour, poll, dot, template.HTMLEscapeString(msg), - ) -} diff --git a/scraper/internal/writer/writer.go b/scraper/internal/writer/writer.go deleted file mode 100644 index 82023d2..0000000 --- a/scraper/internal/writer/writer.go +++ /dev/null @@ -1,476 +0,0 @@ -// Package writer handles persistence of scraped chapters and metadata. -// -// Directory layout: -// -// static/books/ -// ├── {book-slug}/ -// │ ├── metadata.yaml -// │ ├── vol-0/ (no volume grouping) -// │ │ ├── 1-50/ -// │ │ │ ├── chapter-1.md -// │ │ │ └── … -// │ │ └── 51-100/ -// │ │ └── … -// │ └── vol-1/ -// │ └── … -package writer - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "regexp" - "sort" - "strconv" - "strings" - - "github.com/libnovel/scraper/internal/scraper" - "gopkg.in/yaml.v3" -) - -const chaptersPerFolder = 50 - -// Writer persists scraped content under a configurable root directory. -type Writer struct { - root string // e.g. "./static/books" -} - -// New creates a Writer that stores files under root. -func New(root string) *Writer { - return &Writer{root: root} -} - -// ─── Metadata ───────────────────────────────────────────────────────────────── - -// WriteMetadata serialises meta to static/books/{slug}/metadata.yaml. -// It creates the directory if it does not exist and overwrites any existing file. -func (w *Writer) WriteMetadata(meta scraper.BookMeta) error { - dir := w.bookDir(meta.Slug) - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("writer: mkdir %s: %w", dir, err) - } - - path := filepath.Join(dir, "metadata.yaml") - f, err := os.Create(path) - if err != nil { - return fmt.Errorf("writer: create metadata %s: %w", path, err) - } - defer f.Close() - - enc := yaml.NewEncoder(f) - enc.SetIndent(2) - if err := enc.Encode(meta); err != nil { - return fmt.Errorf("writer: encode metadata: %w", err) - } - return enc.Close() -} - -// ReadMetadata reads the metadata.yaml for slug if it exists. -// Returns (zero-value, false, nil) when the file does not exist. -func (w *Writer) ReadMetadata(slug string) (scraper.BookMeta, bool, error) { - path := filepath.Join(w.bookDir(slug), "metadata.yaml") - data, err := os.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return scraper.BookMeta{}, false, nil - } - return scraper.BookMeta{}, false, fmt.Errorf("writer: read metadata %s: %w", path, err) - } - - var meta scraper.BookMeta - if err := yaml.Unmarshal(data, &meta); err != nil { - return scraper.BookMeta{}, true, fmt.Errorf("writer: unmarshal metadata %s: %w", path, err) - } - return meta, true, nil -} - -// MetadataMtime returns the modification time (Unix seconds) of the -// metadata.yaml file for slug, or 0 if the file cannot be stat'd. -func (w *Writer) MetadataMtime(slug string) int64 { - path := filepath.Join(w.bookDir(slug), "metadata.yaml") - fi, err := os.Stat(path) - if err != nil { - return 0 - } - return fi.ModTime().Unix() -} - -// ─── Chapters ───────────────────────────────────────────────────────────────── - -// ChapterExists returns true if the markdown file for ref already exists on disk. -func (w *Writer) ChapterExists(slug string, ref scraper.ChapterRef) bool { - _, err := os.Stat(w.chapterPath(slug, ref)) - return err == nil -} - -// WriteChapter writes chapter.Text to the appropriate markdown file. -// The parent directories are created on demand. -func (w *Writer) WriteChapter(slug string, chapter scraper.Chapter) error { - path := w.chapterPath(slug, chapter.Ref) - dir := filepath.Dir(path) - - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("writer: mkdir %s: %w", dir, err) - } - - // Build the markdown document. - var sb strings.Builder - sb.WriteString("# ") - sb.WriteString(chapter.Ref.Title) - sb.WriteString("\n\n") - sb.WriteString(chapter.Text) - sb.WriteString("\n") - - if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { - return fmt.Errorf("writer: write chapter %s: %w", path, err) - } - return nil -} - -// ─── Catalogue helpers ──────────────────────────────────────────────────────── - -// ListBooks returns metadata for every book that has a metadata.yaml under root. -// Books with unreadable metadata files are silently skipped. -func (w *Writer) ListBooks() ([]scraper.BookMeta, error) { - entries, err := os.ReadDir(w.root) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, fmt.Errorf("writer: list books: %w", err) - } - var books []scraper.BookMeta - for _, e := range entries { - if !e.IsDir() { - continue - } - meta, ok, _ := w.ReadMetadata(e.Name()) - if !ok { - continue - } - books = append(books, meta) - } - sort.Slice(books, func(i, j int) bool { - return books[i].Title < books[j].Title - }) - return books, nil -} - -// LocalSlugs returns the set of book slugs that have a metadata.yaml on disk. -// It is cheaper than ListBooks because it only checks for file existence rather -// than fully parsing every YAML file. -func (w *Writer) LocalSlugs() map[string]bool { - entries, err := os.ReadDir(w.root) - if err != nil { - return map[string]bool{} - } - slugs := make(map[string]bool, len(entries)) - for _, e := range entries { - if !e.IsDir() { - continue - } - metaPath := filepath.Join(w.root, e.Name(), "metadata.yaml") - if _, err := os.Stat(metaPath); err == nil { - slugs[e.Name()] = true - } - } - return slugs -} - -// ChapterInfo is a lightweight chapter descriptor derived from on-disk files. -type ChapterInfo struct { - Number int - Title string // chapter name, cleaned of number prefix and trailing date - Date string // relative date scraped alongside the title, e.g. "1 year ago" -} - -// ListChapters returns all chapters on disk for slug, sorted by number. -func (w *Writer) ListChapters(slug string) ([]ChapterInfo, error) { - bookDir := w.bookDir(slug) - var chapters []ChapterInfo - - // Walk vol-*/range-*/ directories. - volDirs, err := filepath.Glob(filepath.Join(bookDir, "vol-*")) - if err != nil { - return nil, fmt.Errorf("writer: list chapters glob: %w", err) - } - for _, vd := range volDirs { - rangeDirs, _ := filepath.Glob(filepath.Join(vd, "*-*")) - for _, rd := range rangeDirs { - files, _ := filepath.Glob(filepath.Join(rd, "chapter-*.md")) - for _, f := range files { - base := filepath.Base(f) // chapter-N.md - numStr := strings.TrimSuffix(strings.TrimPrefix(base, "chapter-"), ".md") - n, err := strconv.Atoi(numStr) - if err != nil { - continue - } - title, date := chapterTitle(f, n) - chapters = append(chapters, ChapterInfo{Number: n, Title: title, Date: date}) - } - } - } - sort.Slice(chapters, func(i, j int) bool { - return chapters[i].Number < chapters[j].Number - }) - return chapters, nil -} - -// CountChapters returns the number of chapter markdown files on disk for slug. -// It is cheaper than ListChapters because it does not read file contents. -func (w *Writer) CountChapters(slug string) int { - bookDir := w.bookDir(slug) - volDirs, err := filepath.Glob(filepath.Join(bookDir, "vol-*")) - if err != nil { - return 0 - } - count := 0 - for _, vd := range volDirs { - rangeDirs, _ := filepath.Glob(filepath.Join(vd, "*-*")) - for _, rd := range rangeDirs { - files, _ := filepath.Glob(filepath.Join(rd, "chapter-*.md")) - count += len(files) - } - } - return count -} - -// chapterTitle reads the first non-empty line of a markdown file and strips -// the leading "# " heading marker. Falls back to "Chapter N". -func chapterTitle(path string, n int) (title, date string) { - data, err := os.ReadFile(path) - if err != nil { - return fmt.Sprintf("Chapter %d", n), "" - } - for _, line := range strings.SplitN(string(data), "\n", 10) { - line = strings.TrimSpace(line) - if line == "" { - continue - } - line = strings.TrimPrefix(line, "# ") - return SplitChapterTitle(line) - } - return fmt.Sprintf("Chapter %d", n), "" -} - -// SplitChapterTitle separates the human-readable chapter name from the -// trailing relative-date string that novelfire.net appends to the heading. -// Examples of raw heading text (after stripping "# "): -// -// "1 Chapter 1 - 1: The Academy's Weakest1 year ago" -// "2 Chapter 2 - Enter the Storm3 months ago" -// -// The pattern is: optional leading number+whitespace, then the real title, -// then a date that matches /\d+\s+(second|minute|hour|day|week|month|year)s?\s+ago$/ -func SplitChapterTitle(raw string) (title, date string) { - // Strip a leading chapter-number index that novelfire sometimes prepends. - // It looks like "1 " or "12 " at the very start. - raw = strings.TrimSpace(raw) - if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 { - prefix := raw[:idx] - allDigit := true - for _, c := range prefix { - if c < '0' || c > '9' { - allDigit = false - break - } - } - if allDigit { - raw = strings.TrimSpace(raw[idx:]) - } - } - - // Strip "Chapter N - N: " prefix (novelfire double-number format). - // Also handles "Chapter N: " (single number) and "Chapter N - Title" without colon. - chNumRe := regexp.MustCompile(`(?i)^chapter\s+\d+(?:\s*-\s*\d+)?\s*:\s*`) - raw = strings.TrimSpace(chNumRe.ReplaceAllString(raw, "")) - - // Match a trailing relative date: " [s] ago" - dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`) - if m := dateRe.FindStringSubmatchIndex(raw); m != nil { - return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]]) - } - return raw, "" -} - -// ReadChapter returns the raw markdown content for chapter number n of slug. -func (w *Writer) ReadChapter(slug string, n int) (string, error) { - // Reconstruct path using the same bucketing formula as chapterPath. - ref := scraper.ChapterRef{Number: n, Volume: 0} - path := w.chapterPath(slug, ref) - data, err := os.ReadFile(path) - if err != nil { - return "", fmt.Errorf("writer: read chapter %d: %w", n, err) - } - return string(data), nil -} - -// ─── Ranking ───────────────────────────────────────────────────────────────── - -// RankingItem represents a single entry in the ranking. -type RankingItem struct { - Rank int `yaml:"rank" json:"rank"` - Slug string `yaml:"slug" json:"slug"` - Title string `yaml:"title" json:"title"` - Author string `yaml:"author,omitempty" json:"author,omitempty"` - Cover string `yaml:"cover,omitempty" json:"cover,omitempty"` - Status string `yaml:"status,omitempty" json:"status,omitempty"` - Genres []string `yaml:"genres,omitempty" json:"genres,omitempty"` - SourceURL string `yaml:"source_url,omitempty" json:"source_url,omitempty"` -} - -// WriteRanking saves the ranking items as JSON to static/books/ranking.json. -// This replaces the old markdown table format with a structured format that -// is faster to read back (no custom parsing) and safe for titles containing "|". -func (w *Writer) WriteRanking(items []RankingItem) error { - path := filepath.Clean(w.rankingPath()) - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("writer: mkdir %s: %w", dir, err) - } - - data, err := json.MarshalIndent(items, "", " ") - if err != nil { - return fmt.Errorf("writer: marshal ranking: %w", err) - } - if err := os.WriteFile(path, data, 0o644); err != nil { - return fmt.Errorf("writer: write ranking %s: %w", path, err) - } - return nil -} - -// ReadRankingItems parses ranking.json into a slice of RankingItem. -// Returns nil slice (not an error) when the file does not exist yet. -func (w *Writer) ReadRankingItems() ([]RankingItem, error) { - data, err := os.ReadFile(w.rankingPath()) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, fmt.Errorf("writer: read ranking: %w", err) - } - var items []RankingItem - if err := json.Unmarshal(data, &items); err != nil { - return nil, fmt.Errorf("writer: parse ranking json: %w", err) - } - return items, nil -} - -// RankingFileInfo returns os.FileInfo for the ranking.json file, if it exists. -func (w *Writer) RankingFileInfo() (os.FileInfo, error) { - return os.Stat(w.rankingPath()) -} - -func (w *Writer) rankingPath() string { - return filepath.Join(w.root, "ranking.json") -} - -// ─── Ranking page HTML cache ────────────────────────────────────────────────── - -// rankingCacheDir returns the directory that stores per-page HTML caches. -func (w *Writer) rankingCacheDir() string { - return filepath.Join(w.root, "_ranking_cache") -} - -// rankingPageCachePath returns the path for a cached ranking page HTML file. -func (w *Writer) rankingPageCachePath(page int) string { - return filepath.Join(w.rankingCacheDir(), fmt.Sprintf("page-%d.html", page)) -} - -// WriteRankingPageCache persists raw HTML for the given ranking page number. -func (w *Writer) WriteRankingPageCache(page int, html string) error { - dir := w.rankingCacheDir() - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("writer: mkdir ranking cache %s: %w", dir, err) - } - path := w.rankingPageCachePath(page) - if err := os.WriteFile(path, []byte(html), 0o644); err != nil { - return fmt.Errorf("writer: write ranking page cache %s: %w", path, err) - } - return nil -} - -// ReadRankingPageCache reads the cached HTML for the given ranking page. -// Returns ("", nil) when no cache file exists yet. -func (w *Writer) ReadRankingPageCache(page int) (string, error) { - data, err := os.ReadFile(w.rankingPageCachePath(page)) - if err != nil { - if os.IsNotExist(err) { - return "", nil - } - return "", fmt.Errorf("writer: read ranking page cache page %d: %w", page, err) - } - return string(data), nil -} - -// RankingPageCacheInfo returns os.FileInfo for a cached ranking page file. -// Returns (nil, nil) when the file does not exist. -func (w *Writer) RankingPageCacheInfo(page int) (os.FileInfo, error) { - info, err := os.Stat(w.rankingPageCachePath(page)) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - return info, nil -} - -// bookDir returns the root directory for a book slug. -func (w *Writer) bookDir(slug string) string { - return filepath.Join(w.root, slug) -} - -// AudioDir returns the directory used to cache generated MP3 files for a book. -func (w *Writer) AudioDir(slug string) string { - return filepath.Join(w.bookDir(slug), "audio") -} - -// AudioPath returns the full path for a cached chapter audio file. -// The filename is keyed by chapter number, voice, and speed so that different -// settings never collide. Speed is formatted to one decimal place (e.g. "1.0"). -func (w *Writer) AudioPath(slug string, n int, voice string, speed float64) string { - safeVoice := sanitiseVoice(voice) - filename := fmt.Sprintf("ch%d-%s-%.1f.mp3", n, safeVoice, speed) - return filepath.Join(w.AudioDir(slug), filename) -} - -// AudioPartPath returns the path for an individual audio chunk generated during -// chunked TTS. Part files are named ch{n}-{voice}-{speed}.part{p}.mp3 and are -// deleted after they have been merged into the final AudioPath file. -func (w *Writer) AudioPartPath(slug string, n int, voice string, speed float64, part int) string { - safeVoice := sanitiseVoice(voice) - filename := fmt.Sprintf("ch%d-%s-%.1f.part%d.mp3", n, safeVoice, speed, part) - return filepath.Join(w.AudioDir(slug), filename) -} - -// sanitiseVoice converts a voice name into a string that is safe to embed in a -// filename (only a-z, A-Z, 0-9, '_', '-' are kept; everything else becomes '_'). -func sanitiseVoice(voice string) string { - return strings.Map(func(r rune) rune { - if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' { - return r - } - return '_' - }, voice) -} - -// chapterPath computes the full file path for a chapter. -// -// vol-{volume}/{folderRange}/chapter-{number}.md -// -// Example: vol-0/1-50/chapter-1.md, vol-0/51-100/chapter-51.md -func (w *Writer) chapterPath(slug string, ref scraper.ChapterRef) string { - vol := ref.Volume // 0 == no volume grouping - volDir := fmt.Sprintf("vol-%d", vol) - - // Folder group: chapters 1-50 → "1-50", 51-100 → "51-100", … - lo := ((ref.Number-1)/chaptersPerFolder)*chaptersPerFolder + 1 - hi := lo + chaptersPerFolder - 1 - rangeDir := fmt.Sprintf("%d-%d", lo, hi) - - filename := fmt.Sprintf("chapter-%d.md", ref.Number) - - return filepath.Join(w.bookDir(slug), volDir, rangeDir, filename) -} diff --git a/scraper/scraper b/scraper/scraper deleted file mode 100755 index 9e8e1a6..0000000 Binary files a/scraper/scraper and /dev/null differ diff --git a/scripts/e2e-test.mjs b/scripts/e2e-test.mjs new file mode 100644 index 0000000..f6d8f37 --- /dev/null +++ b/scripts/e2e-test.mjs @@ -0,0 +1,423 @@ +#!/usr/bin/env node +/** + * e2e-test.mjs — End-to-end tests for the LibNovel v3 stack. + * + * Hits live services via https://localhost (self-signed cert, TLS verify skipped). + * Requires: Node 18+ (built-in fetch with TLS options via --experimental-fetch or + * native in Node 21+). Run with: node --experimental-vm-modules scripts/e2e-test.mjs + * or simply: node scripts/e2e-test.mjs + * + * Services tested: + * - Caddy / UI https://localhost + * - Go backend via UI proxy routes + * - PocketBase via UI server-side (indirect) + * + * Usage: + * node scripts/e2e-test.mjs + * node scripts/e2e-test.mjs --verbose + */ + +import { createServer } from 'node:https'; +import { request as httpRequest } from 'node:https'; +import { URL } from 'node:url'; + +const BASE = 'https://localhost'; +const VERBOSE = process.argv.includes('--verbose'); + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +let passed = 0; +let failed = 0; +const failures = []; + +function log(...args) { + if (VERBOSE) console.log(...args); +} + +function pass(name) { + passed++; + console.log(` ✓ ${name}`); +} + +function fail(name, reason) { + failed++; + const msg = ` ✗ ${name}: ${reason}`; + console.log(msg); + failures.push({ name, reason }); +} + +/** + * fetch() that ignores TLS certificate errors (self-signed cert on localhost). + */ +async function get(path, { headers = {}, followRedirects = false } = {}) { + const url = path.startsWith('http') ? path : `${BASE}${path}`; + const res = await fetch(url, { + redirect: followRedirects ? 'follow' : 'manual', + headers, + // Node 18/19 uses undici which respects NODE_TLS_REJECT_UNAUTHORIZED + }); + return res; +} + +async function post(path, body, { headers = {}, cookie = '' } = {}) { + const url = path.startsWith('http') ? path : `${BASE}${path}`; + const res = await fetch(url, { + method: 'POST', + redirect: 'manual', + headers: { + 'Content-Type': 'application/json', + ...(cookie ? { Cookie: cookie } : {}), + ...headers, + }, + body: JSON.stringify(body), + }); + return res; +} + +async function del(path, { cookie = '' } = {}) { + const url = path.startsWith('http') ? path : `${BASE}${path}`; + const res = await fetch(url, { + method: 'DELETE', + redirect: 'manual', + headers: cookie ? { Cookie: cookie } : {}, + }); + return res; +} + +/** Extract Set-Cookie header value(s) as a single cookie string. */ +function extractCookies(res) { + const raw = res.headers.getSetCookie?.() ?? []; + return raw.map((c) => c.split(';')[0]).join('; '); +} + +async function assert(name, fn) { + try { + await fn(); + pass(name); + } catch (e) { + fail(name, e.message); + } +} + +function expect(val, label) { + return { + toBe(expected) { + if (val !== expected) throw new Error(`${label}: expected ${expected}, got ${val}`); + }, + toBeOneOf(...options) { + if (!options.includes(val)) throw new Error(`${label}: expected one of [${options.join(', ')}], got ${val}`); + }, + toBeOk() { + if (!val) throw new Error(`${label} was falsy`); + }, + toContainKey(key) { + if (!(key in val)) throw new Error(`${label}: missing key "${key}"`); + }, + toBeArray() { + if (!Array.isArray(val)) throw new Error(`${label}: expected array, got ${typeof val}`); + }, + toBeAbove(n) { + if (!(val > n)) throw new Error(`${label}: expected > ${n}, got ${val}`); + }, + }; +} + +// ─── Test suite ─────────────────────────────────────────────────────────────── + +process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + +// Pick a known book slug from the database (first available) +let TEST_SLUG = null; + +console.log('\nLibNovel v3 — End-to-End Tests'); +console.log('================================\n'); + +// ── 1. Health checks ────────────────────────────────────────────────────────── +console.log('1. Health checks'); + +await assert('UI health endpoint returns 200', async () => { + const res = await get('/health'); + expect(res.status, 'status').toBe(200); +}); + +await assert('Home page returns 200', async () => { + const res = await get('/', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const html = await res.text(); + expect(html.includes(' { + const res = await get('/browse', { followRedirects: true }); + expect(res.status, 'status').toBe(200); +}); + +// ── 2. Home API ─────────────────────────────────────────────────────────────── +console.log('\n2. Home API'); + +let homeData = null; + +await assert('GET /api/home returns continue_reading and recently_updated', async () => { + const res = await get('/api/home', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + homeData = await res.json(); + expect(homeData, 'data').toContainKey('continue_reading'); + expect(homeData, 'data').toContainKey('recently_updated'); + expect(homeData.continue_reading, 'continue_reading').toBeArray(); + expect(homeData.recently_updated, 'recently_updated').toBeArray(); +}); + +await assert('GET /api/home stats has totalBooks and totalChapters', async () => { + if (!homeData) { + const res = await get('/api/home', { followRedirects: true }); + homeData = await res.json(); + } + expect(homeData, 'data').toContainKey('stats'); + expect(typeof homeData.stats.totalBooks, 'totalBooks type').toBe('number'); + expect(typeof homeData.stats.totalChapters, 'totalChapters type').toBe('number'); +}); + +// ── 3. Browse / ranking / search ────────────────────────────────────────────── +console.log('\n3. Browse / ranking / search'); + +await assert('GET /api/browse-page returns novels array', async () => { + const res = await get('/api/browse-page?page=1', { followRedirects: true }); + expect(res.status, 'status').toBeOneOf(200, 503); + if (res.status === 200) { + const data = await res.json(); + // Accepts { novels: [...] } or { error: ... } (when MinIO cache is empty) + expect(typeof data, 'response type').toBe('object'); + } +}); + +await assert('GET /api/ranking returns array or 502 (no data yet)', async () => { + const res = await get('/api/ranking', { followRedirects: true }); + // 200 = ranking data exists; 502 = no ranking data scraped yet — both are expected + expect(res.status, 'status').toBeOneOf(200, 502); + if (res.status === 200) { + const data = await res.json(); + expect(data, 'data').toBeArray(); + } +}); + +await assert('GET /api/search?q=shadow returns results object', async () => { + const res = await get('/api/search?q=shadow', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + expect(typeof data, 'response type').toBe('object'); +}); + +// ── 4. Books ────────────────────────────────────────────────────────────────── +console.log('\n4. Books'); + +// Find a real slug from /api/home +await assert('GET /api/home books are accessible', async () => { + if (!homeData) { + const res = await get('/api/home', { followRedirects: true }); + homeData = await res.json(); + } + const books = homeData?.recently_updated ?? []; + if (books.length > 0) { + TEST_SLUG = books[0].slug; + log(` Using test slug: ${TEST_SLUG}`); + } + // Pass regardless — we just want to find a slug +}); + +if (TEST_SLUG) { + await assert(`GET /api/book/${TEST_SLUG} returns book metadata`, async () => { + const res = await get(`/api/book/${TEST_SLUG}`, { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + // Returns { book: { slug, ... }, chapters: [...] } + expect(data, 'data').toContainKey('book'); + expect(data.book, 'book').toContainKey('slug'); + expect(data.book.slug, 'slug').toBe(TEST_SLUG); + }); + + await assert(`Book detail page /${TEST_SLUG} returns 200`, async () => { + const res = await get(`/${TEST_SLUG}`, { followRedirects: true }); + expect(res.status, 'status').toBe(200); + }); +} else { + console.log(' ⚠ No books in database — skipping book-specific tests'); +} + +// ── 5. Voices ───────────────────────────────────────────────────────────────── +console.log('\n5. Voices'); + +await assert('GET /api/voices returns voices array', async () => { + const res = await get('/api/voices', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + // Returns { voices: [...] } + expect(data, 'data').toContainKey('voices'); + expect(data.voices, 'voices').toBeArray(); + expect(data.voices.length, 'voice count').toBeAbove(0); +}); + +// ── 6. Auth flow ────────────────────────────────────────────────────────────── +console.log('\n6. Auth flow'); + +const TEST_USER = `e2e_test_${Date.now()}`; +const TEST_PASS = 'E2eTestPassword1!'; +let authCookie = ''; + +await assert('POST /api/auth/register creates new user', async () => { + const res = await post('/api/auth/register', { username: TEST_USER, password: TEST_PASS }); + expect(res.status, 'status').toBeOneOf(200, 201); + const data = await res.json(); + // Returns { token: "...", user: { id, username, role } } + expect(data, 'response').toContainKey('user'); + expect(data.user, 'user').toContainKey('username'); + expect(data.user.username, 'username').toBe(TEST_USER); + // Build cookie from token + if (data.token) { + authCookie = `libnovel_auth=${data.token}`; + } else { + authCookie = extractCookies(res); + } + log(` Auth cookie: ${authCookie.slice(0, 40)}...`); +}); + +await assert('GET /api/auth/me returns current user when logged in', async () => { + const res = await get('/api/auth/me', { followRedirects: true, headers: { Cookie: authCookie } }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + expect(data, 'data').toContainKey('username'); + expect(data.username, 'username').toBe(TEST_USER); +}); + +await assert('POST /api/auth/logout clears session', async () => { + const res = await post('/api/auth/logout', {}, { cookie: authCookie }); + expect(res.status, 'status').toBeOneOf(200, 204); +}); + +await assert('POST /api/auth/login works after register', async () => { + const res = await post('/api/auth/login', { username: TEST_USER, password: TEST_PASS }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + // Returns { token: "...", user: { id, username, role } } + expect(data, 'response').toContainKey('user'); + expect(data.user.username, 'username').toBe(TEST_USER); + if (data.token) { + authCookie = `libnovel_auth=${data.token}`; + } else { + authCookie = extractCookies(res); + } +}); + +await assert('GET /api/auth/me unauthenticated returns 401', async () => { + const res = await get('/api/auth/me', { followRedirects: true }); + expect(res.status, 'status').toBe(401); +}); + +// ── 7. Progress ─────────────────────────────────────────────────────────────── +console.log('\n7. Progress'); + +// /api/progress (root) is POST-only. Per-slug is GET/POST/DELETE via /api/progress/[slug]. + +if (TEST_SLUG) { + await assert(`POST /api/progress/${TEST_SLUG} sets progress`, async () => { + const res = await post(`/api/progress/${TEST_SLUG}`, { chapter: 1 }); + expect(res.status, 'status').toBeOneOf(200, 201); + const data = await res.json(); + expect(data, 'data').toContainKey('ok'); + }); + + await assert(`DELETE /api/progress/${TEST_SLUG} removes progress`, async () => { + const res = await del(`/api/progress/${TEST_SLUG}`); + expect(res.status, 'status').toBeOneOf(200, 204); + }); +} else { + console.log(' ⚠ No books — skipping progress tests'); +} + +// ── 8. Library ──────────────────────────────────────────────────────────────── +console.log('\n8. Library'); + +await assert('GET /api/library returns object', async () => { + const res = await get('/api/library', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + expect(typeof data, 'data type').toBe('object'); +}); + +if (TEST_SLUG) { + await assert(`POST /api/library/${TEST_SLUG} saves book`, async () => { + const res = await post(`/api/library/${TEST_SLUG}`, {}); + expect(res.status, 'status').toBeOneOf(200, 201); + }); + + await assert(`DELETE /api/library/${TEST_SLUG} removes book`, async () => { + const res = await del(`/api/library/${TEST_SLUG}`); + expect(res.status, 'status').toBeOneOf(200, 204); + }); +} + +// ── 9. Settings ─────────────────────────────────────────────────────────────── +console.log('\n9. Settings'); + +await assert('GET /api/settings returns settings object', async () => { + const res = await get('/api/settings', { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + expect(typeof data, 'data type').toBe('object'); +}); + +// ── 10. Sessions ────────────────────────────────────────────────────────────── +console.log('\n10. Sessions'); + +await assert('GET /api/sessions (authenticated) returns sessions array', async () => { + const res = await get('/api/sessions', { followRedirects: true, headers: { Cookie: authCookie } }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + // Returns { sessions: [...] } + expect(data, 'data').toContainKey('sessions'); + expect(data.sessions, 'sessions').toBeArray(); +}); + +// ── 11. Comments ────────────────────────────────────────────────────────────── +console.log('\n11. Comments'); + +if (TEST_SLUG) { + await assert(`GET /api/comments/${TEST_SLUG} returns comments`, async () => { + const res = await get(`/api/comments/${TEST_SLUG}`, { followRedirects: true }); + expect(res.status, 'status').toBe(200); + const data = await res.json(); + // Returns { comments: [...], myVotes: {}, avatarUrls: {} } + expect(data, 'data').toContainKey('comments'); + expect(data.comments, 'comments').toBeArray(); + }); +} + +// ── 12. Chapter endpoints ───────────────────────────────────────────────────── +console.log('\n12. Chapter endpoints'); + +if (TEST_SLUG) { + await assert(`GET /api/chapter/${TEST_SLUG}/1 returns 200 or 404`, async () => { + const res = await get(`/api/chapter/${TEST_SLUG}/1`, { followRedirects: true }); + // 200 if chapter exists in MinIO, 404 if not + expect(res.status, 'status').toBeOneOf(200, 404); + }); + + await assert(`GET /api/chapter-text-preview/${TEST_SLUG}/1 returns 200 or error`, async () => { + const res = await get(`/api/chapter-text-preview/${TEST_SLUG}/1`, { followRedirects: true }); + expect(res.status, 'status').toBeOneOf(200, 404, 500, 503); + }); +} + +// ── Summary ─────────────────────────────────────────────────────────────────── +console.log('\n─────────────────────────────────────'); +console.log(`Results: ${passed} passed, ${failed} failed`); + +if (failures.length > 0) { + console.log('\nFailures:'); + for (const f of failures) { + console.log(` ✗ ${f.name}`); + console.log(` ${f.reason}`); + } +} + +console.log('─────────────────────────────────────\n'); +process.exit(failed > 0 ? 1 : 0); diff --git a/scripts/libnovel.sh b/scripts/libnovel.sh new file mode 100644 index 0000000..2b76f98 --- /dev/null +++ b/scripts/libnovel.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ── LibNovel — production management script ─────────────────────────────────── +# Prerequisites on the server: +# - docker + docker compose plugin +# - doppler CLI authenticated (doppler setup run once in the repo directory) +# - docker-compose.yml present in the same directory as this script +# +# Usage: ./libnovel.sh [service] +# +# up Start all services (detached) +# down Stop all services +# restart Stop then start all services +# restart Restart a single service +# pull Pull latest images from Docker Hub (uses GIT_TAG from Doppler) +# update Pull images then recreate containers +# logs Tail all logs +# logs Tail a specific service +# ps Show running containers +# shell Open a shell in a running container +# init Run one-shot init containers +# secrets Print all Doppler secrets (debug) +# ────────────────────────────────────────────────────────────────────────────── + +DC="doppler run -- docker compose" +CMD="${1:-help}" +SVC="${2:-}" + +case "$CMD" in + up) + $DC up -d + ;; + + down) + $DC down + ;; + + restart) + if [[ -n "$SVC" ]]; then + $DC restart "$SVC" + else + $DC down + $DC up -d + fi + ;; + + pull) + $DC pull backend runner ui caddy + ;; + + update) + $DC pull backend runner ui caddy + $DC up -d + ;; + + logs) + if [[ -n "$SVC" ]]; then + $DC logs -f --tail=100 "$SVC" + else + $DC logs -f --tail=50 + fi + ;; + + ps) + $DC ps + ;; + + shell) + [[ -z "$SVC" ]] && { echo "Usage: $0 shell "; exit 1; } + $DC exec "$SVC" sh + ;; + + init) + $DC run --rm minio-init + $DC run --rm pb-init + $DC run --rm postgres-init + ;; + + secrets) + doppler secrets --project libnovel --config prd + ;; + + help|*) + echo "Usage: $0 [service]" + echo "" + echo " up Start all services" + echo " down Stop all services" + echo " restart Full restart" + echo " restart Restart one service" + echo " pull Pull latest images from Docker Hub" + echo " update Pull + recreate containers" + echo " logs Tail all logs" + echo " logs Tail one service" + echo " ps Show running containers" + echo " shell Shell into a container" + echo " init Run init containers (first-time setup)" + echo " secrets Print Doppler secrets" + ;; +esac diff --git a/scripts/pb-init-v3.sh b/scripts/pb-init-v3.sh new file mode 100755 index 0000000..132647a --- /dev/null +++ b/scripts/pb-init-v3.sh @@ -0,0 +1,419 @@ +#!/bin/sh +# pb-init-v3.sh — idempotent PocketBase bootstrap for the v3 stack. +# +# Safe to re-run: existing collections and fields are silently skipped. +# +# Env vars (defaults match docker-compose.yml): +# POCKETBASE_URL http://pocketbase:8090 +# POCKETBASE_ADMIN_EMAIL admin@libnovel.local +# POCKETBASE_ADMIN_PASSWORD changeme123 + +set -e + +PB="${POCKETBASE_URL:-http://pocketbase:8090}" +EMAIL="${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" +PASS="${POCKETBASE_ADMIN_PASSWORD:-changeme123}" + +log() { printf '[pb-init] %s\n' "$*"; } + +# ── 0. Ensure dependencies ──────────────────────────────────────────────────── +command -v curl > /dev/null 2>&1 || apk add --no-cache curl > /dev/null 2>&1 +command -v python3 > /dev/null 2>&1 || apk add --no-cache python3 > /dev/null 2>&1 + +# ── 1. Wait for PocketBase ──────────────────────────────────────────────────── +log "waiting for PocketBase..." +until curl -sf "$PB/api/health" > /dev/null 2>&1; do sleep 2; done +log "PocketBase ready" + +# ── 2. Bootstrap superuser (first-run only) ─────────────────────────────────── +LOCATION=$(curl -sf -o /dev/null -w "%{redirect_url}" "$PB/_/" 2>/dev/null || true) +if echo "$LOCATION" | grep -q "pbinstal/"; then + TOKEN=$(echo "$LOCATION" | sed 's|.*pbinstal/||' | tr -d ' \r\n') + curl -sf -X POST "$PB/api/collections/_superusers/records" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d "{\"email\":\"$EMAIL\",\"password\":\"$PASS\",\"passwordConfirm\":\"$PASS\"}" \ + > /dev/null 2>&1 || true + log "superuser created" +fi + +# ── 3. Authenticate ─────────────────────────────────────────────────────────── +AUTH=$(curl -sf -X POST "$PB/api/collections/_superusers/auth-with-password" \ + -H "Content-Type: application/json" \ + -d "{\"identity\":\"$EMAIL\",\"password\":\"$PASS\"}") +TOK=$(echo "$AUTH" | sed 's/.*"token":"\([^"]*\)".*/\1/') +[ -z "$TOK" ] || [ "$TOK" = "$AUTH" ] && { log "ERROR: auth failed"; exit 1; } +log "authenticated" + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +# create NAME BODY — POST collection; 400/422 = already exists, treated as ok. +create() { + NAME="$1"; BODY="$2" + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST "$PB/api/collections" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOK" \ + -d "$BODY") + case "$STATUS" in + 200|201) log "created: $NAME" ;; + 400|422) log "exists (skip): $NAME" ;; + *) log "WARNING: $NAME returned $STATUS" ;; + esac +} + +# add_index COLLECTION INDEX_NAME SQL_EXPR +# Fetches current schema, adds index if absent by name, PATCHes collection. +add_index() { + COLL="$1"; INAME="$2"; ISQL="$3" + SCHEMA=$(curl -sf -H "Authorization: Bearer $TOK" "$PB/api/collections/$COLL" 2>/dev/null) + PARSED=$(echo "$SCHEMA" | python3 -c " +import sys, json +d = json.load(sys.stdin) +indexes = d.get('indexes', []) +exists = any('$INAME' in idx for idx in indexes) +print('exists=' + str(exists)) +print('id=' + d.get('id', '')) +if not exists: + indexes.append('$ISQL') + print('indexes=' + json.dumps(indexes)) +" 2>/dev/null) + if echo "$PARSED" | grep -q "^exists=True"; then + log "index exists (skip): $COLL.$INAME"; return + fi + COLL_ID=$(echo "$PARSED" | grep "^id=" | sed 's/^id=//') + [ -z "$COLL_ID" ] && { log "WARNING: cannot resolve id for $COLL"; return; } + NEW_INDEXES=$(echo "$PARSED" | grep "^indexes=" | sed 's/^indexes=//') + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -X PATCH "$PB/api/collections/$COLL_ID" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOK" \ + -d "{\"indexes\":${NEW_INDEXES}}") + case "$STATUS" in + 200|201) log "added index: $COLL.$INAME" ;; + *) log "WARNING: add_index $COLL.$INAME returned $STATUS" ;; + esac +} + +# add_field COLLECTION FIELD_NAME FIELD_TYPE +# Fetches current schema, appends field if absent, PATCHes collection. +# Requires python3 for safe JSON manipulation. +add_field() { + COLL="$1"; FIELD="$2"; TYPE="$3" + SCHEMA=$(curl -sf -H "Authorization: Bearer $TOK" "$PB/api/collections/$COLL" 2>/dev/null) + # Check existence and extract collection id + fields via python3 + PARSED=$(echo "$SCHEMA" | python3 -c " +import sys, json +d = json.load(sys.stdin) +fields = d.get('fields', []) +exists = any(f.get('name') == '$FIELD' for f in fields) +print('exists=' + str(exists)) +print('id=' + d.get('id', '')) +if not exists: + fields.append({'name': '$FIELD', 'type': '$TYPE'}) + print('fields=' + json.dumps(fields)) +" 2>/dev/null) + if echo "$PARSED" | grep -q "^exists=True"; then + log "field exists (skip): $COLL.$FIELD"; return + fi + COLL_ID=$(echo "$PARSED" | grep "^id=" | sed 's/^id=//') + [ -z "$COLL_ID" ] && { log "WARNING: cannot resolve id for $COLL"; return; } + NEW_FIELDS=$(echo "$PARSED" | grep "^fields=" | sed 's/^fields=//') + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -X PATCH "$PB/api/collections/$COLL_ID" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOK" \ + -d "{\"fields\":${NEW_FIELDS}}") + case "$STATUS" in + 200|201) log "added field: $COLL.$FIELD ($TYPE)" ;; + *) log "WARNING: add_field $COLL.$FIELD returned $STATUS" ;; + esac +} + +# ── 4. Collections ──────────────────────────────────────────────────────────── + +create "books" '{ + "name":"books","type":"base","fields":[ + {"name":"slug", "type":"text", "required":true}, + {"name":"title", "type":"text", "required":true}, + {"name":"author", "type":"text"}, + {"name":"cover", "type":"text"}, + {"name":"status", "type":"text"}, + {"name":"genres", "type":"json"}, + {"name":"summary", "type":"text"}, + {"name":"total_chapters","type":"number"}, + {"name":"source_url", "type":"text"}, + {"name":"ranking", "type":"number"}, + {"name":"meta_updated", "type":"text"}, + {"name":"archived", "type":"bool"} + ]}' + +create "chapters_idx" '{ + "name":"chapters_idx","type":"base","fields":[ + {"name":"slug", "type":"text", "required":true}, + {"name":"number", "type":"number", "required":true}, + {"name":"title", "type":"text"}, + {"name":"created", "type":"date"} + ]}' + +create "ranking" '{ + "name":"ranking","type":"base","fields":[ + {"name":"rank", "type":"number","required":true}, + {"name":"slug", "type":"text", "required":true}, + {"name":"title", "type":"text"}, + {"name":"author", "type":"text"}, + {"name":"cover", "type":"text"}, + {"name":"status", "type":"text"}, + {"name":"genres", "type":"json"}, + {"name":"source_url","type":"text"} + ]}' + +create "progress" '{ + "name":"progress","type":"base","fields":[ + {"name":"session_id","type":"text", "required":true}, + {"name":"slug", "type":"text", "required":true}, + {"name":"chapter", "type":"number"}, + {"name":"user_id", "type":"text"}, + {"name":"audio_time","type":"number"}, + {"name":"updated", "type":"text"} + ]}' + +create "scraping_tasks" '{ + "name":"scraping_tasks","type":"base","fields":[ + {"name":"kind", "type":"text"}, + {"name":"target_url", "type":"text"}, + {"name":"from_chapter", "type":"number"}, + {"name":"to_chapter", "type":"number"}, + {"name":"worker_id", "type":"text"}, + {"name":"status", "type":"text","required":true}, + {"name":"books_found", "type":"number"}, + {"name":"chapters_scraped", "type":"number"}, + {"name":"chapters_skipped", "type":"number"}, + {"name":"errors", "type":"number"}, + {"name":"error_message", "type":"text"}, + {"name":"started", "type":"date"}, + {"name":"finished", "type":"date"}, + {"name":"heartbeat_at", "type":"date"} + ]}' + +create "audio_jobs" '{ + "name":"audio_jobs","type":"base","fields":[ + {"name":"cache_key", "type":"text", "required":true}, + {"name":"slug", "type":"text", "required":true}, + {"name":"chapter", "type":"number","required":true}, + {"name":"voice", "type":"text"}, + {"name":"worker_id", "type":"text"}, + {"name":"status", "type":"text", "required":true}, + {"name":"error_message","type":"text"}, + {"name":"started", "type":"date"}, + {"name":"finished", "type":"date"}, + {"name":"heartbeat_at", "type":"date"} + ]}' + +create "app_users" '{ + "name":"app_users","type":"base","fields":[ + {"name":"username", "type":"text","required":true}, + {"name":"password_hash", "type":"text"}, + {"name":"role", "type":"text"}, + {"name":"avatar_url", "type":"text"}, + {"name":"created", "type":"text"}, + {"name":"email", "type":"text"}, + {"name":"email_verified", "type":"bool"}, + {"name":"verification_token", "type":"text"}, + {"name":"verification_token_exp","type":"text"}, + {"name":"oauth_provider", "type":"text"}, + {"name":"oauth_id", "type":"text"} + ]}' + + create "user_sessions" '{ + "name":"user_sessions","type":"base","fields":[ + {"name":"user_id", "type":"text","required":true}, + {"name":"session_id", "type":"text","required":true}, + {"name":"user_agent", "type":"text"}, + {"name":"ip", "type":"text"}, + {"name":"device_fingerprint", "type":"text"}, + {"name":"created_at", "type":"text"}, + {"name":"last_seen", "type":"text"} + ]}' + +create "user_library" '{ + "name":"user_library","type":"base","fields":[ + {"name":"session_id","type":"text","required":true}, + {"name":"user_id", "type":"text"}, + {"name":"slug", "type":"text","required":true}, + {"name":"saved_at", "type":"text"} + ]}' + +create "user_settings" '{ + "name":"user_settings","type":"base","fields":[ + {"name":"session_id", "type":"text", "required":true}, + {"name":"user_id", "type":"text"}, + {"name":"auto_next", "type":"bool"}, + {"name":"voice", "type":"text"}, + {"name":"speed", "type":"number"}, + {"name":"theme", "type":"text"}, + {"name":"locale", "type":"text"}, + {"name":"font_family", "type":"text"}, + {"name":"font_size", "type":"number"}, + {"name":"announce_chapter","type":"bool"}, + {"name":"audio_mode", "type":"text"}, + {"name":"updated", "type":"text"} + ]}' + +create "user_subscriptions" '{ + "name":"user_subscriptions","type":"base","fields":[ + {"name":"follower_id","type":"text","required":true}, + {"name":"followee_id","type":"text","required":true}, + {"name":"created", "type":"text"} + ]}' + +create "book_comments" '{ + "name":"book_comments","type":"base","fields":[ + {"name":"slug", "type":"text","required":true}, + {"name":"user_id", "type":"text"}, + {"name":"username", "type":"text"}, + {"name":"body", "type":"text"}, + {"name":"upvotes", "type":"number"}, + {"name":"downvotes","type":"number"}, + {"name":"parent_id","type":"text"}, + {"name":"created", "type":"text"} + ]}' + +create "comment_votes" '{ + "name":"comment_votes","type":"base","fields":[ + {"name":"comment_id","type":"text","required":true}, + {"name":"user_id", "type":"text"}, + {"name":"session_id","type":"text"}, + {"name":"vote", "type":"text"} + ]}' + +create "translation_jobs" '{ + "name":"translation_jobs","type":"base","fields":[ + {"name":"cache_key", "type":"text", "required":true}, + {"name":"slug", "type":"text", "required":true}, + {"name":"chapter", "type":"number","required":true}, + {"name":"lang", "type":"text", "required":true}, + {"name":"worker_id", "type":"text"}, + {"name":"status", "type":"text", "required":true}, + {"name":"error_message","type":"text"}, + {"name":"started", "type":"date"}, + {"name":"finished", "type":"date"}, + {"name":"heartbeat_at", "type":"date"} + ]}' + +create "import_tasks" '{ + "name":"import_tasks","type":"base","fields":[ + {"name":"slug", "type":"text", "required":true}, + {"name":"title", "type":"text", "required":true}, + {"name":"file_name", "type":"text"}, + {"name":"file_type", "type":"text"}, + {"name":"object_key", "type":"text"}, + {"name":"chapters_key", "type":"text"}, + {"name":"author", "type":"text"}, + {"name":"cover_url", "type":"text"}, + {"name":"genres", "type":"text"}, + {"name":"summary", "type":"text"}, + {"name":"book_status", "type":"text"}, + {"name":"worker_id", "type":"text"}, + {"name":"initiator_user_id", "type":"text"}, + {"name":"status", "type":"text", "required":true}, + {"name":"chapters_done", "type":"number"}, + {"name":"chapters_total", "type":"number"}, + {"name":"error_message", "type":"text"}, + {"name":"started", "type":"date"}, + {"name":"finished", "type":"date"}, + {"name":"heartbeat_at", "type":"date"} + ]}' + +create "notifications" '{ + "name":"notifications","type":"base","fields":[ + {"name":"user_id", "type":"text","required":true}, + {"name":"title", "type":"text","required":true}, + {"name":"message", "type":"text"}, + {"name":"link", "type":"text"}, + {"name":"read", "type":"bool"}, + {"name":"created", "type":"date"} + ]}' + +create "push_subscriptions" '{ + "name":"push_subscriptions","type":"base","fields":[ + {"name":"user_id", "type":"text","required":true}, + {"name":"endpoint", "type":"text","required":true}, + {"name":"p256dh", "type":"text","required":true}, + {"name":"auth", "type":"text","required":true} + ]}' + +create "ai_jobs" '{ + "name":"ai_jobs","type":"base","fields":[ + {"name":"kind", "type":"text", "required":true}, + {"name":"slug", "type":"text"}, + {"name":"status", "type":"text", "required":true}, + {"name":"from_item", "type":"number"}, + {"name":"to_item", "type":"number"}, + {"name":"items_done", "type":"number"}, + {"name":"items_total", "type":"number"}, + {"name":"model", "type":"text"}, + {"name":"payload", "type":"text"}, + {"name":"error_message", "type":"text"}, + {"name":"started", "type":"date"}, + {"name":"finished", "type":"date"}, + {"name":"heartbeat_at", "type":"date"} + ]}' + +create "discovery_votes" '{ + "name":"discovery_votes","type":"base","fields":[ + {"name":"session_id","type":"text","required":true}, + {"name":"user_id", "type":"text"}, + {"name":"slug", "type":"text","required":true}, + {"name":"action", "type":"text","required":true} + ]}' + +create "book_ratings" '{ + "name":"book_ratings","type":"base","fields":[ + {"name":"session_id","type":"text", "required":true}, + {"name":"user_id", "type":"text"}, + {"name":"slug", "type":"text", "required":true}, + {"name":"rating", "type":"number", "required":true} + ]}' + +create "site_config" '{ + "name":"site_config","type":"base","fields":[ + {"name":"decoration", "type":"text"}, + {"name":"logoAnimation", "type":"text"}, + {"name":"eventLabel", "type":"text"} + ]}' + +# ── 5. Field migrations (idempotent — adds fields missing from older installs) ─ +add_field "scraping_tasks" "heartbeat_at" "date" +add_field "audio_jobs" "heartbeat_at" "date" +add_field "progress" "user_id" "text" +add_field "progress" "audio_time" "number" +add_field "progress" "updated" "text" +add_field "books" "meta_updated" "text" +add_field "app_users" "email" "text" +add_field "app_users" "email_verified" "bool" +add_field "app_users" "verification_token" "text" +add_field "app_users" "verification_token_exp" "text" +add_field "app_users" "oauth_provider" "text" +add_field "app_users" "oauth_id" "text" +add_field "app_users" "polar_customer_id" "text" +add_field "app_users" "polar_subscription_id" "text" +add_field "user_library" "shelf" "text" +add_field "user_sessions" "device_fingerprint" "text" +add_field "chapters_idx" "created" "date" +add_field "user_settings" "theme" "text" +add_field "user_settings" "locale" "text" +add_field "user_settings" "font_family" "text" +add_field "user_settings" "font_size" "number" +add_field "user_settings" "announce_chapter" "bool" +add_field "user_settings" "audio_mode" "text" +add_field "books" "archived" "bool" +add_field "app_users" "notify_new_chapters" "bool" + +# ── 6. Indexes ──────────────────────────────────────────────────────────────── +add_index "chapters_idx" "idx_chapters_idx_slug_number" \ + "CREATE UNIQUE INDEX idx_chapters_idx_slug_number ON chapters_idx (slug, number)" +add_index "chapters_idx" "idx_chapters_idx_created" \ + "CREATE INDEX idx_chapters_idx_created ON chapters_idx (created)" + +log "done" diff --git a/ui/.dockerignore b/ui/.dockerignore new file mode 100644 index 0000000..62e1cdc --- /dev/null +++ b/ui/.dockerignore @@ -0,0 +1,5 @@ +node_modules +build +.svelte-kit +.env +.env.* diff --git a/ui/.env.example b/ui/.env.example new file mode 100644 index 0000000..9d92739 --- /dev/null +++ b/ui/.env.example @@ -0,0 +1,20 @@ +# libnovel UI — environment variables +# Copy to .env and adjust; do NOT commit with real secrets. + +# Internal URL of the backend API (used by SvelteKit server-side load functions) +# In docker-compose this is the internal service name +BACKEND_API_URL=http://localhost:8080 + +# Public URL of PocketBase (used by SvelteKit server-side load functions) +POCKETBASE_URL=http://localhost:8090 + +# PocketBase admin credentials (server-side only, never exposed to browser) +POCKETBASE_ADMIN_EMAIL=admin@libnovel.local +POCKETBASE_ADMIN_PASSWORD=changeme123 + +# Public-facing MinIO URL (used to rewrite presigned URLs for the browser) +# In dev this is localhost; in prod set to your MinIO public domain +PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000 + +# Secret used to sign auth tokens stored in cookies (generate with: openssl rand -hex 32) +AUTH_SECRET=change_this_to_a_long_random_secret diff --git a/ui/.gitignore b/ui/.gitignore new file mode 100644 index 0000000..b9e091f --- /dev/null +++ b/ui/.gitignore @@ -0,0 +1,25 @@ +node_modules + +# Output +.output +.vercel +.netlify +.wrangler +/.svelte-kit +/build + +# OS +.DS_Store +Thumbs.db + +# Env +.env +.env.* +!.env.example +!.env.test + +# Vite +vite.config.js.timestamp-* +vite.config.ts.timestamp-* + + diff --git a/ui/.npmrc b/ui/.npmrc new file mode 100644 index 0000000..b6f27f1 --- /dev/null +++ b/ui/.npmrc @@ -0,0 +1 @@ +engine-strict=true diff --git a/ui/AGENTS.md b/ui/AGENTS.md new file mode 100644 index 0000000..5dc9be3 --- /dev/null +++ b/ui/AGENTS.md @@ -0,0 +1,71 @@ +# LibNovel UI — Agent Context + +SvelteKit 2 + Svelte 5 frontend. Node adapter for production; served behind Caddy. + +## Design System + +**ACTIVE_STYLE: branded** + +Custom amber + zinc dark palette. No shadcn CLI defaults — primitives are hand-authored to match. + +| Token | Value | +|-------|-------| +| Accent | `#f59e0b` (amber-400) | +| Surface-1 | zinc-900 | +| Surface-2 | zinc-800 | +| Surface-3 | zinc-700 | +| Text-primary | zinc-100 | +| Text-secondary | zinc-400 | +| Destructive | red-400 | + +## Tailwind + +**Version: 4** — configured entirely via `@theme {}` in `src/app.css` and the `@tailwindcss/vite` plugin. + +**There is no `tailwind.config.ts`** — do not create one. +**Do not run `npx shadcn-svelte add ...`** — primitives are hand-authored in `$lib/components/ui/`. + +## shadcn-svelte Primitives + +All in `src/lib/components/ui/`: + +| Component | Variants / Notes | +|-----------|-----------------| +| `button` | default / secondary / outline / ghost / destructive / link; sizes: default / sm / lg / icon | +| `badge` | default / secondary / outline / destructive | +| `card` | Card, CardHeader, CardTitle, CardDescription, CardContent, CardFooter | +| `textarea` | bindable `value` prop | +| `dialog` | Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter | +| `separator` | horizontal / vertical | + +Always use `cn()` from `$lib/utils` — never template-literal class conditionals. + +## Svelte 5 Conventions + +- `@Observable` / runes (`$state`, `$derived`, `$effect`, `$props()`) for all new code. +- Do not add `ObservableObject` / `@Published` — they don't exist in Svelte; don't introduce legacy `writable` stores for new code. +- Navigation: `goto()` from `$app/navigation`; `page` from `$app/state`. + +## iOS/UX Skill + +For any view work, load the `ios-ux` skill at task start: +``` +skill({ name: "ios-ux" }) +``` + +## Key Files + +| File | Role | +|------|------| +| `src/app.css` | Tailwind v4 `@theme` tokens — source of truth for all design tokens | +| `src/lib/utils.ts` | `cn()` helper (clsx + tailwind-merge) | +| `src/lib/types.ts` | Shared client-safe domain types | +| `src/routes/+layout.svelte` | Root layout: sticky nav, persistent `