Compare commits
77 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2deb306419 | ||
|
|
fd283bf6c6 | ||
|
|
3154a22500 | ||
|
|
61e0d98057 | ||
|
|
601c26d436 | ||
|
|
4a267d8fd8 | ||
|
|
c9478a67fb | ||
|
|
1b4835daeb | ||
|
|
c9c12fc4a8 | ||
|
|
dd35024d02 | ||
|
|
4b8104f087 | ||
|
|
5da880d189 | ||
|
|
98631df47a | ||
|
|
83b3dccc41 | ||
|
|
588e455aae | ||
|
|
28ac8d8826 | ||
|
|
0a3a61a3ef | ||
|
|
7a2a4fc755 | ||
|
|
801928aadf | ||
|
|
040072c3f5 | ||
|
|
6a76e97a67 | ||
|
|
71f79c8e02 | ||
|
|
5ee4a06654 | ||
|
|
63b286d0a4 | ||
|
|
d3f06c5c40 | ||
|
|
e71ddc2f8b | ||
|
|
b783dae5f4 | ||
|
|
dcf40197d4 | ||
|
|
9dae5e7cc0 | ||
|
|
908f5679fd | ||
|
|
f75292f531 | ||
|
|
2cf0528730 | ||
|
|
428b57732e | ||
|
|
61e77e3e28 | ||
|
|
b363c151a5 | ||
|
|
aef9e04419 | ||
|
|
58e78cd34d | ||
|
|
c5c167035d | ||
|
|
4a00d953bb | ||
|
|
fe1a933fd0 | ||
|
|
98e4a87432 | ||
|
|
9c8849c6cd | ||
|
|
b30aa23d64 | ||
|
|
fea09e3e23 | ||
|
|
4831c74acc | ||
|
|
7e5e0495cf | ||
|
|
188685e1b6 | ||
|
|
3271a5f3e6 | ||
|
|
ee3ed29316 | ||
|
|
a39f660a37 | ||
|
|
69818089a6 | ||
|
|
09062b8c82 | ||
|
|
d518710cc4 | ||
|
|
e2c15f5931 | ||
|
|
a50b968b95 | ||
|
|
023b1f7fec | ||
|
|
7e99fc6d70 | ||
|
|
12d6d30fb0 | ||
|
|
f9c14685b3 | ||
|
|
4a7009989c | ||
|
|
920ac0d41b | ||
|
|
424f2c5e16 | ||
|
|
8a0f5b6cde | ||
|
|
5fea8f67d0 | ||
|
|
6592d1662c | ||
|
|
59e8cdb19a | ||
|
|
1118392811 | ||
|
|
baa403efa2 | ||
|
|
0ed1112b20 | ||
|
|
16a12ede4d | ||
|
|
b9b69cee44 | ||
|
|
5b27d501af | ||
|
|
a85636d5db | ||
|
|
29d0eeb7e8 | ||
|
|
fabe9724c2 | ||
|
|
4c9bb4adde | ||
|
|
22b6ee824e |
101
.env.example
101
.env.example
@@ -1,101 +0,0 @@
|
||||
# libnovel scraper — environment overrides
|
||||
# Copy to .env and adjust values; do NOT commit this file with real secrets.
|
||||
|
||||
# ── Docker BuildKit ───────────────────────────────────────────────────────────
|
||||
# Required for the backend/Dockerfile cache mounts (--mount=type=cache).
|
||||
# BuildKit is the default in Docker Engine 23+, but Colima users may need this.
|
||||
#
|
||||
# If you see: "the --mount option requires BuildKit", enable it one of two ways:
|
||||
#
|
||||
# Option A — per-project (recommended, zero restart needed):
|
||||
# Uncomment the line below and copy this file to .env.
|
||||
# Docker Compose reads .env automatically, so BuildKit will be active for
|
||||
# every `docker compose build` / `docker compose up --build` in this project.
|
||||
#
|
||||
# Option B — system-wide for Colima (persists across restarts):
|
||||
# echo '{"features":{"buildkit":true}}' > ~/.colima/default/daemon.json
|
||||
# colima stop && colima start
|
||||
#
|
||||
# DOCKER_BUILDKIT=1
|
||||
|
||||
# ── Service ports (host-side) ─────────────────────────────────────────────────
|
||||
# Port the scraper HTTP API listens on (default 8080)
|
||||
SCRAPER_PORT=8080
|
||||
|
||||
# Port PocketBase listens on (default 8090)
|
||||
POCKETBASE_PORT=8090
|
||||
|
||||
# Port MinIO S3 API listens on (default 9000)
|
||||
MINIO_PORT=9000
|
||||
|
||||
# Port MinIO web console listens on (default 9001)
|
||||
MINIO_CONSOLE_PORT=9001
|
||||
|
||||
# Port Browserless Chrome listens on (default 3030)
|
||||
BROWSERLESS_PORT=3030
|
||||
|
||||
# Port the SvelteKit UI listens on (default 3000)
|
||||
UI_PORT=3000
|
||||
|
||||
# ── Browserless ───────────────────────────────────────────────────────────────
|
||||
# Browserless API token (leave empty to disable auth)
|
||||
BROWSERLESS_TOKEN=
|
||||
|
||||
# Number of concurrent browser sessions in Browserless
|
||||
BROWSERLESS_CONCURRENT=10
|
||||
|
||||
# Queue depth before Browserless returns 429
|
||||
BROWSERLESS_QUEUED=100
|
||||
|
||||
# Per-session timeout in ms
|
||||
BROWSERLESS_TIMEOUT=60000
|
||||
|
||||
# Optional webhook URL for Browserless error alerts (leave empty to disable)
|
||||
ERROR_ALERT_URL=
|
||||
|
||||
# Which Browserless strategy the scraper uses: content | scrape | cdp | direct
|
||||
BROWSERLESS_STRATEGY=direct
|
||||
|
||||
# ── Scraper ───────────────────────────────────────────────────────────────────
|
||||
# Chapter worker goroutines (0 = NumCPU inside the container)
|
||||
SCRAPER_WORKERS=0
|
||||
|
||||
# Host path to mount as the static output directory
|
||||
STATIC_ROOT=./static/books
|
||||
|
||||
# ── Kokoro-FastAPI TTS ────────────────────────────────────────────────────────
|
||||
# Base URL for the Kokoro-FastAPI service. When running via docker-compose the
|
||||
# default (http://kokoro:8880) is wired in automatically; override here only if
|
||||
# you are pointing at an external or GPU instance.
|
||||
KOKORO_URL=http://kokoro:8880
|
||||
|
||||
# Default voice used for chapter narration.
|
||||
# Single voices: af_bella, af_sky, af_heart, am_adam, …
|
||||
# Mixed voices: af_bella+af_sky or af_bella(2)+af_sky(1) (weighted blend)
|
||||
KOKORO_VOICE=af_bella
|
||||
|
||||
# ── MinIO / S3 object storage ─────────────────────────────────────────────────
|
||||
MINIO_ROOT_USER=admin
|
||||
MINIO_ROOT_PASSWORD=changeme123
|
||||
MINIO_BUCKET_CHAPTERS=libnovel-chapters
|
||||
MINIO_BUCKET_AUDIO=libnovel-audio
|
||||
MINIO_BUCKET_BROWSE=libnovel-browse
|
||||
|
||||
# ── PocketBase ────────────────────────────────────────────────────────────────
|
||||
# Admin credentials (used by scraper + UI server-side)
|
||||
POCKETBASE_ADMIN_EMAIL=admin@libnovel.local
|
||||
POCKETBASE_ADMIN_PASSWORD=changeme123
|
||||
|
||||
# ── SvelteKit UI ─────────────────────────────────────────────────────────────
|
||||
# Internal URL the SvelteKit server uses to reach the scraper API.
|
||||
# In docker-compose this is http://scraper:8080 (wired automatically).
|
||||
# Override here only if running the UI outside of docker-compose.
|
||||
SCRAPER_API_URL=http://localhost:8080
|
||||
|
||||
# Internal URL the SvelteKit server uses to reach PocketBase.
|
||||
# In docker-compose this is http://pocketbase:8090 (wired automatically).
|
||||
POCKETBASE_URL=http://localhost:8090
|
||||
|
||||
# Public MinIO URL reachable from the browser (for audio/presigned URLs).
|
||||
# In production, point this at your MinIO reverse-proxy or CDN domain.
|
||||
PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000
|
||||
@@ -1,79 +0,0 @@
|
||||
name: CI / Scraper
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main", "master", "v2"]
|
||||
paths:
|
||||
- "scraper/**"
|
||||
- ".gitea/workflows/ci-scraper.yaml"
|
||||
pull_request:
|
||||
branches: ["main", "master", "v2"]
|
||||
paths:
|
||||
- "scraper/**"
|
||||
- ".gitea/workflows/ci-scraper.yaml"
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── lint & vet ───────────────────────────────────────────────────────────────
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: scraper/go.mod
|
||||
cache-dependency-path: scraper/go.sum
|
||||
|
||||
- name: go vet
|
||||
working-directory: scraper
|
||||
run: |
|
||||
go vet ./...
|
||||
go vet -tags integration ./...
|
||||
|
||||
# ── tests ────────────────────────────────────────────────────────────────────
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: scraper/go.mod
|
||||
cache-dependency-path: scraper/go.sum
|
||||
|
||||
- name: Run tests
|
||||
working-directory: scraper
|
||||
run: go test -short -race -count=1 -timeout=60s ./...
|
||||
|
||||
# ── push to Docker Hub ───────────────────────────────────────────────────────
|
||||
docker:
|
||||
name: Docker Push
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint, test]
|
||||
if: gitea.event_name == 'push'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: scraper
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.DOCKER_USER }}/libnovel-scraper:latest
|
||||
${{ secrets.DOCKER_USER }}/libnovel-scraper:${{ gitea.sha }}
|
||||
build-args: |
|
||||
VERSION=${{ gitea.sha }}
|
||||
COMMIT=${{ gitea.sha }}
|
||||
@@ -1,70 +0,0 @@
|
||||
name: CI / UI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main", "master", "v2"]
|
||||
paths:
|
||||
- "ui/**"
|
||||
- ".gitea/workflows/ci-ui.yaml"
|
||||
pull_request:
|
||||
branches: ["main", "master", "v2"]
|
||||
paths:
|
||||
- "ui/**"
|
||||
- ".gitea/workflows/ci-ui.yaml"
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── type-check & build ───────────────────────────────────────────────────────
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
cache-dependency-path: ui/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Type check
|
||||
run: npm run check
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
# ── push to Docker Hub ───────────────────────────────────────────────────────
|
||||
docker:
|
||||
name: Docker Push
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
if: gitea.event_name == 'push'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ui
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.DOCKER_USER }}/libnovel-ui:latest
|
||||
${{ secrets.DOCKER_USER }}/libnovel-ui:${{ gitea.sha }}
|
||||
build-args: |
|
||||
BUILD_VERSION=${{ gitea.sha }}
|
||||
BUILD_COMMIT=${{ gitea.sha }}
|
||||
73
.gitea/workflows/ci.yaml
Normal file
73
.gitea/workflows/ci.yaml
Normal file
@@ -0,0 +1,73 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "backend/**"
|
||||
- "ui/**"
|
||||
- ".gitea/workflows/ci.yaml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "backend/**"
|
||||
- "ui/**"
|
||||
- ".gitea/workflows/ci.yaml"
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── Go: vet + build + test ────────────────────────────────────────────────
|
||||
backend:
|
||||
name: Backend
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: backend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: backend/go.mod
|
||||
cache-dependency-path: backend/go.sum
|
||||
|
||||
- name: go vet
|
||||
run: go vet ./...
|
||||
|
||||
- name: Build backend
|
||||
run: go build -o /dev/null ./cmd/backend
|
||||
|
||||
- name: Build runner
|
||||
run: go build -o /dev/null ./cmd/runner
|
||||
|
||||
- name: Build healthcheck
|
||||
run: go build -o /dev/null ./cmd/healthcheck
|
||||
|
||||
- name: Run tests
|
||||
run: go test -short -race -count=1 -timeout=60s ./...
|
||||
|
||||
# ── UI: type-check + build ────────────────────────────────────────────────
|
||||
ui:
|
||||
name: UI
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
cache-dependency-path: ui/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Type check
|
||||
run: npm run check
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
@@ -1,63 +0,0 @@
|
||||
name: iOS CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["v2", "main"]
|
||||
paths:
|
||||
- "ios/**"
|
||||
- "justfile"
|
||||
- ".gitea/workflows/ios.yaml"
|
||||
- ".gitea/workflows/ios-release.yaml"
|
||||
pull_request:
|
||||
branches: ["v2", "main"]
|
||||
paths:
|
||||
- "ios/**"
|
||||
- "justfile"
|
||||
- ".gitea/workflows/ios.yaml"
|
||||
- ".gitea/workflows/ios-release.yaml"
|
||||
|
||||
concurrency:
|
||||
group: ios-macos-runner
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── build (simulator) ─────────────────────────────────────────────────────
|
||||
build:
|
||||
name: Build
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install just
|
||||
run: command -v just || brew install just
|
||||
|
||||
- name: Build (simulator)
|
||||
env:
|
||||
USER: runner
|
||||
run: just ios-build
|
||||
|
||||
# ── unit tests ────────────────────────────────────────────────────────────
|
||||
test:
|
||||
name: Test
|
||||
runs-on: macos-latest
|
||||
needs: build
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install just
|
||||
run: command -v just || brew install just
|
||||
|
||||
- name: Run unit tests
|
||||
env:
|
||||
USER: runner
|
||||
run: just ios-test
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-results
|
||||
path: ios/LibNovel/test-results.xml
|
||||
retention-days: 7
|
||||
@@ -1,68 +0,0 @@
|
||||
name: Release / Scraper
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── lint & test ──────────────────────────────────────────────────────────────
|
||||
test:
|
||||
name: Test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: scraper/go.mod
|
||||
cache-dependency-path: scraper/go.sum
|
||||
|
||||
- name: go vet
|
||||
working-directory: scraper
|
||||
run: |
|
||||
go vet ./...
|
||||
go vet -tags integration ./...
|
||||
|
||||
- name: Run tests
|
||||
working-directory: scraper
|
||||
run: go test -short -race -count=1 -timeout=60s ./...
|
||||
|
||||
# ── docker build & push ──────────────────────────────────────────────────────
|
||||
docker:
|
||||
name: Docker
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-scraper
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: scraper
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.meta.outputs.version }}
|
||||
COMMIT=${{ gitea.sha }}
|
||||
@@ -1,71 +0,0 @@
|
||||
name: Release / UI
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── type-check & build ───────────────────────────────────────────────────────
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
cache-dependency-path: ui/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Type check
|
||||
run: npm run check
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
# ── docker build & push ──────────────────────────────────────────────────────
|
||||
docker:
|
||||
name: Docker
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-ui
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ui
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
BUILD_VERSION=${{ steps.meta.outputs.version }}
|
||||
BUILD_COMMIT=${{ gitea.sha }}
|
||||
@@ -1,163 +0,0 @@
|
||||
name: Release / v2
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── backend: lint & test ─────────────────────────────────────────────────────
|
||||
test-backend:
|
||||
name: Test backend
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: backend/go.mod
|
||||
cache-dependency-path: backend/go.sum
|
||||
|
||||
- name: go vet
|
||||
working-directory: backend
|
||||
run: go vet ./...
|
||||
|
||||
- name: Run tests
|
||||
working-directory: backend
|
||||
run: go test -short -race -count=1 -timeout=60s ./...
|
||||
|
||||
# ── ui-v2: type-check & build ────────────────────────────────────────────────
|
||||
build-ui:
|
||||
name: Build ui-v2
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui-v2
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
cache-dependency-path: ui-v2/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Type check
|
||||
run: npm run check
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
# ── docker: backend ──────────────────────────────────────────────────────────
|
||||
docker-backend:
|
||||
name: Docker / backend
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test-backend]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-backend
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: backend
|
||||
target: backend
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.meta.outputs.version }}
|
||||
COMMIT=${{ gitea.sha }}
|
||||
|
||||
# ── docker: runner ───────────────────────────────────────────────────────────
|
||||
docker-runner:
|
||||
name: Docker / runner
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test-backend]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-runner
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: backend
|
||||
target: runner
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.meta.outputs.version }}
|
||||
COMMIT=${{ gitea.sha }}
|
||||
|
||||
# ── docker: ui-v2 ────────────────────────────────────────────────────────────
|
||||
docker-ui:
|
||||
name: Docker / ui-v2
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build-ui]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-ui-v2
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ui-v2
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
BUILD_VERSION=${{ steps.meta.outputs.version }}
|
||||
BUILD_COMMIT=${{ gitea.sha }}
|
||||
285
.gitea/workflows/release.yaml
Normal file
285
.gitea/workflows/release.yaml
Normal file
@@ -0,0 +1,285 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*" # e.g. v1.0.0, v1.2.3
|
||||
|
||||
concurrency:
|
||||
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ── backend: vet & test ───────────────────────────────────────────────────────
|
||||
test-backend:
|
||||
name: Test backend
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: backend/go.mod
|
||||
cache-dependency-path: backend/go.sum
|
||||
|
||||
- name: go vet
|
||||
working-directory: backend
|
||||
run: go vet ./...
|
||||
|
||||
- name: Run tests
|
||||
working-directory: backend
|
||||
run: go test -short -race -count=1 -timeout=60s ./...
|
||||
|
||||
# ── ui: type-check & build ────────────────────────────────────────────────────
|
||||
check-ui:
|
||||
name: Check ui
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ui
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
cache-dependency-path: ui/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Type check
|
||||
run: npm run check
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
# ── docker: backend ───────────────────────────────────────────────────────────
|
||||
docker-backend:
|
||||
name: Docker / backend
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test-backend]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-backend
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: backend
|
||||
target: backend
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.meta.outputs.version }}
|
||||
COMMIT=${{ gitea.sha }}
|
||||
cache-from: type=registry,ref=${{ secrets.DOCKER_USER }}/libnovel-backend:latest
|
||||
cache-to: type=inline
|
||||
|
||||
# ── docker: runner ────────────────────────────────────────────────────────────
|
||||
docker-runner:
|
||||
name: Docker / runner
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test-backend]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-runner
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: backend
|
||||
target: runner
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.meta.outputs.version }}
|
||||
COMMIT=${{ gitea.sha }}
|
||||
cache-from: type=registry,ref=${{ secrets.DOCKER_USER }}/libnovel-runner:latest
|
||||
cache-to: type=inline
|
||||
|
||||
# ── ui: source map upload ─────────────────────────────────────────────────────
|
||||
# Commented out: GlitchTip project/auth token needs to be recreated after
|
||||
# the GlitchTip DB wipe. Re-enable once GLITCHTIP_AUTH_TOKEN is updated.
|
||||
# upload-sourcemaps:
|
||||
# name: Upload source maps
|
||||
# runs-on: ubuntu-latest
|
||||
# needs: [check-ui]
|
||||
# defaults:
|
||||
# run:
|
||||
# working-directory: ui
|
||||
# steps:
|
||||
# - uses: actions/checkout@v4
|
||||
#
|
||||
# - uses: actions/setup-node@v4
|
||||
# with:
|
||||
# node-version: "22"
|
||||
# cache: npm
|
||||
# cache-dependency-path: ui/package-lock.json
|
||||
#
|
||||
# - name: Install dependencies
|
||||
# run: npm ci
|
||||
#
|
||||
# - name: Build with source maps
|
||||
# run: npm run build
|
||||
#
|
||||
# - name: Download glitchtip-cli
|
||||
# run: |
|
||||
# curl -L "https://gitlab.com/glitchtip/glitchtip-cli/-/jobs/artifacts/v0.1.0/raw/artifacts/glitchtip-cli-linux-x86_64?job=build-linux-x86_64" \
|
||||
# -o /usr/local/bin/glitchtip-cli
|
||||
# chmod +x /usr/local/bin/glitchtip-cli
|
||||
#
|
||||
# - name: Inject debug IDs into build artifacts
|
||||
# run: glitchtip-cli sourcemaps inject ./build
|
||||
# env:
|
||||
# SENTRY_URL: https://errors.libnovel.cc/
|
||||
# SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }}
|
||||
# SENTRY_ORG: libnovel
|
||||
# SENTRY_PROJECT: libnovel-ui
|
||||
#
|
||||
# - name: Upload source maps to GlitchTip
|
||||
# run: glitchtip-cli sourcemaps upload ./build --release ${{ gitea.ref_name }}
|
||||
# env:
|
||||
# SENTRY_URL: https://errors.libnovel.cc/
|
||||
# SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }}
|
||||
# SENTRY_ORG: libnovel
|
||||
# SENTRY_PROJECT: libnovel-ui
|
||||
|
||||
# ── docker: ui ────────────────────────────────────────────────────────────────
|
||||
docker-ui:
|
||||
name: Docker / ui
|
||||
runs-on: ubuntu-latest
|
||||
needs: [check-ui]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Fetch releases from Gitea API
|
||||
run: |
|
||||
set -euo pipefail
|
||||
RESPONSE=$(curl -sfL \
|
||||
-H "Accept: application/json" \
|
||||
"http://gitea.kalekber.cc/api/v1/repos/kamil/libnovel/releases?limit=50&page=1")
|
||||
# Validate JSON before writing — fails hard if response is not a JSON array
|
||||
COUNT=$(echo "$RESPONSE" | jq 'if type == "array" then length else error("expected array, got \(type)") end')
|
||||
echo "$RESPONSE" > ui/static/releases.json
|
||||
echo "Fetched $COUNT releases"
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-ui
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ui
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
BUILD_VERSION=${{ steps.meta.outputs.version }}
|
||||
BUILD_COMMIT=${{ gitea.sha }}
|
||||
BUILD_TIME=${{ gitea.event.head_commit.timestamp }}
|
||||
cache-from: type=registry,ref=${{ secrets.DOCKER_USER }}/libnovel-ui:latest
|
||||
cache-to: type=inline
|
||||
|
||||
# ── docker: caddy ─────────────────────────────────────────────────────────────
|
||||
docker-caddy:
|
||||
name: Docker / caddy
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USER }}
|
||||
password: ${{ secrets.DOCKER_TOKEN }}
|
||||
|
||||
- name: Docker meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USER }}/libnovel-caddy
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: caddy
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=registry,ref=${{ secrets.DOCKER_USER }}/libnovel-caddy:latest
|
||||
cache-to: type=inline
|
||||
|
||||
# ── Gitea release ─────────────────────────────────────────────────────────────
|
||||
release:
|
||||
name: Gitea Release
|
||||
runs-on: ubuntu-latest
|
||||
needs: [docker-backend, docker-runner, docker-ui, docker-caddy]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Create release
|
||||
uses: https://gitea.com/actions/gitea-release-action@v1
|
||||
with:
|
||||
token: ${{ secrets.GITEA_TOKEN }}
|
||||
generate_release_notes: true
|
||||
12
.gitignore
vendored
12
.gitignore
vendored
@@ -5,16 +5,16 @@
|
||||
/dist/
|
||||
|
||||
# ── Compiled binaries ──────────────────────────────────────────────────────────
|
||||
scraper/bin/
|
||||
scraper/scraper
|
||||
backend/bin/
|
||||
|
||||
# ── Scraped output (large, machine-generated) ──────────────────────────────────
|
||||
|
||||
/static/books
|
||||
# ── Environment & secrets ──────────────────────────────────────────────────────
|
||||
# Secrets are managed by Doppler — never commit .env files.
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
.env.local
|
||||
|
||||
# ── CrowdSec — generated bouncer API key ──────────────────────────────────────
|
||||
crowdsec/.crowdsec.env
|
||||
|
||||
# ── OS artefacts ───────────────────────────────────────────────────────────────
|
||||
.DS_Store
|
||||
|
||||
182
AGENTS.md
182
AGENTS.md
@@ -1,182 +0,0 @@
|
||||
# libnovel Project
|
||||
|
||||
Go web scraper for novelfire.net with TTS support via Kokoro-FastAPI. Structured data in PocketBase, binary blobs (chapters, audio, browse snapshots) in MinIO. SvelteKit frontend.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
scraper/
|
||||
├── cmd/scraper/main.go # Entry point: run | refresh | serve | save-browse
|
||||
├── internal/
|
||||
│ ├── orchestrator/orchestrator.go # Catalogue walk → per-book metadata goroutines → chapter worker pool
|
||||
│ ├── browser/ # BrowserClient interface + direct HTTP (production) + Browserless variants
|
||||
│ ├── novelfire/scraper.go # novelfire.net scraping (catalogue, metadata, chapters, ranking)
|
||||
│ ├── server/ # HTTP API server (server.go + 6 handler files)
|
||||
│ │ ├── server.go # Server struct, route registration, ListenAndServe
|
||||
│ │ ├── handlers_scrape.go # POST /scrape, /scrape/book, /scrape/book/range; job status/tasks
|
||||
│ │ ├── handlers_browse.go # GET /api/browse, /api/search, /api/cover — MinIO-cached browse pages
|
||||
│ │ ├── handlers_preview.go # GET /api/book-preview, /api/chapter-text-preview — live scrape, no store writes
|
||||
│ │ ├── handlers_audio.go # POST /api/audio, GET /api/audio-proxy, voice samples, presign
|
||||
│ │ ├── handlers_progress.go # GET/POST/DELETE /api/progress
|
||||
│ │ ├── handlers_ranking.go # GET /api/ranking, /api/cover
|
||||
│ │ └── helpers.go # stripMarkdown, hardcoded voice list fallback
|
||||
│ ├── storage/ # Persistence layer (PocketBase + MinIO)
|
||||
│ │ ├── store.go # Store interface — single abstraction for server + orchestrator
|
||||
│ │ ├── hybrid.go # HybridStore: routes structured data → PocketBase, blobs → MinIO
|
||||
│ │ ├── pocketbase.go # PocketBase REST admin client (7 collections, auth, schema bootstrap)
|
||||
│ │ ├── minio.go # MinIO client (3 buckets: chapters, audio, browse)
|
||||
│ │ └── coverutil.go # Best-effort cover image downloader → browse bucket
|
||||
│ └── scraper/
|
||||
│ ├── interfaces.go # NovelScraper interface + domain types (BookMeta, ChapterRef, etc.)
|
||||
│ └── htmlutil/htmlutil.go # HTML parsing helpers (NodeToMarkdown, ResolveURL, etc.)
|
||||
```
|
||||
|
||||
## Key Concepts
|
||||
|
||||
- **Orchestrator**: Catalogue stream → per-book goroutines (metadata + chapter list) → shared chapter work channel → N worker goroutines (chapter text). Scrape jobs tracked in PocketBase `scraping_tasks`.
|
||||
- **Storage**: `HybridStore` implements the `Store` interface. PocketBase holds structured records (`books`, `chapters_idx`, `ranking`, `progress`, `audio_cache`, `app_users`, `scraping_tasks`). MinIO holds blobs (chapter markdown, audio MP3s, browse HTML snapshots, cover images).
|
||||
- **Browser Client**: Production uses `NewDirectHTTPClient` (plain HTTP, no Browserless). Browserless variants (content/scrape/cdp) exist in `browser/` but are only wired for the `save-browse` subcommand.
|
||||
- **Preview**: `GET /api/book-preview/{slug}` scrapes metadata + chapter list live without persisting anything — used when a book is not yet in the library. On first visit, metadata and chapter index are auto-saved to PocketBase in the background.
|
||||
- **Server**: 24 HTTP endpoints. Async scrape jobs (mutex, 409 on concurrent), in-flight dedup for audio generation, MinIO-backed browse page cache with mem-cache fallback.
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Build
|
||||
cd scraper && go build -o bin/scraper ./cmd/scraper
|
||||
|
||||
# Full catalogue scrape (one-shot)
|
||||
./bin/scraper run
|
||||
|
||||
# Single book
|
||||
./bin/scraper run --url https://novelfire.net/book/xxx
|
||||
|
||||
# Re-scrape a book already in the DB (uses stored source_url)
|
||||
./bin/scraper refresh <slug>
|
||||
|
||||
# HTTP server
|
||||
./bin/scraper serve
|
||||
|
||||
# Capture browse pages to MinIO via SingleFile CLI (requires SINGLEFILE_PATH + BROWSERLESS_URL)
|
||||
./bin/scraper save-browse
|
||||
|
||||
# Tests (unit only — integration tests require live services)
|
||||
cd scraper && go test ./... -short
|
||||
|
||||
# All tests (requires MinIO + PocketBase + Browserless)
|
||||
cd scraper && go test ./...
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
### Scraper (Go)
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `LOG_LEVEL` | `debug\|info\|warn\|error` | `info` |
|
||||
| `SCRAPER_HTTP_ADDR` | HTTP listen address | `:8080` |
|
||||
| `SCRAPER_WORKERS` | Chapter goroutines | `NumCPU` |
|
||||
| `SCRAPER_TIMEOUT` | Per-request HTTP timeout (seconds) | `90` |
|
||||
| `KOKORO_URL` | Kokoro-FastAPI TTS base URL | `https://kokoro.kalekber.cc` |
|
||||
| `KOKORO_VOICE` | Default TTS voice | `af_bella` |
|
||||
| `MINIO_ENDPOINT` | MinIO S3 API host:port | `localhost:9000` |
|
||||
| `MINIO_PUBLIC_ENDPOINT` | Public MinIO endpoint for presigned URLs | `""` |
|
||||
| `MINIO_ACCESS_KEY` | MinIO access key | `admin` |
|
||||
| `MINIO_SECRET_KEY` | MinIO secret key | `changeme123` |
|
||||
| `MINIO_USE_SSL` | TLS for internal MinIO connection | `false` |
|
||||
| `MINIO_PUBLIC_USE_SSL` | TLS for public presigned URL endpoint | `true` |
|
||||
| `MINIO_BUCKET_CHAPTERS` | Chapter markdown bucket | `libnovel-chapters` |
|
||||
| `MINIO_BUCKET_AUDIO` | Audio MP3 bucket | `libnovel-audio` |
|
||||
| `MINIO_BUCKET_BROWSE` | Browse HTML + cover image bucket | `libnovel-browse` |
|
||||
| `POCKETBASE_URL` | PocketBase base URL | `http://localhost:8090` |
|
||||
| `POCKETBASE_ADMIN_EMAIL` | PocketBase admin email | `admin@libnovel.local` |
|
||||
| `POCKETBASE_ADMIN_PASSWORD` | PocketBase admin password | `changeme123` |
|
||||
| `BROWSERLESS_URL` | Browserless WS endpoint (save-browse only) | `http://localhost:3030` |
|
||||
| `SINGLEFILE_PATH` | SingleFile CLI binary path (save-browse only) | `single-file` |
|
||||
|
||||
### UI (SvelteKit)
|
||||
|
||||
| Variable | Description | Default |
|
||||
|----------|-------------|---------|
|
||||
| `AUTH_SECRET` | HMAC signing secret for auth tokens | `dev_secret_change_in_production` |
|
||||
| `SCRAPER_API_URL` | Internal URL of the Go scraper | `http://localhost:8080` |
|
||||
| `POCKETBASE_URL` | PocketBase base URL | `http://localhost:8090` |
|
||||
| `POCKETBASE_ADMIN_EMAIL` | PocketBase admin email | `admin@libnovel.local` |
|
||||
| `POCKETBASE_ADMIN_PASSWORD` | PocketBase admin password | `changeme123` |
|
||||
| `PUBLIC_MINIO_PUBLIC_URL` | Browser-visible MinIO URL (presigned links) | `http://localhost:9000` |
|
||||
|
||||
## Docker
|
||||
|
||||
```bash
|
||||
docker-compose up -d # Starts: minio, minio-init, pocketbase, pb-init, scraper, ui
|
||||
```
|
||||
|
||||
Services:
|
||||
|
||||
| Service | Port(s) | Role |
|
||||
|---------|---------|------|
|
||||
| `minio` | `9000` (S3 API), `9001` (console) | Object storage |
|
||||
| `minio-init` | — | One-shot bucket creation then exits |
|
||||
| `pocketbase` | `8090` | Structured data store |
|
||||
| `pb-init` | — | One-shot PocketBase collection bootstrap then exits |
|
||||
| `scraper` | `8080` | Go scraper HTTP API |
|
||||
| `ui` | `5252` → internal `3000` | SvelteKit frontend |
|
||||
|
||||
Kokoro and Browserless are **external services** — not in docker-compose.
|
||||
|
||||
## HTTP API Endpoints (Go scraper)
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| `GET` | `/health` | Liveness probe |
|
||||
| `POST` | `/scrape` | Enqueue full catalogue scrape |
|
||||
| `POST` | `/scrape/book` | Enqueue single-book scrape `{url}` |
|
||||
| `POST` | `/scrape/book/range` | Enqueue range scrape `{url, from, to?}` |
|
||||
| `GET` | `/api/scrape/status` | Current scrape job status |
|
||||
| `GET` | `/api/scrape/tasks` | All scrape task records |
|
||||
| `GET` | `/api/browse` | Browse novelfire catalogue (MinIO-cached) |
|
||||
| `GET` | `/api/search` | Search local + remote `?q=` |
|
||||
| `GET` | `/api/ranking` | Ranking list |
|
||||
| `GET` | `/api/cover/{domain}/{slug}` | Proxy cover image from MinIO |
|
||||
| `GET` | `/api/book-preview/{slug}` | Live metadata + chapter list (no store write) |
|
||||
| `GET` | `/api/chapter-text-preview/{slug}/{n}` | Live chapter text (no store write) |
|
||||
| `POST` | `/api/reindex/{slug}` | Rebuild chapters_idx from MinIO |
|
||||
| `GET` | `/api/chapter-text/{slug}/{n}` | Chapter text (markdown stripped) |
|
||||
| `POST` | `/api/audio/{slug}/{n}` | Trigger Kokoro TTS generation |
|
||||
| `GET` | `/api/audio-proxy/{slug}/{n}` | Proxy generated audio |
|
||||
| `POST` | `/api/audio/voice-samples` | Pre-generate voice samples |
|
||||
| `GET` | `/api/voices` | List available Kokoro voices |
|
||||
| `GET` | `/api/presign/chapter/{slug}/{n}` | Presigned MinIO URL for chapter |
|
||||
| `GET` | `/api/presign/audio/{slug}/{n}` | Presigned MinIO URL for audio |
|
||||
| `GET` | `/api/presign/voice-sample/{voice}` | Presigned MinIO URL for voice sample |
|
||||
| `GET` | `/api/progress` | Get reading progress (session-scoped) |
|
||||
| `POST` | `/api/progress/{slug}` | Set reading progress |
|
||||
| `DELETE` | `/api/progress/{slug}` | Delete reading progress |
|
||||
|
||||
## Code Patterns
|
||||
|
||||
- `log/slog` for structured logging throughout
|
||||
- Context-based cancellation on all network calls and goroutines
|
||||
- Worker pool pattern in orchestrator (buffered channel + WaitGroup)
|
||||
- Single async scrape job enforced by mutex; 409 on concurrent requests; job state persisted to `scraping_tasks` in PocketBase
|
||||
- `Store` interface decouples all persistence — pass it around, never touch MinIO/PocketBase clients directly outside `storage/`
|
||||
- Auth: custom HMAC-signed token (`userId:username:role.<sig>`) in `libnovel_auth` cookie; signed with `AUTH_SECRET`
|
||||
|
||||
## AI Context Tips
|
||||
|
||||
- **Primary files to modify**: `orchestrator.go`, `server/handlers_*.go`, `novelfire/scraper.go`, `storage/hybrid.go`, `storage/pocketbase.go`
|
||||
- **To add a new scrape source**: implement `NovelScraper` from `internal/scraper/interfaces.go`
|
||||
- **To add a new API endpoint**: add handler in the appropriate `handlers_*.go` file, register in `server.go` `ListenAndServe()`
|
||||
- **Storage changes**: update `Store` interface in `store.go`, implement on `HybridStore` (hybrid.go) and `PocketBaseStore`/`MinioClient` as needed; update mock in `orchestrator_test.go`
|
||||
- **Skip**: `scraper/bin/` (compiled binary), MinIO/PocketBase data volumes
|
||||
|
||||
## iOS App
|
||||
|
||||
See `ios/AGENTS.md` for full iOS/SwiftUI conventions.
|
||||
|
||||
## Documentation Tools
|
||||
|
||||
This project has two MCP-backed documentation tools available. Use them proactively:
|
||||
|
||||
- **`context7`** — Live Apple SwiftUI/Swift docs, Go stdlib, SvelteKit, and any other library docs. Use before implementing anything non-trivial in Swift/SwiftUI. Example: `use context7 to look up NavigationStack`.
|
||||
- **`gh_grep`** — Search real-world code on GitHub for implementation patterns. Example: `use gh_grep to find examples of background URLSession in Swift`.
|
||||
271
Caddyfile
Normal file
271
Caddyfile
Normal file
@@ -0,0 +1,271 @@
|
||||
# v3/Caddyfile
|
||||
#
|
||||
# Caddy reverse proxy for LibNovel v3.
|
||||
# Custom build includes github.com/mholt/caddy-ratelimit.
|
||||
#
|
||||
# Environment variables consumed (set in docker-compose.yml):
|
||||
# DOMAIN — public hostname, e.g. libnovel.example.com
|
||||
# Use "localhost" for local dev (no TLS cert attempted).
|
||||
# CADDY_ACME_EMAIL — Let's Encrypt notification email (empty = no email)
|
||||
#
|
||||
# Routing rules (main domain):
|
||||
# /health → backend:8080 (liveness probe)
|
||||
# /scrape* → backend:8080 (Go admin scrape endpoints)
|
||||
# /api/book-preview/* → backend:8080 (live scrape, no store write)
|
||||
# /api/chapter-text/* → backend:8080 (chapter markdown from MinIO)
|
||||
# /api/chapter-markdown/* → backend:8080 (chapter markdown from MinIO)
|
||||
# /api/reindex/* → backend:8080 (rebuild chapter index)
|
||||
# /api/cover/* → backend:8080 (proxy cover image)
|
||||
# /api/audio-proxy/* → backend:8080 (proxy generated audio)
|
||||
# /avatars/* → minio:9000 (presigned avatar GETs)
|
||||
# /audio/* → minio:9000 (presigned audio GETs)
|
||||
# /chapters/* → minio:9000 (presigned chapter GETs)
|
||||
# /* (everything else) → ui:3000 (SvelteKit — handles all
|
||||
# remaining /api/* routes)
|
||||
#
|
||||
# Subdomain routing:
|
||||
# feedback.libnovel.cc → fider:3000 (user feedback / feature requests)
|
||||
# errors.libnovel.cc → glitchtip-web:8000 (error tracking)
|
||||
# analytics.libnovel.cc → umami:3000 (page analytics)
|
||||
# logs.libnovel.cc → dozzle:8080 (Docker log viewer)
|
||||
# uptime.libnovel.cc → uptime-kuma:3001 (uptime monitoring)
|
||||
# push.libnovel.cc → gotify:80 (push notifications)
|
||||
# search.libnovel.cc → meilisearch:7700 (search index — homelab runner)
|
||||
#
|
||||
# Routes intentionally removed from direct-to-backend:
|
||||
# /api/scrape/* — SvelteKit has /api/scrape/ counterparts
|
||||
# that enforce auth; routing directly would
|
||||
# bypass SK middleware.
|
||||
# /api/chapter-text-preview/* — Same: SvelteKit owns
|
||||
# /api/chapter-text-preview/[slug]/[n].
|
||||
# /api/browse — Endpoint removed; browse snapshot system
|
||||
# was deleted.
|
||||
{
|
||||
# Email for Let's Encrypt ACME account registration.
|
||||
# When CADDY_ACME_EMAIL is set this expands to e.g. "email you@example.com".
|
||||
# When unset the variable expands to an empty string and Caddy ignores it.
|
||||
email {$CADDY_ACME_EMAIL:}
|
||||
|
||||
# CrowdSec bouncer — streams decisions from the CrowdSec LAPI every 15s.
|
||||
# CROWDSEC_API_KEY is injected at runtime via crowdsec/.crowdsec.env.
|
||||
# The default "disabled" placeholder makes the bouncer fail-open (warn,
|
||||
# pass traffic) when no key is configured — Caddy still starts cleanly.
|
||||
crowdsec {
|
||||
api_url http://crowdsec:8080
|
||||
api_key {$CROWDSEC_API_KEY:disabled}
|
||||
ticker_interval 15s
|
||||
}
|
||||
|
||||
# ── Redis TCP proxy via layer4 ────────────────────────────────────────────
|
||||
# Exposes homelab Redis over TLS for Asynq job enqueueing from the backend.
|
||||
# Listens on :6380 (all interfaces). TLS is terminated here using the cert
|
||||
# for redis.libnovel.cc; traffic is proxied to the homelab Redis instance.
|
||||
# Requires the caddy-l4 module in the custom Caddy build.
|
||||
layer4 {
|
||||
:6380 {
|
||||
route {
|
||||
tls
|
||||
proxy {
|
||||
upstream {$HOMELAB_REDIS_ADDR:192.168.0.109:6379}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(security_headers) {
|
||||
header {
|
||||
# Prevent clickjacking
|
||||
X-Frame-Options "SAMEORIGIN"
|
||||
# Prevent MIME-type sniffing
|
||||
X-Content-Type-Options "nosniff"
|
||||
# Minimal referrer info for cross-origin requests
|
||||
Referrer-Policy "strict-origin-when-cross-origin"
|
||||
# Restrict powerful browser features
|
||||
Permissions-Policy "camera=(), microphone=(), geolocation=(), payment=()"
|
||||
# Enforce HTTPS for 1 year (includeSubDomains)
|
||||
Strict-Transport-Security "max-age=31536000; includeSubDomains"
|
||||
# Enable XSS filter in older browsers
|
||||
X-XSS-Protection "1; mode=block"
|
||||
# Remove server identity header
|
||||
-Server
|
||||
}
|
||||
}
|
||||
|
||||
{$DOMAIN:localhost} {
|
||||
import security_headers
|
||||
|
||||
# ── CrowdSec bouncer ──────────────────────────────────────────────────────
|
||||
# Checks every incoming request against CrowdSec decisions.
|
||||
# Banned IPs receive a 403; all others pass through unchanged.
|
||||
route {
|
||||
crowdsec
|
||||
}
|
||||
|
||||
# ── Rate limiting ─────────────────────────────────────────────────────────
|
||||
# Auth endpoints: strict — 10 req/min per IP
|
||||
rate_limit {
|
||||
zone auth_zone {
|
||||
match {
|
||||
path /api/auth/login /api/auth/register /api/auth/change-password
|
||||
}
|
||||
key {remote_host}
|
||||
window 1m
|
||||
events 10
|
||||
}
|
||||
}
|
||||
|
||||
# Admin scrape endpoints: moderate — 20 req/min per IP
|
||||
rate_limit {
|
||||
zone scrape_zone {
|
||||
match {
|
||||
path /scrape*
|
||||
}
|
||||
key {remote_host}
|
||||
window 1m
|
||||
events 20
|
||||
}
|
||||
}
|
||||
|
||||
# Global: 300 req/min per IP (covers everything)
|
||||
rate_limit {
|
||||
zone global_zone {
|
||||
key {remote_host}
|
||||
window 1m
|
||||
events 300
|
||||
}
|
||||
}
|
||||
|
||||
# ── Liveness probe ────────────────────────────────────────────────────────
|
||||
handle /health {
|
||||
reverse_proxy backend:8080
|
||||
}
|
||||
|
||||
# ── Scrape task creation (Go backend only) ────────────────────────────────
|
||||
handle /scrape* {
|
||||
reverse_proxy backend:8080
|
||||
}
|
||||
|
||||
# ── Backend-only API paths ────────────────────────────────────────────────
|
||||
# These paths are served exclusively by the Go backend and have no
|
||||
# SvelteKit counterpart. Routing them here skips SK intentionally.
|
||||
handle /api/book-preview/* {
|
||||
reverse_proxy backend:8080
|
||||
}
|
||||
handle /api/chapter-text/* {
|
||||
reverse_proxy backend:8080
|
||||
}
|
||||
handle /api/chapter-markdown/* {
|
||||
reverse_proxy backend:8080
|
||||
}
|
||||
handle /api/reindex/* {
|
||||
reverse_proxy backend:8080
|
||||
}
|
||||
handle /api/cover/* {
|
||||
reverse_proxy backend:8080
|
||||
}
|
||||
handle /api/audio-proxy/* {
|
||||
reverse_proxy backend:8080
|
||||
}
|
||||
|
||||
# ── MinIO bucket paths (presigned URLs) ──────────────────────────────────
|
||||
# MinIO path-style presigned URLs include the bucket name as the first
|
||||
# path segment. MINIO_PUBLIC_ENDPOINT points here, so Caddy must proxy
|
||||
# these paths directly to MinIO — no auth layer needed (the presigned
|
||||
# signature itself enforces access and expiry).
|
||||
handle /avatars/* {
|
||||
reverse_proxy minio:9000
|
||||
}
|
||||
handle /audio/* {
|
||||
reverse_proxy minio:9000
|
||||
}
|
||||
handle /chapters/* {
|
||||
reverse_proxy minio:9000
|
||||
}
|
||||
|
||||
# ── SvelteKit UI (catch-all — includes all remaining /api/* routes) ───────
|
||||
handle {
|
||||
reverse_proxy ui:3000 {
|
||||
# Active health check: Caddy polls /health every 5 s and marks the
|
||||
# upstream down immediately when it fails. Combined with
|
||||
# lb_try_duration this means Watchtower container replacements
|
||||
# show the maintenance page within a few seconds instead of
|
||||
# hanging or returning a raw connection error to the browser.
|
||||
health_uri /health
|
||||
health_interval 5s
|
||||
health_timeout 2s
|
||||
health_status 200
|
||||
|
||||
# If the upstream is down, fail fast (don't retry for longer than
|
||||
# 3 s) and let Caddy's handle_errors 502/503 take over.
|
||||
lb_try_duration 3s
|
||||
}
|
||||
}
|
||||
|
||||
# ── Caddy-level error pages ───────────────────────────────────────────────
|
||||
# These fire when the upstream (backend or ui) is completely unreachable.
|
||||
# SvelteKit's own +error.svelte handles application-level errors (404, 500).
|
||||
handle_errors 404 {
|
||||
root * /srv/errors
|
||||
rewrite * /404.html
|
||||
file_server
|
||||
}
|
||||
handle_errors 502 {
|
||||
root * /srv/errors
|
||||
rewrite * /502.html
|
||||
file_server
|
||||
}
|
||||
handle_errors 503 {
|
||||
root * /srv/errors
|
||||
rewrite * /503.html
|
||||
file_server
|
||||
}
|
||||
handle_errors 504 {
|
||||
root * /srv/errors
|
||||
rewrite * /504.html
|
||||
file_server
|
||||
}
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────────────────
|
||||
# JSON log file read by CrowdSec for threat detection.
|
||||
log {
|
||||
output file /var/log/caddy/access.log {
|
||||
roll_size 100MiB
|
||||
roll_keep 5
|
||||
roll_keep_for 720h
|
||||
}
|
||||
format json
|
||||
}
|
||||
}
|
||||
|
||||
# ── Tooling subdomains ────────────────────────────────────────────────────────
|
||||
# feedback.libnovel.cc, errors.libnovel.cc, analytics.libnovel.cc,
|
||||
# logs.libnovel.cc, uptime.libnovel.cc, push.libnovel.cc, grafana.libnovel.cc
|
||||
# are now routed via Cloudflare Tunnel directly to the homelab (192.168.0.109).
|
||||
# No Caddy rules needed here — Cloudflare handles TLS termination and routing.
|
||||
|
||||
# ── PocketBase: exposed for homelab runner task polling ───────────────────────
|
||||
# Allows the homelab runner to claim tasks and write results via the PB API.
|
||||
# Admin UI is also accessible here for convenience.
|
||||
pb.libnovel.cc {
|
||||
import security_headers
|
||||
reverse_proxy pocketbase:8090
|
||||
}
|
||||
|
||||
# ── MinIO S3 API: exposed for homelab runner object writes ────────────────────
|
||||
# The homelab runner connects here as MINIO_ENDPOINT to PutObject audio/chapters.
|
||||
# Also used as MINIO_PUBLIC_ENDPOINT for presigned URL generation.
|
||||
storage.libnovel.cc {
|
||||
import security_headers
|
||||
reverse_proxy minio:9000
|
||||
}
|
||||
|
||||
# ── Meilisearch: exposed for homelab runner search indexing ──────────────────
|
||||
# The homelab runner connects here as MEILI_URL to index books after scraping.
|
||||
# Protected by MEILI_MASTER_KEY bearer token — Meilisearch enforces auth on
|
||||
# every request; Caddy just terminates TLS.
|
||||
search.libnovel.cc {
|
||||
import security_headers
|
||||
reverse_proxy meilisearch:7700
|
||||
}
|
||||
}
|
||||
38
README.md
Normal file
38
README.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# LibNovel
|
||||
|
||||
Self-hosted audiobook platform. Go backend + SvelteKit UI + MinIO/PocketBase/Meilisearch.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Docker + Docker Compose
|
||||
- [just](https://github.com/casey/just)
|
||||
- [Doppler CLI](https://docs.doppler.com/docs/install-cli)
|
||||
|
||||
## Setup
|
||||
|
||||
```sh
|
||||
doppler login
|
||||
doppler setup # project=libnovel, config=prd
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```sh
|
||||
just up # start everything
|
||||
just down # stop
|
||||
just logs # tail all logs
|
||||
just log backend # tail one service
|
||||
just build # rebuild images
|
||||
just restart # down + up
|
||||
just secrets # view/edit secrets
|
||||
```
|
||||
|
||||
## Secrets
|
||||
|
||||
Managed via Doppler (`project=libnovel`, `config=prd`). No `.env` files.
|
||||
|
||||
To add or update a secret:
|
||||
|
||||
```sh
|
||||
doppler secrets set MY_SECRET=value
|
||||
```
|
||||
@@ -30,13 +30,23 @@ RUN --mount=type=cache,target=/root/go/pkg/mod \
|
||||
-o /out/healthcheck ./cmd/healthcheck
|
||||
|
||||
# ── backend service ──────────────────────────────────────────────────────────
|
||||
FROM gcr.io/distroless/static:nonroot AS backend
|
||||
# Uses Alpine (not distroless) so ffmpeg is available for on-demand voice
|
||||
# sample generation via pocket-tts (WAV→MP3 transcoding).
|
||||
FROM alpine:3.21 AS backend
|
||||
RUN apk add --no-cache ffmpeg ca-certificates && \
|
||||
addgroup -S appgroup && adduser -S appuser -G appgroup
|
||||
COPY --from=builder /out/healthcheck /healthcheck
|
||||
COPY --from=builder /out/backend /backend
|
||||
USER appuser
|
||||
ENTRYPOINT ["/backend"]
|
||||
|
||||
# ── runner service ───────────────────────────────────────────────────────────
|
||||
FROM gcr.io/distroless/static:nonroot AS runner
|
||||
# Uses Alpine (not distroless) so ffmpeg is available for WAV→MP3 transcoding
|
||||
# when pocket-tts voices are used.
|
||||
FROM alpine:3.21 AS runner
|
||||
RUN apk add --no-cache ffmpeg ca-certificates && \
|
||||
addgroup -S appgroup && adduser -S appuser -G appgroup
|
||||
COPY --from=builder /out/healthcheck /healthcheck
|
||||
COPY --from=builder /out/runner /runner
|
||||
USER appuser
|
||||
ENTRYPOINT ["/runner"]
|
||||
|
||||
BIN
backend/backend
Executable file
BIN
backend/backend
Executable file
Binary file not shown.
@@ -2,7 +2,7 @@
|
||||
//
|
||||
// It exposes all endpoints consumed by the SvelteKit UI: book/chapter reads,
|
||||
// scrape-task creation, presigned MinIO URLs, audio-task creation, reading
|
||||
// progress, live novelfire.net browse/search, and Kokoro voice list.
|
||||
// progress, live novelfire.net search, and Kokoro voice list.
|
||||
//
|
||||
// All heavy lifting (scraping, TTS generation) is delegated to the runner
|
||||
// binary via PocketBase task records. The backend never scrapes directly.
|
||||
@@ -19,11 +19,19 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/libnovel/backend/internal/asynqqueue"
|
||||
"github.com/libnovel/backend/internal/backend"
|
||||
"github.com/libnovel/backend/internal/config"
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
"github.com/libnovel/backend/internal/meili"
|
||||
"github.com/libnovel/backend/internal/otelsetup"
|
||||
"github.com/libnovel/backend/internal/pockettts"
|
||||
"github.com/libnovel/backend/internal/storage"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
)
|
||||
|
||||
// version and commit are set at build time via -ldflags.
|
||||
@@ -42,6 +50,19 @@ func main() {
|
||||
func run() error {
|
||||
cfg := config.Load()
|
||||
|
||||
// ── Sentry / GlitchTip error tracking ────────────────────────────────────
|
||||
if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" {
|
||||
if err := sentry.Init(sentry.ClientOptions{
|
||||
Dsn: dsn,
|
||||
Release: version + "@" + commit,
|
||||
TracesSampleRate: 0.1,
|
||||
}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "backend: sentry init warning: %v\n", err)
|
||||
} else {
|
||||
defer sentry.Flush(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Logger ───────────────────────────────────────────────────────────────
|
||||
log := buildLogger(cfg.LogLevel)
|
||||
log.Info("backend starting",
|
||||
@@ -54,6 +75,19 @@ func run() error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
// ── OpenTelemetry tracing + logs ──────────────────────────────────────────
|
||||
otelShutdown, otelLog, err := otelsetup.Init(ctx, version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init otel: %w", err)
|
||||
}
|
||||
if otelShutdown != nil {
|
||||
defer otelShutdown()
|
||||
// Replace the plain slog logger with the OTel-bridged one so all
|
||||
// structured log lines are forwarded to Loki with trace IDs attached.
|
||||
log = otelLog
|
||||
log.Info("otel tracing + logs enabled", "endpoint", os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"))
|
||||
}
|
||||
|
||||
// ── Storage ──────────────────────────────────────────────────────────────
|
||||
store, err := storage.NewStore(ctx, cfg, log)
|
||||
if err != nil {
|
||||
@@ -70,6 +104,43 @@ func run() error {
|
||||
kokoroClient = &noopKokoro{}
|
||||
}
|
||||
|
||||
// ── Pocket-TTS (voice list + sample generation; audio generation is the runner's job) ──
|
||||
var pocketTTSClient pockettts.Client
|
||||
if cfg.PocketTTS.URL != "" {
|
||||
pocketTTSClient = pockettts.New(cfg.PocketTTS.URL)
|
||||
log.Info("pocket-tts voices enabled", "url", cfg.PocketTTS.URL)
|
||||
} else {
|
||||
log.Info("POCKET_TTS_URL not set — pocket-tts voices unavailable in backend")
|
||||
}
|
||||
|
||||
// ── Meilisearch (search reads only; indexing is the runner's job) ────────
|
||||
var searchIndex meili.Client
|
||||
if cfg.Meilisearch.URL != "" {
|
||||
searchIndex = meili.New(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey)
|
||||
log.Info("meilisearch search enabled", "url", cfg.Meilisearch.URL)
|
||||
} else {
|
||||
log.Info("MEILI_URL not set — search will use PocketBase substring fallback")
|
||||
searchIndex = meili.NoopClient{}
|
||||
}
|
||||
|
||||
// ── Task Producer ────────────────────────────────────────────────────────
|
||||
// When REDIS_ADDR is set the backend dual-writes: PocketBase record (audit)
|
||||
// + Asynq job (immediate delivery). Otherwise it writes to PocketBase only
|
||||
// and the runner picks up on the next poll tick.
|
||||
var producer taskqueue.Producer = store
|
||||
if cfg.Redis.Addr != "" {
|
||||
redisOpt, parseErr := parseRedisOpt(cfg.Redis)
|
||||
if parseErr != nil {
|
||||
return fmt.Errorf("parse REDIS_ADDR: %w", parseErr)
|
||||
}
|
||||
asynqProducer := asynqqueue.NewProducer(store, redisOpt)
|
||||
defer asynqProducer.Close() //nolint:errcheck
|
||||
producer = asynqProducer
|
||||
log.Info("backend: asynq task dispatch enabled", "addr", cfg.Redis.Addr)
|
||||
} else {
|
||||
log.Info("backend: poll-mode task dispatch (REDIS_ADDR not set)")
|
||||
}
|
||||
|
||||
// ── Backend server ───────────────────────────────────────────────────────
|
||||
srv := backend.New(
|
||||
backend.Config{
|
||||
@@ -79,16 +150,19 @@ func run() error {
|
||||
Commit: commit,
|
||||
},
|
||||
backend.Dependencies{
|
||||
BookReader: store,
|
||||
RankingStore: store,
|
||||
AudioStore: store,
|
||||
PresignStore: store,
|
||||
ProgressStore: store,
|
||||
BrowseStore: store,
|
||||
Producer: store,
|
||||
TaskReader: store,
|
||||
Kokoro: kokoroClient,
|
||||
Log: log,
|
||||
BookReader: store,
|
||||
RankingStore: store,
|
||||
AudioStore: store,
|
||||
TranslationStore: store,
|
||||
PresignStore: store,
|
||||
ProgressStore: store,
|
||||
CoverStore: store,
|
||||
Producer: producer,
|
||||
TaskReader: store,
|
||||
SearchIndex: searchIndex,
|
||||
Kokoro: kokoroClient,
|
||||
PocketTTS: pocketTTSClient,
|
||||
Log: log,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -124,3 +198,16 @@ func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, erro
|
||||
func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// parseRedisOpt converts a config.Redis into an asynq.RedisConnOpt.
|
||||
// Handles full "redis://" / "rediss://" URLs and plain "host:port".
|
||||
func parseRedisOpt(cfg config.Redis) (asynq.RedisConnOpt, error) {
|
||||
addr := cfg.Addr
|
||||
if len(addr) > 7 && (addr[:8] == "redis://" || (len(addr) > 8 && addr[:9] == "rediss://")) {
|
||||
return asynq.ParseRedisURI(addr)
|
||||
}
|
||||
return asynq.RedisClientOpt{
|
||||
Addr: addr,
|
||||
Password: cfg.Password,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -19,12 +19,19 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/libnovel/backend/internal/asynqqueue"
|
||||
"github.com/libnovel/backend/internal/browser"
|
||||
"github.com/libnovel/backend/internal/config"
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
"github.com/libnovel/backend/internal/libretranslate"
|
||||
"github.com/libnovel/backend/internal/meili"
|
||||
"github.com/libnovel/backend/internal/novelfire"
|
||||
"github.com/libnovel/backend/internal/otelsetup"
|
||||
"github.com/libnovel/backend/internal/pockettts"
|
||||
"github.com/libnovel/backend/internal/runner"
|
||||
"github.com/libnovel/backend/internal/storage"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
)
|
||||
|
||||
// version and commit are set at build time via -ldflags.
|
||||
@@ -43,6 +50,19 @@ func main() {
|
||||
func run() error {
|
||||
cfg := config.Load()
|
||||
|
||||
// ── Sentry / GlitchTip error tracking ────────────────────────────────────
|
||||
if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" {
|
||||
if err := sentry.Init(sentry.ClientOptions{
|
||||
Dsn: dsn,
|
||||
Release: version + "@" + commit,
|
||||
TracesSampleRate: 0.1,
|
||||
}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "runner: sentry init warning: %v\n", err)
|
||||
} else {
|
||||
defer sentry.Flush(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Logger ──────────────────────────────────────────────────────────────
|
||||
log := buildLogger(cfg.LogLevel)
|
||||
log.Info("runner starting",
|
||||
@@ -55,6 +75,19 @@ func run() error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
// ── OpenTelemetry tracing + logs ─────────────────────────────────────────
|
||||
otelShutdown, otelLog, err := otelsetup.Init(ctx, version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init otel: %w", err)
|
||||
}
|
||||
if otelShutdown != nil {
|
||||
defer otelShutdown()
|
||||
// Switch to the OTel-bridged logger so all structured log lines are
|
||||
// forwarded to Loki with trace IDs attached.
|
||||
log = otelLog
|
||||
log.Info("otel tracing + logs enabled", "endpoint", os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"))
|
||||
}
|
||||
|
||||
// ── Storage ─────────────────────────────────────────────────────────────
|
||||
store, err := storage.NewStore(ctx, cfg, log)
|
||||
if err != nil {
|
||||
@@ -74,7 +107,6 @@ func run() error {
|
||||
browserClient := browser.NewDirectClient(browser.Config{
|
||||
MaxConcurrent: workers,
|
||||
Timeout: timeout,
|
||||
ProxyURL: cfg.Runner.ProxyURL,
|
||||
})
|
||||
novel := novelfire.New(browserClient, log)
|
||||
|
||||
@@ -84,27 +116,82 @@ func run() error {
|
||||
kokoroClient = kokoro.New(cfg.Kokoro.URL)
|
||||
log.Info("kokoro TTS enabled", "url", cfg.Kokoro.URL)
|
||||
} else {
|
||||
log.Warn("KOKORO_URL not set — audio tasks will fail")
|
||||
log.Warn("KOKORO_URL not set — kokoro voice tasks will fail")
|
||||
kokoroClient = &noopKokoro{}
|
||||
}
|
||||
|
||||
// ── pocket-tts ──────────────────────────────────────────────────────────
|
||||
var pocketTTSClient pockettts.Client
|
||||
if cfg.PocketTTS.URL != "" {
|
||||
pocketTTSClient = pockettts.New(cfg.PocketTTS.URL)
|
||||
log.Info("pocket-tts enabled", "url", cfg.PocketTTS.URL)
|
||||
} else {
|
||||
log.Warn("POCKET_TTS_URL not set — pocket-tts voice tasks will fail")
|
||||
}
|
||||
|
||||
// ── LibreTranslate ──────────────────────────────────────────────────────
|
||||
ltClient := libretranslate.New(cfg.LibreTranslate.URL, cfg.LibreTranslate.APIKey)
|
||||
if ltClient != nil {
|
||||
log.Info("libretranslate enabled", "url", cfg.LibreTranslate.URL)
|
||||
} else {
|
||||
log.Info("LIBRETRANSLATE_URL not set — machine translation disabled")
|
||||
}
|
||||
|
||||
// ── Meilisearch ─────────────────────────────────────────────────────────
|
||||
var searchIndex meili.Client
|
||||
if cfg.Meilisearch.URL != "" {
|
||||
if err := meili.Configure(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey); err != nil {
|
||||
log.Warn("meilisearch configure failed — search indexing disabled", "err", err)
|
||||
searchIndex = meili.NoopClient{}
|
||||
} else {
|
||||
searchIndex = meili.New(cfg.Meilisearch.URL, cfg.Meilisearch.APIKey)
|
||||
log.Info("meilisearch enabled", "url", cfg.Meilisearch.URL)
|
||||
}
|
||||
} else {
|
||||
log.Info("MEILI_URL not set — search indexing disabled")
|
||||
searchIndex = meili.NoopClient{}
|
||||
}
|
||||
|
||||
// ── Runner ──────────────────────────────────────────────────────────────
|
||||
rCfg := runner.Config{
|
||||
WorkerID: cfg.Runner.WorkerID,
|
||||
PollInterval: cfg.Runner.PollInterval,
|
||||
MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape,
|
||||
MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio,
|
||||
OrchestratorWorkers: workers,
|
||||
WorkerID: cfg.Runner.WorkerID,
|
||||
PollInterval: cfg.Runner.PollInterval,
|
||||
MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape,
|
||||
MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio,
|
||||
MaxConcurrentTranslation: cfg.Runner.MaxConcurrentTranslation,
|
||||
OrchestratorWorkers: workers,
|
||||
MetricsAddr: cfg.Runner.MetricsAddr,
|
||||
CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval,
|
||||
CatalogueRequestDelay: cfg.Runner.CatalogueRequestDelay,
|
||||
SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh,
|
||||
RedisAddr: cfg.Redis.Addr,
|
||||
RedisPassword: cfg.Redis.Password,
|
||||
}
|
||||
|
||||
// In Asynq mode the Consumer is a thin wrapper: claim/heartbeat/reap are
|
||||
// no-ops, but FinishAudioTask / FinishScrapeTask / FailTask write back to
|
||||
// PocketBase as before.
|
||||
var consumer taskqueue.Consumer = store
|
||||
if cfg.Redis.Addr != "" {
|
||||
log.Info("runner: asynq mode — using Redis for task dispatch", "addr", cfg.Redis.Addr)
|
||||
consumer = asynqqueue.NewConsumer(store)
|
||||
} else {
|
||||
log.Info("runner: poll mode — using PocketBase for task dispatch")
|
||||
}
|
||||
|
||||
deps := runner.Dependencies{
|
||||
Consumer: store,
|
||||
BookWriter: store,
|
||||
BookReader: store,
|
||||
AudioStore: store,
|
||||
BrowseStore: store,
|
||||
Novel: novel,
|
||||
Kokoro: kokoroClient,
|
||||
Log: log,
|
||||
Consumer: consumer,
|
||||
BookWriter: store,
|
||||
BookReader: store,
|
||||
AudioStore: store,
|
||||
CoverStore: store,
|
||||
TranslationStore: store,
|
||||
SearchIndex: searchIndex,
|
||||
Novel: novel,
|
||||
Kokoro: kokoroClient,
|
||||
PocketTTS: pocketTTSClient,
|
||||
LibreTranslate: ltClient,
|
||||
Log: log,
|
||||
}
|
||||
r := runner.New(rCfg, deps)
|
||||
|
||||
|
||||
@@ -8,22 +8,65 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.1.1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/getsentry/sentry-go v0.43.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
|
||||
github.com/hibiken/asynq v0.26.0 // indirect
|
||||
github.com/hibiken/asynq/x v0.0.0-20260203063626-d704b68a426d // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/meilisearch/meilisearch-go v0.36.1 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/redis/go-redis/v9 v9.18.0 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
github.com/yuin/goldmark v1.8.2 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.17.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.18.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/log v0.18.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/log v0.18.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.42.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect
|
||||
google.golang.org/grpc v1.79.2 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
@@ -1,11 +1,38 @@
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
|
||||
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
|
||||
github.com/hibiken/asynq v0.26.0 h1:1Zxr92MlDnb1Zt/QR5g2vSCqUS03i95lUfqx5X7/wrw=
|
||||
github.com/hibiken/asynq v0.26.0/go.mod h1:Qk4e57bTnWDoyJ67VkchuV6VzSM9IQW2nPvAGuDyw58=
|
||||
github.com/hibiken/asynq/x v0.0.0-20260203063626-d704b68a426d h1:Ld5m8EIK5QVOq/owOexKIbETij3skACg4eU1pArHsrw=
|
||||
github.com/hibiken/asynq/x v0.0.0-20260203063626-d704b68a426d/go.mod h1:hhpStehaxSGg3ib9wJXzw5AXY1YS6lQ9BNavAgPbIhE=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
@@ -13,22 +40,82 @@ github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4O
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/meilisearch/meilisearch-go v0.36.1 h1:mJTCJE5g7tRvaqKco6DfqOuJEjX+rRltDEnkEC02Y0M=
|
||||
github.com/meilisearch/meilisearch-go v0.36.1/go.mod h1:hWcR0MuWLSzHfbz9GGzIr3s9rnXLm1jqkmHkJPbUSvM=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
|
||||
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
|
||||
github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
|
||||
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
|
||||
github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
|
||||
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
|
||||
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.17.0 h1:NFIS6x7wyObQ7cR84x7bt1sr8nYBx89s3x3GwRjw40k=
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.17.0/go.mod h1:39SaByOyDMRMe872AE7uelMuQZidIw7LLFAnQi0FWTE=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
|
||||
go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho=
|
||||
go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.18.0 h1:icqq3Z34UrEFk2u+HMhTtRsvo7Ues+eiJVjaJt62njs=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.18.0/go.mod h1:W2m8P+d5Wn5kipj4/xmbt9uMqezEKfBjzVJadfABSBE=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0 h1:uLXP+3mghfMf7XmV4PkGfFhFKuNWoCvvx5wP/wOXo0o=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0/go.mod h1:v0Tj04armyT59mnURNUJf7RCKcKzq+lgJs6QSjHjaTc=
|
||||
go.opentelemetry.io/otel/log v0.18.0 h1:XgeQIIBjZZrliksMEbcwMZefoOSMI1hdjiLEiiB0bAg=
|
||||
go.opentelemetry.io/otel/log v0.18.0/go.mod h1:KEV1kad0NofR3ycsiDH4Yjcoj0+8206I6Ox2QYFSNgI=
|
||||
go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4=
|
||||
go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo=
|
||||
go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts=
|
||||
go.opentelemetry.io/otel/sdk/log v0.18.0 h1:n8OyZr7t7otkeTnPTbDNom6rW16TBYGtvyy2Gk6buQw=
|
||||
go.opentelemetry.io/otel/sdk/log v0.18.0/go.mod h1:C0+wxkTwKpOCZLrlJ3pewPiiQwpzycPI/u6W0Z9fuYk=
|
||||
go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY=
|
||||
go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
@@ -39,7 +126,18 @@ golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
|
||||
google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU=
|
||||
google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
Binary file not shown.
64
backend/internal/asynqqueue/consumer.go
Normal file
64
backend/internal/asynqqueue/consumer.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package asynqqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
)
|
||||
|
||||
// Consumer wraps the PocketBase-backed Consumer for result write-back only.
|
||||
//
|
||||
// When using Asynq, the runner no longer polls for work — Asynq delivers
|
||||
// tasks via the ServeMux handlers. The only Consumer operations the handlers
|
||||
// need are:
|
||||
// - FinishAudioTask / FinishScrapeTask — write result back to PocketBase
|
||||
// - FailTask — mark PocketBase record as failed
|
||||
//
|
||||
// ClaimNextAudioTask, ClaimNextScrapeTask, HeartbeatTask, and ReapStaleTasks
|
||||
// are all no-ops here because Asynq owns those responsibilities.
|
||||
type Consumer struct {
|
||||
pb taskqueue.Consumer // underlying PocketBase consumer (for write-back)
|
||||
}
|
||||
|
||||
// NewConsumer wraps an existing PocketBase Consumer.
|
||||
func NewConsumer(pb taskqueue.Consumer) *Consumer {
|
||||
return &Consumer{pb: pb}
|
||||
}
|
||||
|
||||
// ── Write-back (delegated to PocketBase) ──────────────────────────────────────
|
||||
|
||||
func (c *Consumer) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
|
||||
return c.pb.FinishScrapeTask(ctx, id, result)
|
||||
}
|
||||
|
||||
func (c *Consumer) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error {
|
||||
return c.pb.FinishAudioTask(ctx, id, result)
|
||||
}
|
||||
|
||||
func (c *Consumer) FinishTranslationTask(ctx context.Context, id string, result domain.TranslationResult) error {
|
||||
return c.pb.FinishTranslationTask(ctx, id, result)
|
||||
}
|
||||
|
||||
func (c *Consumer) FailTask(ctx context.Context, id, errMsg string) error {
|
||||
return c.pb.FailTask(ctx, id, errMsg)
|
||||
}
|
||||
|
||||
// ── No-ops (Asynq owns claiming / heartbeating / reaping) ───────────────────
|
||||
|
||||
func (c *Consumer) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
|
||||
return domain.ScrapeTask{}, false, nil
|
||||
}
|
||||
|
||||
func (c *Consumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
|
||||
return domain.AudioTask{}, false, nil
|
||||
}
|
||||
|
||||
func (c *Consumer) ClaimNextTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) {
|
||||
return domain.TranslationTask{}, false, nil
|
||||
}
|
||||
|
||||
func (c *Consumer) HeartbeatTask(_ context.Context, _ string) error { return nil }
|
||||
|
||||
func (c *Consumer) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) { return 0, nil }
|
||||
96
backend/internal/asynqqueue/producer.go
Normal file
96
backend/internal/asynqqueue/producer.go
Normal file
@@ -0,0 +1,96 @@
|
||||
package asynqqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
)
|
||||
|
||||
// Producer dual-writes every task: first to PocketBase (via pb, for audit /
|
||||
// UI status), then to Redis via Asynq so the runner picks it up immediately.
|
||||
type Producer struct {
|
||||
pb taskqueue.Producer // underlying PocketBase producer
|
||||
client *asynq.Client
|
||||
}
|
||||
|
||||
// NewProducer wraps an existing PocketBase Producer with Asynq dispatch.
|
||||
func NewProducer(pb taskqueue.Producer, redisOpt asynq.RedisConnOpt) *Producer {
|
||||
return &Producer{
|
||||
pb: pb,
|
||||
client: asynq.NewClient(redisOpt),
|
||||
}
|
||||
}
|
||||
|
||||
// Close shuts down the underlying Asynq client connection.
|
||||
func (p *Producer) Close() error {
|
||||
return p.client.Close()
|
||||
}
|
||||
|
||||
// CreateScrapeTask creates a PocketBase record then enqueues an Asynq job.
|
||||
func (p *Producer) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) {
|
||||
id, err := p.pb.CreateScrapeTask(ctx, kind, targetURL, fromChapter, toChapter)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
payload := ScrapePayload{
|
||||
PBTaskID: id,
|
||||
Kind: kind,
|
||||
TargetURL: targetURL,
|
||||
FromChapter: fromChapter,
|
||||
ToChapter: toChapter,
|
||||
}
|
||||
taskType := TypeScrapeBook
|
||||
if kind == "catalogue" {
|
||||
taskType = TypeScrapeCatalogue
|
||||
}
|
||||
if err := p.enqueue(ctx, taskType, payload); err != nil {
|
||||
// Non-fatal: PB record exists; runner will pick it up on next poll.
|
||||
return id, fmt.Errorf("asynq enqueue scrape (task still in PB): %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// CreateAudioTask creates a PocketBase record then enqueues an Asynq job.
|
||||
func (p *Producer) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) {
|
||||
id, err := p.pb.CreateAudioTask(ctx, slug, chapter, voice)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
payload := AudioPayload{
|
||||
PBTaskID: id,
|
||||
Slug: slug,
|
||||
Chapter: chapter,
|
||||
Voice: voice,
|
||||
}
|
||||
if err := p.enqueue(ctx, TypeAudioGenerate, payload); err != nil {
|
||||
return id, fmt.Errorf("asynq enqueue audio (task still in PB): %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// CreateTranslationTask creates a PocketBase record. Translation tasks are
|
||||
// not currently dispatched via Asynq — the runner picks them up via polling.
|
||||
func (p *Producer) CreateTranslationTask(ctx context.Context, slug string, chapter int, lang string) (string, error) {
|
||||
return p.pb.CreateTranslationTask(ctx, slug, chapter, lang)
|
||||
}
|
||||
|
||||
// CancelTask delegates to PocketBase; Asynq jobs may already be running and
|
||||
// cannot be reliably cancelled, so we only update the audit record.
|
||||
func (p *Producer) CancelTask(ctx context.Context, id string) error {
|
||||
return p.pb.CancelTask(ctx, id)
|
||||
}
|
||||
|
||||
// enqueue serialises payload and dispatches it to Asynq.
|
||||
func (p *Producer) enqueue(_ context.Context, taskType string, payload any) error {
|
||||
b, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal payload: %w", err)
|
||||
}
|
||||
_, err = p.client.Enqueue(asynq.NewTask(taskType, b))
|
||||
return err
|
||||
}
|
||||
46
backend/internal/asynqqueue/tasks.go
Normal file
46
backend/internal/asynqqueue/tasks.go
Normal file
@@ -0,0 +1,46 @@
|
||||
// Package asynqqueue provides Asynq-backed implementations of the
|
||||
// taskqueue.Producer and taskqueue.Consumer interfaces.
|
||||
//
|
||||
// Architecture:
|
||||
// - Producer: dual-writes — creates a PocketBase record for audit/UI, then
|
||||
// enqueues an Asynq job so the runner picks it up immediately (sub-ms).
|
||||
// - Consumer: thin wrapper used only for result write-back (FinishAudioTask,
|
||||
// FinishScrapeTask, FailTask). ClaimNext*/Heartbeat/Reap are no-ops because
|
||||
// Asynq owns those responsibilities.
|
||||
// - Handlers: asynq.HandlerFunc wrappers that decode job payloads and invoke
|
||||
// the existing runner logic (runScrapeTask / runAudioTask).
|
||||
//
|
||||
// Fallback: when REDIS_ADDR is empty the caller should use the plain
|
||||
// storage.Store (PocketBase-polling) implementation unchanged.
|
||||
package asynqqueue
|
||||
|
||||
// Queue names — keep all jobs on the default queue for now.
|
||||
// Add separate queues (e.g. "audio", "scrape") later if you need priority.
|
||||
const QueueDefault = "default"
|
||||
|
||||
// Task type constants used for Asynq routing.
|
||||
const (
|
||||
TypeAudioGenerate = "audio:generate"
|
||||
TypeScrapeBook = "scrape:book"
|
||||
TypeScrapeCatalogue = "scrape:catalogue"
|
||||
)
|
||||
|
||||
// AudioPayload is the Asynq job payload for audio generation tasks.
|
||||
type AudioPayload struct {
|
||||
// PBTaskID is the PocketBase record ID created before enqueueing.
|
||||
// The handler uses it to write results back via Consumer.FinishAudioTask.
|
||||
PBTaskID string `json:"pb_task_id"`
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
Voice string `json:"voice"`
|
||||
}
|
||||
|
||||
// ScrapePayload is the Asynq job payload for scrape tasks.
|
||||
type ScrapePayload struct {
|
||||
// PBTaskID is the PocketBase record ID created before enqueueing.
|
||||
PBTaskID string `json:"pb_task_id"`
|
||||
Kind string `json:"kind"` // "catalogue", "book", or "book_range"
|
||||
TargetURL string `json:"target_url"` // empty for catalogue tasks
|
||||
FromChapter int `json:"from_chapter"` // 0 unless Kind=="book_range"
|
||||
ToChapter int `json:"to_chapter"` // 0 unless Kind=="book_range"
|
||||
}
|
||||
@@ -7,8 +7,7 @@ package backend
|
||||
// handleScrapeStatus, handleScrapeTasks
|
||||
// handleBrowse, handleSearch
|
||||
// handleGetRanking, handleGetCover
|
||||
// handleBookPreview, handleChapterText, handleReindex
|
||||
// handleChapterText, handleReindex
|
||||
// handleBookPreview, handleChapterText, handleChapterTextPreview, handleChapterMarkdown, handleReindex
|
||||
// handleAudioGenerate, handleAudioStatus, handleAudioProxy
|
||||
// handleVoices
|
||||
// handlePresignChapter, handlePresignAudio, handlePresignVoiceSample
|
||||
@@ -29,8 +28,11 @@ package backend
|
||||
// by the runner after each catalogue scrape).
|
||||
// - GET /api/book-preview returns stored data when in library, or enqueues a
|
||||
// scrape task and returns 202 when not. The backend never scrapes directly.
|
||||
// - GET /api/chapter-text-preview scrapes a chapter live from novelfire.net
|
||||
// directly (no runner task, no store writes). Used for unscraped books.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -44,6 +46,11 @@ import (
|
||||
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
"github.com/libnovel/backend/internal/meili"
|
||||
"github.com/libnovel/backend/internal/novelfire/htmlutil"
|
||||
"github.com/libnovel/backend/internal/pockettts"
|
||||
"github.com/libnovel/backend/internal/scraper"
|
||||
"github.com/yuin/goldmark"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -172,82 +179,11 @@ type NovelListing struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// handleBrowse handles GET /api/browse.
|
||||
// Fetches novelfire.net live (no MinIO cache in the new backend).
|
||||
// Query params: page (default 1), genre (default "all"), sort (default "popular"),
|
||||
// status (default "all"), type (default "all-novel")
|
||||
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page := q.Get("page")
|
||||
if page == "" {
|
||||
page = "1"
|
||||
}
|
||||
genre := q.Get("genre")
|
||||
if genre == "" {
|
||||
genre = "all"
|
||||
}
|
||||
sortBy := q.Get("sort")
|
||||
if sortBy == "" {
|
||||
sortBy = "popular"
|
||||
}
|
||||
status := q.Get("status")
|
||||
if status == "" {
|
||||
status = "all"
|
||||
}
|
||||
novelType := q.Get("type")
|
||||
if novelType == "" {
|
||||
novelType = "all-novel"
|
||||
}
|
||||
|
||||
pageNum, _ := strconv.Atoi(page)
|
||||
if pageNum <= 0 {
|
||||
pageNum = 1
|
||||
}
|
||||
|
||||
// ── Try MinIO cache first ─────────────────────────────────────────────
|
||||
// Only page 1 is cached; higher pages fall through to live fetch.
|
||||
if pageNum == 1 && s.deps.BrowseStore != nil {
|
||||
if data, ok, err := s.deps.BrowseStore.GetBrowsePage(r.Context(), genre, sortBy, status, novelType, 1); err == nil && ok {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
_, _ = w.Write(data)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fall back to live novelfire.net fetch ──────────────────────────────
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
|
||||
defer cancel()
|
||||
|
||||
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
|
||||
novelFireBase, genre, sortBy, status, novelType, pageNum)
|
||||
|
||||
novels, hasNext, err := s.fetchBrowsePage(ctx, targetURL)
|
||||
if err != nil {
|
||||
// Live fetch also failed — return empty list with cached=false flag so
|
||||
// the UI can show a "not ready yet" state instead of a hard error.
|
||||
s.deps.Log.Error("handleBrowse: fetch failed (no cache)", "url", targetURL, "err", err)
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, 0, map[string]any{
|
||||
"novels": []any{},
|
||||
"page": pageNum,
|
||||
"hasNext": false,
|
||||
"cached": false,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
writeJSON(w, 0, map[string]any{
|
||||
"novels": novels,
|
||||
"page": pageNum,
|
||||
"hasNext": hasNext,
|
||||
"cached": false,
|
||||
})
|
||||
}
|
||||
|
||||
// handleSearch handles GET /api/search.
|
||||
// Query params: q (min 2 chars), source ("local"|"remote"|"all", default "all")
|
||||
//
|
||||
// Local search is powered by Meilisearch when configured; falls back to a
|
||||
// substring match against PocketBase book records otherwise.
|
||||
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query().Get("q")
|
||||
if len([]rune(q)) < 2 {
|
||||
@@ -265,22 +201,35 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
var localResults, remoteResults []NovelListing
|
||||
|
||||
// Local search (PocketBase books)
|
||||
// Local search: Meilisearch → PocketBase substring fallback
|
||||
if source == "local" || source == "all" {
|
||||
books, err := s.deps.BookReader.ListBooks(ctx)
|
||||
if err != nil {
|
||||
s.deps.Log.Warn("search: ListBooks failed", "err", err)
|
||||
meiliBooks, meiliErr := s.deps.SearchIndex.Search(ctx, q, 50)
|
||||
if meiliErr == nil && len(meiliBooks) > 0 {
|
||||
for _, b := range meiliBooks {
|
||||
localResults = append(localResults, NovelListing{
|
||||
Slug: b.Slug,
|
||||
Title: b.Title,
|
||||
Cover: b.Cover,
|
||||
URL: b.SourceURL,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
qLower := strings.ToLower(q)
|
||||
for _, b := range books {
|
||||
if strings.Contains(strings.ToLower(b.Title), qLower) ||
|
||||
strings.Contains(strings.ToLower(b.Author), qLower) {
|
||||
localResults = append(localResults, NovelListing{
|
||||
Slug: b.Slug,
|
||||
Title: b.Title,
|
||||
Cover: b.Cover,
|
||||
URL: b.SourceURL,
|
||||
})
|
||||
// Fallback: substring match against PocketBase
|
||||
books, err := s.deps.BookReader.ListBooks(ctx)
|
||||
if err != nil {
|
||||
s.deps.Log.Warn("search: ListBooks failed", "err", err)
|
||||
} else {
|
||||
qLower := strings.ToLower(q)
|
||||
for _, b := range books {
|
||||
if strings.Contains(strings.ToLower(b.Title), qLower) ||
|
||||
strings.Contains(strings.ToLower(b.Author), qLower) {
|
||||
localResults = append(localResults, NovelListing{
|
||||
Slug: b.Slug,
|
||||
Title: b.Title,
|
||||
Cover: b.Cover,
|
||||
URL: b.SourceURL,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -341,18 +290,34 @@ func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// handleGetCover handles GET /api/cover/{domain}/{slug}.
|
||||
// The new backend does not cache covers in MinIO. Instead it redirects the
|
||||
// client to the novelfire.net source URL. The domain path segment is kept for
|
||||
// API compatibility with the old scraper.
|
||||
// Serves the cover image directly from MinIO when available; falls back to a
|
||||
// redirect to the novelfire CDN when the cover has not yet been downloaded.
|
||||
func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, "missing slug", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// Redirect to the standard novelfire cover CDN URL. If the caller has the
|
||||
// actual cover URL stored in metadata they should use it directly; this
|
||||
// endpoint is a best-effort fallback.
|
||||
|
||||
// Fast path: serve from MinIO if the cover has been downloaded.
|
||||
if s.deps.CoverStore != nil {
|
||||
data, ct, ok, err := s.deps.CoverStore.GetCover(r.Context(), slug)
|
||||
if err != nil {
|
||||
s.deps.Log.Warn("handleGetCover: GetCover error", "slug", slug, "err", err)
|
||||
}
|
||||
if ok && len(data) > 0 {
|
||||
if ct == "" {
|
||||
ct = "image/jpeg"
|
||||
}
|
||||
w.Header().Set("Content-Type", ct)
|
||||
w.Header().Set("Cache-Control", "public, max-age=86400")
|
||||
_, _ = w.Write(data)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: redirect to the CDN. The caller sees a working image; the
|
||||
// cover will be populated on the next catalogue refresh run.
|
||||
coverURL := fmt.Sprintf("https://cdn.novelfire.net/covers/%s.jpg", slug)
|
||||
http.Redirect(w, r, coverURL, http.StatusFound)
|
||||
}
|
||||
@@ -469,6 +434,117 @@ func (s *Server) handleChapterMarkdown(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, raw)
|
||||
}
|
||||
|
||||
// handleChapterTextPreview handles GET /api/chapter-text-preview/{slug}/{n}.
|
||||
//
|
||||
// Fetches a chapter live from novelfire.net and returns its plain text without
|
||||
// writing anything to PocketBase or MinIO. This is the preview path used when
|
||||
// a chapter has not yet been scraped into the library.
|
||||
//
|
||||
// Optional query params:
|
||||
//
|
||||
// chapter_url — the canonical chapter URL (preferred over constructing one)
|
||||
// title — hint for the chapter title (used when the page title is empty)
|
||||
//
|
||||
// Response: {"slug":string,"number":int,"title":string,"text":string,"url":string}
|
||||
func (s *Server) handleChapterTextPreview(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
jsonError(w, http.StatusBadRequest, "invalid slug or chapter number")
|
||||
return
|
||||
}
|
||||
|
||||
// Determine the chapter URL to fetch.
|
||||
chapterURL := r.URL.Query().Get("chapter_url")
|
||||
if chapterURL == "" {
|
||||
// Best-effort: novelfire chapter URLs follow /book/{slug}/chapter-{n}
|
||||
chapterURL = fmt.Sprintf("%s/book/%s/chapter-%d", novelFireBase, slug, n)
|
||||
}
|
||||
|
||||
titleHint := r.URL.Query().Get("title")
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Fetch the chapter page.
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, chapterURL, nil)
|
||||
if err != nil {
|
||||
s.deps.Log.Error("chapter-text-preview: build request failed", "url", chapterURL, "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "failed to build request")
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
s.deps.Log.Warn("chapter-text-preview: fetch failed", "url", chapterURL, "err", err)
|
||||
jsonError(w, http.StatusBadGateway, "failed to fetch chapter")
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
jsonError(w, http.StatusNotFound, "chapter not found")
|
||||
return
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
s.deps.Log.Warn("chapter-text-preview: upstream error",
|
||||
"url", chapterURL, "status", resp.StatusCode, "body_snippet", string(body))
|
||||
jsonError(w, http.StatusBadGateway, fmt.Sprintf("upstream returned %d", resp.StatusCode))
|
||||
return
|
||||
}
|
||||
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
s.deps.Log.Error("chapter-text-preview: read body failed", "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "failed to read response")
|
||||
return
|
||||
}
|
||||
|
||||
// Parse HTML and extract the #content node.
|
||||
root, err := htmlutil.ParseHTML(string(bodyBytes))
|
||||
if err != nil {
|
||||
s.deps.Log.Error("chapter-text-preview: html parse failed", "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "failed to parse chapter HTML")
|
||||
return
|
||||
}
|
||||
|
||||
container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
|
||||
if container == nil {
|
||||
s.deps.Log.Warn("chapter-text-preview: #content not found", "url", chapterURL)
|
||||
jsonError(w, http.StatusNotFound, "chapter content not found on page")
|
||||
return
|
||||
}
|
||||
|
||||
markdownText := htmlutil.NodeToMarkdown(container)
|
||||
plainText := stripMarkdown(markdownText)
|
||||
|
||||
// Extract the chapter title from the page <title> or <h1> if not hinted.
|
||||
chapterTitle := titleHint
|
||||
if chapterTitle == "" {
|
||||
// Try <h1 class="chapter-title"> first, then <h2 class="chapter-title">
|
||||
for _, tag := range []string{"h1", "h2", "h3"} {
|
||||
if node := htmlutil.FindFirst(root, scraper.Selector{Tag: tag, Class: "chapter-title"}); node != nil {
|
||||
chapterTitle = strings.TrimSpace(htmlutil.TextContent(node))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if chapterTitle == "" {
|
||||
chapterTitle = fmt.Sprintf("Chapter %d", n)
|
||||
}
|
||||
|
||||
writeJSON(w, 0, map[string]any{
|
||||
"slug": slug,
|
||||
"number": n,
|
||||
"title": chapterTitle,
|
||||
"text": plainText,
|
||||
"url": chapterURL,
|
||||
})
|
||||
}
|
||||
|
||||
// handleReindex handles POST /api/reindex/{slug}.
|
||||
// Rebuilds the chapters_idx PocketBase collection for a book from MinIO objects.
|
||||
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -627,10 +703,253 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, presignURL, http.StatusFound)
|
||||
}
|
||||
|
||||
// ── Voices ─────────────────────────────────────────────────────────────────────
|
||||
// ── Translation ────────────────────────────────────────────────────────────────
|
||||
|
||||
// handleVoices handles GET /api/voices.
|
||||
// Returns {"voices": [...]} — fetched from Kokoro with built-in fallback.
|
||||
// supportedTranslationLangs is the set of target locales the backend accepts.
|
||||
// Source is always "en".
|
||||
var supportedTranslationLangs = map[string]bool{
|
||||
"ru": true, "id": true, "pt": true, "fr": true,
|
||||
}
|
||||
|
||||
// handleTranslationGenerate handles POST /api/translation/{slug}/{n}.
|
||||
// Query params: lang (required, one of ru|id|pt|fr)
|
||||
//
|
||||
// Returns 200 immediately if translation already exists in MinIO.
|
||||
// Returns 202 with task_id if a new task was created.
|
||||
// Returns 503 if TranslationStore is nil (feature disabled).
|
||||
func (s *Server) handleTranslationGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
if s.deps.TranslationStore == nil {
|
||||
jsonError(w, http.StatusServiceUnavailable, "machine translation not configured")
|
||||
return
|
||||
}
|
||||
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
jsonError(w, http.StatusBadRequest, "invalid chapter")
|
||||
return
|
||||
}
|
||||
|
||||
lang := r.URL.Query().Get("lang")
|
||||
if !supportedTranslationLangs[lang] {
|
||||
jsonError(w, http.StatusBadRequest, "unsupported lang; use ru, id, pt, or fr")
|
||||
return
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, lang)
|
||||
|
||||
// Fast path: translation already in MinIO
|
||||
key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n)
|
||||
if s.deps.TranslationStore.TranslationExists(r.Context(), key) {
|
||||
writeJSON(w, 0, map[string]string{"status": "done", "lang": lang})
|
||||
return
|
||||
}
|
||||
|
||||
// Check if a task is already pending/running
|
||||
task, found, _ := s.deps.TaskReader.GetTranslationTask(r.Context(), cacheKey)
|
||||
if found && (task.Status == domain.TaskStatusPending || task.Status == domain.TaskStatusRunning) {
|
||||
writeJSON(w, http.StatusAccepted, map[string]string{
|
||||
"task_id": task.ID,
|
||||
"status": string(task.Status),
|
||||
"lang": lang,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Create a new translation task
|
||||
taskID, err := s.deps.Producer.CreateTranslationTask(r.Context(), slug, n, lang)
|
||||
if err != nil {
|
||||
s.deps.Log.Error("handleTranslationGenerate: CreateTranslationTask failed", "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "failed to create translation task")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusAccepted, map[string]string{
|
||||
"task_id": taskID,
|
||||
"status": "pending",
|
||||
"lang": lang,
|
||||
})
|
||||
}
|
||||
|
||||
// handleTranslationStatus handles GET /api/translation/status/{slug}/{n}.
|
||||
// Query params: lang (required)
|
||||
func (s *Server) handleTranslationStatus(w http.ResponseWriter, r *http.Request) {
|
||||
if s.deps.TranslationStore == nil {
|
||||
writeJSON(w, 0, map[string]string{"status": "unavailable"})
|
||||
return
|
||||
}
|
||||
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
jsonError(w, http.StatusBadRequest, "invalid params")
|
||||
return
|
||||
}
|
||||
|
||||
lang := r.URL.Query().Get("lang")
|
||||
if !supportedTranslationLangs[lang] {
|
||||
jsonError(w, http.StatusBadRequest, "unsupported lang")
|
||||
return
|
||||
}
|
||||
|
||||
// Fast path: translation exists in MinIO
|
||||
key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n)
|
||||
if s.deps.TranslationStore.TranslationExists(r.Context(), key) {
|
||||
writeJSON(w, 0, map[string]string{"status": "done", "lang": lang})
|
||||
return
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, lang)
|
||||
task, found, _ := s.deps.TaskReader.GetTranslationTask(r.Context(), cacheKey)
|
||||
if !found {
|
||||
writeJSON(w, 0, map[string]string{"status": "idle", "lang": lang})
|
||||
return
|
||||
}
|
||||
|
||||
resp := map[string]string{
|
||||
"status": string(task.Status),
|
||||
"task_id": task.ID,
|
||||
"lang": lang,
|
||||
}
|
||||
if task.Status == domain.TaskStatusFailed && task.ErrorMessage != "" {
|
||||
resp["error"] = task.ErrorMessage
|
||||
}
|
||||
writeJSON(w, 0, resp)
|
||||
}
|
||||
|
||||
// handleTranslationRead handles GET /api/translation/{slug}/{n}.
|
||||
// Query params: lang (required)
|
||||
//
|
||||
// Returns {"html": "<p>...</p>", "lang": "ru"} from the MinIO-cached translation.
|
||||
// Returns 404 when the translation has not been generated yet.
|
||||
func (s *Server) handleTranslationRead(w http.ResponseWriter, r *http.Request) {
|
||||
if s.deps.TranslationStore == nil {
|
||||
http.Error(w, `{"error":"machine translation not configured"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
jsonError(w, http.StatusBadRequest, "invalid params")
|
||||
return
|
||||
}
|
||||
|
||||
lang := r.URL.Query().Get("lang")
|
||||
if !supportedTranslationLangs[lang] {
|
||||
jsonError(w, http.StatusBadRequest, "unsupported lang")
|
||||
return
|
||||
}
|
||||
|
||||
key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n)
|
||||
md, err := s.deps.TranslationStore.GetTranslation(r.Context(), key)
|
||||
if err != nil {
|
||||
s.deps.Log.Warn("handleTranslationRead: translation not found", "slug", slug, "n", n, "lang", lang, "err", err)
|
||||
jsonError(w, http.StatusNotFound, "translation not available")
|
||||
return
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := goldmark.Convert([]byte(md), &buf); err != nil {
|
||||
s.deps.Log.Error("handleTranslationRead: markdown conversion failed", "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "failed to render translation")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, 0, map[string]string{"html": buf.String(), "lang": lang})
|
||||
}
|
||||
|
||||
// handleAdminTranslationJobs handles GET /api/admin/translation/jobs.
|
||||
// Returns the full list of translation jobs sorted by started descending.
|
||||
func (s *Server) handleAdminTranslationJobs(w http.ResponseWriter, r *http.Request) {
|
||||
tasks, err := s.deps.TaskReader.ListTranslationTasks(r.Context())
|
||||
if err != nil {
|
||||
s.deps.Log.Error("handleAdminTranslationJobs: ListTranslationTasks failed", "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "failed to list translation jobs")
|
||||
return
|
||||
}
|
||||
type jobRow struct {
|
||||
ID string `json:"id"`
|
||||
CacheKey string `json:"cache_key"`
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
Lang string `json:"lang"`
|
||||
Status string `json:"status"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
ErrorMessage string `json:"error_message"`
|
||||
Started string `json:"started"`
|
||||
Finished string `json:"finished"`
|
||||
}
|
||||
rows := make([]jobRow, 0, len(tasks))
|
||||
for _, t := range tasks {
|
||||
rows = append(rows, jobRow{
|
||||
ID: t.ID,
|
||||
CacheKey: t.CacheKey,
|
||||
Slug: t.Slug,
|
||||
Chapter: t.Chapter,
|
||||
Lang: t.Lang,
|
||||
Status: string(t.Status),
|
||||
WorkerID: t.WorkerID,
|
||||
ErrorMessage: t.ErrorMessage,
|
||||
Started: t.Started.Format(time.RFC3339),
|
||||
Finished: t.Finished.Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
writeJSON(w, 0, map[string]any{"jobs": rows})
|
||||
}
|
||||
|
||||
// handleAdminTranslationBulk handles POST /api/admin/translation/bulk.
|
||||
// Body: {"slug": "...", "lang": "ru", "from": 1, "to": 50}
|
||||
// Enqueues one translation task per chapter in the range [from, to] inclusive.
|
||||
func (s *Server) handleAdminTranslationBulk(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Slug string `json:"slug"`
|
||||
Lang string `json:"lang"`
|
||||
From int `json:"from"`
|
||||
To int `json:"to"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
jsonError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
if body.Slug == "" {
|
||||
jsonError(w, http.StatusBadRequest, "slug is required")
|
||||
return
|
||||
}
|
||||
if !supportedTranslationLangs[body.Lang] {
|
||||
jsonError(w, http.StatusBadRequest, "unsupported lang; use ru, id, pt, or fr")
|
||||
return
|
||||
}
|
||||
if body.From < 1 || body.To < body.From {
|
||||
jsonError(w, http.StatusBadRequest, "from must be >= 1 and to must be >= from")
|
||||
return
|
||||
}
|
||||
if body.To-body.From > 999 {
|
||||
jsonError(w, http.StatusBadRequest, "range too large; max 1000 chapters per request")
|
||||
return
|
||||
}
|
||||
|
||||
var taskIDs []string
|
||||
for n := body.From; n <= body.To; n++ {
|
||||
id, err := s.deps.Producer.CreateTranslationTask(r.Context(), body.Slug, n, body.Lang)
|
||||
if err != nil {
|
||||
s.deps.Log.Error("handleAdminTranslationBulk: CreateTranslationTask failed",
|
||||
"slug", body.Slug, "chapter", n, "lang", body.Lang, "err", err)
|
||||
jsonError(w, http.StatusInternalServerError,
|
||||
fmt.Sprintf("failed to create task for chapter %d: %s", n, err))
|
||||
return
|
||||
}
|
||||
taskIDs = append(taskIDs, id)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"enqueued": len(taskIDs),
|
||||
"task_ids": taskIDs,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Voices ─────────────────────────────────────────────────────────────────────
|
||||
// Returns {"voices": [...]} — merged list from Kokoro and pocket-tts.
|
||||
func (s *Server) handleVoices(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 0, map[string]any{"voices": s.voices(r.Context())})
|
||||
}
|
||||
@@ -685,7 +1004,13 @@ func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 0, map[string]string{"url": u})
|
||||
}
|
||||
|
||||
// voiceSampleText is the phrase synthesised for every voice sample.
|
||||
const voiceSampleText = "Hello! This is a preview of what I sound like. I hope you enjoy listening to your stories with my voice."
|
||||
|
||||
// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}.
|
||||
// If the sample has not been generated yet it synthesises it on the fly via
|
||||
// the appropriate TTS engine (Kokoro for kokoro voices, pocket-tts for
|
||||
// pocket-tts voices), stores the result in MinIO, and returns the presigned URL.
|
||||
func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) {
|
||||
voice := r.PathValue("voice")
|
||||
if voice == "" {
|
||||
@@ -694,9 +1019,34 @@ func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
|
||||
key := kokoro.VoiceSampleKey(voice)
|
||||
|
||||
// Generate sample on demand when it is not in MinIO yet.
|
||||
if !s.deps.AudioStore.AudioExists(r.Context(), key) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
s.deps.Log.Info("generating voice sample on demand", "voice", voice)
|
||||
|
||||
var (
|
||||
mp3 []byte
|
||||
err error
|
||||
)
|
||||
if pockettts.IsPocketTTSVoice(voice) {
|
||||
if s.deps.PocketTTS == nil {
|
||||
jsonError(w, http.StatusServiceUnavailable, "pocket-tts not configured")
|
||||
return
|
||||
}
|
||||
mp3, err = s.deps.PocketTTS.GenerateAudio(r.Context(), voiceSampleText, voice)
|
||||
} else {
|
||||
mp3, err = s.deps.Kokoro.GenerateAudio(r.Context(), voiceSampleText, voice)
|
||||
}
|
||||
if err != nil {
|
||||
s.deps.Log.Error("voice sample generation failed", "voice", voice, "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "voice sample generation failed")
|
||||
return
|
||||
}
|
||||
if err := s.deps.AudioStore.PutAudio(r.Context(), key, mp3); err != nil {
|
||||
s.deps.Log.Error("voice sample upload failed", "voice", voice, "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "voice sample upload failed")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour)
|
||||
@@ -708,6 +1058,59 @@ func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request
|
||||
writeJSON(w, 0, map[string]string{"url": u})
|
||||
}
|
||||
|
||||
// handleAvatarUpload handles PUT /api/avatar-upload/{userId}.
|
||||
// The request body must be the raw image bytes; Content-Type must be
|
||||
// image/jpeg, image/png, or image/webp.
|
||||
//
|
||||
// This endpoint is called by the SvelteKit server (not the browser directly),
|
||||
// so MinIO credentials and internal networking are not a concern.
|
||||
//
|
||||
// Returns: { "key": "<objectKey>" }
|
||||
func (s *Server) handleAvatarUpload(w http.ResponseWriter, r *http.Request) {
|
||||
userID := r.PathValue("userId")
|
||||
if userID == "" {
|
||||
jsonError(w, http.StatusBadRequest, "missing userId")
|
||||
return
|
||||
}
|
||||
|
||||
ct := r.Header.Get("Content-Type")
|
||||
var ext string
|
||||
switch {
|
||||
case strings.HasPrefix(ct, "image/jpeg"):
|
||||
ext = "jpg"
|
||||
case strings.HasPrefix(ct, "image/png"):
|
||||
ext = "png"
|
||||
case strings.HasPrefix(ct, "image/webp"):
|
||||
ext = "webp"
|
||||
default:
|
||||
jsonError(w, http.StatusBadRequest, "unsupported content-type; use image/jpeg, image/png, or image/webp")
|
||||
return
|
||||
}
|
||||
|
||||
const maxSize = 5 << 20 // 5 MiB
|
||||
data, err := io.ReadAll(io.LimitReader(r.Body, maxSize+1))
|
||||
if err != nil {
|
||||
jsonError(w, http.StatusBadRequest, "failed to read body")
|
||||
return
|
||||
}
|
||||
if len(data) > maxSize {
|
||||
jsonError(w, http.StatusRequestEntityTooLarge, "image too large (max 5 MiB)")
|
||||
return
|
||||
}
|
||||
if len(data) == 0 {
|
||||
jsonError(w, http.StatusBadRequest, "empty body")
|
||||
return
|
||||
}
|
||||
|
||||
key, err := s.deps.PresignStore.PutAvatar(r.Context(), userID, ext, ct, data)
|
||||
if err != nil {
|
||||
s.deps.Log.Error("avatar upload failed", "userId", userID, "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "upload failed")
|
||||
return
|
||||
}
|
||||
writeJSON(w, 0, map[string]string{"key": key})
|
||||
}
|
||||
|
||||
// handlePresignAvatarUpload handles GET /api/presign/avatar-upload/{userId}.
|
||||
// Query params: ext (jpg|png|webp, defaults to jpg)
|
||||
func (s *Server) handlePresignAvatarUpload(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -826,6 +1229,82 @@ func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 0, map[string]string{})
|
||||
}
|
||||
|
||||
// ── Catalogue (Meilisearch-backed browse + search) ────────────────────────────
|
||||
|
||||
// handleCatalogue handles GET /api/catalogue.
|
||||
//
|
||||
// Provides unified browse + search over the locally-indexed book catalogue
|
||||
// via Meilisearch. Unlike /api/browse this never fetches novelfire.net live —
|
||||
// it is entirely served from the Meilisearch index populated by the runner.
|
||||
//
|
||||
// Query params:
|
||||
//
|
||||
// q — full-text search query (optional)
|
||||
// genre — genre filter, e.g. "fantasy" or "all" (default "all")
|
||||
// status — status filter: "ongoing", "completed", or "all" (default "all")
|
||||
// sort — "popular" (default) | "new" | "top-rated" | "rank"
|
||||
// page — 1-indexed page number (default 1)
|
||||
// limit — items per page (default 20, max 100)
|
||||
func (s *Server) handleCatalogue(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
|
||||
genre := q.Get("genre")
|
||||
if genre == "" {
|
||||
genre = "all"
|
||||
}
|
||||
status := q.Get("status")
|
||||
if status == "" {
|
||||
status = "all"
|
||||
}
|
||||
sort := q.Get("sort")
|
||||
if sort == "" {
|
||||
sort = "popular"
|
||||
}
|
||||
|
||||
page, _ := strconv.Atoi(q.Get("page"))
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
limit, _ := strconv.Atoi(q.Get("limit"))
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
if limit > 100 {
|
||||
limit = 100
|
||||
}
|
||||
|
||||
cq := meili.CatalogueQuery{
|
||||
Q: q.Get("q"),
|
||||
Genre: genre,
|
||||
Status: status,
|
||||
Sort: sort,
|
||||
Page: page,
|
||||
Limit: limit,
|
||||
}
|
||||
|
||||
books, total, facets, err := s.deps.SearchIndex.Catalogue(r.Context(), cq)
|
||||
if err != nil {
|
||||
s.deps.Log.Error("handleCatalogue: Catalogue query failed", "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "search failed")
|
||||
return
|
||||
}
|
||||
|
||||
hasNext := int64(page*limit) < total
|
||||
|
||||
w.Header().Set("Cache-Control", "public, max-age=60")
|
||||
writeJSON(w, 0, map[string]any{
|
||||
"books": books,
|
||||
"page": page,
|
||||
"limit": limit,
|
||||
"total": total,
|
||||
"has_next": hasNext,
|
||||
"facets": map[string]any{
|
||||
"genres": facets.Genres,
|
||||
"statuses": facets.Statuses,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ── Browse page parsing helpers ────────────────────────────────────────────────
|
||||
|
||||
// fetchBrowsePage fetches pageURL and parses NovelListings from the HTML.
|
||||
@@ -928,9 +1407,9 @@ func stripMarkdown(src string) string {
|
||||
|
||||
// ── Hardcoded Kokoro voice fallback ───────────────────────────────────────────
|
||||
|
||||
// kokoroVoices is the built-in fallback list used when the Kokoro service is
|
||||
// unavailable. Matches the list in the old scraper helpers.go.
|
||||
var kokoroVoices = []string{
|
||||
// kokoroVoiceIDs is the built-in fallback list of Kokoro voice IDs used when
|
||||
// the Kokoro service is unavailable.
|
||||
var kokoroVoiceIDs = []string{
|
||||
// American English
|
||||
"af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia",
|
||||
"af_jessica", "af_kore", "af_nicole", "af_nova", "af_river",
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
// picks up and executes those tasks asynchronously
|
||||
// - Presigned MinIO URLs for media playback/upload
|
||||
// - Session-scoped reading progress
|
||||
// - Live novelfire.net browse/search (no scraper interface needed; direct HTTP)
|
||||
// - Live novelfire.net search (no scraper interface needed; direct HTTP)
|
||||
// - Kokoro voice list
|
||||
//
|
||||
// The backend never scrapes directly. All scraping (metadata, chapter list,
|
||||
@@ -28,9 +28,14 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sentryhttp "github.com/getsentry/sentry-go/http"
|
||||
"github.com/libnovel/backend/internal/bookstore"
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
"github.com/libnovel/backend/internal/meili"
|
||||
"github.com/libnovel/backend/internal/pockettts"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
)
|
||||
|
||||
// Dependencies holds all external services the backend server depends on.
|
||||
@@ -42,19 +47,28 @@ type Dependencies struct {
|
||||
RankingStore bookstore.RankingStore
|
||||
// AudioStore checks audio object existence and computes MinIO keys.
|
||||
AudioStore bookstore.AudioStore
|
||||
// TranslationStore checks translation existence and reads/writes translated markdown.
|
||||
TranslationStore bookstore.TranslationStore
|
||||
// PresignStore generates short-lived MinIO URLs.
|
||||
PresignStore bookstore.PresignStore
|
||||
// ProgressStore reads/writes per-session reading progress.
|
||||
ProgressStore bookstore.ProgressStore
|
||||
// BrowseStore reads cached browse page snapshots from MinIO.
|
||||
BrowseStore bookstore.BrowseStore
|
||||
// CoverStore reads and writes book cover images from MinIO.
|
||||
// If nil, the cover endpoint falls back to a CDN redirect.
|
||||
CoverStore bookstore.CoverStore
|
||||
// Producer creates scrape/audio tasks in PocketBase.
|
||||
Producer taskqueue.Producer
|
||||
// TaskReader reads scrape/audio task records from PocketBase.
|
||||
TaskReader taskqueue.Reader
|
||||
// Kokoro is the TTS client (used for voice list only in the backend;
|
||||
// SearchIndex provides full-text book search via Meilisearch.
|
||||
// If nil, the local-only fallback search is used.
|
||||
SearchIndex meili.Client
|
||||
// Kokoro is the Kokoro TTS client (used for voice list only in the backend;
|
||||
// audio generation is done by the runner).
|
||||
Kokoro kokoro.Client
|
||||
// PocketTTS is the pocket-tts client (used for voice list only in the backend;
|
||||
// audio generation is done by the runner).
|
||||
PocketTTS pockettts.Client
|
||||
// Log is the structured logger.
|
||||
Log *slog.Logger
|
||||
}
|
||||
@@ -77,7 +91,7 @@ type Server struct {
|
||||
|
||||
// voiceMu guards cachedVoices. Populated lazily on first GET /api/voices.
|
||||
voiceMu sync.RWMutex
|
||||
cachedVoices []string
|
||||
cachedVoices []domain.Voice
|
||||
}
|
||||
|
||||
// New creates a Server from cfg and deps.
|
||||
@@ -88,6 +102,9 @@ func New(cfg Config, deps Dependencies) *Server {
|
||||
if deps.Log == nil {
|
||||
deps.Log = slog.Default()
|
||||
}
|
||||
if deps.SearchIndex == nil {
|
||||
deps.SearchIndex = meili.NoopClient{}
|
||||
}
|
||||
return &Server{cfg: cfg, deps: deps}
|
||||
}
|
||||
|
||||
@@ -112,10 +129,12 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
// Cancel a pending task (scrape or audio)
|
||||
mux.HandleFunc("POST /api/cancel-task/{id}", s.handleCancelTask)
|
||||
|
||||
// Browse & search (live novelfire.net)
|
||||
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
||||
// Browse & search
|
||||
mux.HandleFunc("GET /api/search", s.handleSearch)
|
||||
|
||||
// Catalogue (Meilisearch-backed browse + search — preferred path for UI)
|
||||
mux.HandleFunc("GET /api/catalogue", s.handleCatalogue)
|
||||
|
||||
// Ranking (from PocketBase)
|
||||
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
|
||||
|
||||
@@ -131,6 +150,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
// Use this instead of presign+fetch to avoid SvelteKit→MinIO network path.
|
||||
mux.HandleFunc("GET /api/chapter-markdown/{slug}/{n}", s.handleChapterMarkdown)
|
||||
|
||||
// Chapter text preview — live scrape from novelfire.net, no store writes.
|
||||
// Used when the chapter is not yet in the library (preview mode).
|
||||
mux.HandleFunc("GET /api/chapter-text-preview/{slug}/{n}", s.handleChapterTextPreview)
|
||||
|
||||
// Reindex chapters_idx from MinIO
|
||||
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
|
||||
|
||||
@@ -139,6 +162,15 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus)
|
||||
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
||||
|
||||
// Translation task creation (backend creates task; runner executes via LibreTranslate)
|
||||
mux.HandleFunc("POST /api/translation/{slug}/{n}", s.handleTranslationGenerate)
|
||||
mux.HandleFunc("GET /api/translation/status/{slug}/{n}", s.handleTranslationStatus)
|
||||
mux.HandleFunc("GET /api/translation/{slug}/{n}", s.handleTranslationRead)
|
||||
|
||||
// Admin translation endpoints
|
||||
mux.HandleFunc("GET /api/admin/translation/jobs", s.handleAdminTranslationJobs)
|
||||
mux.HandleFunc("POST /api/admin/translation/bulk", s.handleAdminTranslationBulk)
|
||||
|
||||
// Voices list
|
||||
mux.HandleFunc("GET /api/voices", s.handleVoices)
|
||||
|
||||
@@ -148,15 +180,24 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample)
|
||||
mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload)
|
||||
mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar)
|
||||
mux.HandleFunc("PUT /api/avatar-upload/{userId}", s.handleAvatarUpload)
|
||||
|
||||
// Reading progress
|
||||
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
||||
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
||||
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
|
||||
|
||||
// Wrap mux with OTel tracing (no-op when no TracerProvider is set),
|
||||
// then with Sentry for panic recovery and error reporting.
|
||||
var handler http.Handler = mux
|
||||
handler = otelhttp.NewHandler(handler, "libnovel.backend",
|
||||
otelhttp.WithMessageEvents(otelhttp.ReadEvents, otelhttp.WriteEvents),
|
||||
)
|
||||
handler = sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle(handler)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: s.cfg.Addr,
|
||||
Handler: mux,
|
||||
Handler: handler,
|
||||
ReadTimeout: 15 * time.Second,
|
||||
WriteTimeout: 60 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
@@ -239,10 +280,10 @@ func jsonError(w http.ResponseWriter, status int, msg string) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
// voices returns the list of available Kokoro voices. On the first call it
|
||||
// fetches from the Kokoro service and caches the result. Falls back to the
|
||||
// hardcoded list on error.
|
||||
func (s *Server) voices(ctx context.Context) []string {
|
||||
// voices returns the merged list of available voices from Kokoro and pocket-tts.
|
||||
// On the first call it fetches from both services and caches the result.
|
||||
// Falls back to the hardcoded Kokoro list on error.
|
||||
func (s *Server) voices(ctx context.Context) []domain.Voice {
|
||||
s.voiceMu.RLock()
|
||||
cached := s.cachedVoices
|
||||
s.voiceMu.RUnlock()
|
||||
@@ -250,23 +291,89 @@ func (s *Server) voices(ctx context.Context) []string {
|
||||
return cached
|
||||
}
|
||||
|
||||
if s.deps.Kokoro == nil {
|
||||
return kokoroVoices
|
||||
}
|
||||
|
||||
fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
list, err := s.deps.Kokoro.ListVoices(fetchCtx)
|
||||
if err != nil || len(list) == 0 {
|
||||
s.deps.Log.Warn("backend: could not fetch kokoro voices, using built-in list", "err", err)
|
||||
return kokoroVoices
|
||||
|
||||
var result []domain.Voice
|
||||
|
||||
// ── Kokoro voices ─────────────────────────────────────────────────────────
|
||||
var kokoroIDs []string
|
||||
if s.deps.Kokoro != nil {
|
||||
ids, err := s.deps.Kokoro.ListVoices(fetchCtx)
|
||||
if err != nil || len(ids) == 0 {
|
||||
s.deps.Log.Warn("backend: could not fetch kokoro voices, using built-in list", "err", err)
|
||||
ids = kokoroVoiceIDs
|
||||
} else {
|
||||
s.deps.Log.Info("backend: fetched kokoro voices", "count", len(ids))
|
||||
}
|
||||
kokoroIDs = ids
|
||||
} else {
|
||||
kokoroIDs = kokoroVoiceIDs
|
||||
}
|
||||
for _, id := range kokoroIDs {
|
||||
result = append(result, kokoroVoice(id))
|
||||
}
|
||||
|
||||
// ── Pocket-TTS voices ─────────────────────────────────────────────────────
|
||||
if s.deps.PocketTTS != nil {
|
||||
ids, err := s.deps.PocketTTS.ListVoices(fetchCtx)
|
||||
if err != nil {
|
||||
s.deps.Log.Warn("backend: could not fetch pocket-tts voices", "err", err)
|
||||
} else {
|
||||
for _, id := range ids {
|
||||
result = append(result, pocketTTSVoice(id))
|
||||
}
|
||||
s.deps.Log.Info("backend: fetched pocket-tts voices", "count", len(ids))
|
||||
}
|
||||
}
|
||||
|
||||
s.voiceMu.Lock()
|
||||
s.cachedVoices = list
|
||||
s.cachedVoices = result
|
||||
s.voiceMu.Unlock()
|
||||
s.deps.Log.Info("backend: fetched kokoro voices", "count", len(list))
|
||||
return list
|
||||
return result
|
||||
}
|
||||
|
||||
// kokoroVoice builds a domain.Voice for a Kokoro voice ID.
|
||||
// The two-character prefix encodes language and gender:
|
||||
//
|
||||
// af/am → en-us f/m | bf/bm → en-gb f/m
|
||||
// ef/em → es f/m | ff → fr f
|
||||
// hf/hm → hi f/m | if/im → it f/m
|
||||
// jf/jm → ja f/m | pf/pm → pt f/m
|
||||
// zf/zm → zh f/m
|
||||
func kokoroVoice(id string) domain.Voice {
|
||||
type meta struct{ lang, gender string }
|
||||
prefixMap := map[string]meta{
|
||||
"af": {"en-us", "f"}, "am": {"en-us", "m"},
|
||||
"bf": {"en-gb", "f"}, "bm": {"en-gb", "m"},
|
||||
"ef": {"es", "f"}, "em": {"es", "m"},
|
||||
"ff": {"fr", "f"},
|
||||
"hf": {"hi", "f"}, "hm": {"hi", "m"},
|
||||
"if": {"it", "f"}, "im": {"it", "m"},
|
||||
"jf": {"ja", "f"}, "jm": {"ja", "m"},
|
||||
"pf": {"pt", "f"}, "pm": {"pt", "m"},
|
||||
"zf": {"zh", "f"}, "zm": {"zh", "m"},
|
||||
}
|
||||
if len(id) >= 2 {
|
||||
if m, ok := prefixMap[id[:2]]; ok {
|
||||
return domain.Voice{ID: id, Engine: "kokoro", Lang: m.lang, Gender: m.gender}
|
||||
}
|
||||
}
|
||||
return domain.Voice{ID: id, Engine: "kokoro", Lang: "en", Gender: ""}
|
||||
}
|
||||
|
||||
// pocketTTSVoice builds a domain.Voice for a pocket-tts voice ID.
|
||||
// All pocket-tts voices are English audiobook narrators.
|
||||
func pocketTTSVoice(id string) domain.Voice {
|
||||
femaleVoices := map[string]struct{}{
|
||||
"alba": {}, "fantine": {}, "cosette": {}, "eponine": {},
|
||||
"azelma": {}, "anna": {}, "vera": {}, "mary": {}, "jane": {}, "eve": {},
|
||||
}
|
||||
gender := "m"
|
||||
if _, ok := femaleVoices[id]; ok {
|
||||
gender = "f"
|
||||
}
|
||||
return domain.Voice{ID: id, Engine: "pocket-tts", Lang: "en", Gender: gender}
|
||||
}
|
||||
|
||||
// handleHealth handles GET /health.
|
||||
|
||||
@@ -105,6 +105,10 @@ type PresignStore interface {
|
||||
// Returns ("", false, nil) when no avatar exists.
|
||||
PresignAvatarURL(ctx context.Context, userID string) (string, bool, error)
|
||||
|
||||
// PutAvatar stores raw image bytes for a user avatar directly in MinIO.
|
||||
// ext should be "jpg", "png", or "webp". Returns the object key.
|
||||
PutAvatar(ctx context.Context, userID, ext, contentType string, data []byte) (key string, err error)
|
||||
|
||||
// DeleteAvatar removes all avatar objects for a user.
|
||||
DeleteAvatar(ctx context.Context, userID string) error
|
||||
}
|
||||
@@ -124,14 +128,32 @@ type ProgressStore interface {
|
||||
DeleteProgress(ctx context.Context, sessionID, slug string) error
|
||||
}
|
||||
|
||||
// BrowseStore covers browse page snapshot storage.
|
||||
// The runner writes snapshots; the backend reads them.
|
||||
type BrowseStore interface {
|
||||
// PutBrowsePage stores a raw JSON snapshot for a browse page.
|
||||
// genre, sort, status, novelType and page identify the page.
|
||||
PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error
|
||||
// CoverStore covers book cover image storage in MinIO.
|
||||
// The runner writes covers during catalogue refresh; the backend reads them.
|
||||
type CoverStore interface {
|
||||
// PutCover stores a raw cover image for a book identified by slug.
|
||||
PutCover(ctx context.Context, slug string, data []byte, contentType string) error
|
||||
|
||||
// GetBrowsePage retrieves a raw JSON snapshot. Returns (nil, false, nil)
|
||||
// when no snapshot exists for the given parameters.
|
||||
GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error)
|
||||
// GetCover retrieves the cover image for a book. Returns (nil, false, nil)
|
||||
// when no cover exists for the given slug.
|
||||
GetCover(ctx context.Context, slug string) ([]byte, string, bool, error)
|
||||
|
||||
// CoverExists returns true when a cover image is stored for slug.
|
||||
CoverExists(ctx context.Context, slug string) bool
|
||||
}
|
||||
|
||||
// TranslationStore covers machine-translated chapter storage in MinIO.
|
||||
// The runner writes translations; the backend reads them.
|
||||
type TranslationStore interface {
|
||||
// TranslationObjectKey returns the MinIO object key for a cached translation.
|
||||
TranslationObjectKey(lang, slug string, n int) string
|
||||
|
||||
// TranslationExists returns true when the translation object is present in MinIO.
|
||||
TranslationExists(ctx context.Context, key string) bool
|
||||
|
||||
// PutTranslation stores raw translated markdown under the given MinIO object key.
|
||||
PutTranslation(ctx context.Context, key string, data []byte) error
|
||||
|
||||
// GetTranslation retrieves translated markdown from MinIO.
|
||||
GetTranslation(ctx context.Context, key string) (string, error)
|
||||
}
|
||||
|
||||
@@ -68,6 +68,9 @@ func (m *mockStore) PresignAvatarUpload(_ context.Context, _, _ string) (string,
|
||||
func (m *mockStore) PresignAvatarURL(_ context.Context, _ string) (string, bool, error) {
|
||||
return "", false, nil
|
||||
}
|
||||
func (m *mockStore) PutAvatar(_ context.Context, _, _, _ string, _ []byte) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
func (m *mockStore) DeleteAvatar(_ context.Context, _ string) error { return nil }
|
||||
|
||||
// ProgressStore
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
// Package browser provides a rate-limited HTTP client for web scraping.
|
||||
// The Client interface is the only thing the rest of the codebase depends on;
|
||||
// the concrete DirectClient can be swapped for any other implementation
|
||||
// (e.g. a Browserless-backed client) without touching callers.
|
||||
package browser
|
||||
|
||||
import (
|
||||
@@ -10,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -51,9 +47,6 @@ type Config struct {
|
||||
MaxConcurrent int
|
||||
// Timeout is the per-request deadline. Defaults to 90s when 0.
|
||||
Timeout time.Duration
|
||||
// ProxyURL is an optional outbound proxy, e.g. "http://user:pass@host:3128".
|
||||
// Falls back to HTTP_PROXY / HTTPS_PROXY environment variables when empty.
|
||||
ProxyURL string
|
||||
}
|
||||
|
||||
// DirectClient is a plain net/http-based Client with a concurrency semaphore.
|
||||
@@ -75,14 +68,6 @@ func NewDirectClient(cfg Config) *DirectClient {
|
||||
MaxIdleConnsPerHost: cfg.MaxConcurrent * 2,
|
||||
DisableCompression: false,
|
||||
}
|
||||
if cfg.ProxyURL != "" {
|
||||
proxyParsed, err := url.Parse(cfg.ProxyURL)
|
||||
if err == nil {
|
||||
transport.Proxy = http.ProxyURL(proxyParsed)
|
||||
}
|
||||
} else {
|
||||
transport.Proxy = http.ProxyFromEnvironment
|
||||
}
|
||||
|
||||
return &DirectClient{
|
||||
http: &http.Client{
|
||||
|
||||
@@ -46,23 +46,71 @@ type MinIO struct {
|
||||
BucketAvatars string
|
||||
// BucketBrowse is the bucket that holds cached browse page snapshots (JSON).
|
||||
BucketBrowse string
|
||||
// BucketTranslations is the bucket that holds machine-translated chapter markdown.
|
||||
BucketTranslations string
|
||||
}
|
||||
|
||||
// Kokoro holds connection settings for the Kokoro-FastAPI TTS service.
|
||||
type Kokoro struct {
|
||||
// URL is the base URL of the Kokoro service, e.g. https://kokoro.libnovel.cc
|
||||
// An empty string disables TTS generation.
|
||||
// URL is the base URL of the Kokoro service, e.g. https://tts.libnovel.cc
|
||||
// An empty string disables Kokoro TTS generation.
|
||||
URL string
|
||||
// DefaultVoice is the voice used when none is specified.
|
||||
DefaultVoice string
|
||||
}
|
||||
|
||||
// PocketTTS holds connection settings for the kyutai-labs/pocket-tts service.
|
||||
type PocketTTS struct {
|
||||
// URL is the base URL of the pocket-tts service, e.g. https://pocket-tts.libnovel.cc
|
||||
// An empty string disables pocket-tts generation.
|
||||
URL string
|
||||
}
|
||||
|
||||
// LibreTranslate holds connection settings for a self-hosted LibreTranslate instance.
|
||||
type LibreTranslate struct {
|
||||
// URL is the base URL of the LibreTranslate instance, e.g. https://translate.libnovel.cc
|
||||
// An empty string disables machine translation entirely.
|
||||
URL string
|
||||
// APIKey is the optional API key for the LibreTranslate instance.
|
||||
// Leave empty if the instance runs without authentication.
|
||||
APIKey string
|
||||
}
|
||||
|
||||
// HTTP holds settings for the HTTP server (backend only).
|
||||
type HTTP struct {
|
||||
// Addr is the listen address, e.g. ":8080"
|
||||
Addr string
|
||||
}
|
||||
|
||||
// Meilisearch holds connection settings for the Meilisearch full-text search service.
|
||||
type Meilisearch struct {
|
||||
// URL is the base URL of the Meilisearch instance, e.g. http://localhost:7700
|
||||
// An empty string disables Meilisearch indexing and search.
|
||||
URL string
|
||||
// APIKey is the Meilisearch master/search API key.
|
||||
APIKey string
|
||||
}
|
||||
|
||||
// Valkey holds connection settings for the Valkey/Redis presign URL cache.
|
||||
type Valkey struct {
|
||||
// Addr is the host:port of the Valkey instance, e.g. localhost:6379
|
||||
// An empty string disables the Valkey cache (falls through to MinIO directly).
|
||||
Addr string
|
||||
}
|
||||
|
||||
// Redis holds connection settings for the Asynq task queue Redis instance.
|
||||
// This is separate from Valkey (presign cache) — it may point to the same
|
||||
// Redis or a dedicated one. An empty Addr falls back to PocketBase polling.
|
||||
type Redis struct {
|
||||
// Addr is the host:port (or rediss://... URL) of the Redis instance.
|
||||
// Use rediss:// scheme for TLS (e.g. rediss://:password@redis.libnovel.cc:6380).
|
||||
// An empty string disables Asynq and falls back to PocketBase polling.
|
||||
Addr string
|
||||
// Password is the Redis AUTH password.
|
||||
// Not needed when Addr is a full rediss:// URL that includes the password.
|
||||
Password string
|
||||
}
|
||||
|
||||
// Runner holds settings specific to the runner/worker binary.
|
||||
type Runner struct {
|
||||
// PollInterval is how often the runner checks PocketBase for pending tasks.
|
||||
@@ -71,6 +119,8 @@ type Runner struct {
|
||||
MaxConcurrentScrape int
|
||||
// MaxConcurrentAudio limits simultaneous audio-generation goroutines.
|
||||
MaxConcurrentAudio int
|
||||
// MaxConcurrentTranslation limits simultaneous translation goroutines.
|
||||
MaxConcurrentTranslation int
|
||||
// WorkerID is a unique identifier for this runner instance.
|
||||
// Defaults to the system hostname.
|
||||
WorkerID string
|
||||
@@ -78,17 +128,37 @@ type Runner struct {
|
||||
Workers int
|
||||
// Timeout is the per-request HTTP timeout for scraping.
|
||||
Timeout time.Duration
|
||||
// ProxyURL is an optional outbound proxy for scraper HTTP requests.
|
||||
ProxyURL string
|
||||
// MetricsAddr is the listen address for the runner /metrics HTTP endpoint.
|
||||
// Defaults to ":9091". Set to "" to disable.
|
||||
MetricsAddr string
|
||||
// CatalogueRefreshInterval is how often the runner walks the full catalogue,
|
||||
// scrapes per-book metadata, downloads covers, and re-indexes in Meilisearch.
|
||||
// Defaults to 24h. Set to 0 to use the default.
|
||||
CatalogueRefreshInterval time.Duration
|
||||
// SkipInitialCatalogueRefresh prevents the runner from running a full
|
||||
// catalogue walk on startup. Useful for quick restarts where the catalogue
|
||||
// is already indexed and a 24h walk would be wasteful.
|
||||
// Controlled by RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true.
|
||||
SkipInitialCatalogueRefresh bool
|
||||
// CatalogueRequestDelay is the base delay inserted between per-book metadata
|
||||
// requests during a catalogue refresh. A random jitter of up to 50% is added
|
||||
// on top. Defaults to 2s. Increase to reduce 429 pressure on novelfire.net.
|
||||
// Controlled by RUNNER_CATALOGUE_REQUEST_DELAY (e.g. "3s", "500ms").
|
||||
CatalogueRequestDelay time.Duration
|
||||
}
|
||||
|
||||
// Config is the top-level configuration struct consumed by both binaries.
|
||||
type Config struct {
|
||||
PocketBase PocketBase
|
||||
MinIO MinIO
|
||||
Kokoro Kokoro
|
||||
HTTP HTTP
|
||||
Runner Runner
|
||||
PocketBase PocketBase
|
||||
MinIO MinIO
|
||||
Kokoro Kokoro
|
||||
PocketTTS PocketTTS
|
||||
LibreTranslate LibreTranslate
|
||||
HTTP HTTP
|
||||
Runner Runner
|
||||
Meilisearch Meilisearch
|
||||
Valkey Valkey
|
||||
Redis Redis
|
||||
// LogLevel is one of "debug", "info", "warn", "error".
|
||||
LogLevel string
|
||||
}
|
||||
@@ -111,16 +181,17 @@ func Load() Config {
|
||||
},
|
||||
|
||||
MinIO: MinIO{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
PublicEndpoint: envOr("MINIO_PUBLIC_ENDPOINT", ""),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: envBool("MINIO_USE_SSL", false),
|
||||
PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true),
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"),
|
||||
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
PublicEndpoint: envOr("MINIO_PUBLIC_ENDPOINT", ""),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: envBool("MINIO_USE_SSL", false),
|
||||
PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true),
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "audio"),
|
||||
BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "avatars"),
|
||||
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "catalogue"),
|
||||
BucketTranslations: envOr("MINIO_BUCKET_TRANSLATIONS", "translations"),
|
||||
},
|
||||
|
||||
Kokoro: Kokoro{
|
||||
@@ -128,18 +199,40 @@ func Load() Config {
|
||||
DefaultVoice: envOr("KOKORO_VOICE", "af_bella"),
|
||||
},
|
||||
|
||||
PocketTTS: PocketTTS{
|
||||
URL: envOr("POCKET_TTS_URL", ""),
|
||||
},
|
||||
|
||||
HTTP: HTTP{
|
||||
Addr: envOr("BACKEND_HTTP_ADDR", ":8080"),
|
||||
},
|
||||
|
||||
Runner: Runner{
|
||||
PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second),
|
||||
MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1),
|
||||
MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1),
|
||||
WorkerID: envOr("RUNNER_WORKER_ID", workerID),
|
||||
Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU()
|
||||
Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second),
|
||||
ProxyURL: envOr("SCRAPER_PROXY", ""),
|
||||
PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second),
|
||||
MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1),
|
||||
MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1),
|
||||
MaxConcurrentTranslation: envInt("RUNNER_MAX_CONCURRENT_TRANSLATION", 1),
|
||||
WorkerID: envOr("RUNNER_WORKER_ID", workerID),
|
||||
Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU()
|
||||
Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second),
|
||||
MetricsAddr: envOr("RUNNER_METRICS_ADDR", ":9091"),
|
||||
CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0),
|
||||
SkipInitialCatalogueRefresh: envBool("RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH", false),
|
||||
CatalogueRequestDelay: envDuration("RUNNER_CATALOGUE_REQUEST_DELAY", 2*time.Second),
|
||||
},
|
||||
|
||||
Meilisearch: Meilisearch{
|
||||
URL: envOr("MEILI_URL", ""),
|
||||
APIKey: envOr("MEILI_API_KEY", ""),
|
||||
},
|
||||
|
||||
Valkey: Valkey{
|
||||
Addr: envOr("VALKEY_ADDR", ""),
|
||||
},
|
||||
|
||||
Redis: Redis{
|
||||
Addr: envOr("REDIS_ADDR", ""),
|
||||
Password: envOr("REDIS_PASSWORD", ""),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ func TestLoad_Defaults(t *testing.T) {
|
||||
"KOKORO_URL", "KOKORO_VOICE",
|
||||
"BACKEND_HTTP_ADDR",
|
||||
"RUNNER_POLL_INTERVAL", "RUNNER_MAX_CONCURRENT_SCRAPE", "RUNNER_MAX_CONCURRENT_AUDIO",
|
||||
"RUNNER_WORKER_ID", "RUNNER_WORKERS", "RUNNER_TIMEOUT", "SCRAPER_PROXY",
|
||||
"RUNNER_WORKER_ID", "RUNNER_WORKERS", "RUNNER_TIMEOUT",
|
||||
}
|
||||
for _, k := range unset {
|
||||
t.Setenv(k, "")
|
||||
@@ -33,8 +33,8 @@ func TestLoad_Defaults(t *testing.T) {
|
||||
if cfg.PocketBase.URL != "http://localhost:8090" {
|
||||
t.Errorf("PocketBase.URL: want http://localhost:8090, got %q", cfg.PocketBase.URL)
|
||||
}
|
||||
if cfg.MinIO.BucketChapters != "libnovel-chapters" {
|
||||
t.Errorf("MinIO.BucketChapters: want libnovel-chapters, got %q", cfg.MinIO.BucketChapters)
|
||||
if cfg.MinIO.BucketChapters != "chapters" {
|
||||
t.Errorf("MinIO.BucketChapters: want chapters, got %q", cfg.MinIO.BucketChapters)
|
||||
}
|
||||
if cfg.MinIO.UseSSL != false {
|
||||
t.Errorf("MinIO.UseSSL: want false, got %v", cfg.MinIO.UseSSL)
|
||||
|
||||
@@ -19,10 +19,16 @@ type BookMeta struct {
|
||||
TotalChapters int `json:"total_chapters,omitempty"`
|
||||
SourceURL string `json:"source_url"`
|
||||
Ranking int `json:"ranking,omitempty"`
|
||||
Rating float64 `json:"rating,omitempty"`
|
||||
// MetaUpdated is the Unix timestamp (seconds) when the book record was last
|
||||
// updated in PocketBase. Populated on read; not sent on write (PocketBase
|
||||
// manages its own updated field).
|
||||
MetaUpdated int64 `json:"meta_updated,omitempty"`
|
||||
}
|
||||
|
||||
// CatalogueEntry is a lightweight book reference returned by catalogue pages.
|
||||
type CatalogueEntry struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
@@ -54,6 +60,20 @@ type RankingItem struct {
|
||||
Updated time.Time `json:"updated,omitempty"`
|
||||
}
|
||||
|
||||
// ── Voice types ───────────────────────────────────────────────────────────────
|
||||
|
||||
// Voice describes a single text-to-speech voice available in the system.
|
||||
type Voice struct {
|
||||
// ID is the voice identifier passed to TTS clients (e.g. "af_bella", "alba").
|
||||
ID string `json:"id"`
|
||||
// Engine is "kokoro" or "pocket-tts".
|
||||
Engine string `json:"engine"`
|
||||
// Lang is the primary language tag (e.g. "en-us", "en-gb", "en", "es", "fr").
|
||||
Lang string `json:"lang"`
|
||||
// Gender is "f" or "m".
|
||||
Gender string `json:"gender"`
|
||||
}
|
||||
|
||||
// ── Storage record types ──────────────────────────────────────────────────────
|
||||
|
||||
// ChapterInfo is a lightweight chapter descriptor stored in the index.
|
||||
@@ -129,3 +149,23 @@ type AudioResult struct {
|
||||
ObjectKey string `json:"object_key,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
}
|
||||
|
||||
// TranslationTask represents a machine-translation job stored in PocketBase.
|
||||
type TranslationTask struct {
|
||||
ID string `json:"id"`
|
||||
CacheKey string `json:"cache_key"` // "{slug}/{chapter}/{lang}"
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
Lang string `json:"lang"`
|
||||
WorkerID string `json:"worker_id,omitempty"`
|
||||
Status TaskStatus `json:"status"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
Started time.Time `json:"started"`
|
||||
Finished time.Time `json:"finished,omitempty"`
|
||||
}
|
||||
|
||||
// TranslationResult is the outcome reported by the runner after finishing a TranslationTask.
|
||||
type TranslationResult struct {
|
||||
ObjectKey string `json:"object_key,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
}
|
||||
|
||||
181
backend/internal/libretranslate/client.go
Normal file
181
backend/internal/libretranslate/client.go
Normal file
@@ -0,0 +1,181 @@
|
||||
// Package libretranslate provides an HTTP client for a self-hosted
|
||||
// LibreTranslate instance. It handles text chunking, concurrent translation,
|
||||
// and reassembly so callers can pass arbitrarily long markdown strings.
|
||||
package libretranslate
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// maxChunkBytes is the target maximum size of each chunk sent to
|
||||
// LibreTranslate. LibreTranslate's default limit is 5000 characters;
|
||||
// we stay comfortably below that.
|
||||
maxChunkBytes = 4500
|
||||
// concurrency is the number of simultaneous translation requests per chapter.
|
||||
concurrency = 3
|
||||
)
|
||||
|
||||
// Client translates text via LibreTranslate.
|
||||
// A nil Client is valid — all calls return the original text unchanged.
|
||||
type Client interface {
|
||||
// Translate translates text from sourceLang to targetLang.
|
||||
// text is a raw markdown string. The returned string is the translated
|
||||
// markdown, reassembled in original paragraph order.
|
||||
Translate(ctx context.Context, text, sourceLang, targetLang string) (string, error)
|
||||
}
|
||||
|
||||
// New returns a Client for the given LibreTranslate URL.
|
||||
// Returns nil when url is empty, which disables translation.
|
||||
func New(url, apiKey string) Client {
|
||||
if url == "" {
|
||||
return nil
|
||||
}
|
||||
return &httpClient{
|
||||
url: strings.TrimRight(url, "/"),
|
||||
apiKey: apiKey,
|
||||
http: &http.Client{Timeout: 60 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
type httpClient struct {
|
||||
url string
|
||||
apiKey string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// Translate splits text into paragraph chunks, translates them concurrently
|
||||
// (up to concurrency goroutines), and reassembles in order.
|
||||
func (c *httpClient) Translate(ctx context.Context, text, sourceLang, targetLang string) (string, error) {
|
||||
paragraphs := splitParagraphs(text)
|
||||
if len(paragraphs) == 0 {
|
||||
return text, nil
|
||||
}
|
||||
chunks := binChunks(paragraphs, maxChunkBytes)
|
||||
|
||||
translated := make([]string, len(chunks))
|
||||
errs := make([]error, len(chunks))
|
||||
|
||||
sem := make(chan struct{}, concurrency)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i, chunk := range chunks {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(idx int, chunkText string) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
result, err := c.translateChunk(ctx, chunkText, sourceLang, targetLang)
|
||||
translated[idx] = result
|
||||
errs[idx] = err
|
||||
}(i, chunk)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(translated, "\n\n"), nil
|
||||
}
|
||||
|
||||
// translateChunk sends a single POST /translate request.
|
||||
func (c *httpClient) translateChunk(ctx context.Context, text, sourceLang, targetLang string) (string, error) {
|
||||
reqBody := map[string]string{
|
||||
"q": text,
|
||||
"source": sourceLang,
|
||||
"target": targetLang,
|
||||
"format": "html",
|
||||
}
|
||||
if c.apiKey != "" {
|
||||
reqBody["api_key"] = c.apiKey
|
||||
}
|
||||
|
||||
b, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("libretranslate: marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url+"/translate", bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("libretranslate: build request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("libretranslate: request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
var errBody struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
_ = json.NewDecoder(resp.Body).Decode(&errBody)
|
||||
return "", fmt.Errorf("libretranslate: status %d: %s", resp.StatusCode, errBody.Error)
|
||||
}
|
||||
|
||||
var result struct {
|
||||
TranslatedText string `json:"translatedText"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return "", fmt.Errorf("libretranslate: decode response: %w", err)
|
||||
}
|
||||
return result.TranslatedText, nil
|
||||
}
|
||||
|
||||
// splitParagraphs splits markdown text on blank lines, preserving non-empty paragraphs.
|
||||
func splitParagraphs(text string) []string {
|
||||
// Normalise line endings.
|
||||
text = strings.ReplaceAll(text, "\r\n", "\n")
|
||||
// Split on double newlines (blank lines between paragraphs).
|
||||
parts := strings.Split(text, "\n\n")
|
||||
var paragraphs []string
|
||||
for _, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if p != "" {
|
||||
paragraphs = append(paragraphs, p)
|
||||
}
|
||||
}
|
||||
return paragraphs
|
||||
}
|
||||
|
||||
// binChunks groups paragraphs into chunks each at most maxBytes in length.
|
||||
// Each chunk is a single string with paragraphs joined by "\n\n".
|
||||
func binChunks(paragraphs []string, maxBytes int) []string {
|
||||
var chunks []string
|
||||
var current strings.Builder
|
||||
|
||||
for _, p := range paragraphs {
|
||||
needed := len(p)
|
||||
if current.Len() > 0 {
|
||||
needed += 2 // for the "\n\n" separator
|
||||
}
|
||||
|
||||
if current.Len()+needed > maxBytes && current.Len() > 0 {
|
||||
// Flush current chunk.
|
||||
chunks = append(chunks, current.String())
|
||||
current.Reset()
|
||||
}
|
||||
|
||||
if current.Len() > 0 {
|
||||
current.WriteString("\n\n")
|
||||
}
|
||||
current.WriteString(p)
|
||||
}
|
||||
|
||||
if current.Len() > 0 {
|
||||
chunks = append(chunks, current.String())
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
327
backend/internal/meili/client.go
Normal file
327
backend/internal/meili/client.go
Normal file
@@ -0,0 +1,327 @@
|
||||
// Package meili provides a thin Meilisearch client for indexing and searching
|
||||
// locally scraped books.
|
||||
//
|
||||
// Index:
|
||||
// - Name: "books"
|
||||
// - Primary key: "slug"
|
||||
// - Searchable attributes: title, author, genres, summary
|
||||
// - Filterable attributes: status, genres
|
||||
// - Sortable attributes: rank, rating, total_chapters, meta_updated
|
||||
//
|
||||
// The client is intentionally simple: UpsertBook and Search only. All
|
||||
// Meilisearch-specific details (index management, attribute configuration)
|
||||
// are handled once in Configure(), called at startup.
|
||||
package meili
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/meilisearch/meilisearch-go"
|
||||
)
|
||||
|
||||
const indexName = "books"
|
||||
|
||||
// Client is the interface for Meilisearch operations used by runner and backend.
|
||||
type Client interface {
|
||||
// UpsertBook adds or updates a book document in the search index.
|
||||
UpsertBook(ctx context.Context, book domain.BookMeta) error
|
||||
// BookExists reports whether a book with the given slug is already in the
|
||||
// index. Used by the catalogue refresh to skip re-indexing known books.
|
||||
BookExists(ctx context.Context, slug string) bool
|
||||
// Search returns up to limit books matching query.
|
||||
Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error)
|
||||
// Catalogue queries books with optional filters, sort, and pagination.
|
||||
// Returns books, the total hit count for pagination, and a FacetResult
|
||||
// with available genre and status values from the index.
|
||||
Catalogue(ctx context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error)
|
||||
}
|
||||
|
||||
// CatalogueQuery holds parameters for the /api/catalogue endpoint.
|
||||
type CatalogueQuery struct {
|
||||
Q string // full-text query (may be empty for browse)
|
||||
Genre string // genre filter, e.g. "fantasy" or "all"
|
||||
Status string // status filter, e.g. "ongoing", "completed", or "all"
|
||||
Sort string // sort field: "popular", "new", "update", "top-rated", "rank", ""
|
||||
Page int // 1-indexed
|
||||
Limit int // items per page, default 20
|
||||
}
|
||||
|
||||
// FacetResult holds the available filter values discovered from the index.
|
||||
// Values are sorted alphabetically and include only those present in the index.
|
||||
type FacetResult struct {
|
||||
Genres []string // distinct genre values
|
||||
Statuses []string // distinct status values
|
||||
}
|
||||
|
||||
// MeiliClient wraps the meilisearch-go SDK.
|
||||
type MeiliClient struct {
|
||||
idx meilisearch.IndexManager
|
||||
}
|
||||
|
||||
// New creates a MeiliClient. Call Configure() once at startup to ensure the
|
||||
// index exists and has the correct attribute settings.
|
||||
func New(host, apiKey string) *MeiliClient {
|
||||
cli := meilisearch.New(host, meilisearch.WithAPIKey(apiKey))
|
||||
return &MeiliClient{idx: cli.Index(indexName)}
|
||||
}
|
||||
|
||||
// Configure creates the index if absent and sets searchable/filterable
|
||||
// attributes. It is idempotent — safe to call on every startup.
|
||||
func Configure(host, apiKey string) error {
|
||||
cli := meilisearch.New(host, meilisearch.WithAPIKey(apiKey))
|
||||
|
||||
// Create index with primary key. Returns 202 if exists — ignore.
|
||||
task, err := cli.CreateIndex(&meilisearch.IndexConfig{
|
||||
Uid: indexName,
|
||||
PrimaryKey: "slug",
|
||||
})
|
||||
if err != nil {
|
||||
// 400 "index_already_exists" is not an error here; the SDK returns
|
||||
// an error with Code "index_already_exists" which we can ignore.
|
||||
// Any other error is fatal.
|
||||
if apiErr, ok := err.(*meilisearch.Error); ok && apiErr.MeilisearchApiError.Code == "index_already_exists" {
|
||||
// already exists — continue
|
||||
} else {
|
||||
return fmt.Errorf("meili: create index: %w", err)
|
||||
}
|
||||
} else {
|
||||
_ = task // task is async; we don't wait for it
|
||||
}
|
||||
|
||||
idx := cli.Index(indexName)
|
||||
|
||||
searchable := []string{"title", "author", "genres", "summary"}
|
||||
if _, err := idx.UpdateSearchableAttributes(&searchable); err != nil {
|
||||
return fmt.Errorf("meili: update searchable attributes: %w", err)
|
||||
}
|
||||
|
||||
filterable := []interface{}{"status", "genres"}
|
||||
if _, err := idx.UpdateFilterableAttributes(&filterable); err != nil {
|
||||
return fmt.Errorf("meili: update filterable attributes: %w", err)
|
||||
}
|
||||
|
||||
sortable := []string{"rank", "rating", "total_chapters", "meta_updated"}
|
||||
if _, err := idx.UpdateSortableAttributes(&sortable); err != nil {
|
||||
return fmt.Errorf("meili: update sortable attributes: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// bookDoc is the Meilisearch document shape for a book.
|
||||
type bookDoc struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Cover string `json:"cover"`
|
||||
Status string `json:"status"`
|
||||
Genres []string `json:"genres"`
|
||||
Summary string `json:"summary"`
|
||||
TotalChapters int `json:"total_chapters"`
|
||||
SourceURL string `json:"source_url"`
|
||||
Rank int `json:"rank"`
|
||||
Rating float64 `json:"rating"`
|
||||
// MetaUpdated is the Unix timestamp (seconds) of the last PocketBase update.
|
||||
// Used for sort=update ("recently updated" ordering).
|
||||
MetaUpdated int64 `json:"meta_updated"`
|
||||
}
|
||||
|
||||
func toDoc(b domain.BookMeta) bookDoc {
|
||||
return bookDoc{
|
||||
Slug: b.Slug,
|
||||
Title: b.Title,
|
||||
Author: b.Author,
|
||||
Cover: b.Cover,
|
||||
Status: b.Status,
|
||||
Genres: b.Genres,
|
||||
Summary: b.Summary,
|
||||
TotalChapters: b.TotalChapters,
|
||||
SourceURL: b.SourceURL,
|
||||
Rank: b.Ranking,
|
||||
Rating: b.Rating,
|
||||
MetaUpdated: b.MetaUpdated,
|
||||
}
|
||||
}
|
||||
|
||||
func fromDoc(d bookDoc) domain.BookMeta {
|
||||
return domain.BookMeta{
|
||||
Slug: d.Slug,
|
||||
Title: d.Title,
|
||||
Author: d.Author,
|
||||
Cover: d.Cover,
|
||||
Status: d.Status,
|
||||
Genres: d.Genres,
|
||||
Summary: d.Summary,
|
||||
TotalChapters: d.TotalChapters,
|
||||
SourceURL: d.SourceURL,
|
||||
Ranking: d.Rank,
|
||||
Rating: d.Rating,
|
||||
MetaUpdated: d.MetaUpdated,
|
||||
}
|
||||
}
|
||||
|
||||
// UpsertBook adds or replaces the book document in Meilisearch. The operation
|
||||
// is fire-and-forget (Meilisearch processes tasks asynchronously).
|
||||
func (c *MeiliClient) UpsertBook(_ context.Context, book domain.BookMeta) error {
|
||||
docs := []bookDoc{toDoc(book)}
|
||||
pk := "slug"
|
||||
if _, err := c.idx.AddDocuments(docs, &meilisearch.DocumentOptions{PrimaryKey: &pk}); err != nil {
|
||||
return fmt.Errorf("meili: upsert book %q: %w", book.Slug, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BookExists reports whether the slug is already present in the index.
|
||||
// It fetches the document by primary key; a 404 or any error is treated as
|
||||
// "not present" (safe default: re-index rather than silently skip).
|
||||
func (c *MeiliClient) BookExists(_ context.Context, slug string) bool {
|
||||
var doc bookDoc
|
||||
err := c.idx.GetDocument(slug, nil, &doc)
|
||||
return err == nil && doc.Slug != ""
|
||||
}
|
||||
|
||||
// Search returns books matching query, up to limit results.
|
||||
func (c *MeiliClient) Search(_ context.Context, query string, limit int) ([]domain.BookMeta, error) {
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
res, err := c.idx.Search(query, &meilisearch.SearchRequest{
|
||||
Limit: int64(limit),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("meili: search %q: %w", query, err)
|
||||
}
|
||||
|
||||
books := make([]domain.BookMeta, 0, len(res.Hits))
|
||||
for _, hit := range res.Hits {
|
||||
// Hit is map[string]json.RawMessage — unmarshal directly into bookDoc.
|
||||
var doc bookDoc
|
||||
raw, err := json.Marshal(hit)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
continue
|
||||
}
|
||||
books = append(books, fromDoc(doc))
|
||||
}
|
||||
return books, nil
|
||||
}
|
||||
|
||||
// Catalogue queries books with optional full-text search, genre/status filters,
|
||||
// sort order, and pagination. Returns matching books, the total estimate, and
|
||||
// a FacetResult containing available genre and status values from the index.
|
||||
func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) {
|
||||
if q.Limit <= 0 {
|
||||
q.Limit = 20
|
||||
}
|
||||
if q.Page <= 0 {
|
||||
q.Page = 1
|
||||
}
|
||||
|
||||
req := &meilisearch.SearchRequest{
|
||||
Limit: int64(q.Limit),
|
||||
Offset: int64((q.Page - 1) * q.Limit),
|
||||
// Request facet distribution so the UI can build filter options
|
||||
// dynamically without hardcoding genre/status lists.
|
||||
Facets: []string{"genres", "status"},
|
||||
}
|
||||
|
||||
// Build filter
|
||||
var filters []string
|
||||
if q.Genre != "" && q.Genre != "all" {
|
||||
filters = append(filters, fmt.Sprintf("genres = %q", q.Genre))
|
||||
}
|
||||
if q.Status != "" && q.Status != "all" {
|
||||
filters = append(filters, fmt.Sprintf("status = %q", q.Status))
|
||||
}
|
||||
if len(filters) > 0 {
|
||||
req.Filter = strings.Join(filters, " AND ")
|
||||
}
|
||||
|
||||
// Map UI sort tokens to Meilisearch sort expressions.
|
||||
switch q.Sort {
|
||||
case "rank":
|
||||
req.Sort = []string{"rank:asc"}
|
||||
case "top-rated":
|
||||
req.Sort = []string{"rating:desc"}
|
||||
case "new":
|
||||
req.Sort = []string{"total_chapters:desc"}
|
||||
case "update":
|
||||
req.Sort = []string{"meta_updated:desc"}
|
||||
// "popular" and "" → relevance (no explicit sort)
|
||||
}
|
||||
|
||||
res, err := c.idx.Search(q.Q, req)
|
||||
if err != nil {
|
||||
return nil, 0, FacetResult{}, fmt.Errorf("meili: catalogue query: %w", err)
|
||||
}
|
||||
|
||||
books := make([]domain.BookMeta, 0, len(res.Hits))
|
||||
for _, hit := range res.Hits {
|
||||
var doc bookDoc
|
||||
raw, err := json.Marshal(hit)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
continue
|
||||
}
|
||||
books = append(books, fromDoc(doc))
|
||||
}
|
||||
|
||||
facets := parseFacets(res.FacetDistribution)
|
||||
return books, res.EstimatedTotalHits, facets, nil
|
||||
}
|
||||
|
||||
// parseFacets extracts sorted genre and status slices from a Meilisearch
|
||||
// facetDistribution raw JSON value.
|
||||
// The JSON shape is: {"genres":{"fantasy":12,"action":5},"status":{"ongoing":7}}
|
||||
func parseFacets(raw json.RawMessage) FacetResult {
|
||||
var result FacetResult
|
||||
if len(raw) == 0 {
|
||||
return result
|
||||
}
|
||||
var dist map[string]map[string]int64
|
||||
if err := json.Unmarshal(raw, &dist); err != nil {
|
||||
return result
|
||||
}
|
||||
if m, ok := dist["genres"]; ok {
|
||||
for k := range m {
|
||||
result.Genres = append(result.Genres, k)
|
||||
}
|
||||
sortStrings(result.Genres)
|
||||
}
|
||||
if m, ok := dist["status"]; ok {
|
||||
for k := range m {
|
||||
result.Statuses = append(result.Statuses, k)
|
||||
}
|
||||
sortStrings(result.Statuses)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// sortStrings sorts a slice of strings in place.
|
||||
func sortStrings(s []string) {
|
||||
for i := 1; i < len(s); i++ {
|
||||
for j := i; j > 0 && s[j] < s[j-1]; j-- {
|
||||
s[j], s[j-1] = s[j-1], s[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NoopClient is a no-op Client used when Meilisearch is not configured.
|
||||
type NoopClient struct{}
|
||||
|
||||
func (NoopClient) UpsertBook(_ context.Context, _ domain.BookMeta) error { return nil }
|
||||
func (NoopClient) BookExists(_ context.Context, _ string) bool { return false }
|
||||
func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (NoopClient) Catalogue(_ context.Context, _ CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) {
|
||||
return nil, 0, FacetResult{}, nil
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/rand"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
@@ -55,6 +56,9 @@ func (s *Scraper) SourceName() string { return "novelfire.net" }
|
||||
// ── CatalogueProvider ─────────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages.
|
||||
// Each page fetch uses retryGet with 429-aware exponential backoff.
|
||||
// A small inter-page delay (cataloguePageDelay) is inserted between requests to
|
||||
// avoid hammering the server when paging through hundreds of catalogue pages.
|
||||
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
|
||||
entries := make(chan domain.CatalogueEntry, 64)
|
||||
errs := make(chan error, 16)
|
||||
@@ -73,8 +77,18 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
|
||||
default:
|
||||
}
|
||||
|
||||
// Polite inter-page delay — skipped on the very first page.
|
||||
if page > 1 {
|
||||
jitter := time.Duration(500+rand.Intn(1000)) * time.Millisecond
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(jitter):
|
||||
}
|
||||
}
|
||||
|
||||
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
|
||||
raw, err := s.client.GetContent(ctx, pageURL)
|
||||
raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 10*time.Second)
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
|
||||
return
|
||||
@@ -111,7 +125,7 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case entries <- domain.CatalogueEntry{Title: title, URL: bookURL}:
|
||||
case entries <- domain.CatalogueEntry{Slug: slugFromURL(bookURL), Title: title, URL: bookURL}:
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,10 +153,11 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE
|
||||
// ── MetadataProvider ──────────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeMetadata fetches and parses book metadata from the book's landing page.
|
||||
// Uses retryGet with 429-aware exponential backoff (up to 9 attempts).
|
||||
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) {
|
||||
s.log.Debug("metadata fetch starting", "url", bookURL)
|
||||
|
||||
raw, err := s.client.GetContent(ctx, bookURL)
|
||||
raw, err := retryGet(ctx, s.log, s.client, bookURL, 9, 10*time.Second)
|
||||
if err != nil {
|
||||
return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
|
||||
}
|
||||
@@ -163,12 +178,26 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.Bo
|
||||
}
|
||||
}
|
||||
|
||||
status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})
|
||||
// Status: novelfire renders <strong class="ongoing">Ongoing</strong> (or
|
||||
// "completed", "hiatus") inside the .header-stats block. We take the text
|
||||
// content and lowercase it so the index value is always canonical lowercase.
|
||||
var status string
|
||||
for _, cls := range []string{"ongoing", "completed", "hiatus"} {
|
||||
if v := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "strong", Class: cls}); v != "" {
|
||||
status = strings.ToLower(strings.TrimSpace(v))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
|
||||
// Genres: novelfire renders <div class="categories"><ul><li><a class="property-item">Genre</a>
|
||||
// Each <a class="property-item"> is one genre tag. Lowercase for index consistency.
|
||||
var genres []string
|
||||
if genresNode != nil {
|
||||
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
|
||||
if categoriesNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "categories"}); categoriesNode != nil {
|
||||
for _, v := range htmlutil.ExtractAll(categoriesNode, scraper.Selector{Tag: "a", Class: "property-item", Multiple: true}) {
|
||||
if v != "" {
|
||||
genres = append(genres, strings.ToLower(strings.TrimSpace(v)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"})
|
||||
@@ -194,8 +223,12 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.Bo
|
||||
|
||||
// ── ChapterListProvider ───────────────────────────────────────────────────────
|
||||
|
||||
// ScrapeChapterList returns all chapter references for a book, ordered ascending.
|
||||
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) {
|
||||
// ScrapeChapterList returns chapter references for a book, ordered ascending.
|
||||
// upTo > 0 stops pagination as soon as at least upTo chapter numbers have been
|
||||
// collected — use this for range scrapes so we don't paginate 100 pages just
|
||||
// to discover refs we'll never scrape. upTo == 0 fetches all pages.
|
||||
// Each page fetch uses retryGet with 429-aware exponential backoff.
|
||||
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error) {
|
||||
var refs []domain.ChapterRef
|
||||
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
|
||||
page := 1
|
||||
@@ -210,7 +243,7 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma
|
||||
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
|
||||
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
|
||||
|
||||
raw, err := s.client.GetContent(ctx, pageURL)
|
||||
raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second)
|
||||
if err != nil {
|
||||
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
|
||||
}
|
||||
@@ -255,6 +288,13 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma
|
||||
})
|
||||
}
|
||||
|
||||
// Early-stop: if we have seen at least upTo chapter numbers, we have
|
||||
// enough refs to cover the requested range — no need to paginate further.
|
||||
if upTo > 0 && len(refs) > 0 && refs[len(refs)-1].Number >= upTo {
|
||||
s.log.Debug("chapter list early-stop reached", "upTo", upTo, "collected", len(refs))
|
||||
break
|
||||
}
|
||||
|
||||
page++
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package novelfire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -100,6 +101,56 @@ func TestRetryGet_EventualSuccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseMetadataSelectors verifies that the status and genres selectors
|
||||
// match the current novelfire.net HTML structure.
|
||||
func TestParseMetadataSelectors(t *testing.T) {
|
||||
// Minimal HTML reproducing the relevant novelfire.net book page structure.
|
||||
const html = `<!DOCTYPE html>
|
||||
<html><body>
|
||||
<h1 class="novel-title">Shadow Slave</h1>
|
||||
<span class="author">Guiltythree</span>
|
||||
<figure class="cover"><img src="https://cdn.example.com/cover.jpg"></figure>
|
||||
<div class="header-stats">
|
||||
<span><strong>123</strong><small>Chapters</small></span>
|
||||
<span> <strong class="ongoing">Ongoing</strong> <small>Status</small></span>
|
||||
</div>
|
||||
<div class="categories">
|
||||
<h4>Genres</h4>
|
||||
<ul>
|
||||
<li><a href="/genre-fantasy/..." class="property-item">Fantasy</a></li>
|
||||
<li><a href="/genre-action/..." class="property-item">Action</a></li>
|
||||
<li><a href="/genre-adventure/..." class="property-item">Adventure</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
<span class="chapter-count">123 Chapters</span>
|
||||
</body></html>`
|
||||
|
||||
stub := newStubClient()
|
||||
stub.setFn("https://novelfire.net/book/shadow-slave", func() (string, error) {
|
||||
return html, nil
|
||||
})
|
||||
|
||||
s := &Scraper{client: stub, log: slog.Default()}
|
||||
meta, err := s.ScrapeMetadata(t.Context(), "https://novelfire.net/book/shadow-slave")
|
||||
if err != nil {
|
||||
t.Fatalf("ScrapeMetadata: %v", err)
|
||||
}
|
||||
|
||||
if meta.Status != "ongoing" {
|
||||
t.Errorf("status = %q, want %q", meta.Status, "ongoing")
|
||||
}
|
||||
|
||||
wantGenres := []string{"fantasy", "action", "adventure"}
|
||||
if len(meta.Genres) != len(wantGenres) {
|
||||
t.Fatalf("genres = %v, want %v", meta.Genres, wantGenres)
|
||||
}
|
||||
for i, g := range meta.Genres {
|
||||
if g != wantGenres[i] {
|
||||
t.Errorf("genres[%d] = %q, want %q", i, g, wantGenres[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── minimal stub client for tests ─────────────────────────────────────────────
|
||||
|
||||
type stubClient struct {
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
// - RunBook scrapes one book (metadata + chapter list + chapter texts) end-to-end.
|
||||
// - N worker goroutines pull chapter refs from a shared queue and call ScrapeChapterText.
|
||||
// - The caller (runner poll loop) owns the outer task-claim / finish cycle.
|
||||
// - An optional PostMetadata hook (set in Config) is called after WriteMetadata
|
||||
// succeeds. The runner uses this to upsert books into Meilisearch.
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
@@ -25,14 +27,19 @@ type Config struct {
|
||||
// Workers is the number of goroutines used to scrape chapters in parallel.
|
||||
// Defaults to runtime.NumCPU() when 0.
|
||||
Workers int
|
||||
// PostMetadata is an optional hook called with the scraped BookMeta after
|
||||
// WriteMetadata succeeds. Errors from the hook are logged but not fatal.
|
||||
// Used by the runner to index books in Meilisearch.
|
||||
PostMetadata func(ctx context.Context, meta domain.BookMeta)
|
||||
}
|
||||
|
||||
// Orchestrator runs a single-book scrape pipeline.
|
||||
type Orchestrator struct {
|
||||
novel scraper.NovelScraper
|
||||
store bookstore.BookWriter
|
||||
log *slog.Logger
|
||||
workers int
|
||||
novel scraper.NovelScraper
|
||||
store bookstore.BookWriter
|
||||
log *slog.Logger
|
||||
workers int
|
||||
postMetadata func(ctx context.Context, meta domain.BookMeta)
|
||||
}
|
||||
|
||||
// New returns a new Orchestrator.
|
||||
@@ -44,7 +51,13 @@ func New(cfg Config, novel scraper.NovelScraper, store bookstore.BookWriter, log
|
||||
if workers <= 0 {
|
||||
workers = runtime.NumCPU()
|
||||
}
|
||||
return &Orchestrator{novel: novel, store: store, log: log, workers: workers}
|
||||
return &Orchestrator{
|
||||
novel: novel,
|
||||
store: store,
|
||||
log: log,
|
||||
workers: workers,
|
||||
postMetadata: cfg.PostMetadata,
|
||||
}
|
||||
}
|
||||
|
||||
// RunBook scrapes a single book described by task. It handles:
|
||||
@@ -84,12 +97,16 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma
|
||||
result.Errors++
|
||||
} else {
|
||||
result.BooksFound = 1
|
||||
// Fire optional post-metadata hook (e.g. Meilisearch indexing).
|
||||
if o.postMetadata != nil {
|
||||
o.postMetadata(ctx, meta)
|
||||
}
|
||||
}
|
||||
|
||||
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
|
||||
|
||||
// ── Step 2: Chapter list ──────────────────────────────────────────────────
|
||||
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL)
|
||||
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL, task.ToChapter)
|
||||
if err != nil {
|
||||
o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err)
|
||||
result.ErrorMessage = fmt.Sprintf("chapter list: %v", err)
|
||||
|
||||
@@ -34,7 +34,7 @@ func (s *stubScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMe
|
||||
return s.meta, s.metaErr
|
||||
}
|
||||
|
||||
func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
|
||||
func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) {
|
||||
return s.refs, s.refsErr
|
||||
}
|
||||
|
||||
|
||||
120
backend/internal/otelsetup/otelsetup.go
Normal file
120
backend/internal/otelsetup/otelsetup.go
Normal file
@@ -0,0 +1,120 @@
|
||||
// Package otelsetup initialises the OpenTelemetry SDK for the LibNovel backend.
|
||||
//
|
||||
// It reads two environment variables:
|
||||
//
|
||||
// OTEL_EXPORTER_OTLP_ENDPOINT — OTLP/HTTP endpoint; accepts either a full
|
||||
// URL ("https://otel.example.com") or a bare
|
||||
// host[:port] ("otel-collector:4318").
|
||||
// TLS is used when the value starts with "https://".
|
||||
// OTEL_SERVICE_NAME — service name reported in traces (default: "backend")
|
||||
//
|
||||
// When OTEL_EXPORTER_OTLP_ENDPOINT is empty the function is a no-op: it
|
||||
// returns a nil shutdown func and the default slog.Logger, so callers never
|
||||
// need to branch on it.
|
||||
//
|
||||
// Usage in main.go:
|
||||
//
|
||||
// shutdown, log, err := otelsetup.Init(ctx, version)
|
||||
// if err != nil { return err }
|
||||
// if shutdown != nil { defer shutdown() }
|
||||
package otelsetup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/contrib/bridges/otelslog"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
||||
otellog "go.opentelemetry.io/otel/log/global"
|
||||
"go.opentelemetry.io/otel/sdk/log"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
|
||||
)
|
||||
|
||||
// Init sets up TracerProvider and LoggerProvider that export via OTLP/HTTP.
|
||||
//
|
||||
// Returns:
|
||||
// - shutdown: flushes and stops both providers (nil when OTel is disabled).
|
||||
// - logger: an slog.Logger bridged to OTel logs (falls back to default when disabled).
|
||||
// - err: non-nil only on SDK initialisation failure.
|
||||
func Init(ctx context.Context, version string) (shutdown func(), logger *slog.Logger, err error) {
|
||||
rawEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
|
||||
if rawEndpoint == "" {
|
||||
return nil, slog.Default(), nil // OTel disabled — not an error
|
||||
}
|
||||
|
||||
// WithEndpoint expects a host[:port] value — no scheme.
|
||||
// Support both "https://otel.example.com" and "otel-collector:4318".
|
||||
useTLS := strings.HasPrefix(rawEndpoint, "https://")
|
||||
endpoint := strings.TrimPrefix(rawEndpoint, "https://")
|
||||
endpoint = strings.TrimPrefix(endpoint, "http://")
|
||||
|
||||
serviceName := os.Getenv("OTEL_SERVICE_NAME")
|
||||
if serviceName == "" {
|
||||
serviceName = "backend"
|
||||
}
|
||||
|
||||
// ── Shared resource ───────────────────────────────────────────────────────
|
||||
res, err := resource.New(ctx,
|
||||
resource.WithAttributes(
|
||||
semconv.ServiceName(serviceName),
|
||||
semconv.ServiceVersion(version),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, slog.Default(), fmt.Errorf("otelsetup: create resource: %w", err)
|
||||
}
|
||||
|
||||
// ── Trace provider ────────────────────────────────────────────────────────
|
||||
traceOpts := []otlptracehttp.Option{otlptracehttp.WithEndpoint(endpoint)}
|
||||
if !useTLS {
|
||||
traceOpts = append(traceOpts, otlptracehttp.WithInsecure())
|
||||
}
|
||||
traceExp, err := otlptracehttp.New(ctx, traceOpts...)
|
||||
if err != nil {
|
||||
return nil, slog.Default(), fmt.Errorf("otelsetup: create OTLP trace exporter: %w", err)
|
||||
}
|
||||
|
||||
tp := sdktrace.NewTracerProvider(
|
||||
sdktrace.WithBatcher(traceExp),
|
||||
sdktrace.WithResource(res),
|
||||
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.2))),
|
||||
)
|
||||
otel.SetTracerProvider(tp)
|
||||
|
||||
// ── Log provider ──────────────────────────────────────────────────────────
|
||||
logOpts := []otlploghttp.Option{otlploghttp.WithEndpoint(endpoint)}
|
||||
if !useTLS {
|
||||
logOpts = append(logOpts, otlploghttp.WithInsecure())
|
||||
}
|
||||
logExp, err := otlploghttp.New(ctx, logOpts...)
|
||||
if err != nil {
|
||||
return nil, slog.Default(), fmt.Errorf("otelsetup: create OTLP log exporter: %w", err)
|
||||
}
|
||||
|
||||
lp := log.NewLoggerProvider(
|
||||
log.WithProcessor(log.NewBatchProcessor(logExp)),
|
||||
log.WithResource(res),
|
||||
)
|
||||
otellog.SetLoggerProvider(lp)
|
||||
|
||||
// Bridge slog → OTel logs. Structured fields and trace IDs are forwarded
|
||||
// automatically; Grafana can correlate log lines with Tempo traces.
|
||||
otelLogger := otelslog.NewLogger(serviceName)
|
||||
|
||||
shutdown = func() {
|
||||
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = tp.Shutdown(shutCtx)
|
||||
_ = lp.Shutdown(shutCtx)
|
||||
}
|
||||
|
||||
return shutdown, otelLogger, nil
|
||||
}
|
||||
159
backend/internal/pockettts/client.go
Normal file
159
backend/internal/pockettts/client.go
Normal file
@@ -0,0 +1,159 @@
|
||||
// Package pockettts provides a client for the kyutai-labs/pocket-tts TTS service.
|
||||
//
|
||||
// pocket-tts exposes a non-OpenAI API:
|
||||
//
|
||||
// POST /tts (multipart form: text, voice_url) → streaming WAV
|
||||
// GET /health → {"status":"healthy"}
|
||||
//
|
||||
// GenerateAudio streams the WAV response and transcodes it to MP3 using ffmpeg,
|
||||
// so callers receive MP3 bytes — the same format as the kokoro client — and the
|
||||
// rest of the pipeline does not need to care which TTS engine was used.
|
||||
//
|
||||
// Predefined voices (pass the bare name as the voice parameter):
|
||||
//
|
||||
// alba, marius, javert, jean, fantine, cosette, eponine, azelma,
|
||||
// anna, vera, charles, paul, george, mary, jane, michael, eve,
|
||||
// bill_boerst, peter_yearsley, stuart_bell
|
||||
package pockettts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PredefinedVoices is the set of voice names built into pocket-tts.
|
||||
// The runner uses this to decide which TTS engine to route a task to.
|
||||
var PredefinedVoices = map[string]struct{}{
|
||||
"alba": {}, "marius": {}, "javert": {}, "jean": {},
|
||||
"fantine": {}, "cosette": {}, "eponine": {}, "azelma": {},
|
||||
"anna": {}, "vera": {}, "charles": {}, "paul": {},
|
||||
"george": {}, "mary": {}, "jane": {}, "michael": {},
|
||||
"eve": {}, "bill_boerst": {}, "peter_yearsley": {}, "stuart_bell": {},
|
||||
}
|
||||
|
||||
// IsPocketTTSVoice reports whether voice is served by pocket-tts.
|
||||
func IsPocketTTSVoice(voice string) bool {
|
||||
_, ok := PredefinedVoices[voice]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Client is the interface for interacting with the pocket-tts service.
|
||||
type Client interface {
|
||||
// GenerateAudio synthesises text using the given voice and returns MP3 bytes.
|
||||
// Voice must be one of the predefined pocket-tts voice names.
|
||||
GenerateAudio(ctx context.Context, text, voice string) ([]byte, error)
|
||||
|
||||
// ListVoices returns the available predefined voice names.
|
||||
ListVoices(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// httpClient is the concrete pocket-tts HTTP client.
|
||||
type httpClient struct {
|
||||
baseURL string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// New returns a Client targeting baseURL (e.g. "https://pocket-tts.libnovel.cc").
|
||||
func New(baseURL string) Client {
|
||||
return &httpClient{
|
||||
baseURL: strings.TrimRight(baseURL, "/"),
|
||||
http: &http.Client{Timeout: 10 * time.Minute},
|
||||
}
|
||||
}
|
||||
|
||||
// GenerateAudio posts to POST /tts and transcodes the WAV response to MP3
|
||||
// using the system ffmpeg binary. Requires ffmpeg to be on PATH (available in
|
||||
// the runner Docker image via Alpine's ffmpeg package).
|
||||
func (c *httpClient) GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) {
|
||||
if text == "" {
|
||||
return nil, fmt.Errorf("pockettts: empty text")
|
||||
}
|
||||
if voice == "" {
|
||||
voice = "alba"
|
||||
}
|
||||
|
||||
// ── Build multipart form ──────────────────────────────────────────────────
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
|
||||
if err := mw.WriteField("text", text); err != nil {
|
||||
return nil, fmt.Errorf("pockettts: write text field: %w", err)
|
||||
}
|
||||
// pocket-tts accepts a predefined voice name as voice_url.
|
||||
if err := mw.WriteField("voice_url", voice); err != nil {
|
||||
return nil, fmt.Errorf("pockettts: write voice_url field: %w", err)
|
||||
}
|
||||
if err := mw.Close(); err != nil {
|
||||
return nil, fmt.Errorf("pockettts: close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
c.baseURL+"/tts", &body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pockettts: build request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pockettts: request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("pockettts: server returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
wavData, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pockettts: read response body: %w", err)
|
||||
}
|
||||
|
||||
// ── Transcode WAV → MP3 via ffmpeg ────────────────────────────────────────
|
||||
mp3Data, err := wavToMP3(ctx, wavData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pockettts: transcode to mp3: %w", err)
|
||||
}
|
||||
return mp3Data, nil
|
||||
}
|
||||
|
||||
// ListVoices returns the statically known predefined voice names.
|
||||
// pocket-tts has no REST endpoint for listing voices.
|
||||
func (c *httpClient) ListVoices(_ context.Context) ([]string, error) {
|
||||
voices := make([]string, 0, len(PredefinedVoices))
|
||||
for v := range PredefinedVoices {
|
||||
voices = append(voices, v)
|
||||
}
|
||||
return voices, nil
|
||||
}
|
||||
|
||||
// wavToMP3 converts raw WAV bytes to MP3 using ffmpeg.
|
||||
// ffmpeg reads from stdin (pipe:0) and writes to stdout (pipe:1).
|
||||
func wavToMP3(ctx context.Context, wav []byte) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx,
|
||||
"ffmpeg",
|
||||
"-hide_banner", "-loglevel", "error",
|
||||
"-i", "pipe:0", // read WAV from stdin
|
||||
"-f", "mp3", // output format
|
||||
"-q:a", "2", // VBR quality ~190 kbps
|
||||
"pipe:1", // write MP3 to stdout
|
||||
)
|
||||
cmd.Stdin = bytes.NewReader(wav)
|
||||
|
||||
var out, stderr bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("ffmpeg: %w (stderr: %s)", err, stderr.String())
|
||||
}
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
96
backend/internal/presigncache/cache.go
Normal file
96
backend/internal/presigncache/cache.go
Normal file
@@ -0,0 +1,96 @@
|
||||
// Package presigncache provides a Valkey (Redis-compatible) backed cache for
|
||||
// MinIO presigned URLs. The backend generates presigned URLs and stores them
|
||||
// here with a TTL; subsequent requests for the same key return the cached URL
|
||||
// without re-contacting MinIO.
|
||||
//
|
||||
// Design:
|
||||
// - Cache is intentionally best-effort: Get returns ("", false, nil) on any
|
||||
// Valkey error, so callers always have a fallback path to regenerate.
|
||||
// - Set silently drops errors — a miss on the next request is acceptable.
|
||||
// - TTL should be set shorter than the actual presigned URL lifetime so that
|
||||
// cached URLs are always valid when served. Recommended: 55 minutes for a
|
||||
// 1-hour presigned URL.
|
||||
package presigncache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// Cache is the interface for presign URL caching.
|
||||
// Implementations must be safe for concurrent use.
|
||||
type Cache interface {
|
||||
// Get returns the cached URL for key. ok is false on cache miss or error.
|
||||
Get(ctx context.Context, key string) (url string, ok bool, err error)
|
||||
// Set stores url under key with the given TTL.
|
||||
Set(ctx context.Context, key, url string, ttl time.Duration) error
|
||||
// Delete removes key from the cache.
|
||||
Delete(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
// ValkeyCache is a Cache backed by Valkey / Redis via go-redis.
|
||||
type ValkeyCache struct {
|
||||
rdb *redis.Client
|
||||
}
|
||||
|
||||
// New creates a ValkeyCache connecting to addr (e.g. "valkey:6379").
|
||||
// The connection is not established until the first command; use Ping to
|
||||
// verify connectivity at startup.
|
||||
func New(addr string) *ValkeyCache {
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: addr,
|
||||
DialTimeout: 2 * time.Second,
|
||||
ReadTimeout: 1 * time.Second,
|
||||
WriteTimeout: 1 * time.Second,
|
||||
})
|
||||
return &ValkeyCache{rdb: rdb}
|
||||
}
|
||||
|
||||
// Ping checks connectivity. Call once at startup.
|
||||
func (c *ValkeyCache) Ping(ctx context.Context) error {
|
||||
if err := c.rdb.Ping(ctx).Err(); err != nil {
|
||||
return fmt.Errorf("presigncache: ping valkey: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns (url, true, nil) on hit, ("", false, nil) on miss, and
|
||||
// ("", false, err) only on unexpected errors (not redis.Nil).
|
||||
func (c *ValkeyCache) Get(ctx context.Context, key string) (string, bool, error) {
|
||||
val, err := c.rdb.Get(ctx, key).Result()
|
||||
if err == redis.Nil {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("presigncache: get %q: %w", key, err)
|
||||
}
|
||||
return val, true, nil
|
||||
}
|
||||
|
||||
// Set stores url under key with ttl. Errors are returned but are non-fatal
|
||||
// for callers — a Set failure means the next request will miss and regenerate.
|
||||
func (c *ValkeyCache) Set(ctx context.Context, key, url string, ttl time.Duration) error {
|
||||
if err := c.rdb.Set(ctx, key, url, ttl).Err(); err != nil {
|
||||
return fmt.Errorf("presigncache: set %q: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes key from the cache. It is not an error if the key does not exist.
|
||||
func (c *ValkeyCache) Delete(ctx context.Context, key string) error {
|
||||
if err := c.rdb.Del(ctx, key).Err(); err != nil {
|
||||
return fmt.Errorf("presigncache: delete %q: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NoopCache is a no-op Cache that always returns a miss. Used when Valkey is
|
||||
// not configured (e.g. local development without Docker).
|
||||
type NoopCache struct{}
|
||||
|
||||
func (NoopCache) Get(_ context.Context, _ string) (string, bool, error) { return "", false, nil }
|
||||
func (NoopCache) Set(_ context.Context, _, _ string, _ time.Duration) error { return nil }
|
||||
func (NoopCache) Delete(_ context.Context, _ string) error { return nil }
|
||||
236
backend/internal/runner/asynq_runner.go
Normal file
236
backend/internal/runner/asynq_runner.go
Normal file
@@ -0,0 +1,236 @@
|
||||
package runner
|
||||
|
||||
// asynq_runner.go — Asynq-based task dispatch for the runner.
|
||||
//
|
||||
// When cfg.RedisAddr is set, Run() calls runAsynq() instead of runPoll().
|
||||
// The Asynq server replaces the polling loop: it listens on Redis for tasks
|
||||
// enqueued by the backend Producer and delivers them immediately.
|
||||
//
|
||||
// Handlers in this file decode Asynq job payloads and call the existing
|
||||
// runScrapeTask / runAudioTask methods, keeping all execution logic in one place.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
asynqmetrics "github.com/hibiken/asynq/x/metrics"
|
||||
"github.com/libnovel/backend/internal/asynqqueue"
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
)
|
||||
|
||||
// runAsynq starts an Asynq server that replaces the PocketBase poll loop.
|
||||
// It also starts the periodic catalogue refresh ticker.
|
||||
// Blocks until ctx is cancelled.
|
||||
func (r *Runner) runAsynq(ctx context.Context) error {
|
||||
redisOpt, err := r.redisConnOpt()
|
||||
if err != nil {
|
||||
return fmt.Errorf("runner: parse redis addr: %w", err)
|
||||
}
|
||||
|
||||
srv := asynq.NewServer(redisOpt, asynq.Config{
|
||||
// Allocate concurrency slots for each task type.
|
||||
// Total concurrency = scrape + audio slots.
|
||||
Concurrency: r.cfg.MaxConcurrentScrape + r.cfg.MaxConcurrentAudio,
|
||||
Queues: map[string]int{
|
||||
asynqqueue.QueueDefault: 1,
|
||||
},
|
||||
// Let Asynq handle retries with exponential back-off.
|
||||
RetryDelayFunc: asynq.DefaultRetryDelayFunc,
|
||||
// Log errors from handlers via the existing structured logger.
|
||||
ErrorHandler: asynq.ErrorHandlerFunc(func(_ context.Context, task *asynq.Task, err error) {
|
||||
r.deps.Log.Error("runner: asynq task failed",
|
||||
"type", task.Type(),
|
||||
"err", err,
|
||||
)
|
||||
}),
|
||||
})
|
||||
|
||||
mux := asynq.NewServeMux()
|
||||
mux.HandleFunc(asynqqueue.TypeAudioGenerate, r.handleAudioTask)
|
||||
mux.HandleFunc(asynqqueue.TypeScrapeBook, r.handleScrapeTask)
|
||||
mux.HandleFunc(asynqqueue.TypeScrapeCatalogue, r.handleScrapeTask)
|
||||
|
||||
// Register Asynq queue metrics with the default Prometheus registry so
|
||||
// the /metrics endpoint (metrics.go) can expose them.
|
||||
inspector := asynq.NewInspector(redisOpt)
|
||||
collector := asynqmetrics.NewQueueMetricsCollector(inspector)
|
||||
if err := r.metricsRegistry.Register(collector); err != nil {
|
||||
r.deps.Log.Warn("runner: could not register asynq prometheus collector", "err", err)
|
||||
}
|
||||
|
||||
// Start the periodic catalogue refresh.
|
||||
catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval)
|
||||
defer catalogueTick.Stop()
|
||||
if !r.cfg.SkipInitialCatalogueRefresh {
|
||||
go r.runCatalogueRefresh(ctx)
|
||||
} else {
|
||||
r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)")
|
||||
}
|
||||
|
||||
r.deps.Log.Info("runner: asynq mode active", "redis_addr", r.cfg.RedisAddr)
|
||||
|
||||
// ── Heartbeat goroutine ──────────────────────────────────────────────
|
||||
// Write /tmp/runner.alive every 30s so Docker healthcheck passes in asynq mode.
|
||||
// This mirrors the heartbeat file behavior from the poll() loop.
|
||||
go func() {
|
||||
heartbeatTick := time.NewTicker(r.cfg.StaleTaskThreshold)
|
||||
defer heartbeatTick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-heartbeatTick.C:
|
||||
if f, err := os.Create("/tmp/runner.alive"); err != nil {
|
||||
r.deps.Log.Warn("runner: could not write heartbeat file", "err", err)
|
||||
} else {
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// ── Translation polling goroutine ────────────────────────────────────
|
||||
// Translation tasks live in PocketBase (not Redis), so we need a separate
|
||||
// poll loop to claim and dispatch them. This runs alongside the Asynq server.
|
||||
translationSem := make(chan struct{}, r.cfg.MaxConcurrentTranslation)
|
||||
var translationWg sync.WaitGroup
|
||||
go func() {
|
||||
tick := time.NewTicker(r.cfg.PollInterval)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
r.pollTranslationTasks(ctx, translationSem, &translationWg)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Run catalogue refresh ticker in the background.
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-catalogueTick.C:
|
||||
go r.runCatalogueRefresh(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Start Asynq server (non-blocking).
|
||||
if err := srv.Start(mux); err != nil {
|
||||
return fmt.Errorf("runner: asynq server start: %w", err)
|
||||
}
|
||||
|
||||
// Block until context is cancelled, then gracefully stop.
|
||||
<-ctx.Done()
|
||||
r.deps.Log.Info("runner: context cancelled, shutting down asynq server")
|
||||
srv.Shutdown()
|
||||
|
||||
// Wait for translation tasks to complete.
|
||||
translationWg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// redisConnOpt parses cfg.RedisAddr into an asynq.RedisConnOpt.
|
||||
// Supports full "redis://" / "rediss://" URLs and plain "host:port".
|
||||
func (r *Runner) redisConnOpt() (asynq.RedisConnOpt, error) {
|
||||
addr := r.cfg.RedisAddr
|
||||
// ParseRedisURI handles redis:// and rediss:// schemes.
|
||||
if len(addr) > 7 && (addr[:8] == "redis://" || addr[:9] == "rediss://") {
|
||||
return asynq.ParseRedisURI(addr)
|
||||
}
|
||||
// Plain "host:port" — use RedisClientOpt directly.
|
||||
return asynq.RedisClientOpt{
|
||||
Addr: addr,
|
||||
Password: r.cfg.RedisPassword,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// handleScrapeTask is the Asynq handler for TypeScrapeBook and TypeScrapeCatalogue.
|
||||
func (r *Runner) handleScrapeTask(ctx context.Context, t *asynq.Task) error {
|
||||
var p asynqqueue.ScrapePayload
|
||||
if err := json.Unmarshal(t.Payload(), &p); err != nil {
|
||||
return fmt.Errorf("unmarshal scrape payload: %w", err)
|
||||
}
|
||||
task := domain.ScrapeTask{
|
||||
ID: p.PBTaskID,
|
||||
Kind: p.Kind,
|
||||
TargetURL: p.TargetURL,
|
||||
FromChapter: p.FromChapter,
|
||||
ToChapter: p.ToChapter,
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runScrapeTask(ctx, task)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleAudioTask is the Asynq handler for TypeAudioGenerate.
|
||||
func (r *Runner) handleAudioTask(ctx context.Context, t *asynq.Task) error {
|
||||
var p asynqqueue.AudioPayload
|
||||
if err := json.Unmarshal(t.Payload(), &p); err != nil {
|
||||
return fmt.Errorf("unmarshal audio payload: %w", err)
|
||||
}
|
||||
task := domain.AudioTask{
|
||||
ID: p.PBTaskID,
|
||||
Slug: p.Slug,
|
||||
Chapter: p.Chapter,
|
||||
Voice: p.Voice,
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runAudioTask(ctx, task)
|
||||
return nil
|
||||
}
|
||||
|
||||
// pollTranslationTasks claims all available translation tasks from PocketBase
|
||||
// and dispatches them to goroutines. Translation tasks don't go through Redis/Asynq
|
||||
// because they're stored in PocketBase, so we need this separate poll loop.
|
||||
func (r *Runner) pollTranslationTasks(ctx context.Context, translationSem chan struct{}, wg *sync.WaitGroup) {
|
||||
// Reap orphaned tasks (same logic as poll() in runner.go).
|
||||
if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
|
||||
r.deps.Log.Warn("runner: reap stale translation tasks failed", "err", err)
|
||||
} else if n > 0 {
|
||||
r.deps.Log.Info("runner: reaped stale translation tasks", "count", n)
|
||||
}
|
||||
|
||||
translationLoop:
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case translationSem <- struct{}{}:
|
||||
// Slot acquired — proceed to claim a task.
|
||||
default:
|
||||
// All slots busy; leave remaining pending tasks for next tick.
|
||||
break translationLoop
|
||||
}
|
||||
task, ok, err := r.deps.Consumer.ClaimNextTranslationTask(ctx, r.cfg.WorkerID)
|
||||
if err != nil {
|
||||
<-translationSem
|
||||
r.deps.Log.Error("runner: ClaimNextTranslationTask failed", "err", err)
|
||||
break
|
||||
}
|
||||
if !ok {
|
||||
<-translationSem
|
||||
break
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
wg.Add(1)
|
||||
go func(t domain.TranslationTask) {
|
||||
defer wg.Done()
|
||||
defer func() { <-translationSem }()
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runTranslationTask(ctx, t)
|
||||
}(task)
|
||||
}
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
package runner
|
||||
|
||||
// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
|
||||
// browse page snapshots and stores them in MinIO.
|
||||
//
|
||||
// Design:
|
||||
// - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
|
||||
// - Fetches page 1 for each combination of the standard genre/sort/status
|
||||
// filter values and stores the parsed JSON blob in MinIO via BrowseStore.
|
||||
// - The backend's handleBrowse then serves from MinIO instead of calling
|
||||
// novelfire.net live, which avoids IP-based rate-limiting on the server.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
|
||||
type browseNovelListing struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Cover string `json:"cover"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// browseSnapshot is the JSON structure stored in MinIO.
|
||||
type browseSnapshot struct {
|
||||
Novels []browseNovelListing `json:"novels"`
|
||||
Page int `json:"page"`
|
||||
HasNext bool `json:"hasNext"`
|
||||
// CachedAt is the UTC time the snapshot was written (ISO 8601).
|
||||
CachedAt string `json:"cachedAt"`
|
||||
}
|
||||
|
||||
// browseCombos lists the filter combinations to pre-fetch.
|
||||
// Each entry is (genre, sort, status, novelType).
|
||||
var browseCombos = []struct{ genre, sort, status, novelType string }{
|
||||
{"all", "popular", "all", "all-novel"},
|
||||
{"all", "popular", "ongoing", "all-novel"},
|
||||
{"all", "popular", "completed", "all-novel"},
|
||||
{"all", "new", "all", "all-novel"},
|
||||
{"all", "new", "ongoing", "all-novel"},
|
||||
{"all", "new", "completed", "all-novel"},
|
||||
{"all", "top-rated", "all", "all-novel"},
|
||||
{"all", "top-rated", "ongoing", "all-novel"},
|
||||
{"all", "top-rated", "completed", "all-novel"},
|
||||
}
|
||||
|
||||
const novelFireBrowseBase = "https://novelfire.net"
|
||||
|
||||
// runBrowseRefresh fetches all browse combos from novelfire.net and stores
|
||||
// the results in MinIO. Errors per-combo are logged but do not abort the
|
||||
// whole refresh cycle.
|
||||
func (r *Runner) runBrowseRefresh(ctx context.Context) {
|
||||
if r.deps.BrowseStore == nil {
|
||||
r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
|
||||
return
|
||||
}
|
||||
|
||||
log := r.deps.Log.With("op", "browse_refresh")
|
||||
log.Info("runner: browse refresh starting", "combos", len(browseCombos))
|
||||
|
||||
ok, fail := 0, 0
|
||||
for _, c := range browseCombos {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
|
||||
if err != nil {
|
||||
log.Warn("runner: browse fetch failed",
|
||||
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
|
||||
fail++
|
||||
continue
|
||||
}
|
||||
|
||||
snap := browseSnapshot{
|
||||
Novels: novels,
|
||||
Page: 1,
|
||||
HasNext: hasNext,
|
||||
CachedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
data, _ := json.Marshal(snap)
|
||||
if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
|
||||
log.Warn("runner: browse put failed",
|
||||
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
|
||||
fail++
|
||||
continue
|
||||
}
|
||||
ok++
|
||||
}
|
||||
|
||||
log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
|
||||
}
|
||||
|
||||
// fetchBrowsePage calls novelfire.net and returns a list of novel listings
|
||||
// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
|
||||
func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
|
||||
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
|
||||
novelFireBrowseBase, genre, sort, status, novelType)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
|
||||
httpClient := &http.Client{Timeout: 45 * time.Second}
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
|
||||
}
|
||||
|
||||
return parseBrowseHTML(resp.Body)
|
||||
}
|
||||
|
||||
// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
|
||||
// in backend/handlers.go — kept separate to avoid coupling packages.
|
||||
func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
body := string(data)
|
||||
|
||||
hasNext := strings.Contains(body, `rel="next"`) ||
|
||||
strings.Contains(body, `aria-label="Next"`) ||
|
||||
strings.Contains(body, `class="next"`)
|
||||
|
||||
slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
|
||||
titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
|
||||
coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
|
||||
|
||||
slugMatches := slugRe.FindAllStringSubmatch(body, -1)
|
||||
titleMatches := titleRe.FindAllStringSubmatch(body, -1)
|
||||
coverMatches := coverRe.FindAllStringSubmatch(body, -1)
|
||||
|
||||
var novels []browseNovelListing
|
||||
seen := make(map[string]bool)
|
||||
for i, sm := range slugMatches {
|
||||
slug := sm[1]
|
||||
if seen[slug] {
|
||||
continue
|
||||
}
|
||||
seen[slug] = true
|
||||
|
||||
item := browseNovelListing{
|
||||
Slug: slug,
|
||||
URL: novelFireBrowseBase + "/book/" + slug,
|
||||
}
|
||||
if i < len(titleMatches) {
|
||||
item.Title = strings.TrimSpace(titleMatches[i][1])
|
||||
}
|
||||
if i < len(coverMatches) {
|
||||
item.Cover = coverMatches[i][1]
|
||||
}
|
||||
if item.Title != "" {
|
||||
novels = append(novels, item)
|
||||
}
|
||||
}
|
||||
|
||||
return novels, hasNext, nil
|
||||
}
|
||||
186
backend/internal/runner/catalogue_refresh.go
Normal file
186
backend/internal/runner/catalogue_refresh.go
Normal file
@@ -0,0 +1,186 @@
|
||||
package runner
|
||||
|
||||
// catalogue_refresh.go — independent loop that walks the full novelfire.net
|
||||
// catalogue, scrapes per-book metadata, downloads cover images to MinIO, and
|
||||
// indexes every book in Meilisearch.
|
||||
//
|
||||
// Design:
|
||||
// - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run().
|
||||
// - Also fires once on startup (unless SkipInitialCatalogueRefresh is set).
|
||||
// - ScrapeCatalogue streams CatalogueEntry values over a channel — already has
|
||||
// its own inter-page jitter + retryGet (see scraper.go).
|
||||
// - Per-book: only metadata is scraped here (not chapters). Chapters are scraped
|
||||
// on-demand when a user opens a book or via an explicit scrape task.
|
||||
// - Between each metadata request a configurable base delay plus up to 50%
|
||||
// random jitter is applied (CatalogueRequestDelay, default 2s). This keeps
|
||||
// the request rate well below novelfire.net's rate limit even for ~15k books.
|
||||
// - ScrapeMetadata itself uses retryGet with 429-aware exponential backoff
|
||||
// (up to 9 attempts), so transient rate limits are handled gracefully.
|
||||
// - Cover images are fetched and stored in MinIO on first sight; subsequent
|
||||
// refreshes skip covers that already exist (CoverExists check).
|
||||
// - Books already present in Meilisearch are skipped entirely (fast path).
|
||||
// - Errors for individual books are logged and skipped; the loop never aborts.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// runCatalogueRefresh performs one full catalogue walk: scrapes metadata for
|
||||
// every book on novelfire.net, downloads covers to MinIO, and upserts to
|
||||
// Meilisearch. Individual book failures are logged and skipped.
|
||||
func (r *Runner) runCatalogueRefresh(ctx context.Context) {
|
||||
if r.deps.Novel == nil {
|
||||
r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured")
|
||||
return
|
||||
}
|
||||
if r.deps.BookWriter == nil {
|
||||
r.deps.Log.Warn("runner: catalogue refresh skipped — BookWriter not configured")
|
||||
return
|
||||
}
|
||||
|
||||
delay := r.cfg.CatalogueRequestDelay
|
||||
log := r.deps.Log.With("op", "catalogue_refresh")
|
||||
log.Info("runner: catalogue refresh starting", "request_delay", delay)
|
||||
|
||||
entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
|
||||
|
||||
ok, skipped, errCount := 0, 0, 0
|
||||
for entry := range entries {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
// Fast path: skip books already indexed in Meilisearch.
|
||||
if r.deps.SearchIndex.BookExists(ctx, entry.Slug) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
// Polite delay between metadata requests: base + up to 50% jitter.
|
||||
// This applies before every fetch so we never fire bursts.
|
||||
jitter := time.Duration(rand.Int63n(int64(delay / 2)))
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break
|
||||
case <-time.After(delay + jitter):
|
||||
}
|
||||
|
||||
// ScrapeMetadata internally retries on 429 with exponential back-off.
|
||||
meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL)
|
||||
if err != nil {
|
||||
log.Warn("runner: catalogue refresh: metadata scrape failed — skipping book",
|
||||
"slug", entry.Slug, "url", entry.URL, "err", err)
|
||||
errCount++
|
||||
continue
|
||||
}
|
||||
|
||||
// Rewrite cover URL to backend proxy path so UI never hits CDN directly.
|
||||
originalCover := meta.Cover
|
||||
meta.Cover = fmt.Sprintf("/api/cover/novelfire.net/%s", meta.Slug)
|
||||
|
||||
// Persist to PocketBase.
|
||||
if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil {
|
||||
log.Warn("runner: catalogue refresh: WriteMetadata failed — skipping book",
|
||||
"slug", meta.Slug, "err", err)
|
||||
errCount++
|
||||
continue
|
||||
}
|
||||
|
||||
// Index in Meilisearch (non-fatal).
|
||||
if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
|
||||
log.Warn("runner: catalogue refresh: UpsertBook failed",
|
||||
"slug", meta.Slug, "err", err)
|
||||
}
|
||||
|
||||
// Download cover to MinIO if not already cached (non-fatal).
|
||||
if r.deps.CoverStore != nil && originalCover != "" {
|
||||
if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) {
|
||||
if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil {
|
||||
log.Warn("runner: catalogue refresh: cover download failed",
|
||||
"slug", meta.Slug, "url", originalCover, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ok++
|
||||
if ok%50 == 0 {
|
||||
log.Info("runner: catalogue refresh progress",
|
||||
"scraped", ok, "skipped", skipped, "errors", errCount)
|
||||
}
|
||||
}
|
||||
|
||||
if err := <-errCh; err != nil {
|
||||
log.Warn("runner: catalogue refresh: catalogue stream error", "err", err)
|
||||
}
|
||||
|
||||
log.Info("runner: catalogue refresh finished",
|
||||
"ok", ok, "skipped", skipped, "errors", errCount)
|
||||
}
|
||||
|
||||
// downloadCover fetches the cover image from coverURL and stores it in MinIO
|
||||
// under covers/{slug}.jpg. It retries up to 3 times with exponential backoff
|
||||
// on transient errors (5xx, network failures).
|
||||
func (r *Runner) downloadCover(ctx context.Context, slug, coverURL string) error {
|
||||
const maxRetries = 3
|
||||
delay := 2 * time.Second
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if attempt > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
delay *= 2
|
||||
}
|
||||
|
||||
data, err := fetchCoverBytes(ctx, coverURL)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
if err := r.deps.CoverStore.PutCover(ctx, slug, data, ""); err != nil {
|
||||
return fmt.Errorf("put cover: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("download cover after %d retries: %w", maxRetries, lastErr)
|
||||
}
|
||||
|
||||
// fetchCoverBytes performs a single HTTP GET for coverURL and returns the body.
|
||||
func fetchCoverBytes(ctx context.Context, coverURL string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, coverURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
|
||||
req.Header.Set("Referer", "https://novelfire.net/")
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http get: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 500 {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("upstream %d for %s", resp.StatusCode, coverURL)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, coverURL)
|
||||
}
|
||||
|
||||
return io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5 MiB cap
|
||||
}
|
||||
113
backend/internal/runner/metrics.go
Normal file
113
backend/internal/runner/metrics.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package runner
|
||||
|
||||
// metrics.go — Prometheus metrics HTTP endpoint for the runner.
|
||||
//
|
||||
// GET /metrics returns a Prometheus text/plain scrape response.
|
||||
// Exposes:
|
||||
// - Standard Go runtime metrics (via promhttp)
|
||||
// - Runner task counters (tasks_running, tasks_completed, tasks_failed)
|
||||
// - Asynq queue metrics (registered in asynq_runner.go when Redis is enabled)
|
||||
//
|
||||
// GET /health — simple liveness probe.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
// metricsServer serves GET /metrics and GET /health for the runner process.
|
||||
type metricsServer struct {
|
||||
addr string
|
||||
r *Runner
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
func newMetricsServer(addr string, r *Runner, log *slog.Logger) *metricsServer {
|
||||
ms := &metricsServer{addr: addr, r: r, log: log}
|
||||
ms.registerCollectors()
|
||||
return ms
|
||||
}
|
||||
|
||||
// registerCollectors registers runner-specific Prometheus collectors.
|
||||
// Called once at construction; Asynq queue collector is registered separately
|
||||
// in asynq_runner.go after the Redis connection is established.
|
||||
func (ms *metricsServer) registerCollectors() {
|
||||
// Runner task gauges / counters backed by the atomic fields on Runner.
|
||||
ms.r.metricsRegistry.MustRegister(prometheus.NewGaugeFunc(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "runner",
|
||||
Name: "tasks_running",
|
||||
Help: "Number of tasks currently being processed.",
|
||||
},
|
||||
func() float64 { return float64(ms.r.tasksRunning.Load()) },
|
||||
))
|
||||
ms.r.metricsRegistry.MustRegister(prometheus.NewCounterFunc(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "runner",
|
||||
Name: "tasks_completed_total",
|
||||
Help: "Total number of tasks completed successfully since startup.",
|
||||
},
|
||||
func() float64 { return float64(ms.r.tasksCompleted.Load()) },
|
||||
))
|
||||
ms.r.metricsRegistry.MustRegister(prometheus.NewCounterFunc(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "runner",
|
||||
Name: "tasks_failed_total",
|
||||
Help: "Total number of tasks that ended in failure since startup.",
|
||||
},
|
||||
func() float64 { return float64(ms.r.tasksFailed.Load()) },
|
||||
))
|
||||
ms.r.metricsRegistry.MustRegister(prometheus.NewGaugeFunc(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "runner",
|
||||
Name: "uptime_seconds",
|
||||
Help: "Seconds since the runner process started.",
|
||||
},
|
||||
func() float64 { return time.Since(ms.r.startedAt).Seconds() },
|
||||
))
|
||||
}
|
||||
|
||||
// ListenAndServe starts the HTTP server and blocks until ctx is cancelled or
|
||||
// a fatal listen error occurs.
|
||||
func (ms *metricsServer) ListenAndServe(ctx context.Context) error {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("GET /metrics", promhttp.HandlerFor(ms.r.metricsRegistry, promhttp.HandlerOpts{}))
|
||||
mux.HandleFunc("GET /health", ms.handleHealth)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: ms.addr,
|
||||
Handler: mux,
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
BaseContext: func(_ net.Listener) context.Context { return ctx },
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
ms.log.Info("runner: metrics server listening", "addr", ms.addr)
|
||||
errCh <- srv.ListenAndServe()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = srv.Shutdown(shutCtx)
|
||||
return nil
|
||||
case err := <-errCh:
|
||||
return fmt.Errorf("runner: metrics server: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handleHealth handles GET /health — simple liveness probe.
|
||||
func (ms *metricsServer) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"status":"ok"}`))
|
||||
}
|
||||
@@ -8,6 +8,9 @@
|
||||
// - Audio tasks fetch chapter text, call Kokoro, upload to MinIO, and report
|
||||
// the result back (up to MaxConcurrentAudio goroutines).
|
||||
// - The runner is stateless between ticks; all state lives in PocketBase.
|
||||
// - Atomic task counters are exposed via /metrics (see metrics.go).
|
||||
// - Books are indexed in Meilisearch via an orchestrator.Config.PostMetadata
|
||||
// hook injected at construction time.
|
||||
package runner
|
||||
|
||||
import (
|
||||
@@ -16,14 +19,23 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
"github.com/libnovel/backend/internal/bookstore"
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
"github.com/libnovel/backend/internal/libretranslate"
|
||||
"github.com/libnovel/backend/internal/meili"
|
||||
"github.com/libnovel/backend/internal/orchestrator"
|
||||
"github.com/libnovel/backend/internal/pockettts"
|
||||
"github.com/libnovel/backend/internal/scraper"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// Config tunes the runner behaviour.
|
||||
@@ -31,22 +43,49 @@ type Config struct {
|
||||
// WorkerID uniquely identifies this runner instance in PocketBase records.
|
||||
WorkerID string
|
||||
// PollInterval is how often the runner checks for new tasks.
|
||||
// Only used in PocketBase-polling mode (RedisAddr == "").
|
||||
PollInterval time.Duration
|
||||
// MaxConcurrentScrape limits simultaneous book-scrape goroutines.
|
||||
MaxConcurrentScrape int
|
||||
// MaxConcurrentAudio limits simultaneous audio-generation goroutines.
|
||||
MaxConcurrentAudio int
|
||||
// MaxConcurrentTranslation limits simultaneous translation goroutines.
|
||||
MaxConcurrentTranslation int
|
||||
// OrchestratorWorkers is the chapter-scraping parallelism inside each book run.
|
||||
OrchestratorWorkers int
|
||||
// HeartbeatInterval is how often active tasks PATCH their heartbeat_at
|
||||
// timestamp to signal they are still alive. Defaults to 30s when 0.
|
||||
// Only used in PocketBase-polling mode.
|
||||
HeartbeatInterval time.Duration
|
||||
// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
|
||||
// task is considered orphaned and reset to pending. Defaults to 2m when 0.
|
||||
// Only used in PocketBase-polling mode.
|
||||
StaleTaskThreshold time.Duration
|
||||
// BrowseRefreshInterval is how often the runner pre-fetches browse page
|
||||
// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
|
||||
BrowseRefreshInterval time.Duration
|
||||
// CatalogueRefreshInterval is how often the runner walks the full catalogue,
|
||||
// scrapes per-book metadata, downloads covers, and re-indexes everything in
|
||||
// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
|
||||
CatalogueRefreshInterval time.Duration
|
||||
// CatalogueRequestDelay is the base inter-request pause during a catalogue
|
||||
// refresh metadata walk. Jitter of up to 50% is added on top.
|
||||
// Defaults to 2s. Set via RUNNER_CATALOGUE_REQUEST_DELAY.
|
||||
CatalogueRequestDelay time.Duration
|
||||
// SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that
|
||||
// otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval)
|
||||
// still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for
|
||||
// quick restarts where the catalogue is already up to date.
|
||||
SkipInitialCatalogueRefresh bool
|
||||
// MetricsAddr is the HTTP listen address for the /metrics endpoint.
|
||||
// Defaults to ":9091". Set to "" to disable.
|
||||
MetricsAddr string
|
||||
// RedisAddr is the address of the Redis instance used for Asynq task
|
||||
// dispatch. When set the runner switches from PocketBase-polling mode to
|
||||
// Asynq ServeMux mode (immediate task delivery, no polling).
|
||||
// Supports plain "host:port" or a full "rediss://..." URL.
|
||||
// When empty the runner falls back to PocketBase polling.
|
||||
RedisAddr string
|
||||
// RedisPassword is the Redis AUTH password.
|
||||
// Not required when RedisAddr is a full URL that includes credentials.
|
||||
RedisPassword string
|
||||
}
|
||||
|
||||
// Dependencies are the external services the runner depends on.
|
||||
@@ -59,12 +98,23 @@ type Dependencies struct {
|
||||
BookReader bookstore.BookReader
|
||||
// AudioStore persists generated audio and checks key existence.
|
||||
AudioStore bookstore.AudioStore
|
||||
// BrowseStore stores browse page snapshots in MinIO.
|
||||
BrowseStore bookstore.BrowseStore
|
||||
// TranslationStore persists translated markdown and checks key existence.
|
||||
TranslationStore bookstore.TranslationStore
|
||||
// CoverStore stores book cover images in MinIO.
|
||||
CoverStore bookstore.CoverStore
|
||||
// SearchIndex indexes books in Meilisearch after scraping.
|
||||
// If nil a no-op is used.
|
||||
SearchIndex meili.Client
|
||||
// Novel is the scraper implementation.
|
||||
Novel scraper.NovelScraper
|
||||
// Kokoro is the TTS client.
|
||||
// Kokoro is the Kokoro-FastAPI TTS client (GPU, OpenAI-compatible voices).
|
||||
Kokoro kokoro.Client
|
||||
// PocketTTS is the pocket-tts client (CPU, kyutai voices: alba, marius, etc.).
|
||||
// If nil, pocket-tts voice tasks will fail with a clear error.
|
||||
PocketTTS pockettts.Client
|
||||
// LibreTranslate is the machine translation client.
|
||||
// If nil, translation tasks will fail with a clear error.
|
||||
LibreTranslate libretranslate.Client
|
||||
// Log is the structured logger.
|
||||
Log *slog.Logger
|
||||
}
|
||||
@@ -73,10 +123,18 @@ type Dependencies struct {
|
||||
type Runner struct {
|
||||
cfg Config
|
||||
deps Dependencies
|
||||
|
||||
metricsRegistry *prometheus.Registry
|
||||
|
||||
// Atomic task counters — read by /metrics without locking.
|
||||
tasksRunning atomic.Int64
|
||||
tasksCompleted atomic.Int64
|
||||
tasksFailed atomic.Int64
|
||||
|
||||
startedAt time.Time
|
||||
}
|
||||
|
||||
// New creates a Runner from cfg and deps.
|
||||
// Any zero/nil field in deps will cause a panic at construction time to fail fast.
|
||||
func New(cfg Config, deps Dependencies) *Runner {
|
||||
if cfg.PollInterval <= 0 {
|
||||
cfg.PollInterval = 30 * time.Second
|
||||
@@ -87,6 +145,9 @@ func New(cfg Config, deps Dependencies) *Runner {
|
||||
if cfg.MaxConcurrentAudio <= 0 {
|
||||
cfg.MaxConcurrentAudio = 1
|
||||
}
|
||||
if cfg.MaxConcurrentTranslation <= 0 {
|
||||
cfg.MaxConcurrentTranslation = 1
|
||||
}
|
||||
if cfg.WorkerID == "" {
|
||||
cfg.WorkerID = "runner"
|
||||
}
|
||||
@@ -96,63 +157,90 @@ func New(cfg Config, deps Dependencies) *Runner {
|
||||
if cfg.StaleTaskThreshold <= 0 {
|
||||
cfg.StaleTaskThreshold = 2 * time.Minute
|
||||
}
|
||||
if cfg.BrowseRefreshInterval <= 0 {
|
||||
cfg.BrowseRefreshInterval = 6 * time.Hour
|
||||
if cfg.CatalogueRefreshInterval <= 0 {
|
||||
cfg.CatalogueRefreshInterval = 24 * time.Hour
|
||||
}
|
||||
if cfg.CatalogueRequestDelay <= 0 {
|
||||
cfg.CatalogueRequestDelay = 2 * time.Second
|
||||
}
|
||||
if cfg.MetricsAddr == "" {
|
||||
cfg.MetricsAddr = ":9091"
|
||||
}
|
||||
if deps.Log == nil {
|
||||
deps.Log = slog.Default()
|
||||
}
|
||||
return &Runner{cfg: cfg, deps: deps}
|
||||
}
|
||||
|
||||
// livenessFile is the path written on every successful poll so that the Docker
|
||||
// healthcheck (CMD /healthcheck file /tmp/runner.alive <max_age>) can verify
|
||||
// the runner is still making progress.
|
||||
const livenessFile = "/tmp/runner.alive"
|
||||
|
||||
// touchAlive writes the current UTC time to livenessFile. Errors are logged but
|
||||
// never fatal — liveness is best-effort and should not crash the runner.
|
||||
func (r *Runner) touchAlive() {
|
||||
data := []byte(time.Now().UTC().Format(time.RFC3339))
|
||||
if err := os.WriteFile(livenessFile, data, 0o644); err != nil {
|
||||
r.deps.Log.Warn("runner: failed to write liveness file", "err", err)
|
||||
if deps.SearchIndex == nil {
|
||||
deps.SearchIndex = meili.NoopClient{}
|
||||
}
|
||||
return &Runner{cfg: cfg, deps: deps, startedAt: time.Now(), metricsRegistry: prometheus.NewRegistry()}
|
||||
}
|
||||
|
||||
// Run starts the poll loop, blocking until ctx is cancelled.
|
||||
// On each tick it claims and executes all available pending tasks.
|
||||
// Scrape and audio tasks run in separate goroutine pools bounded by
|
||||
// MaxConcurrentScrape and MaxConcurrentAudio respectively.
|
||||
// Run starts the worker loop and the metrics HTTP server, blocking until ctx
|
||||
// is cancelled.
|
||||
//
|
||||
// When cfg.RedisAddr is set the runner uses Asynq (immediate task delivery).
|
||||
// Otherwise it falls back to PocketBase polling (legacy mode).
|
||||
func (r *Runner) Run(ctx context.Context) error {
|
||||
r.deps.Log.Info("runner: starting",
|
||||
"worker_id", r.cfg.WorkerID,
|
||||
"poll_interval", r.cfg.PollInterval,
|
||||
"mode", r.mode(),
|
||||
"max_scrape", r.cfg.MaxConcurrentScrape,
|
||||
"max_audio", r.cfg.MaxConcurrentAudio,
|
||||
"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
|
||||
"max_translation", r.cfg.MaxConcurrentTranslation,
|
||||
"catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval,
|
||||
"metrics_addr", r.cfg.MetricsAddr,
|
||||
)
|
||||
|
||||
// Start metrics HTTP server in background if configured.
|
||||
if r.cfg.MetricsAddr != "" {
|
||||
ms := newMetricsServer(r.cfg.MetricsAddr, r, r.deps.Log)
|
||||
go func() {
|
||||
if err := ms.ListenAndServe(ctx); err != nil {
|
||||
r.deps.Log.Error("runner: metrics server error", "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if r.cfg.RedisAddr != "" {
|
||||
return r.runAsynq(ctx)
|
||||
}
|
||||
return r.runPoll(ctx)
|
||||
}
|
||||
|
||||
// mode returns a short string describing the active dispatch mode.
|
||||
func (r *Runner) mode() string {
|
||||
if r.cfg.RedisAddr != "" {
|
||||
return "asynq"
|
||||
}
|
||||
return "poll"
|
||||
}
|
||||
|
||||
// runPoll is the legacy PocketBase-polling dispatch loop.
|
||||
// Used when cfg.RedisAddr is empty.
|
||||
func (r *Runner) runPoll(ctx context.Context) error {
|
||||
scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
|
||||
audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio)
|
||||
translationSem := make(chan struct{}, r.cfg.MaxConcurrentTranslation)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Write liveness file immediately so the first healthcheck passes before
|
||||
// the first poll completes.
|
||||
r.touchAlive()
|
||||
|
||||
tick := time.NewTicker(r.cfg.PollInterval)
|
||||
defer tick.Stop()
|
||||
|
||||
browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
|
||||
defer browseTick.Stop()
|
||||
catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval)
|
||||
defer catalogueTick.Stop()
|
||||
|
||||
// Run one browse refresh and one poll immediately on startup.
|
||||
go r.runBrowseRefresh(ctx)
|
||||
// Run one catalogue refresh immediately on startup (unless skipped by flag).
|
||||
if !r.cfg.SkipInitialCatalogueRefresh {
|
||||
go r.runCatalogueRefresh(ctx)
|
||||
} else {
|
||||
r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)")
|
||||
}
|
||||
|
||||
r.deps.Log.Info("runner: poll mode active", "poll_interval", r.cfg.PollInterval)
|
||||
|
||||
// Run one poll immediately on startup, then on each tick.
|
||||
for {
|
||||
r.poll(ctx, scrapeSem, audioSem, &wg)
|
||||
r.touchAlive()
|
||||
r.poll(ctx, scrapeSem, audioSem, translationSem, &wg)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -169,16 +257,24 @@ func (r *Runner) Run(ctx context.Context) error {
|
||||
r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
|
||||
}
|
||||
return nil
|
||||
case <-browseTick.C:
|
||||
go r.runBrowseRefresh(ctx)
|
||||
case <-catalogueTick.C:
|
||||
go r.runCatalogueRefresh(ctx)
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// poll claims all available pending tasks and dispatches them to goroutines.
|
||||
// It claims tasks in a tight loop until no more are available.
|
||||
func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) {
|
||||
func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem, translationSem chan struct{}, wg *sync.WaitGroup) {
|
||||
// ── Heartbeat file ────────────────────────────────────────────────────
|
||||
// Touch /tmp/runner.alive so the Docker health check can confirm the
|
||||
// runner is actively polling. Failure is non-fatal — just log it.
|
||||
if f, err := os.Create("/tmp/runner.alive"); err != nil {
|
||||
r.deps.Log.Warn("runner: could not write heartbeat file", "err", err)
|
||||
} else {
|
||||
f.Close()
|
||||
}
|
||||
|
||||
// ── Reap orphaned tasks ───────────────────────────────────────────────
|
||||
if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
|
||||
r.deps.Log.Warn("runner: reap stale tasks failed", "err", err)
|
||||
@@ -197,63 +293,123 @@ func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg
|
||||
break
|
||||
}
|
||||
if !ok {
|
||||
break // queue empty
|
||||
break
|
||||
}
|
||||
// Acquire semaphore (non-blocking when full — leave task running).
|
||||
select {
|
||||
case scrapeSem <- struct{}{}:
|
||||
default:
|
||||
// Too many concurrent scrapes — the task stays claimed but we can't
|
||||
// run it right now. Log and break; the next poll will pick it up if
|
||||
// still running (it won't be re-claimed while status=running).
|
||||
r.deps.Log.Warn("runner: scrape semaphore full, will retry next tick",
|
||||
"task_id", task.ID)
|
||||
break
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
wg.Add(1)
|
||||
go func(t domain.ScrapeTask) {
|
||||
defer wg.Done()
|
||||
defer func() { <-scrapeSem }()
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runScrapeTask(ctx, t)
|
||||
}(task)
|
||||
}
|
||||
|
||||
// ── Audio tasks ───────────────────────────────────────────────────────
|
||||
// Only claim tasks when there is a free slot in the semaphore.
|
||||
// This avoids the old bug where we claimed (status→running) a task and
|
||||
// then couldn't dispatch it, leaving it orphaned until the reaper fired.
|
||||
audioLoop:
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
// Check capacity before claiming to avoid orphaning tasks.
|
||||
select {
|
||||
case audioSem <- struct{}{}:
|
||||
// Slot acquired — proceed to claim a task.
|
||||
default:
|
||||
// All slots busy; leave remaining pending tasks for next tick.
|
||||
break audioLoop
|
||||
}
|
||||
task, ok, err := r.deps.Consumer.ClaimNextAudioTask(ctx, r.cfg.WorkerID)
|
||||
if err != nil {
|
||||
<-audioSem // release the pre-acquired slot
|
||||
r.deps.Log.Error("runner: ClaimNextAudioTask failed", "err", err)
|
||||
break
|
||||
}
|
||||
if !ok {
|
||||
break // queue empty
|
||||
}
|
||||
select {
|
||||
case audioSem <- struct{}{}:
|
||||
default:
|
||||
r.deps.Log.Warn("runner: audio semaphore full, will retry next tick",
|
||||
"task_id", task.ID)
|
||||
<-audioSem // release the pre-acquired slot; queue empty
|
||||
break
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
wg.Add(1)
|
||||
go func(t domain.AudioTask) {
|
||||
defer wg.Done()
|
||||
defer func() { <-audioSem }()
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runAudioTask(ctx, t)
|
||||
}(task)
|
||||
}
|
||||
|
||||
// ── Translation tasks ─────────────────────────────────────────────────
|
||||
translationLoop:
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case translationSem <- struct{}{}:
|
||||
// Slot acquired — proceed to claim a task.
|
||||
default:
|
||||
// All slots busy; leave remaining pending tasks for next tick.
|
||||
break translationLoop
|
||||
}
|
||||
task, ok, err := r.deps.Consumer.ClaimNextTranslationTask(ctx, r.cfg.WorkerID)
|
||||
if err != nil {
|
||||
<-translationSem
|
||||
r.deps.Log.Error("runner: ClaimNextTranslationTask failed", "err", err)
|
||||
break
|
||||
}
|
||||
if !ok {
|
||||
<-translationSem
|
||||
break
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
wg.Add(1)
|
||||
go func(t domain.TranslationTask) {
|
||||
defer wg.Done()
|
||||
defer func() { <-translationSem }()
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runTranslationTask(ctx, t)
|
||||
}(task)
|
||||
}
|
||||
}
|
||||
|
||||
// newOrchestrator builds an orchestrator with the Meilisearch post-hook wired in.
|
||||
func (r *Runner) newOrchestrator() *orchestrator.Orchestrator {
|
||||
oCfg := orchestrator.Config{
|
||||
Workers: r.cfg.OrchestratorWorkers,
|
||||
PostMetadata: func(ctx context.Context, meta domain.BookMeta) {
|
||||
if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
|
||||
r.deps.Log.Warn("runner: meilisearch upsert failed",
|
||||
"slug", meta.Slug, "err", err)
|
||||
}
|
||||
},
|
||||
}
|
||||
return orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log)
|
||||
}
|
||||
|
||||
// runScrapeTask executes one scrape task end-to-end and reports the result.
|
||||
func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
|
||||
ctx, span := otel.Tracer("runner").Start(ctx, "runner.scrape_task")
|
||||
defer span.End()
|
||||
span.SetAttributes(
|
||||
attribute.String("task.id", task.ID),
|
||||
attribute.String("task.kind", task.Kind),
|
||||
attribute.String("task.url", task.TargetURL),
|
||||
)
|
||||
|
||||
log := r.deps.Log.With("task_id", task.ID, "kind", task.Kind, "url", task.TargetURL)
|
||||
log.Info("runner: scrape task starting")
|
||||
|
||||
// Heartbeat goroutine: periodically PATCH heartbeat_at so the reaper knows
|
||||
// this task is still alive. Cancelled when the task finishes.
|
||||
hbCtx, hbCancel := context.WithCancel(ctx)
|
||||
defer hbCancel()
|
||||
go func() {
|
||||
@@ -271,9 +427,7 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
|
||||
}
|
||||
}()
|
||||
|
||||
oCfg := orchestrator.Config{Workers: r.cfg.OrchestratorWorkers}
|
||||
o := orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log)
|
||||
|
||||
o := r.newOrchestrator()
|
||||
var result domain.ScrapeResult
|
||||
|
||||
switch task.Kind {
|
||||
@@ -289,6 +443,15 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
|
||||
if err := r.deps.Consumer.FinishScrapeTask(ctx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishScrapeTask failed", "err", err)
|
||||
}
|
||||
|
||||
if result.ErrorMessage != "" {
|
||||
r.tasksFailed.Add(1)
|
||||
span.SetStatus(codes.Error, result.ErrorMessage)
|
||||
} else {
|
||||
r.tasksCompleted.Add(1)
|
||||
span.SetStatus(codes.Ok, "")
|
||||
}
|
||||
|
||||
log.Info("runner: scrape task finished",
|
||||
"scraped", result.ChaptersScraped,
|
||||
"skipped", result.ChaptersSkipped,
|
||||
@@ -296,8 +459,7 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
|
||||
)
|
||||
}
|
||||
|
||||
// runCatalogueTask runs a full catalogue scrape by iterating catalogue entries
|
||||
// and running a book task for each one.
|
||||
// runCatalogueTask runs a full catalogue scrape.
|
||||
func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o *orchestrator.Orchestrator, log *slog.Logger) domain.ScrapeResult {
|
||||
entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
|
||||
var result domain.ScrapeResult
|
||||
@@ -328,17 +490,20 @@ func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o
|
||||
return result
|
||||
}
|
||||
|
||||
// runAudioTask executes one audio-generation task:
|
||||
// 1. Read chapter text from MinIO.
|
||||
// 2. Call Kokoro to generate audio.
|
||||
// 3. Upload MP3 to MinIO under the standard audio object key.
|
||||
// 4. Report result back to PocketBase.
|
||||
// runAudioTask executes one audio-generation task.
|
||||
func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
|
||||
ctx, span := otel.Tracer("runner").Start(ctx, "runner.audio_task")
|
||||
defer span.End()
|
||||
span.SetAttributes(
|
||||
attribute.String("task.id", task.ID),
|
||||
attribute.String("book.slug", task.Slug),
|
||||
attribute.Int("chapter.number", task.Chapter),
|
||||
attribute.String("audio.voice", task.Voice),
|
||||
)
|
||||
|
||||
log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "voice", task.Voice)
|
||||
log.Info("runner: audio task starting")
|
||||
|
||||
// Heartbeat goroutine: periodically PATCH heartbeat_at so the reaper knows
|
||||
// this task is still alive. Cancelled when the task finishes.
|
||||
hbCtx, hbCancel := context.WithCancel(ctx)
|
||||
defer hbCancel()
|
||||
go func() {
|
||||
@@ -358,13 +523,14 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
|
||||
|
||||
fail := func(msg string) {
|
||||
log.Error("runner: audio task failed", "reason", msg)
|
||||
r.tasksFailed.Add(1)
|
||||
span.SetStatus(codes.Error, msg)
|
||||
result := domain.AudioResult{ErrorMessage: msg}
|
||||
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishAudioTask failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 1: read chapter text.
|
||||
raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter)
|
||||
if err != nil {
|
||||
fail(fmt.Sprintf("read chapter: %v", err))
|
||||
@@ -376,25 +542,41 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
|
||||
return
|
||||
}
|
||||
|
||||
// Step 2: generate audio.
|
||||
if r.deps.Kokoro == nil {
|
||||
fail("kokoro client not configured")
|
||||
return
|
||||
}
|
||||
audioData, err := r.deps.Kokoro.GenerateAudio(ctx, text, task.Voice)
|
||||
if err != nil {
|
||||
fail(fmt.Sprintf("kokoro generate: %v", err))
|
||||
return
|
||||
var audioData []byte
|
||||
if pockettts.IsPocketTTSVoice(task.Voice) {
|
||||
if r.deps.PocketTTS == nil {
|
||||
fail("pocket-tts client not configured (POCKET_TTS_URL is empty)")
|
||||
return
|
||||
}
|
||||
var genErr error
|
||||
audioData, genErr = r.deps.PocketTTS.GenerateAudio(ctx, text, task.Voice)
|
||||
if genErr != nil {
|
||||
fail(fmt.Sprintf("pocket-tts generate: %v", genErr))
|
||||
return
|
||||
}
|
||||
log.Info("runner: audio generated via pocket-tts", "voice", task.Voice)
|
||||
} else {
|
||||
if r.deps.Kokoro == nil {
|
||||
fail("kokoro client not configured (KOKORO_URL is empty)")
|
||||
return
|
||||
}
|
||||
var genErr error
|
||||
audioData, genErr = r.deps.Kokoro.GenerateAudio(ctx, text, task.Voice)
|
||||
if genErr != nil {
|
||||
fail(fmt.Sprintf("kokoro generate: %v", genErr))
|
||||
return
|
||||
}
|
||||
log.Info("runner: audio generated via kokoro-fastapi", "voice", task.Voice)
|
||||
}
|
||||
|
||||
// Step 3: upload to MinIO.
|
||||
key := r.deps.AudioStore.AudioObjectKey(task.Slug, task.Chapter, task.Voice)
|
||||
if err := r.deps.AudioStore.PutAudio(ctx, key, audioData); err != nil {
|
||||
fail(fmt.Sprintf("put audio: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// Step 4: report success.
|
||||
r.tasksCompleted.Add(1)
|
||||
span.SetStatus(codes.Ok, "")
|
||||
result := domain.AudioResult{ObjectKey: key}
|
||||
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishAudioTask failed", "err", err)
|
||||
|
||||
@@ -48,6 +48,10 @@ func (s *stubConsumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.A
|
||||
return t, true, nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) ClaimNextTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) {
|
||||
return domain.TranslationTask{}, false, nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) FinishScrapeTask(_ context.Context, id string, _ domain.ScrapeResult) error {
|
||||
s.finished = append(s.finished, id)
|
||||
return nil
|
||||
@@ -58,6 +62,11 @@ func (s *stubConsumer) FinishAudioTask(_ context.Context, id string, _ domain.Au
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) FinishTranslationTask(_ context.Context, id string, _ domain.TranslationResult) error {
|
||||
s.finished = append(s.finished, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) FailTask(_ context.Context, id, _ string) error {
|
||||
s.failCalled = append(s.failCalled, id)
|
||||
return nil
|
||||
@@ -146,7 +155,7 @@ func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.B
|
||||
return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil
|
||||
}
|
||||
|
||||
func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
|
||||
func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) {
|
||||
return s.chapters, nil
|
||||
}
|
||||
|
||||
|
||||
97
backend/internal/runner/translation.go
Normal file
97
backend/internal/runner/translation.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
)
|
||||
|
||||
// runTranslationTask executes one machine-translation task end-to-end and
|
||||
// reports the result back to PocketBase.
|
||||
func (r *Runner) runTranslationTask(ctx context.Context, task domain.TranslationTask) {
|
||||
ctx, span := otel.Tracer("runner").Start(ctx, "runner.translation_task")
|
||||
defer span.End()
|
||||
span.SetAttributes(
|
||||
attribute.String("task.id", task.ID),
|
||||
attribute.String("book.slug", task.Slug),
|
||||
attribute.Int("chapter.number", task.Chapter),
|
||||
attribute.String("translation.lang", task.Lang),
|
||||
)
|
||||
|
||||
log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "lang", task.Lang)
|
||||
log.Info("runner: translation task starting")
|
||||
|
||||
// Heartbeat goroutine — keeps the task alive while translation runs.
|
||||
hbCtx, hbCancel := context.WithCancel(ctx)
|
||||
defer hbCancel()
|
||||
go func() {
|
||||
tick := time.NewTicker(r.cfg.HeartbeatInterval)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-hbCtx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil {
|
||||
log.Warn("runner: heartbeat failed", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
fail := func(msg string) {
|
||||
log.Error("runner: translation task failed", "reason", msg)
|
||||
r.tasksFailed.Add(1)
|
||||
span.SetStatus(codes.Error, msg)
|
||||
result := domain.TranslationResult{ErrorMessage: msg}
|
||||
if err := r.deps.Consumer.FinishTranslationTask(ctx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishTranslationTask failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Guard: LibreTranslate must be configured.
|
||||
if r.deps.LibreTranslate == nil {
|
||||
fail("libretranslate client not configured (LIBRETRANSLATE_URL is empty)")
|
||||
return
|
||||
}
|
||||
|
||||
// 1. Read raw markdown chapter.
|
||||
raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter)
|
||||
if err != nil {
|
||||
fail(fmt.Sprintf("read chapter: %v", err))
|
||||
return
|
||||
}
|
||||
if raw == "" {
|
||||
fail("chapter text is empty")
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Translate (chunked, concurrent).
|
||||
translated, err := r.deps.LibreTranslate.Translate(ctx, raw, "en", task.Lang)
|
||||
if err != nil {
|
||||
fail(fmt.Sprintf("translate: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Store translated markdown in MinIO.
|
||||
key := r.deps.TranslationStore.TranslationObjectKey(task.Lang, task.Slug, task.Chapter)
|
||||
if err := r.deps.TranslationStore.PutTranslation(ctx, key, []byte(translated)); err != nil {
|
||||
fail(fmt.Sprintf("put translation: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// 4. Report success.
|
||||
r.tasksCompleted.Add(1)
|
||||
span.SetStatus(codes.Ok, "")
|
||||
result := domain.TranslationResult{ObjectKey: key}
|
||||
if err := r.deps.Consumer.FinishTranslationTask(ctx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishTranslationTask failed", "err", err)
|
||||
}
|
||||
log.Info("runner: translation task finished", "key", key)
|
||||
}
|
||||
@@ -20,8 +20,10 @@ type MetadataProvider interface {
|
||||
}
|
||||
|
||||
// ChapterListProvider can enumerate all chapters of a book.
|
||||
// upTo > 0 stops pagination once at least upTo chapter numbers have been
|
||||
// collected (early-exit optimisation for range scrapes). upTo == 0 fetches all pages.
|
||||
type ChapterListProvider interface {
|
||||
ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error)
|
||||
ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error)
|
||||
}
|
||||
|
||||
// ChapterTextProvider can extract the readable text from a single chapter page.
|
||||
|
||||
@@ -17,12 +17,13 @@ import (
|
||||
|
||||
// minioClient wraps the official minio-go client with bucket names.
|
||||
type minioClient struct {
|
||||
client *minio.Client // internal — all read/write operations
|
||||
pubClient *minio.Client // presign-only — initialised against the public endpoint
|
||||
bucketChapters string
|
||||
bucketAudio string
|
||||
bucketAvatars string
|
||||
bucketBrowse string
|
||||
client *minio.Client // internal — all read/write operations
|
||||
pubClient *minio.Client // presign-only — initialised against the public endpoint
|
||||
bucketChapters string
|
||||
bucketAudio string
|
||||
bucketAvatars string
|
||||
bucketBrowse string
|
||||
bucketTranslations string
|
||||
}
|
||||
|
||||
func newMinioClient(cfg config.MinIO) (*minioClient, error) {
|
||||
@@ -74,18 +75,19 @@ func newMinioClient(cfg config.MinIO) (*minioClient, error) {
|
||||
}
|
||||
|
||||
return &minioClient{
|
||||
client: internal,
|
||||
pubClient: pub,
|
||||
bucketChapters: cfg.BucketChapters,
|
||||
bucketAudio: cfg.BucketAudio,
|
||||
bucketAvatars: cfg.BucketAvatars,
|
||||
bucketBrowse: cfg.BucketBrowse,
|
||||
client: internal,
|
||||
pubClient: pub,
|
||||
bucketChapters: cfg.BucketChapters,
|
||||
bucketAudio: cfg.BucketAudio,
|
||||
bucketAvatars: cfg.BucketAvatars,
|
||||
bucketBrowse: cfg.BucketBrowse,
|
||||
bucketTranslations: cfg.BucketTranslations,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ensureBuckets creates all required buckets if they don't already exist.
|
||||
func (m *minioClient) ensureBuckets(ctx context.Context) error {
|
||||
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse} {
|
||||
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse, m.bucketTranslations} {
|
||||
exists, err := m.client.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: check bucket %q: %w", bucket, err)
|
||||
@@ -119,10 +121,16 @@ func AvatarObjectKey(userID, ext string) string {
|
||||
return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
|
||||
}
|
||||
|
||||
// BrowseObjectKey returns the MinIO object key for a cached browse page snapshot.
|
||||
// Format: browse/{genre}/{sort}/{status}/{type}/page-{n}.json
|
||||
func BrowseObjectKey(genre, sort, status, novelType string, page int) string {
|
||||
return fmt.Sprintf("browse/%s/%s/%s/%s/page-%d.json", genre, sort, status, novelType, page)
|
||||
// CoverObjectKey returns the MinIO object key for a book cover image.
|
||||
// Format: covers/{slug}.jpg
|
||||
func CoverObjectKey(slug string) string {
|
||||
return fmt.Sprintf("covers/%s.jpg", slug)
|
||||
}
|
||||
|
||||
// TranslationObjectKey returns the MinIO object key for a translated chapter.
|
||||
// Format: {lang}/{slug}/{n:06d}.md
|
||||
func TranslationObjectKey(lang, slug string, n int) string {
|
||||
return fmt.Sprintf("%s/%s/%06d.md", lang, slug, n)
|
||||
}
|
||||
|
||||
// chapterNumberFromKey extracts the chapter number from a MinIO object key.
|
||||
@@ -201,16 +209,16 @@ func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string)
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// ── Browse operations ─────────────────────────────────────────────────────────
|
||||
// ── Cover operations ──────────────────────────────────────────────────────────
|
||||
|
||||
// putBrowse stores raw JSON bytes for a browse page snapshot.
|
||||
func (m *minioClient) putBrowse(ctx context.Context, key string, data []byte) error {
|
||||
return m.putObject(ctx, m.bucketBrowse, key, "application/json", data)
|
||||
// putCover stores a raw cover image in the browse bucket under covers/{slug}.jpg.
|
||||
func (m *minioClient) putCover(ctx context.Context, key, contentType string, data []byte) error {
|
||||
return m.putObject(ctx, m.bucketBrowse, key, contentType, data)
|
||||
}
|
||||
|
||||
// getBrowse retrieves a browse page snapshot. Returns (nil, false, nil) when
|
||||
// the object does not exist.
|
||||
func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool, error) {
|
||||
// getCover retrieves a cover image. Returns (nil, "", false, nil) when the
|
||||
// object does not exist.
|
||||
func (m *minioClient) getCover(ctx context.Context, key string) ([]byte, bool, error) {
|
||||
if !m.objectExists(ctx, m.bucketBrowse, key) {
|
||||
return nil, false, nil
|
||||
}
|
||||
@@ -220,3 +228,25 @@ func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool,
|
||||
}
|
||||
return data, true, nil
|
||||
}
|
||||
|
||||
// coverExists returns true when the cover image object exists.
|
||||
func (m *minioClient) coverExists(ctx context.Context, key string) bool {
|
||||
return m.objectExists(ctx, m.bucketBrowse, key)
|
||||
}
|
||||
|
||||
// coverContentType inspects the first bytes of data to determine if it is
|
||||
// a JPEG or PNG image. Falls back to "image/jpeg".
|
||||
func coverContentType(data []byte) string {
|
||||
if len(data) >= 4 {
|
||||
// PNG magic: 0x89 0x50 0x4E 0x47
|
||||
if data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E && data[3] == 0x47 {
|
||||
return "image/png"
|
||||
}
|
||||
// WebP: starts with "RIFF" at 0..3 and "WEBP" at 8..11
|
||||
if len(data) >= 12 && data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' &&
|
||||
data[8] == 'W' && data[9] == 'E' && data[10] == 'B' && data[11] == 'P' {
|
||||
return "image/webp"
|
||||
}
|
||||
}
|
||||
return "image/jpeg"
|
||||
}
|
||||
|
||||
@@ -247,8 +247,9 @@ func (c *pbClient) claimRecord(ctx context.Context, collection, workerID string,
|
||||
}
|
||||
|
||||
claim := map[string]any{
|
||||
"status": string(domain.TaskStatusRunning),
|
||||
"worker_id": workerID,
|
||||
"status": string(domain.TaskStatusRunning),
|
||||
"worker_id": workerID,
|
||||
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
for k, v := range extraClaim {
|
||||
claim[k] = v
|
||||
|
||||
@@ -50,7 +50,8 @@ var _ bookstore.RankingStore = (*Store)(nil)
|
||||
var _ bookstore.AudioStore = (*Store)(nil)
|
||||
var _ bookstore.PresignStore = (*Store)(nil)
|
||||
var _ bookstore.ProgressStore = (*Store)(nil)
|
||||
var _ bookstore.BrowseStore = (*Store)(nil)
|
||||
var _ bookstore.CoverStore = (*Store)(nil)
|
||||
var _ bookstore.TranslationStore = (*Store)(nil)
|
||||
var _ taskqueue.Producer = (*Store)(nil)
|
||||
var _ taskqueue.Consumer = (*Store)(nil)
|
||||
var _ taskqueue.Reader = (*Store)(nil)
|
||||
@@ -69,6 +70,7 @@ func (s *Store) WriteMetadata(ctx context.Context, meta domain.BookMeta) error {
|
||||
"total_chapters": meta.TotalChapters,
|
||||
"source_url": meta.SourceURL,
|
||||
"ranking": meta.Ranking,
|
||||
"rating": meta.Rating,
|
||||
}
|
||||
// Upsert via filter: if exists PATCH, otherwise POST.
|
||||
existing, err := s.getBookBySlug(ctx, meta.Slug)
|
||||
@@ -138,10 +140,15 @@ type pbBook struct {
|
||||
TotalChapters int `json:"total_chapters"`
|
||||
SourceURL string `json:"source_url"`
|
||||
Ranking int `json:"ranking"`
|
||||
Rating float64 `json:"rating"`
|
||||
Updated string `json:"updated"`
|
||||
}
|
||||
|
||||
func (b pbBook) toDomain() domain.BookMeta {
|
||||
var metaUpdated int64
|
||||
if t, err := time.Parse(time.RFC3339, b.Updated); err == nil {
|
||||
metaUpdated = t.Unix()
|
||||
}
|
||||
return domain.BookMeta{
|
||||
Slug: b.Slug,
|
||||
Title: b.Title,
|
||||
@@ -153,6 +160,8 @@ func (b pbBook) toDomain() domain.BookMeta {
|
||||
TotalChapters: b.TotalChapters,
|
||||
SourceURL: b.SourceURL,
|
||||
Ranking: b.Ranking,
|
||||
Rating: b.Rating,
|
||||
MetaUpdated: metaUpdated,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -401,6 +410,17 @@ func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bo
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
func (s *Store) PutAvatar(ctx context.Context, userID, ext, contentType string, data []byte) (string, error) {
|
||||
// Delete existing avatar objects for this user before writing the new one
|
||||
// so old extensions don't linger (e.g. old .png after uploading a .jpg).
|
||||
_ = s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
|
||||
key := AvatarObjectKey(userID, ext)
|
||||
if err := s.mc.putObject(ctx, s.mc.bucketAvatars, key, contentType, data); err != nil {
|
||||
return "", fmt.Errorf("put avatar: %w", err)
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func (s *Store) DeleteAvatar(ctx context.Context, userID string) error {
|
||||
return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
|
||||
}
|
||||
@@ -516,13 +536,36 @@ func (s *Store) CreateAudioTask(ctx context.Context, slug string, chapter int, v
|
||||
return rec.ID, nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateTranslationTask(ctx context.Context, slug string, chapter int, lang string) (string, error) {
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, lang)
|
||||
payload := map[string]any{
|
||||
"cache_key": cacheKey,
|
||||
"slug": slug,
|
||||
"chapter": chapter,
|
||||
"lang": lang,
|
||||
"status": string(domain.TaskStatusPending),
|
||||
"started": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := s.pb.post(ctx, "/api/collections/translation_jobs/records", payload, &rec); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rec.ID, nil
|
||||
}
|
||||
|
||||
func (s *Store) CancelTask(ctx context.Context, id string) error {
|
||||
// Try scraping_tasks first, then audio_jobs.
|
||||
// Try scraping_tasks first, then audio_jobs, then translation_jobs.
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),
|
||||
map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id),
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id),
|
||||
map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id),
|
||||
map[string]string{"status": string(domain.TaskStatusCancelled)})
|
||||
}
|
||||
|
||||
@@ -552,6 +595,18 @@ func (s *Store) ClaimNextAudioTask(ctx context.Context, workerID string) (domain
|
||||
return task, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) ClaimNextTranslationTask(ctx context.Context, workerID string) (domain.TranslationTask, bool, error) {
|
||||
raw, err := s.pb.claimRecord(ctx, "translation_jobs", workerID, nil)
|
||||
if err != nil {
|
||||
return domain.TranslationTask{}, false, err
|
||||
}
|
||||
if raw == nil {
|
||||
return domain.TranslationTask{}, false, nil
|
||||
}
|
||||
task, err := parseTranslationTask(raw)
|
||||
return task, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
|
||||
status := string(domain.TaskStatusDone)
|
||||
if result.ErrorMessage != "" {
|
||||
@@ -580,6 +635,18 @@ func (s *Store) FinishAudioTask(ctx context.Context, id string, result domain.Au
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) FinishTranslationTask(ctx context.Context, id string, result domain.TranslationResult) error {
|
||||
status := string(domain.TaskStatusDone)
|
||||
if result.ErrorMessage != "" {
|
||||
status = string(domain.TaskStatusFailed)
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), map[string]any{
|
||||
"status": status,
|
||||
"error_message": result.ErrorMessage,
|
||||
"finished": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
|
||||
payload := map[string]any{
|
||||
"status": string(domain.TaskStatusFailed),
|
||||
@@ -589,11 +656,14 @@ func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), payload)
|
||||
}
|
||||
|
||||
// HeartbeatTask updates the heartbeat_at field on a running task.
|
||||
// Tries scraping_tasks first, then audio_jobs (same pattern as FailTask).
|
||||
// Tries scraping_tasks first, then audio_jobs, then translation_jobs.
|
||||
func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
|
||||
payload := map[string]any{
|
||||
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
|
||||
@@ -601,7 +671,10 @@ func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), payload)
|
||||
}
|
||||
|
||||
// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing
|
||||
@@ -619,7 +692,7 @@ func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (i
|
||||
}
|
||||
|
||||
total := 0
|
||||
for _, collection := range []string{"scraping_tasks", "audio_jobs"} {
|
||||
for _, collection := range []string{"scraping_tasks", "audio_jobs", "translation_jobs"} {
|
||||
items, err := s.pb.listAll(ctx, collection, filter, "")
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err)
|
||||
@@ -687,7 +760,7 @@ func (s *Store) ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error)
|
||||
}
|
||||
|
||||
func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) {
|
||||
filter := fmt.Sprintf(`cache_key=%q`, cacheKey)
|
||||
filter := fmt.Sprintf(`cache_key='%s'`, cacheKey)
|
||||
items, err := s.pb.listAll(ctx, "audio_jobs", filter, "-started")
|
||||
if err != nil || len(items) == 0 {
|
||||
return domain.AudioTask{}, false, err
|
||||
@@ -696,6 +769,31 @@ func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.Audio
|
||||
return t, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) ListTranslationTasks(ctx context.Context) ([]domain.TranslationTask, error) {
|
||||
items, err := s.pb.listAll(ctx, "translation_jobs", "", "-started")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks := make([]domain.TranslationTask, 0, len(items))
|
||||
for _, raw := range items {
|
||||
t, err := parseTranslationTask(raw)
|
||||
if err == nil {
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetTranslationTask(ctx context.Context, cacheKey string) (domain.TranslationTask, bool, error) {
|
||||
filter := fmt.Sprintf(`cache_key='%s'`, cacheKey)
|
||||
items, err := s.pb.listAll(ctx, "translation_jobs", filter, "-started")
|
||||
if err != nil || len(items) == 0 {
|
||||
return domain.TranslationTask{}, false, err
|
||||
}
|
||||
t, err := parseTranslationTask(items[0])
|
||||
return t, err == nil, err
|
||||
}
|
||||
|
||||
// ── Parsers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) {
|
||||
@@ -770,21 +868,86 @@ func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ── BrowseStore ────────────────────────────────────────────────────────────────
|
||||
func parseTranslationTask(raw json.RawMessage) (domain.TranslationTask, error) {
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
CacheKey string `json:"cache_key"`
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
Lang string `json:"lang"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
Status string `json:"status"`
|
||||
ErrorMessage string `json:"error_message"`
|
||||
Started string `json:"started"`
|
||||
Finished string `json:"finished"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &rec); err != nil {
|
||||
return domain.TranslationTask{}, err
|
||||
}
|
||||
started, _ := time.Parse(time.RFC3339, rec.Started)
|
||||
finished, _ := time.Parse(time.RFC3339, rec.Finished)
|
||||
return domain.TranslationTask{
|
||||
ID: rec.ID,
|
||||
CacheKey: rec.CacheKey,
|
||||
Slug: rec.Slug,
|
||||
Chapter: rec.Chapter,
|
||||
Lang: rec.Lang,
|
||||
WorkerID: rec.WorkerID,
|
||||
Status: domain.TaskStatus(rec.Status),
|
||||
ErrorMessage: rec.ErrorMessage,
|
||||
Started: started,
|
||||
Finished: finished,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error {
|
||||
key := BrowseObjectKey(genre, sort, status, novelType, page)
|
||||
if err := s.mc.putBrowse(ctx, key, data); err != nil {
|
||||
return fmt.Errorf("PutBrowsePage: %w", err)
|
||||
// ── CoverStore ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) PutCover(ctx context.Context, slug string, data []byte, contentType string) error {
|
||||
key := CoverObjectKey(slug)
|
||||
if contentType == "" {
|
||||
contentType = coverContentType(data)
|
||||
}
|
||||
if err := s.mc.putCover(ctx, key, contentType, data); err != nil {
|
||||
return fmt.Errorf("PutCover: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) {
|
||||
key := BrowseObjectKey(genre, sort, status, novelType, page)
|
||||
data, ok, err := s.mc.getBrowse(ctx, key)
|
||||
func (s *Store) GetCover(ctx context.Context, slug string) ([]byte, string, bool, error) {
|
||||
key := CoverObjectKey(slug)
|
||||
data, ok, err := s.mc.getCover(ctx, key)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("GetBrowsePage: %w", err)
|
||||
return nil, "", false, fmt.Errorf("GetCover: %w", err)
|
||||
}
|
||||
return data, ok, nil
|
||||
if !ok {
|
||||
return nil, "", false, nil
|
||||
}
|
||||
ct := coverContentType(data)
|
||||
return data, ct, true, nil
|
||||
}
|
||||
|
||||
func (s *Store) CoverExists(ctx context.Context, slug string) bool {
|
||||
return s.mc.coverExists(ctx, CoverObjectKey(slug))
|
||||
}
|
||||
|
||||
// ── TranslationStore ───────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) TranslationObjectKey(lang, slug string, n int) string {
|
||||
return TranslationObjectKey(lang, slug, n)
|
||||
}
|
||||
|
||||
func (s *Store) TranslationExists(ctx context.Context, key string) bool {
|
||||
return s.mc.objectExists(ctx, s.mc.bucketTranslations, key)
|
||||
}
|
||||
|
||||
func (s *Store) PutTranslation(ctx context.Context, key string, data []byte) error {
|
||||
return s.mc.putObject(ctx, s.mc.bucketTranslations, key, "text/markdown; charset=utf-8", data)
|
||||
}
|
||||
|
||||
func (s *Store) GetTranslation(ctx context.Context, key string) (string, error) {
|
||||
data, err := s.mc.getObject(ctx, s.mc.bucketTranslations, key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("GetTranslation: %w", err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
@@ -29,6 +29,10 @@ type Producer interface {
|
||||
// returns the assigned PocketBase record ID.
|
||||
CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error)
|
||||
|
||||
// CreateTranslationTask inserts a new translation task with status=pending and
|
||||
// returns the assigned PocketBase record ID.
|
||||
CreateTranslationTask(ctx context.Context, slug string, chapter int, lang string) (string, error)
|
||||
|
||||
// CancelTask transitions a pending task to status=cancelled.
|
||||
// Returns ErrNotFound if the task does not exist.
|
||||
CancelTask(ctx context.Context, id string) error
|
||||
@@ -46,13 +50,21 @@ type Consumer interface {
|
||||
// Returns (zero, false, nil) when the queue is empty.
|
||||
ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error)
|
||||
|
||||
// ClaimNextTranslationTask atomically finds the oldest pending translation task,
|
||||
// sets its status=running and worker_id=workerID, and returns it.
|
||||
// Returns (zero, false, nil) when the queue is empty.
|
||||
ClaimNextTranslationTask(ctx context.Context, workerID string) (domain.TranslationTask, bool, error)
|
||||
|
||||
// FinishScrapeTask marks a running scrape task as done and records the result.
|
||||
FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error
|
||||
|
||||
// FinishAudioTask marks a running audio task as done and records the result.
|
||||
FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error
|
||||
|
||||
// FailTask marks a task (scrape or audio) as failed with an error message.
|
||||
// FinishTranslationTask marks a running translation task as done and records the result.
|
||||
FinishTranslationTask(ctx context.Context, id string, result domain.TranslationResult) error
|
||||
|
||||
// FailTask marks a task (scrape, audio, or translation) as failed with an error message.
|
||||
FailTask(ctx context.Context, id, errMsg string) error
|
||||
|
||||
// HeartbeatTask updates the heartbeat_at timestamp on a running task.
|
||||
@@ -81,4 +93,11 @@ type Reader interface {
|
||||
// GetAudioTask returns the most recent audio task for cacheKey.
|
||||
// Returns (zero, false, nil) if not found.
|
||||
GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error)
|
||||
|
||||
// ListTranslationTasks returns all translation tasks sorted by started descending.
|
||||
ListTranslationTasks(ctx context.Context) ([]domain.TranslationTask, error)
|
||||
|
||||
// GetTranslationTask returns the most recent translation task for cacheKey.
|
||||
// Returns (zero, false, nil) if not found.
|
||||
GetTranslationTask(ctx context.Context, cacheKey string) (domain.TranslationTask, bool, error)
|
||||
}
|
||||
|
||||
@@ -23,6 +23,9 @@ func (s *stubStore) CreateScrapeTask(_ context.Context, _, _ string, _, _ int) (
|
||||
func (s *stubStore) CreateAudioTask(_ context.Context, _ string, _ int, _ string) (string, error) {
|
||||
return "audio-1", nil
|
||||
}
|
||||
func (s *stubStore) CreateTranslationTask(_ context.Context, _ string, _ int, _ string) (string, error) {
|
||||
return "translation-1", nil
|
||||
}
|
||||
func (s *stubStore) CancelTask(_ context.Context, _ string) error { return nil }
|
||||
|
||||
func (s *stubStore) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
|
||||
@@ -31,12 +34,18 @@ func (s *stubStore) ClaimNextScrapeTask(_ context.Context, _ string) (domain.Scr
|
||||
func (s *stubStore) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
|
||||
return domain.AudioTask{ID: "audio-1", Status: domain.TaskStatusRunning}, true, nil
|
||||
}
|
||||
func (s *stubStore) ClaimNextTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) {
|
||||
return domain.TranslationTask{ID: "translation-1", Status: domain.TaskStatusRunning}, true, nil
|
||||
}
|
||||
func (s *stubStore) FinishScrapeTask(_ context.Context, _ string, _ domain.ScrapeResult) error {
|
||||
return nil
|
||||
}
|
||||
func (s *stubStore) FinishAudioTask(_ context.Context, _ string, _ domain.AudioResult) error {
|
||||
return nil
|
||||
}
|
||||
func (s *stubStore) FinishTranslationTask(_ context.Context, _ string, _ domain.TranslationResult) error {
|
||||
return nil
|
||||
}
|
||||
func (s *stubStore) FailTask(_ context.Context, _, _ string) error { return nil }
|
||||
|
||||
func (s *stubStore) HeartbeatTask(_ context.Context, _ string) error { return nil }
|
||||
@@ -53,6 +62,12 @@ func (s *stubStore) ListAudioTasks(_ context.Context) ([]domain.AudioTask, error
|
||||
func (s *stubStore) GetAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
|
||||
return domain.AudioTask{}, false, nil
|
||||
}
|
||||
func (s *stubStore) ListTranslationTasks(_ context.Context) ([]domain.TranslationTask, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *stubStore) GetTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) {
|
||||
return domain.TranslationTask{}, false, nil
|
||||
}
|
||||
|
||||
// Verify the stub satisfies all three interfaces at compile time.
|
||||
var _ taskqueue.Producer = (*stubStore)(nil)
|
||||
|
||||
BIN
backend/runner
Executable file
BIN
backend/runner
Executable file
Binary file not shown.
9
caddy/Dockerfile
Normal file
9
caddy/Dockerfile
Normal file
@@ -0,0 +1,9 @@
|
||||
FROM caddy:2-builder AS builder
|
||||
|
||||
RUN xcaddy build \
|
||||
--with github.com/mholt/caddy-ratelimit \
|
||||
--with github.com/hslatman/caddy-crowdsec-bouncer/http \
|
||||
--with github.com/mholt/caddy-l4
|
||||
|
||||
FROM caddy:2-alpine
|
||||
COPY --from=builder /usr/bin/caddy /usr/bin/caddy
|
||||
139
caddy/errors/404.html
Normal file
139
caddy/errors/404.html
Normal file
@@ -0,0 +1,139 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>404 — Page Not Found — LibNovel</title>
|
||||
<style>
|
||||
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
|
||||
html, body {
|
||||
height: 100%;
|
||||
background: #09090b;
|
||||
}
|
||||
|
||||
body {
|
||||
min-height: 100svh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
font-family: ui-sans-serif, system-ui, sans-serif;
|
||||
color: #a1a1aa;
|
||||
}
|
||||
|
||||
header {
|
||||
padding: 1.5rem 2rem;
|
||||
border-bottom: 1px solid #27272a;
|
||||
}
|
||||
.logo {
|
||||
font-size: 1.125rem;
|
||||
font-weight: 700;
|
||||
color: #e4e4e7;
|
||||
letter-spacing: -0.02em;
|
||||
text-decoration: none;
|
||||
}
|
||||
.logo span { color: #f59e0b; }
|
||||
|
||||
main {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 3rem 2rem;
|
||||
text-align: center;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
.watermark {
|
||||
font-size: clamp(5rem, 22vw, 9rem);
|
||||
font-weight: 800;
|
||||
color: #18181b;
|
||||
line-height: 1;
|
||||
letter-spacing: -0.04em;
|
||||
user-select: none;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.status-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 1.25rem;
|
||||
}
|
||||
.dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: #71717a;
|
||||
}
|
||||
.status-label {
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.08em;
|
||||
color: #71717a;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 1.5rem;
|
||||
font-weight: 700;
|
||||
color: #e4e4e7;
|
||||
letter-spacing: -0.02em;
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
p {
|
||||
font-size: 0.9375rem;
|
||||
max-width: 38ch;
|
||||
line-height: 1.65;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.btn {
|
||||
display: inline-block;
|
||||
padding: 0.625rem 1.5rem;
|
||||
border-radius: 0.5rem;
|
||||
background: #f59e0b;
|
||||
color: #000;
|
||||
font-weight: 600;
|
||||
font-size: 0.875rem;
|
||||
text-decoration: none;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
.btn:hover { background: #d97706; }
|
||||
|
||||
footer {
|
||||
padding: 1.5rem 2rem;
|
||||
border-top: 1px solid #27272a;
|
||||
text-align: center;
|
||||
font-size: 0.8rem;
|
||||
color: #3f3f46;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<a class="logo" href="/">Lib<span>Novel</span></a>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="watermark">404</div>
|
||||
|
||||
<div class="status-row">
|
||||
<div class="dot"></div>
|
||||
<span class="status-label">Page not found</span>
|
||||
</div>
|
||||
|
||||
<h1>Nothing here</h1>
|
||||
<p>The page you're looking for doesn't exist or has been moved.</p>
|
||||
|
||||
<a class="btn" href="/">Go home</a>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
© LibNovel
|
||||
</footer>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
162
caddy/errors/502.html
Normal file
162
caddy/errors/502.html
Normal file
@@ -0,0 +1,162 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>502 — Service Unavailable — LibNovel</title>
|
||||
<style>
|
||||
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
|
||||
html, body {
|
||||
height: 100%;
|
||||
background: #09090b;
|
||||
}
|
||||
|
||||
body {
|
||||
min-height: 100svh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
font-family: ui-sans-serif, system-ui, sans-serif;
|
||||
color: #a1a1aa;
|
||||
}
|
||||
|
||||
header {
|
||||
padding: 1.5rem 2rem;
|
||||
border-bottom: 1px solid #27272a;
|
||||
}
|
||||
.logo {
|
||||
font-size: 1.125rem;
|
||||
font-weight: 700;
|
||||
color: #e4e4e7;
|
||||
letter-spacing: -0.02em;
|
||||
text-decoration: none;
|
||||
}
|
||||
.logo span { color: #f59e0b; }
|
||||
|
||||
main {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 3rem 2rem;
|
||||
text-align: center;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
.watermark {
|
||||
font-size: clamp(5rem, 22vw, 9rem);
|
||||
font-weight: 800;
|
||||
color: #18181b;
|
||||
line-height: 1;
|
||||
letter-spacing: -0.04em;
|
||||
user-select: none;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.status-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 1.25rem;
|
||||
}
|
||||
.dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: #f59e0b;
|
||||
animation: pulse 2s ease-in-out infinite;
|
||||
}
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.4; transform: scale(0.75); }
|
||||
}
|
||||
.status-label {
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.08em;
|
||||
color: #f59e0b;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 1.5rem;
|
||||
font-weight: 700;
|
||||
color: #e4e4e7;
|
||||
letter-spacing: -0.02em;
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
p {
|
||||
font-size: 0.9375rem;
|
||||
max-width: 38ch;
|
||||
line-height: 1.65;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.btn {
|
||||
display: inline-block;
|
||||
padding: 0.625rem 1.5rem;
|
||||
border-radius: 0.5rem;
|
||||
background: #f59e0b;
|
||||
color: #000;
|
||||
font-weight: 600;
|
||||
font-size: 0.875rem;
|
||||
text-decoration: none;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
.btn:hover { background: #d97706; }
|
||||
|
||||
.refresh-note {
|
||||
margin-top: 1.25rem;
|
||||
font-size: 0.8rem;
|
||||
color: #52525b;
|
||||
}
|
||||
#countdown { color: #71717a; }
|
||||
|
||||
footer {
|
||||
padding: 1.5rem 2rem;
|
||||
border-top: 1px solid #27272a;
|
||||
text-align: center;
|
||||
font-size: 0.8rem;
|
||||
color: #3f3f46;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<a class="logo" href="/">Lib<span>Novel</span></a>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="watermark">502</div>
|
||||
|
||||
<div class="status-row">
|
||||
<div class="dot"></div>
|
||||
<span class="status-label">Service unavailable</span>
|
||||
</div>
|
||||
|
||||
<h1>Something went wrong</h1>
|
||||
<p>The server is temporarily unreachable. This usually resolves itself quickly.</p>
|
||||
|
||||
<a class="btn" href="/">Try again</a>
|
||||
<p class="refresh-note">Page refreshes automatically in <span id="countdown">20</span>s</p>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
© LibNovel
|
||||
</footer>
|
||||
|
||||
<script>
|
||||
var s = 20;
|
||||
var el = document.getElementById('countdown');
|
||||
var t = setInterval(function () {
|
||||
s--;
|
||||
el.textContent = s;
|
||||
if (s <= 0) { clearInterval(t); location.reload(); }
|
||||
}, 1000);
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
165
caddy/errors/503.html
Normal file
165
caddy/errors/503.html
Normal file
@@ -0,0 +1,165 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Under Maintenance — LibNovel</title>
|
||||
<style>
|
||||
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
|
||||
html, body {
|
||||
height: 100%;
|
||||
background: #09090b;
|
||||
}
|
||||
|
||||
body {
|
||||
min-height: 100svh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
font-family: ui-sans-serif, system-ui, sans-serif;
|
||||
color: #a1a1aa;
|
||||
}
|
||||
|
||||
/* ── Header ── */
|
||||
header {
|
||||
padding: 1.5rem 2rem;
|
||||
border-bottom: 1px solid #27272a;
|
||||
}
|
||||
.logo {
|
||||
font-size: 1.125rem;
|
||||
font-weight: 700;
|
||||
color: #e4e4e7;
|
||||
letter-spacing: -0.02em;
|
||||
text-decoration: none;
|
||||
}
|
||||
.logo span { color: #f59e0b; }
|
||||
|
||||
/* ── Main ── */
|
||||
main {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 3rem 2rem;
|
||||
text-align: center;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
.watermark {
|
||||
font-size: clamp(5rem, 22vw, 9rem);
|
||||
font-weight: 800;
|
||||
color: #18181b;
|
||||
line-height: 1;
|
||||
letter-spacing: -0.04em;
|
||||
user-select: none;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.status-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 1.25rem;
|
||||
}
|
||||
.dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: #f59e0b;
|
||||
animation: pulse 2s ease-in-out infinite;
|
||||
}
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.4; transform: scale(0.75); }
|
||||
}
|
||||
.status-label {
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.08em;
|
||||
color: #f59e0b;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 1.5rem;
|
||||
font-weight: 700;
|
||||
color: #e4e4e7;
|
||||
letter-spacing: -0.02em;
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
p {
|
||||
font-size: 0.9375rem;
|
||||
max-width: 38ch;
|
||||
line-height: 1.65;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.btn {
|
||||
display: inline-block;
|
||||
padding: 0.625rem 1.5rem;
|
||||
border-radius: 0.5rem;
|
||||
background: #f59e0b;
|
||||
color: #000;
|
||||
font-weight: 600;
|
||||
font-size: 0.875rem;
|
||||
text-decoration: none;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
.btn:hover { background: #d97706; }
|
||||
|
||||
.refresh-note {
|
||||
margin-top: 1.25rem;
|
||||
font-size: 0.8rem;
|
||||
color: #52525b;
|
||||
}
|
||||
#countdown { color: #71717a; }
|
||||
|
||||
/* ── Footer ── */
|
||||
footer {
|
||||
padding: 1.5rem 2rem;
|
||||
border-top: 1px solid #27272a;
|
||||
text-align: center;
|
||||
font-size: 0.8rem;
|
||||
color: #3f3f46;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<a class="logo" href="/">Lib<span>Novel</span></a>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="watermark">503</div>
|
||||
|
||||
<div class="status-row">
|
||||
<div class="dot"></div>
|
||||
<span class="status-label">Maintenance in progress</span>
|
||||
</div>
|
||||
|
||||
<h1>We'll be right back</h1>
|
||||
<p>LibNovel is briefly offline for scheduled maintenance. No data is being changed — hang tight.</p>
|
||||
|
||||
<a class="btn" href="/">Try again</a>
|
||||
<p class="refresh-note">Page refreshes automatically in <span id="countdown">30</span>s</p>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
© LibNovel
|
||||
</footer>
|
||||
|
||||
<script>
|
||||
var s = 30;
|
||||
var el = document.getElementById('countdown');
|
||||
var t = setInterval(function () {
|
||||
s--;
|
||||
el.textContent = s;
|
||||
if (s <= 0) { clearInterval(t); location.reload(); }
|
||||
}, 1000);
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
162
caddy/errors/504.html
Normal file
162
caddy/errors/504.html
Normal file
@@ -0,0 +1,162 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>504 — Gateway Timeout — LibNovel</title>
|
||||
<style>
|
||||
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
|
||||
html, body {
|
||||
height: 100%;
|
||||
background: #09090b;
|
||||
}
|
||||
|
||||
body {
|
||||
min-height: 100svh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
font-family: ui-sans-serif, system-ui, sans-serif;
|
||||
color: #a1a1aa;
|
||||
}
|
||||
|
||||
header {
|
||||
padding: 1.5rem 2rem;
|
||||
border-bottom: 1px solid #27272a;
|
||||
}
|
||||
.logo {
|
||||
font-size: 1.125rem;
|
||||
font-weight: 700;
|
||||
color: #e4e4e7;
|
||||
letter-spacing: -0.02em;
|
||||
text-decoration: none;
|
||||
}
|
||||
.logo span { color: #f59e0b; }
|
||||
|
||||
main {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 3rem 2rem;
|
||||
text-align: center;
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
.watermark {
|
||||
font-size: clamp(5rem, 22vw, 9rem);
|
||||
font-weight: 800;
|
||||
color: #18181b;
|
||||
line-height: 1;
|
||||
letter-spacing: -0.04em;
|
||||
user-select: none;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.status-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-bottom: 1.25rem;
|
||||
}
|
||||
.dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: #f59e0b;
|
||||
animation: pulse 2s ease-in-out infinite;
|
||||
}
|
||||
@keyframes pulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.4; transform: scale(0.75); }
|
||||
}
|
||||
.status-label {
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.08em;
|
||||
color: #f59e0b;
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: 1.5rem;
|
||||
font-weight: 700;
|
||||
color: #e4e4e7;
|
||||
letter-spacing: -0.02em;
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
p {
|
||||
font-size: 0.9375rem;
|
||||
max-width: 38ch;
|
||||
line-height: 1.65;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.btn {
|
||||
display: inline-block;
|
||||
padding: 0.625rem 1.5rem;
|
||||
border-radius: 0.5rem;
|
||||
background: #f59e0b;
|
||||
color: #000;
|
||||
font-weight: 600;
|
||||
font-size: 0.875rem;
|
||||
text-decoration: none;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
.btn:hover { background: #d97706; }
|
||||
|
||||
.refresh-note {
|
||||
margin-top: 1.25rem;
|
||||
font-size: 0.8rem;
|
||||
color: #52525b;
|
||||
}
|
||||
#countdown { color: #71717a; }
|
||||
|
||||
footer {
|
||||
padding: 1.5rem 2rem;
|
||||
border-top: 1px solid #27272a;
|
||||
text-align: center;
|
||||
font-size: 0.8rem;
|
||||
color: #3f3f46;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<a class="logo" href="/">Lib<span>Novel</span></a>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="watermark">504</div>
|
||||
|
||||
<div class="status-row">
|
||||
<div class="dot"></div>
|
||||
<span class="status-label">Gateway timeout</span>
|
||||
</div>
|
||||
|
||||
<h1>Request timed out</h1>
|
||||
<p>The server took too long to respond. Please refresh and try again.</p>
|
||||
|
||||
<a class="btn" href="/">Try again</a>
|
||||
<p class="refresh-note">Page refreshes automatically in <span id="countdown">20</span>s</p>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
© LibNovel
|
||||
</footer>
|
||||
|
||||
<script>
|
||||
var s = 20;
|
||||
var el = document.getElementById('countdown');
|
||||
var t = setInterval(function () {
|
||||
s--;
|
||||
el.textContent = s;
|
||||
if (s <= 0) { clearInterval(t); location.reload(); }
|
||||
}, 1000);
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
12
crowdsec/acquis.yaml
Normal file
12
crowdsec/acquis.yaml
Normal file
@@ -0,0 +1,12 @@
|
||||
# CrowdSec log acquisition — tells the CrowdSec agent which logs to parse.
|
||||
#
|
||||
# Caddy writes JSON access logs to /var/log/caddy/access.log (mounted from the
|
||||
# caddy_logs Docker volume). CrowdSec reads the same volume at the same path.
|
||||
#
|
||||
# The `crowdsecurity/caddy` collection (installed via COLLECTIONS env var)
|
||||
# provides the parser that understands Caddy's JSON log format.
|
||||
|
||||
filenames:
|
||||
- /var/log/caddy/access.log
|
||||
labels:
|
||||
type: caddy
|
||||
@@ -1,211 +0,0 @@
|
||||
services:
|
||||
# ─── MinIO (object storage: chapters, audio, avatars) ────────────────────────
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
restart: unless-stopped
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "${MINIO_PORT:-9000}:9000" # S3 API
|
||||
- "${MINIO_CONSOLE_PORT:-9001}:9001" # Web console
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── MinIO bucket initialisation ─────────────────────────────────────────────
|
||||
minio-init:
|
||||
image: minio/mc:latest
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
|
||||
mc mb --ignore-existing local/libnovel-chapters;
|
||||
mc mb --ignore-existing local/libnovel-audio;
|
||||
mc mb --ignore-existing local/libnovel-avatars;
|
||||
mc mb --ignore-existing local/libnovel-browse;
|
||||
echo 'buckets ready';
|
||||
"
|
||||
environment:
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
|
||||
# ─── PocketBase (auth + structured data) ─────────────────────────────────────
|
||||
pocketbase:
|
||||
image: ghcr.io/muchobien/pocketbase:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
PB_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
PB_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "${POCKETBASE_PORT:-8090}:8090"
|
||||
volumes:
|
||||
- pb_data:/pb_data
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── PocketBase collection bootstrap ─────────────────────────────────────────
|
||||
pb-init:
|
||||
image: alpine:3.19
|
||||
depends_on:
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
volumes:
|
||||
- ./scripts/pb-init.sh:/pb-init.sh:ro
|
||||
entrypoint: ["sh", "/pb-init.sh"]
|
||||
|
||||
# ─── Backend API ──────────────────────────────────────────────────────────────
|
||||
backend:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
target: backend
|
||||
args:
|
||||
VERSION: "${GIT_TAG:-dev}"
|
||||
COMMIT: "${GIT_COMMIT:-unknown}"
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 35s
|
||||
depends_on:
|
||||
pb-init:
|
||||
condition: service_completed_successfully
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
BACKEND_HTTP_ADDR: ":8080"
|
||||
LOG_LEVEL: "${LOG_LEVEL:-info}"
|
||||
# MinIO
|
||||
MINIO_ENDPOINT: "minio:9000"
|
||||
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
MINIO_USE_SSL: "false"
|
||||
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
|
||||
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
|
||||
MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
|
||||
MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
|
||||
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}"
|
||||
# PocketBase
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "${BACKEND_PORT:-8080}:8080"
|
||||
healthcheck:
|
||||
test: ["CMD", "/healthcheck", "http://localhost:8080/health"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# ─── Runner (background task worker) ─────────────────────────────────────────
|
||||
runner:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
target: runner
|
||||
args:
|
||||
VERSION: "${GIT_TAG:-dev}"
|
||||
COMMIT: "${GIT_COMMIT:-unknown}"
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 135s
|
||||
depends_on:
|
||||
pb-init:
|
||||
condition: service_completed_successfully
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
LOG_LEVEL: "${LOG_LEVEL:-info}"
|
||||
# Runner tuning
|
||||
RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL:-30s}"
|
||||
# RUNNER_MAX_CONCURRENT_SCRAPE controls how many books are scraped in parallel.
|
||||
# Default is 1 (sequential). Increase for faster catalogue scrapes at the
|
||||
# cost of higher CPU/network load on the novelfire.net target.
|
||||
RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE:-1}"
|
||||
RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO:-1}"
|
||||
RUNNER_WORKER_ID: "${RUNNER_WORKER_ID:-runner-1}"
|
||||
RUNNER_WORKERS: "${RUNNER_WORKERS:-0}"
|
||||
RUNNER_TIMEOUT: "${RUNNER_TIMEOUT:-90s}"
|
||||
SCRAPER_PROXY: "${SCRAPER_PROXY:-}"
|
||||
# Kokoro-FastAPI TTS endpoint
|
||||
KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc}"
|
||||
KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}"
|
||||
# MinIO
|
||||
MINIO_ENDPOINT: "minio:9000"
|
||||
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
MINIO_USE_SSL: "false"
|
||||
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
|
||||
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
|
||||
MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
|
||||
MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
|
||||
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}"
|
||||
# PocketBase
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
healthcheck:
|
||||
# The runner has no HTTP server. It writes /tmp/runner.alive on every poll.
|
||||
# 120s = 2× the default 30s poll interval with generous headroom.
|
||||
test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# ─── SvelteKit UI ─────────────────────────────────────────────────────────────
|
||||
ui:
|
||||
build:
|
||||
context: ./ui-v2
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
BUILD_VERSION: "${GIT_TAG:-dev}"
|
||||
BUILD_COMMIT: "${GIT_COMMIT:-unknown}"
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 35s
|
||||
depends_on:
|
||||
pb-init:
|
||||
condition: service_completed_successfully
|
||||
backend:
|
||||
condition: service_healthy
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
# ORIGIN must match the URL the browser uses to reach the UI.
|
||||
# adapter-node uses this for SvelteKit's built-in CSRF origin check.
|
||||
# When running behind a reverse proxy or non-standard port, set this via
|
||||
# the ORIGIN env var (e.g. https://libnovel.example.com).
|
||||
ORIGIN: "${ORIGIN:-http://localhost:5252}"
|
||||
SCRAPER_API_URL: "http://backend:8080"
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
AUTH_SECRET: "${AUTH_SECRET:-dev_secret_change_in_production}"
|
||||
PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||
ports:
|
||||
- "${UI_PORT:-5252}:3000"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/health"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
minio_data:
|
||||
pb_data:
|
||||
@@ -1,18 +1,39 @@
|
||||
version: "3.9"
|
||||
# ── Shared environment fragments ──────────────────────────────────────────────
|
||||
# These YAML anchors eliminate duplication between backend and runner.
|
||||
# All values come from Doppler — no fallbacks needed here.
|
||||
# Run commands via: just up / just build / etc. (see justfile)
|
||||
x-infra-env: &infra-env
|
||||
# MinIO
|
||||
MINIO_ENDPOINT: "minio:9000"
|
||||
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}"
|
||||
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD}"
|
||||
MINIO_USE_SSL: "false"
|
||||
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT}"
|
||||
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL}"
|
||||
# PocketBase
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}"
|
||||
# Meilisearch
|
||||
MEILI_URL: "${MEILI_URL:-http://meilisearch:7700}"
|
||||
MEILI_API_KEY: "${MEILI_MASTER_KEY}"
|
||||
# Valkey
|
||||
VALKEY_ADDR: "valkey:6379"
|
||||
|
||||
services:
|
||||
# ─── MinIO (object storage for chapter .md files + audio cache) ─────────────
|
||||
# ─── MinIO (object storage: chapters, audio, avatars, browse) ────────────────
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
#container_name: libnovel-minio
|
||||
restart: unless-stopped
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "${MINIO_PORT:-9000}:9000" # S3 API
|
||||
- "${MINIO_CONSOLE_PORT:-9001}:9001" # Web console
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD}"
|
||||
# No public port — all presigned URL traffic goes through backend or a
|
||||
# separately-exposed MINIO_PUBLIC_ENDPOINT (e.g. storage.libnovel.cc).
|
||||
expose:
|
||||
- "9000"
|
||||
- "9001"
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
@@ -22,37 +43,34 @@ services:
|
||||
retries: 5
|
||||
|
||||
# ─── MinIO bucket initialisation ─────────────────────────────────────────────
|
||||
# Runs once to create the default buckets and then exits.
|
||||
minio-init:
|
||||
image: minio/mc:latest
|
||||
#container_name: libnovel-minio-init
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
|
||||
mc mb --ignore-existing local/libnovel-chapters;
|
||||
mc mb --ignore-existing local/libnovel-audio;
|
||||
mc mb --ignore-existing local/libnovel-browse;
|
||||
mc mb --ignore-existing local/libnovel-avatars;
|
||||
mc alias set local http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD};
|
||||
mc mb --ignore-existing local/chapters;
|
||||
mc mb --ignore-existing local/audio;
|
||||
mc mb --ignore-existing local/avatars;
|
||||
mc mb --ignore-existing local/catalogue;
|
||||
echo 'buckets ready';
|
||||
"
|
||||
environment:
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD}"
|
||||
|
||||
# ─── PocketBase (auth + structured data: books, chapters index, ranking, progress) ──
|
||||
# ─── PocketBase (auth + structured data) ─────────────────────────────────────
|
||||
pocketbase:
|
||||
image: ghcr.io/muchobien/pocketbase:latest
|
||||
#container_name: libnovel-pocketbase
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# Auto-create superuser on first boot (used by entrypoint.sh)
|
||||
PB_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
PB_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "${POCKETBASE_PORT:-8090}:8090"
|
||||
PB_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}"
|
||||
PB_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}"
|
||||
# No public port — accessed only by backend/runner on the internal network.
|
||||
expose:
|
||||
- "8090"
|
||||
volumes:
|
||||
- pb_data:/pb_data
|
||||
healthcheck:
|
||||
@@ -61,9 +79,7 @@ services:
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── PocketBase collection bootstrap ────────────────────────────────────────
|
||||
# One-shot init container: creates all required collections via the admin API
|
||||
# and exits. Idempotent — safe to run on every `docker compose up`.
|
||||
# ─── PocketBase collection bootstrap ─────────────────────────────────────────
|
||||
pb-init:
|
||||
image: alpine:3.19
|
||||
depends_on:
|
||||
@@ -71,22 +87,59 @@ services:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}"
|
||||
volumes:
|
||||
- ./scripts/pb-init.sh:/pb-init.sh:ro
|
||||
- ./scripts/pb-init-v3.sh:/pb-init.sh:ro
|
||||
entrypoint: ["sh", "/pb-init.sh"]
|
||||
|
||||
# ─── Scraper ─────────────────────────────────────────────────────────────────
|
||||
scraper:
|
||||
build:
|
||||
context: ./scraper
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
VERSION: "${GIT_TAG:-dev}"
|
||||
COMMIT: "${GIT_COMMIT:-unknown}"
|
||||
#container_name: libnovel-scraper
|
||||
# ─── Meilisearch (full-text search) ──────────────────────────────────────────
|
||||
meilisearch:
|
||||
image: getmeili/meilisearch:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MEILI_MASTER_KEY: "${MEILI_MASTER_KEY}"
|
||||
MEILI_ENV: "${MEILI_ENV}"
|
||||
# No public port — backend/runner reach it via internal network.
|
||||
expose:
|
||||
- "7700"
|
||||
volumes:
|
||||
- meili_data:/meili_data
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:7700/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── Valkey (presign URL cache) ───────────────────────────────────────────────
|
||||
valkey:
|
||||
image: valkey/valkey:7-alpine
|
||||
restart: unless-stopped
|
||||
# No public port — backend/runner/ui reach it via internal network.
|
||||
expose:
|
||||
- "6379"
|
||||
volumes:
|
||||
- valkey_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── Backend API ──────────────────────────────────────────────────────────────
|
||||
backend:
|
||||
image: kalekber/libnovel-backend:${GIT_TAG:-latest}
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
target: backend
|
||||
args:
|
||||
VERSION: "${GIT_TAG}"
|
||||
COMMIT: "${GIT_COMMIT}"
|
||||
labels:
|
||||
com.centurylinklabs.watchtower.enable: "true"
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 35s
|
||||
depends_on:
|
||||
pb-init:
|
||||
condition: service_completed_successfully
|
||||
@@ -94,72 +147,281 @@ services:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
meilisearch:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
# No public port — all traffic is routed via Caddy.
|
||||
expose:
|
||||
- "8080"
|
||||
environment:
|
||||
# 0 → defaults to NumCPU inside the container.
|
||||
SCRAPER_WORKERS: "${SCRAPER_WORKERS:-0}"
|
||||
SCRAPER_HTTP_ADDR: ":8080"
|
||||
LOG_LEVEL: "debug"
|
||||
# Kokoro-FastAPI TTS endpoint.
|
||||
KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc}"
|
||||
KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}"
|
||||
# MinIO / S3 object storage
|
||||
MINIO_ENDPOINT: "minio:9000"
|
||||
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
MINIO_USE_SSL: "false"
|
||||
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
|
||||
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
|
||||
MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
|
||||
MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
|
||||
# Public endpoint used to sign presigned audio URLs so browsers can reach them.
|
||||
# Leave empty to use MINIO_ENDPOINT (fine for local dev).
|
||||
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-}"
|
||||
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-true}"
|
||||
# SingleFile CLI path for save-browse subcommand
|
||||
SINGLEFILE_PATH: "${SINGLEFILE_PATH:-single-file}"
|
||||
# PocketBase
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "${SCRAPER_PORT:-8080}:8080"
|
||||
<<: *infra-env
|
||||
BACKEND_HTTP_ADDR: ":8080"
|
||||
LOG_LEVEL: "${LOG_LEVEL}"
|
||||
KOKORO_URL: "${KOKORO_URL}"
|
||||
KOKORO_VOICE: "${KOKORO_VOICE}"
|
||||
POCKET_TTS_URL: "${POCKET_TTS_URL}"
|
||||
GLITCHTIP_DSN: "${GLITCHTIP_DSN}"
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: "${OTEL_EXPORTER_OTLP_ENDPOINT}"
|
||||
OTEL_SERVICE_NAME: "backend"
|
||||
# Asynq task queue — backend enqueues jobs to homelab Redis via Caddy TLS proxy.
|
||||
# Set to "rediss://:password@redis.libnovel.cc:6380" in Doppler prd config.
|
||||
# Leave empty to fall back to PocketBase polling.
|
||||
REDIS_ADDR: "${REDIS_ADDR}"
|
||||
REDIS_PASSWORD: "${REDIS_PASSWORD}"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:8080/health"]
|
||||
test: ["CMD", "/healthcheck", "http://localhost:8080/health"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# ─── SvelteKit UI ────────────────────────────────────────────────────────────
|
||||
# ─── Runner (background task worker) ─────────────────────────────────────────
|
||||
# profiles: [runner] prevents accidental restart on `docker compose up -d`.
|
||||
# The homelab runner (192.168.0.109) is the sole worker in production.
|
||||
# To start explicitly: doppler run -- docker compose --profile runner up -d runner
|
||||
runner:
|
||||
profiles: [runner]
|
||||
image: kalekber/libnovel-runner:${GIT_TAG:-latest}
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
target: runner
|
||||
args:
|
||||
VERSION: "${GIT_TAG}"
|
||||
COMMIT: "${GIT_COMMIT}"
|
||||
labels:
|
||||
com.centurylinklabs.watchtower.enable: "true"
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 135s
|
||||
depends_on:
|
||||
pb-init:
|
||||
condition: service_completed_successfully
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
meilisearch:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
# Metrics endpoint — internal only; expose publicly via Caddy if needed.
|
||||
expose:
|
||||
- "9091"
|
||||
environment:
|
||||
<<: *infra-env
|
||||
LOG_LEVEL: "${LOG_LEVEL}"
|
||||
# Runner tuning
|
||||
RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL}"
|
||||
RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE}"
|
||||
RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO}"
|
||||
RUNNER_WORKER_ID: "${RUNNER_WORKER_ID}"
|
||||
RUNNER_TIMEOUT: "${RUNNER_TIMEOUT}"
|
||||
RUNNER_METRICS_ADDR: "${RUNNER_METRICS_ADDR}"
|
||||
# Suppress the on-startup catalogue walk — catalogue_refresh now skips
|
||||
# books already in Meilisearch, so a full walk on every restart is wasteful.
|
||||
# The 24h periodic ticker (CatalogueRefreshInterval) still fires normally.
|
||||
RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH: "true"
|
||||
# Kokoro-FastAPI TTS endpoint
|
||||
KOKORO_URL: "${KOKORO_URL}"
|
||||
KOKORO_VOICE: "${KOKORO_VOICE}"
|
||||
POCKET_TTS_URL: "${POCKET_TTS_URL}"
|
||||
GLITCHTIP_DSN: "${GLITCHTIP_DSN}"
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: "${OTEL_EXPORTER_OTLP_ENDPOINT}"
|
||||
OTEL_SERVICE_NAME: "runner"
|
||||
healthcheck:
|
||||
# 120s = 2× the default 30s poll interval with generous headroom.
|
||||
test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# ─── SvelteKit UI ─────────────────────────────────────────────────────────────
|
||||
ui:
|
||||
image: kalekber/libnovel-ui:${GIT_TAG:-latest}
|
||||
build:
|
||||
context: ./ui
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
BUILD_VERSION: "${GIT_TAG:-dev}"
|
||||
BUILD_COMMIT: "${GIT_COMMIT:-unknown}"
|
||||
# container_name: libnovel-ui
|
||||
BUILD_VERSION: "${GIT_TAG}"
|
||||
BUILD_COMMIT: "${GIT_COMMIT}"
|
||||
labels:
|
||||
com.centurylinklabs.watchtower.enable: "true"
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 35s
|
||||
depends_on:
|
||||
pb-init:
|
||||
condition: service_completed_successfully
|
||||
scraper:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
# No public port — all traffic via Caddy.
|
||||
expose:
|
||||
- "3000"
|
||||
environment:
|
||||
SCRAPER_API_URL: "http://scraper:8080"
|
||||
# ORIGIN must match the public URL Caddy serves on.
|
||||
# adapter-node uses this for SvelteKit's built-in CSRF origin check.
|
||||
ORIGIN: "${ORIGIN}"
|
||||
BACKEND_API_URL: "http://backend:8080"
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||
ports:
|
||||
- "${UI_PORT:-5252}:3000"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}"
|
||||
AUTH_SECRET: "${AUTH_SECRET}"
|
||||
PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT}"
|
||||
# Valkey
|
||||
VALKEY_ADDR: "valkey:6379"
|
||||
# Umami analytics
|
||||
PUBLIC_UMAMI_WEBSITE_ID: "${PUBLIC_UMAMI_WEBSITE_ID}"
|
||||
PUBLIC_UMAMI_SCRIPT_URL: "${PUBLIC_UMAMI_SCRIPT_URL}"
|
||||
# GlitchTip client + server-side error tracking
|
||||
PUBLIC_GLITCHTIP_DSN: "${PUBLIC_GLITCHTIP_DSN}"
|
||||
# OpenTelemetry tracing
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: "${OTEL_EXPORTER_OTLP_ENDPOINT}"
|
||||
OTEL_SERVICE_NAME: "ui"
|
||||
# OAuth2 providers
|
||||
GOOGLE_CLIENT_ID: "${GOOGLE_CLIENT_ID}"
|
||||
GOOGLE_CLIENT_SECRET: "${GOOGLE_CLIENT_SECRET}"
|
||||
GITHUB_CLIENT_ID: "${GITHUB_CLIENT_ID}"
|
||||
GITHUB_CLIENT_SECRET: "${GITHUB_CLIENT_SECRET}"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/health"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# ─── CrowdSec (threat detection + IP blocking) ───────────────────────────────
|
||||
# Reads Caddy JSON access logs from the shared caddy_logs volume and enforces
|
||||
# decisions via the Caddy bouncer plugin.
|
||||
crowdsec:
|
||||
image: crowdsecurity/crowdsec:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
GID: "1000"
|
||||
COLLECTIONS: "crowdsecurity/caddy crowdsecurity/http-dos crowdsecurity/base-http-scenarios"
|
||||
volumes:
|
||||
- crowdsec_data:/var/lib/crowdsec/data
|
||||
- ./crowdsec/acquis.yaml:/etc/crowdsec/acquis.yaml:ro
|
||||
- caddy_logs:/var/log/caddy:ro
|
||||
expose:
|
||||
- "8080"
|
||||
healthcheck:
|
||||
test: ["CMD", "cscli", "version"]
|
||||
interval: 20s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
|
||||
# ─── Dozzle agent ────────────────────────────────────────────────────────────
|
||||
# Exposes prod container logs to the Dozzle instance on the homelab.
|
||||
# The homelab Dozzle connects here via DOZZLE_REMOTE_AGENT.
|
||||
# Port 7007 is bound to localhost only — not reachable from the internet.
|
||||
dozzle-agent:
|
||||
image: amir20/dozzle:latest
|
||||
restart: unless-stopped
|
||||
command: agent
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
ports:
|
||||
- "127.0.0.1:7007:7007"
|
||||
|
||||
# ─── CrowdSec bouncer registration ───────────────────────────────────────────
|
||||
# One-shot: registers the Caddy bouncer with the CrowdSec LAPI and writes the
|
||||
# generated API key to crowdsec/.crowdsec.env, which Caddy reads via env_file.
|
||||
# Uses the Docker socket to exec cscli inside the running crowdsec container.
|
||||
crowdsec-init:
|
||||
image: docker:cli
|
||||
depends_on:
|
||||
crowdsec:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- ./crowdsec:/crowdsec-out
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
out=/crowdsec-out/.crowdsec.env;
|
||||
existing=$$(grep -s '^CROWDSEC_API_KEY=.' \"$$out\" | cut -d= -f2-);
|
||||
if [ -n \"$$existing\" ]; then
|
||||
echo 'crowdsec-init: key already present, skipping registration';
|
||||
exit 0;
|
||||
fi;
|
||||
container=$$(docker ps --filter name=crowdsec --filter status=running --format '{{.Names}}' | grep -v init | head -1);
|
||||
echo \"crowdsec-init: using container $$container\";
|
||||
docker exec $$container cscli bouncers delete caddy-bouncer 2>/dev/null || true;
|
||||
key=$$(docker exec $$container cscli bouncers add caddy-bouncer -o raw 2>&1);
|
||||
if [ -z \"$$key\" ]; then
|
||||
echo 'crowdsec-init: ERROR — failed to obtain bouncer key' >&2;
|
||||
exit 1;
|
||||
fi;
|
||||
printf 'CROWDSEC_API_KEY=%s\n' \"$$key\" > \"$$out\";
|
||||
echo \"crowdsec-init: bouncer key written (key length: $${#key})\";
|
||||
"
|
||||
restart: "no"
|
||||
|
||||
|
||||
# ─── Caddy (reverse proxy + automatic HTTPS) ──────────────────────────────────
|
||||
# Custom build includes github.com/mholt/caddy-ratelimit,
|
||||
# github.com/hslatman/caddy-crowdsec-bouncer/http, and
|
||||
# github.com/mholt/caddy-l4 (TCP layer4 proxy for Redis).
|
||||
caddy:
|
||||
image: kalekber/libnovel-caddy:${GIT_TAG:-latest}
|
||||
build:
|
||||
context: ./caddy
|
||||
dockerfile: Dockerfile
|
||||
labels:
|
||||
com.centurylinklabs.watchtower.enable: "true"
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
ui:
|
||||
condition: service_healthy
|
||||
crowdsec-init:
|
||||
condition: service_completed_successfully
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
- "443:443/udp" # HTTP/3 (QUIC)
|
||||
- "6380:6380" # Redis TCP proxy (TLS) for homelab → Asynq
|
||||
environment:
|
||||
DOMAIN: "${DOMAIN}"
|
||||
CADDY_ACME_EMAIL: "${CADDY_ACME_EMAIL}"
|
||||
# Homelab Redis address — Caddy TCP-proxies inbound :6380 to this.
|
||||
HOMELAB_REDIS_ADDR: "${HOMELAB_REDIS_ADDR:?HOMELAB_REDIS_ADDR required for Redis TCP proxy}"
|
||||
env_file:
|
||||
- path: ./crowdsec/.crowdsec.env
|
||||
required: false
|
||||
volumes:
|
||||
- ./Caddyfile:/etc/caddy/Caddyfile:ro
|
||||
- ./caddy/errors:/srv/errors:ro
|
||||
- caddy_data:/data
|
||||
- caddy_config:/config
|
||||
- caddy_logs:/var/log/caddy
|
||||
|
||||
# ─── Watchtower (auto-redeploy custom services on new images) ────────────────
|
||||
# Only watches services labelled com.centurylinklabs.watchtower.enable=true.
|
||||
# Third-party infra images (minio, pocketbase, meilisearch, etc.) are excluded.
|
||||
# doppler binary is mounted from the host so watchtower fetches fresh secrets
|
||||
# on every start (notification URL, credentials) without baking them in.
|
||||
watchtower:
|
||||
image: containrrr/watchtower:latest
|
||||
restart: unless-stopped
|
||||
entrypoint: ["/usr/bin/doppler", "run", "--project", "libnovel", "--config", "prd", "--"]
|
||||
command: ["/watchtower", "--label-enable", "--interval", "300", "--cleanup"]
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- /usr/bin/doppler:/usr/bin/doppler:ro
|
||||
- /root/.doppler:/root/.doppler:ro
|
||||
environment:
|
||||
HOME: "/root"
|
||||
DOCKER_API_VERSION: "1.44"
|
||||
|
||||
volumes:
|
||||
minio_data:
|
||||
pb_data:
|
||||
meili_data:
|
||||
valkey_data:
|
||||
caddy_data:
|
||||
caddy_config:
|
||||
caddy_logs:
|
||||
crowdsec_data:
|
||||
|
||||
82
docs/api-endpoints.md
Normal file
82
docs/api-endpoints.md
Normal file
@@ -0,0 +1,82 @@
|
||||
# API Endpoint Reference
|
||||
|
||||
> **Routing ownership map**: see [`docs/d2/api-routing.svg`](d2/api-routing.svg) (source: [`docs/d2/api-routing.d2`](d2/api-routing.d2)) for a visual overview of which paths Caddy sends to the backend directly vs. through SvelteKit, with auth levels colour-coded.
|
||||
|
||||
All traffic enters through **Caddy :443**. Caddy routes a subset of paths directly to the Go backend (bypassing SvelteKit); everything else goes to SvelteKit, which enforces auth before proxying onward.
|
||||
|
||||
## Health / Version
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| `GET` | `/health` | — | Liveness probe. Returns `{"ok":true}`. |
|
||||
| `GET` | `/api/version` | — | Build version + commit hash. |
|
||||
|
||||
## Scrape Jobs (admin)
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| `POST` | `/scrape` | admin | Enqueue full catalogue scrape. |
|
||||
| `POST` | `/scrape/book` | admin | Enqueue single-book scrape `{url}`. |
|
||||
| `POST` | `/scrape/book/range` | admin | Enqueue range scrape `{url, from, to?}`. |
|
||||
| `GET` | `/api/scrape/status` | admin | Current job status. |
|
||||
| `GET` | `/api/scrape/tasks` | admin | All scrape task records. |
|
||||
| `POST` | `/api/cancel-task/{id}` | admin | Cancel a pending task. |
|
||||
|
||||
## Browse / Catalogue
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| `GET` | `/api/browse` | — | Live novelfire.net browse (MinIO page-1 cache). Legacy — used by save-browse subcommand. |
|
||||
| `GET` | `/api/catalogue` | — | **Primary browse endpoint.** Meilisearch-backed, paginated. Params: `q`, `page`, `limit`, `genre`, `status`, `sort` (`popular`\|`new`\|`update`\|`rank`\|`top-rated`). Falls back to empty when Meilisearch is not configured. |
|
||||
| `GET` | `/api/search` | — | Full-text search: Meilisearch local results merged with live novelfire.net remote results. Param: `q` (≥ 2 chars). Used by iOS app. |
|
||||
| `GET` | `/api/ranking` | — | Top-ranked novels from PocketBase. |
|
||||
| `GET` | `/api/cover/{domain}/{slug}` | — | Proxy cover image from MinIO (redirect to presigned URL). |
|
||||
|
||||
## Book / Chapter Content
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| `GET` | `/api/book-preview/{slug}` | — | Returns stored metadata + chapter list, or enqueues a scrape task (202) if unknown. |
|
||||
| `GET` | `/api/chapter-text/{slug}/{n}` | — | Chapter content as plain text (markdown stripped). |
|
||||
| `GET` | `/api/chapter-markdown/{slug}/{n}` | — | Chapter content as raw markdown from MinIO. |
|
||||
| `POST` | `/api/reindex/{slug}` | admin | Rebuild `chapters_idx` from MinIO objects. |
|
||||
|
||||
## Audio
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| `POST` | `/api/audio/{slug}/{n}` | — | Trigger Kokoro TTS generation. Body: `{voice?}`. Returns `200 {status:"done"}` if cached, `202 {task_id, status}` if enqueued. |
|
||||
| `GET` | `/api/audio/status/{slug}/{n}` | — | Poll audio generation status. Param: `voice`. Returns `{status, task_id?, error?}`. |
|
||||
| `GET` | `/api/audio-proxy/{slug}/{n}` | — | Redirect to presigned MinIO audio URL. |
|
||||
| `GET` | `/api/voices` | — | List available Kokoro voices. Returns `{voices:[]}` on error. |
|
||||
|
||||
## Presigned URLs
|
||||
|
||||
All presign endpoints return a `302` redirect to a short-lived MinIO presigned
|
||||
URL. The URL is cached in Valkey (TTL ~55 min) to avoid regenerating on every
|
||||
request.
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| `GET` | `/api/presign/chapter/{slug}/{n}` | — | Presigned URL for chapter markdown object. |
|
||||
| `GET` | `/api/presign/audio/{slug}/{n}` | — | Presigned URL for audio MP3. Param: `voice`. |
|
||||
| `GET` | `/api/presign/voice-sample/{voice}` | — | Presigned URL for voice sample MP3. |
|
||||
| `GET` | `/api/presign/avatar-upload/{userId}` | user | Presigned PUT URL for avatar upload. |
|
||||
| `GET` | `/api/presign/avatar/{userId}` | — | Presigned GET URL for avatar image. |
|
||||
|
||||
## Reading Progress
|
||||
|
||||
Session-scoped (anonymous via cookie session ID, or tied to authenticated user).
|
||||
|
||||
| Method | Path | Auth | Description |
|
||||
|--------|------|------|-------------|
|
||||
| `GET` | `/api/progress` | — | Get all reading progress for the current session/user. |
|
||||
| `POST` | `/api/progress/{slug}` | — | Set progress. Body: `{chapter}`. |
|
||||
| `DELETE` | `/api/progress/{slug}` | — | Delete progress for a book. |
|
||||
|
||||
## Notes
|
||||
|
||||
- **Auth**: The backend does not enforce auth itself — the SvelteKit UI layer enforces admin/user guards before proxying requests. The backend trusts all incoming requests.
|
||||
- **`/api/catalogue` vs `/api/browse`**: `/api/catalogue` is the primary UI endpoint (Meilisearch, always-local, fast). `/api/browse` hits or caches the live novelfire.net browse page and is only used internally by the `save-browse` subcommand.
|
||||
- **Meilisearch fallback**: When `MEILI_URL` is unset, `/api/catalogue` returns `{books:[], has_next:false}` and `/api/search` falls back to a PocketBase substring scan.
|
||||
- **`BACKEND_API_URL`**: The SvelteKit UI reads this env var (default `http://localhost:8080`) to reach the backend server-side. In docker-compose it is set to `http://backend:8080`.
|
||||
208
docs/d2/api-routing.d2
Normal file
208
docs/d2/api-routing.d2
Normal file
@@ -0,0 +1,208 @@
|
||||
direction: right
|
||||
|
||||
# ─── Legend ───────────────────────────────────────────────────────────────────
|
||||
|
||||
legend: Legend {
|
||||
style.fill: "#fafafa"
|
||||
style.stroke: "#d4d4d8"
|
||||
|
||||
pub: public {
|
||||
style.fill: "#f0fdf4"
|
||||
style.font-color: "#15803d"
|
||||
style.stroke: "#86efac"
|
||||
}
|
||||
user: user auth {
|
||||
style.fill: "#eff6ff"
|
||||
style.font-color: "#1d4ed8"
|
||||
style.stroke: "#93c5fd"
|
||||
}
|
||||
adm: admin only {
|
||||
style.fill: "#fff7ed"
|
||||
style.font-color: "#c2410c"
|
||||
style.stroke: "#fdba74"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── Client ───────────────────────────────────────────────────────────────────
|
||||
|
||||
client: Browser / iOS App {
|
||||
shape: person
|
||||
style.fill: "#fff9e6"
|
||||
}
|
||||
|
||||
# ─── Caddy ────────────────────────────────────────────────────────────────────
|
||||
|
||||
caddy: Caddy :443 {
|
||||
shape: rectangle
|
||||
style.fill: "#f1f5f9"
|
||||
label: "Caddy :443\ncustom build · caddy-l4 · caddy-ratelimit\nCrowdSec bouncer · security headers\nrate limiting · static error pages\nRedis TCP proxy :6380"
|
||||
}
|
||||
|
||||
# ─── SvelteKit UI ─────────────────────────────────────────────────────────────
|
||||
# All routes here pass through SvelteKit — auth is enforced server-side.
|
||||
|
||||
sk: SvelteKit UI :3000 {
|
||||
style.fill: "#fef3c7"
|
||||
|
||||
auth: Auth {
|
||||
style.fill: "#fde68a"
|
||||
style.stroke: "#f59e0b"
|
||||
label: "POST /api/auth/login\nPOST /api/auth/register\nPOST /api/auth/change-password\nGET /api/auth/session"
|
||||
}
|
||||
|
||||
catalogue_sk: Catalogue {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "GET /api/catalogue-page (infinite scroll)\nGET /api/search"
|
||||
}
|
||||
|
||||
book_sk: Book {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "GET /api/book/{slug}\nGET /api/chapter/{slug}/{n}\nGET /api/chapter-text-preview/{slug}/{n}"
|
||||
}
|
||||
|
||||
scrape_sk: Scrape (admin) {
|
||||
style.fill: "#fff7ed"
|
||||
style.stroke: "#fdba74"
|
||||
label: "GET /api/scrape/status\nGET /api/scrape/tasks\nPOST /api/scrape\nPOST /api/scrape/book\nPOST /api/scrape/book/range\nPOST /api/scrape/cancel/{id}"
|
||||
}
|
||||
|
||||
audio_sk: Audio {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "POST /api/audio/{slug}/{n}\nGET /api/audio/status/{slug}/{n}\nGET /api/voices"
|
||||
}
|
||||
|
||||
presign_sk: Presigned URLs (public) {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "GET /api/presign/chapter/{slug}/{n}\nGET /api/presign/audio/{slug}/{n}\nGET /api/presign/voice-sample/{voice}"
|
||||
}
|
||||
|
||||
presign_user: Presigned URLs (user) {
|
||||
style.fill: "#eff6ff"
|
||||
style.stroke: "#93c5fd"
|
||||
label: "GET /api/presign/avatar-upload/{userId}\nGET /api/presign/avatar/{userId}"
|
||||
}
|
||||
|
||||
progress_sk: Progress {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "GET /api/progress\nPOST /api/progress/{slug}\nDELETE /api/progress/{slug}"
|
||||
}
|
||||
|
||||
library_sk: Library {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "GET /api/library\nPOST /api/library/{slug}\nDELETE /api/library/{slug}"
|
||||
}
|
||||
|
||||
comments_sk: Comments {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "GET /api/comments/{slug}\nPOST /api/comments/{slug}"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── Go Backend ───────────────────────────────────────────────────────────────
|
||||
# Caddy proxies these paths directly — bypasses SvelteKit entirely.
|
||||
|
||||
be: Backend API :8080 {
|
||||
style.fill: "#eef3ff"
|
||||
|
||||
health_be: Health / Version {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "GET /health\nGET /api/version"
|
||||
}
|
||||
|
||||
scrape_be: Scrape admin (direct) {
|
||||
style.fill: "#fff7ed"
|
||||
style.stroke: "#fdba74"
|
||||
label: "POST /scrape\nPOST /scrape/book\nPOST /scrape/book/range"
|
||||
}
|
||||
|
||||
catalogue_be: Catalogue {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "GET /api/catalogue (Meilisearch)\nGET /api/browse (legacy MinIO cache)\nGET /api/ranking\nGET /api/cover/{domain}/{slug}"
|
||||
}
|
||||
|
||||
book_be: Book / Chapter {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "GET /api/book-preview/{slug}\nGET /api/chapter-text/{slug}/{n}\nGET /api/chapter-markdown/{slug}/{n}\nPOST /api/reindex/{slug} ⚠ admin"
|
||||
}
|
||||
|
||||
audio_be: Audio {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "POST /api/audio/{slug}/{n}\nGET /api/audio/status/{slug}/{n}\nGET /api/audio-proxy/{slug}/{n}\nGET /api/voices"
|
||||
}
|
||||
|
||||
presign_be: Presigned URLs {
|
||||
style.fill: "#f0fdf4"
|
||||
style.stroke: "#86efac"
|
||||
label: "GET /api/presign/chapter/{slug}/{n}\nGET /api/presign/audio/{slug}/{n}\nGET /api/presign/voice-sample/{voice}\nGET /api/presign/avatar-upload/{userId}\nGET /api/presign/avatar/{userId}"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── Storage ──────────────────────────────────────────────────────────────────
|
||||
|
||||
storage: Storage {
|
||||
style.fill: "#eaf7ea"
|
||||
|
||||
pb: PocketBase :8090 {
|
||||
shape: cylinder
|
||||
label: "auth · books · progress\ncomments · library\nscrape_jobs · audio_cache\nranking"
|
||||
}
|
||||
mn: MinIO :9000 {
|
||||
shape: cylinder
|
||||
label: "chapters · audio\navatars · catalogue (browse)"
|
||||
}
|
||||
ms: Meilisearch :7700 {
|
||||
shape: cylinder
|
||||
label: "index: books\nfilterable: status · genres\nsortable: rank · rating\n total_chapters · meta_updated"
|
||||
}
|
||||
vk: Valkey :6379 {
|
||||
shape: cylinder
|
||||
label: "presign URL cache (TTL ~55 min)\nAsynq job queue (runner)"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── Caddy routing ────────────────────────────────────────────────────────────
|
||||
|
||||
client -> caddy: HTTPS :443
|
||||
|
||||
caddy -> sk: "/* (catch-all)\n→ SvelteKit enforces auth"
|
||||
caddy -> be: "/health /scrape*\n/api/browse /api/catalogue /api/ranking\n/api/version /api/book-preview/*\n/api/chapter-text/* /api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio* /api/voices /api/presign/*"
|
||||
caddy -> storage.mn: "/avatars/* /audio/* /chapters/*\n(presigned MinIO GETs)"
|
||||
|
||||
# ─── SvelteKit → Backend (server-side proxy) ──────────────────────────────────
|
||||
|
||||
sk.catalogue_sk -> be.catalogue_be: internal proxy
|
||||
sk.book_sk -> be.book_be: internal proxy
|
||||
sk.audio_sk -> be.audio_be: internal proxy
|
||||
sk.presign_sk -> be.presign_be: internal proxy
|
||||
sk.presign_user -> be.presign_be: internal proxy
|
||||
|
||||
# ─── SvelteKit → Storage (direct) ────────────────────────────────────────────
|
||||
|
||||
sk.auth -> storage.pb: sessions / users
|
||||
sk.scrape_sk -> storage.pb: scrape job records
|
||||
sk.progress_sk -> storage.pb
|
||||
sk.library_sk -> storage.pb
|
||||
sk.comments_sk -> storage.pb
|
||||
|
||||
# ─── Backend → Storage ────────────────────────────────────────────────────────
|
||||
|
||||
be.catalogue_be -> storage.ms: full-text search + facets
|
||||
be.catalogue_be -> storage.pb: ranking records
|
||||
be.catalogue_be -> storage.mn: cover presign
|
||||
be.book_be -> storage.mn: chapter objects
|
||||
be.book_be -> storage.pb: book metadata
|
||||
be.audio_be -> storage.mn: audio presign
|
||||
be.audio_be -> storage.vk: presign cache
|
||||
be.presign_be -> storage.vk: check / set presign cache
|
||||
be.presign_be -> storage.mn: generate presigned URL
|
||||
128
docs/d2/api-routing.svg
Normal file
128
docs/d2/api-routing.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 60 KiB |
180
docs/d2/architecture.d2
Normal file
180
docs/d2/architecture.d2
Normal file
@@ -0,0 +1,180 @@
|
||||
direction: right
|
||||
|
||||
# ─── External ─────────────────────────────────────────────────────────────────
|
||||
|
||||
novelfire: novelfire.net {
|
||||
shape: cloud
|
||||
style.fill: "#f0f4ff"
|
||||
label: "novelfire.net\n(scrape source)"
|
||||
}
|
||||
|
||||
kokoro: Kokoro-FastAPI TTS {
|
||||
shape: cloud
|
||||
style.fill: "#f0f4ff"
|
||||
label: "Kokoro-FastAPI TTS\n(self-hosted · homelab)\nchapter audio"
|
||||
}
|
||||
|
||||
pockettts: pocket-tts {
|
||||
shape: cloud
|
||||
style.fill: "#f0f4ff"
|
||||
label: "pocket-tts\n(self-hosted · homelab)\nvoice sample MP3s"
|
||||
}
|
||||
|
||||
letsencrypt: Let's Encrypt {
|
||||
shape: cloud
|
||||
style.fill: "#f0f4ff"
|
||||
label: "Let's Encrypt\n(ACME TLS-ALPN-01)"
|
||||
}
|
||||
|
||||
browser: Browser / iOS App {
|
||||
shape: person
|
||||
style.fill: "#fff9e6"
|
||||
}
|
||||
|
||||
# ─── Init containers (one-shot) ───────────────────────────────────────────────
|
||||
|
||||
init: Init containers {
|
||||
style.fill: "#f5f5f5"
|
||||
style.stroke-dash: 4
|
||||
|
||||
minio-init: minio-init {
|
||||
shape: rectangle
|
||||
label: "minio-init\n(mc: create buckets\n chapters · audio\n avatars · catalogue)"
|
||||
}
|
||||
|
||||
pb-init: pb-init {
|
||||
shape: rectangle
|
||||
label: "pb-init\n(bootstrap PocketBase\n collections + schema)"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── Storage ──────────────────────────────────────────────────────────────────
|
||||
|
||||
storage: Storage {
|
||||
style.fill: "#eaf7ea"
|
||||
|
||||
minio: MinIO {
|
||||
shape: cylinder
|
||||
label: "MinIO :9000\nbuckets:\n chapters · audio\n avatars · catalogue"
|
||||
}
|
||||
|
||||
pocketbase: PocketBase {
|
||||
shape: cylinder
|
||||
label: "PocketBase :8090\ncollections:\n books · chapters_idx\n audio_cache · progress\n scrape_jobs · app_users\n ranking · library\n comments"
|
||||
}
|
||||
|
||||
valkey: Valkey {
|
||||
shape: cylinder
|
||||
label: "Valkey :6379\npresign URL cache (TTL ~55 min)\nAsynq job queue (runner tasks)"
|
||||
}
|
||||
|
||||
meilisearch: Meilisearch {
|
||||
shape: cylinder
|
||||
label: "Meilisearch :7700\nindex: books\n(filterable: status · genres\n sortable: rank · rating\n total_chapters · meta_updated)"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── Application — prod VPS (165.22.70.138) ───────────────────────────────────
|
||||
|
||||
app: Application — prod (165.22.70.138) {
|
||||
style.fill: "#eef3ff"
|
||||
|
||||
caddy: caddy {
|
||||
shape: rectangle
|
||||
label: "Caddy :443 / :80 / :6380\ncustom build\n+ caddy-l4 (Redis TCP proxy)\n+ caddy-ratelimit\nauto-HTTPS · security headers\nrate limiting (per-IP)\nstatic error pages (404/502/503/504)\nCrowdSec bouncer"
|
||||
}
|
||||
|
||||
backend: backend {
|
||||
shape: rectangle
|
||||
label: "Backend API :8080\n(Go)\nHTTP API server\nffmpeg (audio sample conv.)\nOpenTelemetry tracing\nSentry / GlitchTip errors"
|
||||
}
|
||||
|
||||
ui: ui {
|
||||
shape: rectangle
|
||||
label: "SvelteKit UI :3000\n(adapter-node)\nSSR · session auth\nserver-side API proxy"
|
||||
}
|
||||
|
||||
crowdsec: CrowdSec {
|
||||
shape: rectangle
|
||||
label: "CrowdSec :8080\nsecurity engine\nreads Caddy JSON logs\nbouncer integrated in Caddy"
|
||||
}
|
||||
|
||||
dozzle: Dozzle agent {
|
||||
shape: rectangle
|
||||
label: "Dozzle agent\n127.0.0.1:7007\nlog relay → homelab dashboard"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── Runner — homelab (192.168.0.109) ────────────────────────────────────────
|
||||
|
||||
homelab: Runner — homelab (192.168.0.109) {
|
||||
style.fill: "#fef9ec"
|
||||
|
||||
runner: runner {
|
||||
shape: rectangle
|
||||
label: "Runner :9091\n(Go background worker)\nscrape pipeline\nTTS audio job queue\nPrometheus /metrics\ncron: catalogue refresh\nAsynq worker → Valkey"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── Ops ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
ops: Ops {
|
||||
style.fill: "#f5f5f5"
|
||||
|
||||
watchtower: Watchtower {
|
||||
shape: rectangle
|
||||
label: "Watchtower\n(containrrr/watchtower)\npolls Docker Hub every 5 min\nautopulls + redeploys:\n backend · ui\n(runner: label-disabled on prod)"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── CI / CD ──────────────────────────────────────────────────────────────────
|
||||
|
||||
cicd: CI / CD {
|
||||
style.fill: "#f0f9ff"
|
||||
|
||||
gitea: Gitea Actions {
|
||||
shape: rectangle
|
||||
label: "Gitea Actions\n(homelab runner)\ntag v* trigger:\n test-backend\n check-ui (type-check + build)\n docker-backend\n docker-runner\n docker-ui (bakes releases.json)\n docker-caddy\n → push Docker Hub\n → Gitea Release"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── Init → Storage ───────────────────────────────────────────────────────────
|
||||
|
||||
init.minio-init -> storage.minio: create buckets {style.stroke-dash: 4}
|
||||
init.pb-init -> storage.pocketbase: bootstrap schema {style.stroke-dash: 4}
|
||||
|
||||
# ─── App internal ─────────────────────────────────────────────────────────────
|
||||
|
||||
app.caddy -> app.ui: "/* (catch-all)\nSvelteKit — auth enforced"
|
||||
app.caddy -> app.backend: "/health /scrape*\n/api/browse /api/catalogue\n/api/ranking /api/version\n/api/book-preview/*\n/api/chapter-text/*\n/api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/* /api/voices\n/api/audio* /api/presign/*"
|
||||
app.caddy -> storage.minio: "/avatars/* /audio/*\n/chapters/*\n(presigned GETs)"
|
||||
app.caddy -> app.crowdsec: bouncer check (15 s poll)
|
||||
app.caddy -> letsencrypt: ACME cert (TLS-ALPN-01)
|
||||
|
||||
app.ui -> app.backend: "internal REST proxy\n(server-side only)"
|
||||
app.ui -> storage.pocketbase: "auth · sessions\nprogress · library\ncomments"
|
||||
|
||||
app.backend -> storage.minio: "chapter objs · audio MP3s\navatars · browse cache"
|
||||
app.backend -> storage.pocketbase: "books · scrape_jobs\naudio_cache · ranking"
|
||||
app.backend -> storage.valkey: "presign URL cache\n(SET/GET TTL ~55 min)"
|
||||
app.backend -> storage.meilisearch: "catalogue search\nfacets: genres · status"
|
||||
app.backend -> pockettts: "voice sample gen.\n(on-demand · ffmpeg conv.)"
|
||||
|
||||
# ─── Runner → deps ────────────────────────────────────────────────────────────
|
||||
|
||||
homelab.runner -> novelfire: "HTTP scrape\nHTML → Markdown"
|
||||
homelab.runner -> kokoro: "TTS generation\ntext → MP3"
|
||||
homelab.runner -> storage.minio: "write chapters\n& audio MP3s"
|
||||
homelab.runner -> storage.pocketbase: "read/update scrape_jobs\nwrite book records"
|
||||
homelab.runner -> storage.meilisearch: "index books\n(on scrape completion)"
|
||||
homelab.runner -> storage.valkey: "Asynq job queue\n(task consume)"
|
||||
|
||||
# ─── Client ───────────────────────────────────────────────────────────────────
|
||||
|
||||
browser -> app.caddy: HTTPS :443\n(single entry point)
|
||||
|
||||
# ─── Ops / CI ─────────────────────────────────────────────────────────────────
|
||||
|
||||
ops.watchtower -> app.backend: watch (label-enabled)
|
||||
ops.watchtower -> app.ui: watch (label-enabled)
|
||||
cicd.gitea -> ops.watchtower: push to Docker Hub\n→ Watchtower detects new tag
|
||||
132
docs/d2/architecture.svg
Normal file
132
docs/d2/architecture.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 65 KiB |
72
docs/mermaid/architecture.mermaid.md
Normal file
72
docs/mermaid/architecture.mermaid.md
Normal file
@@ -0,0 +1,72 @@
|
||||
# Architecture Overview
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
%% ── External ──────────────────────────────────────────────────────────
|
||||
NF([novelfire.net])
|
||||
KK([Kokoro-FastAPI TTS])
|
||||
LE([Let's Encrypt])
|
||||
CL([Browser / iOS App])
|
||||
|
||||
%% ── Init containers ───────────────────────────────────────────────────
|
||||
subgraph INIT["Init containers (one-shot)"]
|
||||
MI[minio-init\nmc: create buckets]
|
||||
PI[pb-init\nbootstrap collections]
|
||||
end
|
||||
|
||||
%% ── Storage ───────────────────────────────────────────────────────────
|
||||
subgraph STORAGE["Storage"]
|
||||
MN[(MinIO :9000\nchapters · audio\navatars · browse)]
|
||||
PB[(PocketBase :8090\nbooks · chapters_idx\naudio_cache · progress\nscrape_jobs · app_users · ranking)]
|
||||
VK[(Valkey :6379\npresign URL cache\nTTL-based · shared)]
|
||||
MS[(Meilisearch :7700\nindex: books)]
|
||||
end
|
||||
|
||||
%% ── Application ───────────────────────────────────────────────────────
|
||||
subgraph APP["Application"]
|
||||
CD["Caddy :443/:80\ncustom build + caddy-ratelimit\nauto-HTTPS · security headers\nrate limiting · error pages"]
|
||||
BE[Backend API :8080\nGo HTTP server]
|
||||
RN[Runner :9091\nGo background worker\n/metrics endpoint]
|
||||
UI[SvelteKit UI :3000\nadapter-node]
|
||||
end
|
||||
|
||||
%% ── Ops ───────────────────────────────────────────────────────────────
|
||||
subgraph OPS["Ops"]
|
||||
WT[Watchtower\npolls every 5 min\nautopull + redeploy\nbackend · runner · ui]
|
||||
end
|
||||
|
||||
%% ── Init → Storage ────────────────────────────────────────────────────
|
||||
MI -.->|create buckets| MN
|
||||
PI -.->|bootstrap schema| PB
|
||||
|
||||
%% ── App → Storage ─────────────────────────────────────────────────────
|
||||
BE -->|blobs| MN
|
||||
BE -->|structured records| PB
|
||||
BE -->|cache presigned URLs| VK
|
||||
RN -->|chapter markdown & audio| MN
|
||||
RN -->|read/update jobs & books| PB
|
||||
RN -->|index books on scrape| MS
|
||||
UI -->|read presign cache| VK
|
||||
UI -->|auth · progress · comments| PB
|
||||
|
||||
%% ── App internal ──────────────────────────────────────────────────────
|
||||
UI -->|"REST API (server-side)\n/api/catalogue /api/book-preview\n/api/chapter-text /api/audio"| BE
|
||||
|
||||
%% ── Caddy routing ─────────────────────────────────────────────────────
|
||||
CD -->|"/* catch-all\n/api/scrape/*\n/api/chapter-text-preview/*\n→ SvelteKit (auth enforced)"| UI
|
||||
CD -->|"/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*"| BE
|
||||
CD -->|/avatars/* presigned GETs| MN
|
||||
|
||||
%% ── Runner → External ─────────────────────────────────────────────────
|
||||
RN -->|scrape HTTP GET| NF
|
||||
RN -->|TTS HTTP POST| KK
|
||||
CD -->|ACME certificate| LE
|
||||
|
||||
%% ── Ops ───────────────────────────────────────────────────────────────
|
||||
WT -->|watch label-enabled| BE
|
||||
WT -->|watch label-enabled| RN
|
||||
WT -->|watch label-enabled| UI
|
||||
|
||||
%% ── Client ────────────────────────────────────────────────────────────
|
||||
CL -->|HTTPS :443 single entry| CD
|
||||
```
|
||||
102
docs/mermaid/data-flow.mermaid.md
Normal file
102
docs/mermaid/data-flow.mermaid.md
Normal file
@@ -0,0 +1,102 @@
|
||||
# Data Flow — Scrape & TTS Job Pipeline
|
||||
|
||||
How content moves from novelfire.net through the runner into storage, and how
|
||||
audio is generated on-demand via the backend.
|
||||
|
||||
## Catalogue Scrape Pipeline
|
||||
|
||||
The runner performs a background catalogue walk on startup and then on a
|
||||
configurable interval (`RUNNER_CATALOGUE_REFRESH_INTERVAL`, default 24 h).
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A([Runner starts / refresh tick]) --> B[Walk novelfire.net catalogue\npages 1…N]
|
||||
B --> C{Book already\nin PocketBase?}
|
||||
C -- no --> D[Scrape book metadata\ntitle · author · genres\ncover · summary · status]
|
||||
C -- yes --> E[Check for new chapters\ncompare total_chapters]
|
||||
D --> F[Write BookMeta\nto PocketBase books]
|
||||
E --> G{New chapters\nfound?}
|
||||
G -- no --> Z([Done — next book])
|
||||
G -- yes --> H
|
||||
F --> H[Scrape chapter list with upTo limit\n→ chapters_idx in PocketBase\nretries on 429 with Retry-After backoff]
|
||||
H --> I[Worker pool — N goroutines\nRUNNER_MAX_CONCURRENT_SCRAPE]
|
||||
I --> J[For each missing chapter:\nGET chapter HTML from novelfire.net]
|
||||
J --> K[Parse HTML → Markdown\nhtmlutil.NodeToMarkdown]
|
||||
K --> L[PUT object to MinIO\nchapters/{slug}/{n}.md]
|
||||
L --> M[Upsert book doc\nto Meilisearch index: books]
|
||||
M --> Z
|
||||
F --> M
|
||||
```
|
||||
|
||||
## On-Demand Single-Book Scrape
|
||||
|
||||
Triggered when a user visits `/books/{slug}` and the book is not in PocketBase.
|
||||
The UI calls `GET /api/book-preview/{slug}` → backend enqueues a scrape task.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
actor U as User
|
||||
participant UI as SvelteKit UI
|
||||
participant BE as Backend API
|
||||
participant TQ as Task Queue (PocketBase)
|
||||
participant RN as Runner
|
||||
participant NF as novelfire.net
|
||||
participant PB as PocketBase
|
||||
participant MN as MinIO
|
||||
participant MS as Meilisearch
|
||||
|
||||
U->>UI: Visit /books/{slug}
|
||||
UI->>BE: GET /api/book-preview/{slug}
|
||||
BE->>PB: getBook(slug) — not found
|
||||
BE->>TQ: INSERT scrape_task (slug, status=pending)
|
||||
BE-->>UI: 202 {task_id, message}
|
||||
UI-->>U: "Scraping…" placeholder
|
||||
|
||||
RN->>TQ: Poll for pending tasks
|
||||
TQ-->>RN: scrape_task (slug)
|
||||
RN->>NF: GET novelfire.net/book/{slug}
|
||||
NF-->>RN: HTML
|
||||
RN->>PB: upsert book + chapters_idx
|
||||
RN->>MN: PUT chapter objects
|
||||
RN->>MS: UpsertBook doc
|
||||
RN->>TQ: UPDATE task status=done
|
||||
|
||||
U->>UI: Poll GET /api/scrape/tasks/{task_id}
|
||||
UI->>BE: GET /api/scrape/status
|
||||
BE->>TQ: get task
|
||||
TQ-->>BE: status=done
|
||||
BE-->>UI: {status:"done"}
|
||||
UI-->>U: Redirect to /books/{slug}
|
||||
```
|
||||
|
||||
## TTS Audio Generation Pipeline
|
||||
|
||||
Audio is generated lazily: on first request the job is enqueued; subsequent
|
||||
requests poll for completion and then stream from MinIO via presigned URL.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A(["POST /api/audio/{slug}/{n}\nbody: voice=af_bella"]) --> B{Audio already\nin MinIO?}
|
||||
B -- yes --> C[200 status: done]
|
||||
B -- no --> D{Job already\nin queue?}
|
||||
D -- "yes pending/generating" --> E[202 task_id + status]
|
||||
D -- no --> F[INSERT audio_task\nstatus=pending\nin PocketBase]
|
||||
F --> E
|
||||
|
||||
G([Runner polls task queue]) --> H[Claim audio_task\nstatus=generating]
|
||||
H --> I["GET /api/chapter-text/{slug}/{n}\nfrom backend — plain text"]
|
||||
I --> J[POST /v1/audio/speech\nto Kokoro-FastAPI\nbody: text + voice]
|
||||
J --> K[Stream MP3 response]
|
||||
K --> L[PUT object to MinIO\naudio/{slug}/{n}/{voice}.mp3]
|
||||
L --> M[UPDATE audio_task\nstatus=done]
|
||||
|
||||
N(["Client polls\nGET /api/audio/status/{slug}/{n}"]) --> O{status?}
|
||||
O -- "pending/generating" --> N
|
||||
O -- done --> P["GET /api/presign/audio/{slug}/{n}"]
|
||||
P --> Q{Valkey cache hit?}
|
||||
Q -- yes --> R[302 → presigned URL]
|
||||
Q -- no --> S[GeneratePresignedURL\nfrom MinIO — TTL 1h]
|
||||
S --> T[Cache in Valkey\nTTL 3500s]
|
||||
T --> R
|
||||
R --> U([Client streams audio\ndirectly from MinIO])
|
||||
```
|
||||
111
docs/mermaid/request-flow.mermaid.md
Normal file
111
docs/mermaid/request-flow.mermaid.md
Normal file
@@ -0,0 +1,111 @@
|
||||
# Request Flow
|
||||
|
||||
Two representative request paths through the stack: a **page load** (SSR) and a
|
||||
**media playback** (presigned URL → direct MinIO stream).
|
||||
|
||||
## SSR Page Load — Catalogue / Book Detail
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
actor C as Browser / iOS App
|
||||
participant CD as Caddy :443
|
||||
participant UI as SvelteKit UI :3000
|
||||
participant BE as Backend API :8080
|
||||
participant MS as Meilisearch :7700
|
||||
participant PB as PocketBase :8090
|
||||
participant VK as Valkey :6379
|
||||
|
||||
C->>CD: HTTPS GET /catalogue
|
||||
CD->>UI: proxy /* (SvelteKit catch-all)
|
||||
UI->>BE: GET /api/catalogue?page=1&sort=popular
|
||||
BE->>MS: search(query, filters, sort)
|
||||
MS-->>BE: [{slug, title, …}, …]
|
||||
BE-->>UI: {books[], page, total, has_next}
|
||||
UI-->>CD: SSR HTML
|
||||
CD-->>C: 200 HTML
|
||||
|
||||
Note over C,UI: Infinite scroll — client fetches next page via SvelteKit API route
|
||||
C->>CD: HTTPS GET /api/catalogue-page?page=2
|
||||
CD->>UI: proxy /* (SvelteKit /api/catalogue-page server route)
|
||||
UI->>BE: GET /api/catalogue?page=2
|
||||
BE->>MS: search(…)
|
||||
MS-->>BE: next page
|
||||
BE-->>UI: {books[], …}
|
||||
UI-->>C: JSON
|
||||
```
|
||||
|
||||
## Audio Playback — Presigned URL Flow
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
actor C as Browser / iOS App
|
||||
participant CD as Caddy :443
|
||||
participant UI as SvelteKit UI :3000
|
||||
participant BE as Backend API :8080
|
||||
participant VK as Valkey :6379
|
||||
participant MN as MinIO :9000
|
||||
|
||||
C->>CD: GET /api/presign/audio/{slug}/{n}?voice=af_bella
|
||||
CD->>UI: proxy /* (SvelteKit /api/presign/audio route)
|
||||
UI->>BE: GET /api/presign/audio/{slug}/{n}?voice=af_bella
|
||||
BE->>VK: GET presign:audio:{slug}:{n}:{voice}
|
||||
alt cache hit
|
||||
VK-->>BE: presigned URL (TTL remaining)
|
||||
BE-->>UI: 302 redirect → presigned URL
|
||||
UI-->>C: 302 redirect
|
||||
else cache miss
|
||||
BE->>MN: GeneratePresignedURL(audio-bucket, key, 1h)
|
||||
MN-->>BE: presigned URL
|
||||
BE->>VK: SET presign:audio:… EX 3500
|
||||
BE-->>UI: 302 redirect → presigned URL
|
||||
UI-->>C: 302 redirect
|
||||
end
|
||||
C->>MN: GET presigned URL (direct, no proxy)
|
||||
MN-->>C: audio/mpeg stream
|
||||
```
|
||||
|
||||
## Chapter Read — SSR + Content Fetch
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
actor C as Browser / iOS App
|
||||
participant CD as Caddy :443
|
||||
participant UI as SvelteKit UI :3000
|
||||
participant BE as Backend API :8080
|
||||
participant PB as PocketBase :8090
|
||||
participant MN as MinIO :9000
|
||||
|
||||
C->>CD: HTTPS GET /books/{slug}/chapters/{n}
|
||||
CD->>UI: proxy /* (SvelteKit catch-all)
|
||||
UI->>PB: getBook(slug) + listChapterIdx(slug)
|
||||
PB-->>UI: book meta + chapter list
|
||||
UI->>BE: GET /api/chapter-text/{slug}/{n}
|
||||
BE->>MN: GetObject(chapters-bucket, {slug}/{n}.md)
|
||||
MN-->>BE: markdown text
|
||||
BE-->>UI: plain text (markdown stripped)
|
||||
Note over UI: marked() → HTML
|
||||
UI-->>CD: SSR HTML
|
||||
CD-->>C: 200 HTML
|
||||
```
|
||||
|
||||
## Caddy Request Lifecycle
|
||||
|
||||
Shows how security hardening applies before a request reaches any upstream.
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A([Incoming HTTPS request]) --> B[TLS termination\nLet's Encrypt cert]
|
||||
B --> C{Rate limit check\ncaddy-ratelimit}
|
||||
C -- over limit --> D[429 Too Many Requests]
|
||||
C -- ok --> E[Add security headers\nX-Frame-Options · X-Content-Type-Options\nReferrer-Policy · Permissions-Policy\nHSTS · X-XSS-Protection\nremove Server header]
|
||||
E --> F{Route match}
|
||||
F -- "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*" --> G[reverse_proxy → backend:8080]
|
||||
F -- "/avatars/*" --> H[reverse_proxy → minio:9000]
|
||||
F -- "/* everything else\n(incl. /api/scrape/*\n/api/chapter-text-preview/*)" --> I[reverse_proxy → ui:3000\nSvelteKit auth middleware runs]
|
||||
G --> J{Upstream healthy?}
|
||||
H --> J
|
||||
I --> J
|
||||
J -- yes --> K([Response to client])
|
||||
J -- "502/503/504" --> L[handle_errors\nstatic HTML from /srv/errors/]
|
||||
L --> K
|
||||
```
|
||||
5
dozzle/users.yml
Normal file
5
dozzle/users.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
users:
|
||||
admin:
|
||||
name: admin
|
||||
email: admin@libnovel.cc
|
||||
password: "$2y$10$4jqLza2grpxnQn0EGux2C.UmlSxRmOvH/J1ySzOBxMZgW6cA2TnmK"
|
||||
471
homelab/docker-compose.yml
Normal file
471
homelab/docker-compose.yml
Normal file
@@ -0,0 +1,471 @@
|
||||
# LibNovel homelab
|
||||
#
|
||||
# Runs on 192.168.0.109. Hosts:
|
||||
# - libnovel runner (background task worker)
|
||||
# - tooling: GlitchTip, Umami, Fider, Dozzle, Uptime Kuma, Gotify
|
||||
# - observability: OTel Collector, Tempo, Loki, Prometheus, Grafana
|
||||
# - cloudflared tunnel (public subdomains via Cloudflare Zero Trust)
|
||||
# - shared Postgres for tooling DBs
|
||||
#
|
||||
# All secrets come from Doppler (project=libnovel, config=prd_homelab).
|
||||
# Run with: doppler run -- docker compose up -d
|
||||
#
|
||||
# Public subdomains (via Cloudflare Tunnel — no ports exposed to internet):
|
||||
# errors.libnovel.cc → glitchtip-web:8000
|
||||
# analytics.libnovel.cc → umami:3000
|
||||
# feedback.libnovel.cc → fider:3000
|
||||
# logs.libnovel.cc → dozzle:8080
|
||||
# uptime.libnovel.cc → uptime-kuma:3001
|
||||
# push.libnovel.cc → gotify:80
|
||||
# grafana.libnovel.cc → grafana:3000
|
||||
|
||||
services:
|
||||
|
||||
# ── Cloudflare Tunnel ───────────────────────────────────────────────────────
|
||||
# Outbound-only encrypted tunnel to Cloudflare.
|
||||
# Routes all public subdomains to their respective containers on this network.
|
||||
# No inbound ports needed — cloudflared initiates all connections outward.
|
||||
cloudflared:
|
||||
image: cloudflare/cloudflared:latest
|
||||
restart: unless-stopped
|
||||
command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN}
|
||||
environment:
|
||||
CLOUDFLARE_TUNNEL_TOKEN: "${CLOUDFLARE_TUNNEL_TOKEN}"
|
||||
|
||||
# ── LibNovel Runner ─────────────────────────────────────────────────────────
|
||||
# Background task worker. Connects to prod PocketBase, MinIO, Meilisearch
|
||||
# via their public subdomains (pb.libnovel.cc, storage.libnovel.cc, etc.)
|
||||
runner:
|
||||
image: kalekber/libnovel-runner:latest
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 135s
|
||||
labels:
|
||||
com.centurylinklabs.watchtower.enable: "true"
|
||||
environment:
|
||||
POCKETBASE_URL: "https://pb.libnovel.cc"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}"
|
||||
|
||||
MINIO_ENDPOINT: "storage.libnovel.cc"
|
||||
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}"
|
||||
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD}"
|
||||
MINIO_USE_SSL: "true"
|
||||
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT}"
|
||||
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL}"
|
||||
|
||||
MEILI_URL: "${MEILI_URL}"
|
||||
MEILI_API_KEY: "${MEILI_API_KEY}"
|
||||
VALKEY_ADDR: ""
|
||||
GODEBUG: "preferIPv4=1"
|
||||
|
||||
KOKORO_URL: "http://kokoro-fastapi:8880"
|
||||
KOKORO_VOICE: "${KOKORO_VOICE}"
|
||||
|
||||
POCKET_TTS_URL: "http://pocket-tts:8000"
|
||||
|
||||
RUNNER_WORKER_ID: "${RUNNER_WORKER_ID}"
|
||||
RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL}"
|
||||
RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE}"
|
||||
RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO}"
|
||||
RUNNER_TIMEOUT: "${RUNNER_TIMEOUT}"
|
||||
RUNNER_METRICS_ADDR: "${RUNNER_METRICS_ADDR}"
|
||||
RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH: "true"
|
||||
|
||||
LOG_LEVEL: "${LOG_LEVEL}"
|
||||
GLITCHTIP_DSN: "${GLITCHTIP_DSN}"
|
||||
|
||||
# OTel — send runner traces/metrics to the local collector (HTTP)
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318"
|
||||
OTEL_SERVICE_NAME: "runner"
|
||||
|
||||
healthcheck:
|
||||
test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# ── Shared Postgres ─────────────────────────────────────────────────────────
|
||||
# Hosts glitchtip, umami, and fider databases.
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: "${POSTGRES_USER}"
|
||||
POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
|
||||
POSTGRES_DB: postgres
|
||||
expose:
|
||||
- "5432"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── Postgres database initialisation ────────────────────────────────────────
|
||||
postgres-init:
|
||||
image: postgres:16-alpine
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
PGPASSWORD: "${POSTGRES_PASSWORD}"
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='fider'\" | grep -q 1 ||
|
||||
psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE fider\";
|
||||
psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='glitchtip'\" | grep -q 1 ||
|
||||
psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE glitchtip\";
|
||||
psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='umami'\" | grep -q 1 ||
|
||||
psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE umami\";
|
||||
echo 'postgres-init: databases ready';
|
||||
"
|
||||
restart: "no"
|
||||
|
||||
# ── GlitchTip DB migration ──────────────────────────────────────────────────
|
||||
glitchtip-migrate:
|
||||
image: glitchtip/glitchtip:latest
|
||||
depends_on:
|
||||
postgres-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip"
|
||||
SECRET_KEY: "${GLITCHTIP_SECRET_KEY}"
|
||||
GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}"
|
||||
EMAIL_URL: "${GLITCHTIP_EMAIL_URL}"
|
||||
DEFAULT_FROM_EMAIL: "noreply@libnovel.cc"
|
||||
VALKEY_URL: "redis://valkey:6379/1"
|
||||
command: "./manage.py migrate"
|
||||
restart: "no"
|
||||
|
||||
# ── GlitchTip web ───────────────────────────────────────────────────────────
|
||||
glitchtip-web:
|
||||
image: glitchtip/glitchtip:latest
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
glitchtip-migrate:
|
||||
condition: service_completed_successfully
|
||||
expose:
|
||||
- "8000"
|
||||
environment:
|
||||
DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip"
|
||||
SECRET_KEY: "${GLITCHTIP_SECRET_KEY}"
|
||||
GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}"
|
||||
EMAIL_URL: "${GLITCHTIP_EMAIL_URL}"
|
||||
DEFAULT_FROM_EMAIL: "noreply@libnovel.cc"
|
||||
VALKEY_URL: "redis://valkey:6379/1"
|
||||
PORT: "8000"
|
||||
ENABLE_USER_REGISTRATION: "false"
|
||||
healthcheck:
|
||||
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/0/')"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── GlitchTip worker ────────────────────────────────────────────────────────
|
||||
glitchtip-worker:
|
||||
image: glitchtip/glitchtip:latest
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
glitchtip-migrate:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip"
|
||||
SECRET_KEY: "${GLITCHTIP_SECRET_KEY}"
|
||||
GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}"
|
||||
EMAIL_URL: "${GLITCHTIP_EMAIL_URL}"
|
||||
DEFAULT_FROM_EMAIL: "noreply@libnovel.cc"
|
||||
VALKEY_URL: "redis://valkey:6379/1"
|
||||
SERVER_ROLE: "worker"
|
||||
|
||||
# ── Umami ───────────────────────────────────────────────────────────────────
|
||||
umami:
|
||||
image: ghcr.io/umami-software/umami:postgresql-latest
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
postgres-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
expose:
|
||||
- "3000"
|
||||
environment:
|
||||
DATABASE_URL: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/umami"
|
||||
APP_SECRET: "${UMAMI_APP_SECRET}"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://localhost:3000/api/heartbeat"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── Fider ───────────────────────────────────────────────────────────────────
|
||||
fider:
|
||||
image: getfider/fider:stable
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
postgres-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
expose:
|
||||
- "3000"
|
||||
environment:
|
||||
BASE_URL: "${FIDER_BASE_URL}"
|
||||
DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/fider?sslmode=disable"
|
||||
JWT_SECRET: "${FIDER_JWT_SECRET}"
|
||||
EMAIL_NOREPLY: "noreply@libnovel.cc"
|
||||
EMAIL_SMTP_HOST: "${FIDER_SMTP_HOST}"
|
||||
EMAIL_SMTP_PORT: "${FIDER_SMTP_PORT}"
|
||||
EMAIL_SMTP_USERNAME: "${FIDER_SMTP_USER}"
|
||||
EMAIL_SMTP_PASSWORD: "${FIDER_SMTP_PASSWORD}"
|
||||
EMAIL_SMTP_ENABLE_STARTTLS: "${FIDER_SMTP_ENABLE_STARTTLS}"
|
||||
OAUTH_GOOGLE_CLIENTID: "${OAUTH_GOOGLE_CLIENTID}"
|
||||
OAUTH_GOOGLE_SECRET: "${OAUTH_GOOGLE_SECRET}"
|
||||
OAUTH_GITHUB_CLIENTID: "${OAUTH_GITHUB_CLIENTID}"
|
||||
OAUTH_GITHUB_SECRET: "${OAUTH_GITHUB_SECRET}"
|
||||
|
||||
# ── Dozzle ──────────────────────────────────────────────────────────────────
|
||||
# Watches both homelab and prod containers.
|
||||
# Prod agent runs on 165.22.70.138:7007 (added separately to prod compose).
|
||||
dozzle:
|
||||
image: amir20/dozzle:latest
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- ./dozzle/users.yml:/data/users.yml:ro
|
||||
expose:
|
||||
- "8080"
|
||||
environment:
|
||||
DOZZLE_AUTH_PROVIDER: simple
|
||||
DOZZLE_HOSTNAME: "logs.libnovel.cc"
|
||||
DOZZLE_REMOTE_AGENT: "prod@165.22.70.138:7007"
|
||||
healthcheck:
|
||||
test: ["CMD", "/dozzle", "healthcheck"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── Uptime Kuma ─────────────────────────────────────────────────────────────
|
||||
uptime-kuma:
|
||||
image: louislam/uptime-kuma:1
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- uptime_kuma_data:/app/data
|
||||
expose:
|
||||
- "3001"
|
||||
healthcheck:
|
||||
test: ["CMD", "extra/healthcheck"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── Gotify ──────────────────────────────────────────────────────────────────
|
||||
gotify:
|
||||
image: gotify/server:latest
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- gotify_data:/app/data
|
||||
expose:
|
||||
- "80"
|
||||
environment:
|
||||
GOTIFY_DEFAULTUSER_NAME: "${GOTIFY_ADMIN_USER}"
|
||||
GOTIFY_DEFAULTUSER_PASS: "${GOTIFY_ADMIN_PASS}"
|
||||
GOTIFY_SERVER_PORT: "80"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://localhost:80/health"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── Valkey ──────────────────────────────────────────────────────────────────
|
||||
# Used by GlitchTip for task queuing.
|
||||
valkey:
|
||||
image: valkey/valkey:7-alpine
|
||||
restart: unless-stopped
|
||||
expose:
|
||||
- "6379"
|
||||
volumes:
|
||||
- valkey_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── OTel Collector ──────────────────────────────────────────────────────────
|
||||
# Receives OTLP from backend/ui/runner, fans out to Tempo + Prometheus + Loki.
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:latest
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
|
||||
expose:
|
||||
- "4317" # OTLP gRPC
|
||||
- "4318" # OTLP HTTP
|
||||
- "8888" # Collector self-metrics (scraped by Prometheus)
|
||||
depends_on:
|
||||
- tempo
|
||||
- prometheus
|
||||
- loki
|
||||
# No healthcheck — distroless image has no shell or curl
|
||||
|
||||
# ── Tempo ───────────────────────────────────────────────────────────────────
|
||||
# Distributed trace storage. Receives OTLP from the collector.
|
||||
tempo:
|
||||
image: grafana/tempo:2.6.1
|
||||
restart: unless-stopped
|
||||
command: ["-config.file=/etc/tempo.yaml"]
|
||||
volumes:
|
||||
- ./otel/tempo.yaml:/etc/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
expose:
|
||||
- "3200" # Tempo query API (queried by Grafana)
|
||||
- "4317" # OTLP gRPC ingest (collector → tempo)
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:3200/ready"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── Prometheus ──────────────────────────────────────────────────────────────
|
||||
# Scrapes metrics from backend (via prod), runner, and otel-collector.
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- "--config.file=/etc/prometheus/prometheus.yaml"
|
||||
- "--storage.tsdb.path=/prometheus"
|
||||
- "--storage.tsdb.retention.time=30d"
|
||||
- "--web.enable-remote-write-receiver"
|
||||
volumes:
|
||||
- ./otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
expose:
|
||||
- "9090"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:9090/-/healthy"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── Loki ────────────────────────────────────────────────────────────────────
|
||||
# Log aggregation. Receives logs from OTel collector. Replaces manual Dozzle
|
||||
# tailing for structured log search.
|
||||
loki:
|
||||
image: grafana/loki:latest
|
||||
restart: unless-stopped
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./otel/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki_data:/loki
|
||||
expose:
|
||||
- "3100"
|
||||
# No healthcheck — distroless image has no shell or curl
|
||||
|
||||
# ── Grafana ─────────────────────────────────────────────────────────────────
|
||||
# Single UI for traces (Tempo), metrics (Prometheus), and logs (Loki).
|
||||
# Accessible at grafana.libnovel.cc via Cloudflare Tunnel.
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- tempo
|
||||
- prometheus
|
||||
- loki
|
||||
expose:
|
||||
- "3000"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
- ./otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
environment:
|
||||
GF_SERVER_ROOT_URL: "https://grafana.libnovel.cc"
|
||||
GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER}"
|
||||
GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD}"
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||
GF_FEATURE_TOGGLES_ENABLE: "traceqlEditor"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── Kokoro-FastAPI (GPU TTS) ────────────────────────────────────────────────
|
||||
# OpenAI-compatible TTS service backed by the Kokoro model, running on the
|
||||
# homelab RTX 3050 (8 GB VRAM). Replaces the broken kokoro.kalekber.cc DNS.
|
||||
# Voices match existing IDs: af_bella, af_sky, af_heart, etc.
|
||||
# The runner reaches it at http://kokoro-fastapi:8880 via the Docker network.
|
||||
kokoro-fastapi:
|
||||
image: kokoro-fastapi:latest
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu]
|
||||
expose:
|
||||
- "8880"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://localhost:8880/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
|
||||
# ── pocket-tts (CPU TTS) ────────────────────────────────────────────────────
|
||||
# Lightweight CPU-only TTS using kyutai-labs/pocket-tts.
|
||||
# Image is built locally on homelab from https://github.com/kyutai-labs/pocket-tts
|
||||
# (no prebuilt image published): cd /tmp && git clone --depth=1 https://github.com/kyutai-labs/pocket-tts.git && docker build -t pocket-tts:latest /tmp/pocket-tts
|
||||
# OpenAI-compatible: POST /tts (multipart form) on port 8000.
|
||||
# Voices: alba, marius, javert, jean, fantine, cosette, eponine, azelma, etc.
|
||||
# Not currently used by the runner (runner uses kokoro-fastapi), but available
|
||||
# for experimentation / fallback.
|
||||
pocket-tts:
|
||||
image: pocket-tts:latest
|
||||
restart: unless-stopped
|
||||
command: ["uv", "run", "pocket-tts", "serve", "--host", "0.0.0.0"]
|
||||
expose:
|
||||
- "8000"
|
||||
volumes:
|
||||
- pocket_tts_cache:/root/.cache/pocket_tts
|
||||
- hf_cache:/root/.cache/huggingface
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 120s
|
||||
|
||||
# ── Watchtower ──────────────────────────────────────────────────────────────
|
||||
# Auto-updates runner image when CI pushes a new tag.
|
||||
# Only watches services with the watchtower label.
|
||||
# doppler binary is mounted from the host so watchtower fetches fresh secrets
|
||||
# on every start (notification URL, credentials) without baking them in.
|
||||
watchtower:
|
||||
image: containrrr/watchtower:latest
|
||||
restart: unless-stopped
|
||||
entrypoint: ["/usr/bin/doppler", "run", "--project", "libnovel", "--config", "prd_homelab", "--"]
|
||||
command: ["/watchtower", "--label-enable", "--interval", "300", "--cleanup"]
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- /usr/bin/doppler:/usr/bin/doppler:ro
|
||||
- /root/.doppler:/root/.doppler:ro
|
||||
environment:
|
||||
HOME: "/root"
|
||||
DOCKER_API_VERSION: "1.44"
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
valkey_data:
|
||||
uptime_kuma_data:
|
||||
gotify_data:
|
||||
tempo_data:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
grafana_data:
|
||||
pocket_tts_cache:
|
||||
hf_cache:
|
||||
5
homelab/dozzle/users.yml
Normal file
5
homelab/dozzle/users.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
users:
|
||||
admin:
|
||||
name: admin
|
||||
email: admin@libnovel.cc
|
||||
password: "$2y$10$4jqLza2grpxnQn0EGux2C.UmlSxRmOvH/J1ySzOBxMZgW6cA2TnmK"
|
||||
68
homelab/otel/collector.yaml
Normal file
68
homelab/otel/collector.yaml
Normal file
@@ -0,0 +1,68 @@
|
||||
# OTel Collector config
|
||||
#
|
||||
# Receivers: OTLP (gRPC + HTTP) from backend, ui, runner
|
||||
# Processors: batch for efficiency, resource detection for host metadata
|
||||
# Exporters: Tempo (traces), Prometheus (metrics), Loki (logs)
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
batch:
|
||||
timeout: 5s
|
||||
send_batch_size: 512
|
||||
|
||||
# Attach host metadata to all telemetry
|
||||
resourcedetection:
|
||||
detectors: [env, system]
|
||||
timeout: 5s
|
||||
|
||||
exporters:
|
||||
# Traces → Tempo
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
|
||||
# Metrics → Prometheus (remote write)
|
||||
prometheusremotewrite:
|
||||
endpoint: "http://prometheus:9090/api/v1/write"
|
||||
tls:
|
||||
insecure_skip_verify: true
|
||||
|
||||
# Logs → Loki (via OTLP HTTP endpoint)
|
||||
otlphttp/loki:
|
||||
endpoint: "http://loki:3100/otlp"
|
||||
tls:
|
||||
insecure: true
|
||||
|
||||
# Collector self-observability (optional debug)
|
||||
debug:
|
||||
verbosity: basic
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
pprof:
|
||||
endpoint: 0.0.0.0:1777
|
||||
|
||||
service:
|
||||
extensions: [health_check, pprof]
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [resourcedetection, batch]
|
||||
exporters: [otlp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [resourcedetection, batch]
|
||||
exporters: [prometheusremotewrite]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [resourcedetection, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,16 @@
|
||||
# Grafana alerting provisioning — contact points
|
||||
# Sends all alerts to Gotify (self-hosted push notifications).
|
||||
apiVersion: 1
|
||||
|
||||
contactPoints:
|
||||
- orgId: 1
|
||||
name: Gotify
|
||||
receivers:
|
||||
- uid: gotify-webhook
|
||||
type: webhook
|
||||
settings:
|
||||
url: "http://gotify/message?token=ABZrZgCY-4ivcmt"
|
||||
httpMethod: POST
|
||||
title: "{{ .CommonLabels.alertname }}"
|
||||
message: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ .Annotations.description }}{{ end }}"
|
||||
disableResolveMessage: false
|
||||
@@ -0,0 +1,15 @@
|
||||
# Grafana alerting provisioning — notification policies
|
||||
# Routes all alerts to Gotify by default.
|
||||
apiVersion: 1
|
||||
|
||||
policies:
|
||||
- orgId: 1
|
||||
receiver: Gotify
|
||||
group_by: ["alertname", "service"]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
routes:
|
||||
- receiver: Gotify
|
||||
matchers:
|
||||
- severity =~ "critical|warning"
|
||||
214
homelab/otel/grafana/provisioning/alerting/rules.yaml
Normal file
214
homelab/otel/grafana/provisioning/alerting/rules.yaml
Normal file
@@ -0,0 +1,214 @@
|
||||
# Grafana alerting provisioning — alert rules
|
||||
# Covers: runner down, high task failure rate, audio error spike, backend error spike.
|
||||
apiVersion: 1
|
||||
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: LibNovel Runner
|
||||
folder: LibNovel
|
||||
interval: 1m
|
||||
rules:
|
||||
|
||||
- uid: runner-down
|
||||
title: Runner Down
|
||||
condition: C
|
||||
for: 2m
|
||||
annotations:
|
||||
summary: "LibNovel runner is not reachable"
|
||||
description: "The Prometheus scrape of runner:9091 has been failing for >2 minutes. Tasks are not being processed."
|
||||
labels:
|
||||
severity: critical
|
||||
service: runner
|
||||
data:
|
||||
- refId: A
|
||||
datasourceUid: prometheus
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
model:
|
||||
expr: "up{job=\"libnovel-runner\"}"
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
relativeTimeRange: { from: 300, to: 0 }
|
||||
model:
|
||||
type: classic_conditions
|
||||
conditions:
|
||||
- evaluator: { params: [1], type: lt }
|
||||
operator: { type: and }
|
||||
query: { params: [A] }
|
||||
reducer: { params: [], type: last }
|
||||
|
||||
- uid: runner-high-failure-rate
|
||||
title: Runner High Task Failure Rate
|
||||
condition: C
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "Runner task failure rate is above 20%"
|
||||
description: "More than 20% of runner tasks have been failing for the last 5 minutes. Check runner logs."
|
||||
labels:
|
||||
severity: warning
|
||||
service: runner
|
||||
data:
|
||||
- refId: A
|
||||
datasourceUid: prometheus
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
model:
|
||||
expr: "rate(libnovel_runner_tasks_failed_total[5m]) / clamp_min(rate(libnovel_runner_tasks_completed_total[5m]) + rate(libnovel_runner_tasks_failed_total[5m]), 0.001)"
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
model:
|
||||
type: classic_conditions
|
||||
conditions:
|
||||
- evaluator: { params: [0.2], type: gt }
|
||||
operator: { type: and }
|
||||
query: { params: [A] }
|
||||
reducer: { params: [], type: last }
|
||||
|
||||
- uid: runner-tasks-stalled
|
||||
title: Runner Tasks Stalled
|
||||
condition: C
|
||||
for: 10m
|
||||
annotations:
|
||||
summary: "Runner has tasks running for >10 minutes with no completions"
|
||||
description: "tasks_running > 0 but rate(tasks_completed) is 0. Tasks may be stuck or the runner is in a crash loop."
|
||||
labels:
|
||||
severity: warning
|
||||
service: runner
|
||||
data:
|
||||
- refId: Running
|
||||
datasourceUid: prometheus
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
model:
|
||||
expr: "libnovel_runner_tasks_running"
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
- refId: Rate
|
||||
datasourceUid: prometheus
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
model:
|
||||
expr: "rate(libnovel_runner_tasks_completed_total[10m])"
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
relativeTimeRange: { from: 900, to: 0 }
|
||||
model:
|
||||
type: classic_conditions
|
||||
conditions:
|
||||
- evaluator: { params: [0], type: gt }
|
||||
operator: { type: and }
|
||||
query: { params: [Running] }
|
||||
reducer: { params: [], type: last }
|
||||
- evaluator: { params: [0.001], type: lt }
|
||||
operator: { type: and }
|
||||
query: { params: [Rate] }
|
||||
reducer: { params: [], type: last }
|
||||
|
||||
- orgId: 1
|
||||
name: LibNovel Backend
|
||||
folder: LibNovel
|
||||
interval: 1m
|
||||
rules:
|
||||
|
||||
- uid: backend-high-error-rate
|
||||
title: Backend High Error Rate
|
||||
condition: C
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "Backend API error rate above 5%"
|
||||
description: "More than 5% of backend HTTP requests are returning 5xx status codes (as seen from UI OTel instrumentation)."
|
||||
labels:
|
||||
severity: warning
|
||||
service: backend
|
||||
data:
|
||||
- refId: A
|
||||
datasourceUid: prometheus
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
model:
|
||||
expr: "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\", http_response_status_code=~\"5..\"}[5m])) / clamp_min(sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\"}[5m])), 0.001)"
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
model:
|
||||
type: classic_conditions
|
||||
conditions:
|
||||
- evaluator: { params: [0.05], type: gt }
|
||||
operator: { type: and }
|
||||
query: { params: [A] }
|
||||
reducer: { params: [], type: last }
|
||||
|
||||
- uid: backend-high-p95-latency
|
||||
title: Backend High p95 Latency
|
||||
condition: C
|
||||
for: 5m
|
||||
annotations:
|
||||
summary: "Backend p95 latency above 2s"
|
||||
description: "95th percentile latency of backend spans has exceeded 2 seconds for >5 minutes."
|
||||
labels:
|
||||
severity: warning
|
||||
service: backend
|
||||
data:
|
||||
- refId: A
|
||||
datasourceUid: prometheus
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
model:
|
||||
expr: "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))"
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
model:
|
||||
type: classic_conditions
|
||||
conditions:
|
||||
- evaluator: { params: [2], type: gt }
|
||||
operator: { type: and }
|
||||
query: { params: [A] }
|
||||
reducer: { params: [], type: last }
|
||||
|
||||
- orgId: 1
|
||||
name: LibNovel OTel Pipeline
|
||||
folder: LibNovel
|
||||
interval: 2m
|
||||
rules:
|
||||
|
||||
- uid: otel-collector-down
|
||||
title: OTel Collector Down
|
||||
condition: C
|
||||
for: 3m
|
||||
annotations:
|
||||
summary: "OTel collector is not reachable"
|
||||
description: "Prometheus cannot scrape otel-collector:8888. Traces and logs may be dropping."
|
||||
labels:
|
||||
severity: warning
|
||||
service: otel-collector
|
||||
data:
|
||||
- refId: A
|
||||
datasourceUid: prometheus
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
model:
|
||||
expr: "up{job=\"otel-collector\"}"
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
relativeTimeRange: { from: 600, to: 0 }
|
||||
model:
|
||||
type: classic_conditions
|
||||
conditions:
|
||||
- evaluator: { params: [1], type: lt }
|
||||
operator: { type: and }
|
||||
query: { params: [A] }
|
||||
reducer: { params: [], type: last }
|
||||
338
homelab/otel/grafana/provisioning/dashboards/backend.json
Normal file
338
homelab/otel/grafana/provisioning/dashboards/backend.json
Normal file
@@ -0,0 +1,338 @@
|
||||
{
|
||||
"uid": "libnovel-backend",
|
||||
"title": "Backend API",
|
||||
"description": "Request rate, error rate, and latency for the LibNovel backend. Powered by Tempo span metrics and UI OTel instrumentation.",
|
||||
"tags": ["libnovel", "backend", "api"],
|
||||
"timezone": "browser",
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-3h", "to": "now" },
|
||||
"schemaVersion": 39,
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Request Rate (RPS)",
|
||||
"gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 },
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"textMode": "auto"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"backend\"}[5m]))",
|
||||
"legendFormat": "rps",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Error Rate",
|
||||
"gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 },
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "background",
|
||||
"graphMode": "none"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.01 },
|
||||
{ "color": "red", "value": 0.05 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"backend\", status_code=\"STATUS_CODE_ERROR\"}[5m])) / clamp_min(sum(rate(traces_spanmetrics_calls_total{service=\"backend\"}[5m])), 0.001)",
|
||||
"legendFormat": "error rate",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "p50 Latency",
|
||||
"gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 },
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.2 },
|
||||
{ "color": "red", "value": 1 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))",
|
||||
"legendFormat": "p50",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "p95 Latency",
|
||||
"gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 },
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.5 },
|
||||
{ "color": "red", "value": 2 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))",
|
||||
"legendFormat": "p95",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "p99 Latency",
|
||||
"gridPos": { "x": 16, "y": 0, "w": 4, "h": 4 },
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "red", "value": 5 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))",
|
||||
"legendFormat": "p99",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "stat",
|
||||
"title": "5xx Errors / min",
|
||||
"gridPos": { "x": 20, "y": 0, "w": 4, "h": 4 },
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "red", "value": 5 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\", http_response_status_code=~\"5..\"}[5m])) * 60",
|
||||
"legendFormat": "5xx/min",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Request Rate by Status",
|
||||
"gridPos": { "x": 0, "y": 4, "w": 12, "h": 8 },
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "reqps", "custom": { "lineWidth": 2, "fillOpacity": 10 } },
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byFrameRefID", "options": "errors" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "success",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\", http_response_status_code=~\"2..\"}[5m]))",
|
||||
"legendFormat": "2xx"
|
||||
},
|
||||
{
|
||||
"refId": "notfound",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\", http_response_status_code=~\"4..\"}[5m]))",
|
||||
"legendFormat": "4xx"
|
||||
},
|
||||
{
|
||||
"refId": "errors",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\", http_response_status_code=~\"5..\"}[5m]))",
|
||||
"legendFormat": "5xx"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Latency Percentiles (backend spans)",
|
||||
"gridPos": { "x": 12, "y": 4, "w": 12, "h": 8 },
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "s", "custom": { "lineWidth": 2, "fillOpacity": 10 } }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Requests / min by HTTP method (UI → Backend)",
|
||||
"gridPos": { "x": 0, "y": 12, "w": 12, "h": 8 },
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "short", "custom": { "lineWidth": 2, "fillOpacity": 5 } }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\"}[5m])) by (http_request_method) * 60",
|
||||
"legendFormat": "{{http_request_method}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "timeseries",
|
||||
"title": "Requests / min — UI → PocketBase",
|
||||
"gridPos": { "x": 12, "y": 12, "w": 12, "h": 8 },
|
||||
"description": "Traffic from SvelteKit server to PocketBase (auth, collections, etc.).",
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "short", "custom": { "lineWidth": 2, "fillOpacity": 5 } }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"pocketbase\"}[5m])) by (http_request_method, http_response_status_code) * 60",
|
||||
"legendFormat": "{{http_request_method}} {{http_response_status_code}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "timeseries",
|
||||
"title": "UI → Backend Latency (p50 / p95)",
|
||||
"gridPos": { "x": 0, "y": 20, "w": 12, "h": 8 },
|
||||
"description": "HTTP client latency as seen from the SvelteKit SSR layer calling backend.",
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "s", "custom": { "lineWidth": 2, "fillOpacity": 5 } }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(http_client_request_duration_seconds_bucket{job=\"ui\", server_address=\"backend\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_client_request_duration_seconds_bucket{job=\"ui\", server_address=\"backend\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "logs",
|
||||
"title": "Backend Errors",
|
||||
"gridPos": { "x": 0, "y": 28, "w": 24, "h": 10 },
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": true,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=\"backend\"} | json | level =~ `(WARN|ERROR|error|warn)`",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
275
homelab/otel/grafana/provisioning/dashboards/catalogue.json
Normal file
275
homelab/otel/grafana/provisioning/dashboards/catalogue.json
Normal file
@@ -0,0 +1,275 @@
|
||||
{
|
||||
"uid": "libnovel-catalogue",
|
||||
"title": "Catalogue & Content Progress",
|
||||
"description": "Scraping progress, audio generation coverage, and catalogue health derived from runner structured logs.",
|
||||
"tags": ["libnovel", "catalogue", "content"],
|
||||
"timezone": "browser",
|
||||
"refresh": "1m",
|
||||
"time": { "from": "now-24h", "to": "now" },
|
||||
"schemaVersion": 39,
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Books Scraped (last 24h)",
|
||||
"description": "Count of unique book slugs appearing in successful scrape task completions.",
|
||||
"gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 },
|
||||
"options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "value", "graphMode": "none" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "fixedColor": "blue", "mode": "fixed" },
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "sum_over_time({service_name=\"runner\"} | json | msg=`scrape task done` [24h])",
|
||||
"legendFormat": "books scraped"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Chapters Scraped (last 24h)",
|
||||
"gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 },
|
||||
"options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "value", "graphMode": "none" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "fixedColor": "blue", "mode": "fixed" },
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "sum_over_time({service_name=\"runner\"} | json | unwrap scraped [24h])",
|
||||
"legendFormat": "chapters scraped"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Audio Jobs Completed (last 24h)",
|
||||
"gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 },
|
||||
"options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "value", "graphMode": "none" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "fixedColor": "green", "mode": "fixed" },
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "sum_over_time({service_name=\"runner\"} | json | msg=`audio task done` [24h])",
|
||||
"legendFormat": "audio done"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Audio Jobs Failed (last 24h)",
|
||||
"gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 },
|
||||
"options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background", "graphMode": "none" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "red", "value": 5 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "sum_over_time({service_name=\"runner\"} | json | msg=`audio task failed` [24h])",
|
||||
"legendFormat": "audio failed"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Scrape Errors (last 24h)",
|
||||
"gridPos": { "x": 16, "y": 0, "w": 4, "h": 4 },
|
||||
"options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background", "graphMode": "none" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "red", "value": 10 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "sum_over_time({service_name=\"runner\"} | json | msg=`scrape task failed` [24h])",
|
||||
"legendFormat": "scrape errors"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "stat",
|
||||
"title": "Catalogue Refresh — Books Indexed",
|
||||
"description": "Total books indexed in the last catalogue refresh cycle (from the ok field in the summary log).",
|
||||
"gridPos": { "x": 20, "y": 0, "w": 4, "h": 4 },
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "fixedColor": "purple", "mode": "fixed" },
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "last_over_time({service_name=\"runner\"} | json | op=`catalogue_refresh` | msg=`catalogue refresh done` | unwrap ok [7d])",
|
||||
"legendFormat": "indexed"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Audio Generation Rate (tasks/min)",
|
||||
"gridPos": { "x": 0, "y": 4, "w": 12, "h": 8 },
|
||||
"description": "Rate of audio task completions and failures over time.",
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "short", "custom": { "lineWidth": 2, "fillOpacity": 10 } },
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byName", "options": "failed" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] },
|
||||
{ "matcher": { "id": "byName", "options": "completed" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] }
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"runner\", span_name=\"runner.audio_task\", status_code!=\"STATUS_CODE_ERROR\"}[5m])) * 60",
|
||||
"legendFormat": "completed"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"runner\", span_name=\"runner.audio_task\", status_code=\"STATUS_CODE_ERROR\"}[5m])) * 60",
|
||||
"legendFormat": "failed"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Scraping Rate (tasks/min)",
|
||||
"gridPos": { "x": 12, "y": 4, "w": 12, "h": 8 },
|
||||
"description": "Rate of scrape task completions and failures over time.",
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": { "unit": "short", "custom": { "lineWidth": 2, "fillOpacity": 10 } },
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byName", "options": "failed" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] },
|
||||
{ "matcher": { "id": "byName", "options": "completed" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"runner\", span_name=\"runner.scrape_task\", status_code!=\"STATUS_CODE_ERROR\"}[5m])) * 60",
|
||||
"legendFormat": "completed"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"runner\", span_name=\"runner.scrape_task\", status_code=\"STATUS_CODE_ERROR\"}[5m])) * 60",
|
||||
"legendFormat": "failed"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "logs",
|
||||
"title": "Scrape Task Events",
|
||||
"description": "One log line per completed or failed scrape task. Fields: task_id, kind, url, scraped, skipped, errors.",
|
||||
"gridPos": { "x": 0, "y": 12, "w": 24, "h": 10 },
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"wrapLogMessage": false,
|
||||
"prettifyLogMessage": true,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=\"runner\"} | json | msg =~ `scrape task (done|failed|starting)`",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "logs",
|
||||
"title": "Audio Task Events",
|
||||
"description": "One log line per completed or failed audio task. Fields: task_id, slug, chapter, voice, key (on success), reason (on failure).",
|
||||
"gridPos": { "x": 0, "y": 22, "w": 24, "h": 10 },
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"wrapLogMessage": false,
|
||||
"prettifyLogMessage": true,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=\"runner\"} | json | msg =~ `audio task (done|failed|starting)`",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"type": "logs",
|
||||
"title": "Catalogue Refresh Progress",
|
||||
"description": "Progress logs from the background catalogue refresh (every 24h). Fields: op=catalogue_refresh, scraped, ok, skipped, errors.",
|
||||
"gridPos": { "x": 0, "y": 32, "w": 24, "h": 8 },
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"wrapLogMessage": false,
|
||||
"prettifyLogMessage": true,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=\"runner\"} | json | op=`catalogue_refresh`",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
13
homelab/otel/grafana/provisioning/dashboards/dashboards.yaml
Normal file
13
homelab/otel/grafana/provisioning/dashboards/dashboards.yaml
Normal file
@@ -0,0 +1,13 @@
|
||||
# Grafana dashboard provisioning
|
||||
# Points Grafana at the local dashboards directory.
|
||||
# Drop any .json dashboard file into homelab/otel/grafana/provisioning/dashboards/
|
||||
# and it will appear in Grafana automatically on restart.
|
||||
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: libnovel
|
||||
folder: LibNovel
|
||||
type: file
|
||||
options:
|
||||
path: /etc/grafana/provisioning/dashboards
|
||||
377
homelab/otel/grafana/provisioning/dashboards/runner.json
Normal file
377
homelab/otel/grafana/provisioning/dashboards/runner.json
Normal file
@@ -0,0 +1,377 @@
|
||||
{
|
||||
"uid": "libnovel-runner",
|
||||
"title": "Runner Operations",
|
||||
"description": "Task queue health, throughput, TTS routing, and live logs for the homelab runner.",
|
||||
"tags": ["libnovel", "runner"],
|
||||
"timezone": "browser",
|
||||
"refresh": "30s",
|
||||
"time": { "from": "now-3h", "to": "now" },
|
||||
"schemaVersion": 39,
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Tasks Running",
|
||||
"gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 },
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "red", "value": 3 }
|
||||
]
|
||||
},
|
||||
"mappings": []
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "libnovel_runner_tasks_running",
|
||||
"legendFormat": "running",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Tasks Completed (total)",
|
||||
"gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 },
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"textMode": "auto"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "fixedColor": "green", "mode": "fixed" },
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "libnovel_runner_tasks_completed_total",
|
||||
"legendFormat": "completed",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Tasks Failed (total)",
|
||||
"gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 },
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "red", "value": 5 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "libnovel_runner_tasks_failed_total",
|
||||
"legendFormat": "failed",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Runner Uptime",
|
||||
"gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 },
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "yellow", "value": 60 },
|
||||
{ "color": "green", "value": 300 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "libnovel_runner_uptime_seconds",
|
||||
"legendFormat": "uptime",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Task Failure Rate",
|
||||
"gridPos": { "x": 16, "y": 0, "w": 4, "h": 4 },
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.05 },
|
||||
{ "color": "red", "value": 0.2 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "libnovel_runner_tasks_failed_total / clamp_min(libnovel_runner_tasks_completed_total + libnovel_runner_tasks_failed_total, 1)",
|
||||
"legendFormat": "failure rate",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "stat",
|
||||
"title": "Runner Alive",
|
||||
"gridPos": { "x": 20, "y": 0, "w": 4, "h": 4 },
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"] },
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "1": { "text": "UP", "color": "green" }, "0": { "text": "DOWN", "color": "red" } } }
|
||||
],
|
||||
"thresholds": { "mode": "absolute", "steps": [] }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "up{job=\"libnovel-runner\"}",
|
||||
"legendFormat": "runner",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Task Throughput (per minute)",
|
||||
"gridPos": { "x": 0, "y": 4, "w": 12, "h": 8 },
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops",
|
||||
"custom": { "lineWidth": 2, "fillOpacity": 10 }
|
||||
},
|
||||
"overrides": [
|
||||
{ "matcher": { "id": "byName", "options": "failed" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] },
|
||||
{ "matcher": { "id": "byName", "options": "completed" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] }
|
||||
]
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "rate(libnovel_runner_tasks_completed_total[5m]) * 60",
|
||||
"legendFormat": "completed"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "rate(libnovel_runner_tasks_failed_total[5m]) * 60",
|
||||
"legendFormat": "failed"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "libnovel_runner_tasks_running",
|
||||
"legendFormat": "running"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Audio Task Span Latency (p50 / p95 / p99)",
|
||||
"gridPos": { "x": 12, "y": 4, "w": 12, "h": 8 },
|
||||
"description": "End-to-end latency of runner.audio_task spans from Tempo span metrics.",
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"custom": { "lineWidth": 2, "fillOpacity": 10 }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.audio_task\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.audio_task\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.audio_task\"}[5m])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "timeseries",
|
||||
"title": "Scrape Task Span Latency (p50 / p95 / p99)",
|
||||
"gridPos": { "x": 0, "y": 12, "w": 12, "h": 8 },
|
||||
"description": "End-to-end latency of runner.scrape_task spans from Tempo span metrics.",
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"custom": { "lineWidth": 2, "fillOpacity": 10 }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.scrape_task\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.scrape_task\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service=\"runner\", span_name=\"runner.scrape_task\"}[5m])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "timeseries",
|
||||
"title": "Audio vs Scrape Task Rate",
|
||||
"gridPos": { "x": 12, "y": 12, "w": 12, "h": 8 },
|
||||
"description": "Relative throughput of audio generation vs book scraping.",
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom" }
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops",
|
||||
"custom": { "lineWidth": 2, "fillOpacity": 10 }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"runner\", span_name=\"runner.audio_task\"}[5m]))",
|
||||
"legendFormat": "audio tasks/s"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"runner\", span_name=\"runner.scrape_task\"}[5m]))",
|
||||
"legendFormat": "scrape tasks/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "logs",
|
||||
"title": "Runner Logs (errors & warnings)",
|
||||
"gridPos": { "x": 0, "y": 20, "w": 24, "h": 10 },
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"showCommonLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": true,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=\"runner\"} | json | level =~ `(WARN|ERROR|error|warn)`",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "logs",
|
||||
"title": "Runner Logs (all)",
|
||||
"gridPos": { "x": 0, "y": 30, "w": 24, "h": 10 },
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"showCommonLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": true,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=\"runner\"} | json",
|
||||
"legendFormat": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
# Grafana datasource provisioning
|
||||
# Auto-configures Tempo, Prometheus, and Loki on first start.
|
||||
# No manual setup needed in the UI.
|
||||
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Tempo
|
||||
type: tempo
|
||||
uid: tempo
|
||||
url: http://tempo:3200
|
||||
access: proxy
|
||||
isDefault: false
|
||||
jsonData:
|
||||
httpMethod: GET
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
traceQuery:
|
||||
timeShiftEnabled: true
|
||||
spanStartTimeShift: "1h"
|
||||
spanEndTimeShift: "-1h"
|
||||
spanBar:
|
||||
type: "Tag"
|
||||
tag: "http.url"
|
||||
lokiSearch:
|
||||
datasourceUid: loki
|
||||
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
url: http://prometheus:9090
|
||||
access: proxy
|
||||
isDefault: true
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- name: traceID
|
||||
datasourceUid: tempo
|
||||
|
||||
- name: Loki
|
||||
type: loki
|
||||
uid: loki
|
||||
url: http://loki:3100
|
||||
access: proxy
|
||||
isDefault: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- datasourceUid: tempo
|
||||
matcherRegex: '"traceID":"(\w+)"'
|
||||
name: TraceID
|
||||
url: "$${__value.raw}"
|
||||
38
homelab/otel/loki.yaml
Normal file
38
homelab/otel/loki.yaml
Normal file
@@ -0,0 +1,38 @@
|
||||
# Loki config — minimal single-node setup
|
||||
# Receives logs from OTel Collector. 30-day retention.
|
||||
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
retention_period: 720h # 30 days
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
delete_request_store: filesystem
|
||||
retention_enabled: true
|
||||
22
homelab/otel/prometheus.yaml
Normal file
22
homelab/otel/prometheus.yaml
Normal file
@@ -0,0 +1,22 @@
|
||||
# Prometheus config
|
||||
# Scrapes OTel collector self-metrics and runner metrics endpoint.
|
||||
# Backend metrics come in via OTel remote-write — no direct scrape needed.
|
||||
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
environment: production
|
||||
|
||||
scrape_configs:
|
||||
# OTel Collector self-metrics
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8888"]
|
||||
|
||||
# Runner JSON metrics endpoint (native format, no Prometheus client yet)
|
||||
# Will be replaced by OTLP once runner is instrumented with OTel SDK.
|
||||
- job_name: libnovel-runner
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: ["runner:9091"]
|
||||
45
homelab/otel/tempo.yaml
Normal file
45
homelab/otel/tempo.yaml
Normal file
@@ -0,0 +1,45 @@
|
||||
# Tempo config — minimal single-node setup
|
||||
# Stores traces locally. Grafana queries via the HTTP API on port 3200.
|
||||
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
|
||||
ingester:
|
||||
trace_idle_period: 10s
|
||||
max_block_bytes: 104857600 # 100MB
|
||||
max_block_duration: 30m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 720h # 30 days
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
|
||||
metrics_generator:
|
||||
registry:
|
||||
external_labels:
|
||||
source: tempo
|
||||
storage:
|
||||
path: /var/tempo/generator/wal
|
||||
remote_write:
|
||||
- url: http://prometheus:9090/api/v1/write
|
||||
send_exemplars: true
|
||||
|
||||
overrides:
|
||||
defaults:
|
||||
metrics_generator:
|
||||
processors: [service-graphs, span-metrics]
|
||||
generate_native_histograms: both
|
||||
122
homelab/runner/docker-compose.yml
Normal file
122
homelab/runner/docker-compose.yml
Normal file
@@ -0,0 +1,122 @@
|
||||
# LibNovel homelab runner
|
||||
#
|
||||
# Connects to production PocketBase and MinIO via public subdomains.
|
||||
# All secrets come from Doppler (project=libnovel, config=prd_homelab).
|
||||
# Run with: doppler run -- docker compose up -d
|
||||
#
|
||||
# Differs from prod runner:
|
||||
# - RUNNER_WORKER_ID=homelab-runner-1 (unique, avoids task claiming conflicts)
|
||||
# - MINIO_ENDPOINT/USE_SSL → storage.libnovel.cc over HTTPS
|
||||
# - POCKETBASE_URL → https://pb.libnovel.cc
|
||||
# - MEILI_URL → https://search.libnovel.cc (Caddy-proxied)
|
||||
# - VALKEY_ADDR → unset (not exposed publicly)
|
||||
# - RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true
|
||||
# - Redis service for Asynq task queue (local to homelab, exposed to prod via Caddy TCP proxy)
|
||||
# - LibreTranslate service for machine translation (internal network only)
|
||||
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
command: >
|
||||
redis-server
|
||||
--appendonly yes
|
||||
--requirepass "${REDIS_PASSWORD}"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
libretranslate:
|
||||
image: libretranslate/libretranslate:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
LT_API_KEYS: "true"
|
||||
LT_API_KEYS_DB_PATH: "/app/db/api_keys.db"
|
||||
# Limit to source→target pairs the runner actually uses
|
||||
LT_LOAD_ONLY: "en,ru,id,pt,fr"
|
||||
LT_DISABLE_WEB_UI: "true"
|
||||
LT_UPDATE_MODELS: "false"
|
||||
volumes:
|
||||
- libretranslate_models:/home/libretranslate/.local/share/argos-translate
|
||||
- libretranslate_db:/app/db
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://localhost:5000/languages || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 120s
|
||||
|
||||
runner:
|
||||
image: kalekber/libnovel-runner:latest
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 135s
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
libretranslate:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
# ── PocketBase ──────────────────────────────────────────────────────────
|
||||
POCKETBASE_URL: "https://pb.libnovel.cc"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}"
|
||||
|
||||
# ── MinIO (S3 API via public subdomain) ─────────────────────────────────
|
||||
MINIO_ENDPOINT: "storage.libnovel.cc"
|
||||
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}"
|
||||
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD}"
|
||||
MINIO_USE_SSL: "true"
|
||||
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT}"
|
||||
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL}"
|
||||
|
||||
# ── Meilisearch (via search.libnovel.cc Caddy proxy) ────────────────────
|
||||
MEILI_URL: "${MEILI_URL}"
|
||||
MEILI_API_KEY: "${MEILI_API_KEY}"
|
||||
VALKEY_ADDR: ""
|
||||
# Force IPv4 DNS resolution — homelab has no IPv6 route to search.libnovel.cc
|
||||
GODEBUG: "preferIPv4=1"
|
||||
|
||||
# ── Kokoro TTS ──────────────────────────────────────────────────────────
|
||||
KOKORO_URL: "${KOKORO_URL}"
|
||||
KOKORO_VOICE: "${KOKORO_VOICE}"
|
||||
|
||||
# ── Pocket TTS ──────────────────────────────────────────────────────────
|
||||
POCKET_TTS_URL: "${POCKET_TTS_URL}"
|
||||
|
||||
# ── LibreTranslate (internal Docker network) ────────────────────────────
|
||||
LIBRETRANSLATE_URL: "http://libretranslate:5000"
|
||||
LIBRETRANSLATE_API_KEY: "${LIBRETRANSLATE_API_KEY}"
|
||||
|
||||
# ── Asynq / Redis (local service) ───────────────────────────────────────
|
||||
# The runner connects to the local Redis sidecar.
|
||||
REDIS_ADDR: "redis:6379"
|
||||
REDIS_PASSWORD: "${REDIS_PASSWORD}"
|
||||
|
||||
# ── Runner tuning ───────────────────────────────────────────────────────
|
||||
RUNNER_WORKER_ID: "${RUNNER_WORKER_ID}"
|
||||
RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL}"
|
||||
RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE}"
|
||||
RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO}"
|
||||
RUNNER_MAX_CONCURRENT_TRANSLATION: "${RUNNER_MAX_CONCURRENT_TRANSLATION}"
|
||||
RUNNER_TIMEOUT: "${RUNNER_TIMEOUT}"
|
||||
RUNNER_METRICS_ADDR: "${RUNNER_METRICS_ADDR}"
|
||||
RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH: "true"
|
||||
|
||||
# ── Observability ───────────────────────────────────────────────────────
|
||||
LOG_LEVEL: "${LOG_LEVEL}"
|
||||
GLITCHTIP_DSN: "${GLITCHTIP_DSN}"
|
||||
|
||||
healthcheck:
|
||||
test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"]
|
||||
interval: 60s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
redis_data:
|
||||
libretranslate_models:
|
||||
libretranslate_db:
|
||||
10
ios/LibNovel/.gitignore
vendored
10
ios/LibNovel/.gitignore
vendored
@@ -1,10 +0,0 @@
|
||||
# Fastlane
|
||||
fastlane/report.xml
|
||||
fastlane/Preview.html
|
||||
fastlane/screenshots/**/*.png
|
||||
fastlane/test_output
|
||||
fastlane/README.md
|
||||
|
||||
# Bundler
|
||||
.bundle
|
||||
vendor/bundle
|
||||
@@ -1,21 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>method</key>
|
||||
<string>app-store</string>
|
||||
<key>teamID</key>
|
||||
<string>GHZXC6FVMU</string>
|
||||
<key>uploadBitcode</key>
|
||||
<false/>
|
||||
<key>uploadSymbols</key>
|
||||
<true/>
|
||||
<key>signingStyle</key>
|
||||
<string>manual</string>
|
||||
<key>provisioningProfiles</key>
|
||||
<dict>
|
||||
<key>com.kalekber.LibNovel</key>
|
||||
<string>LibNovel Distribution</string>
|
||||
</dict>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -1,3 +0,0 @@
|
||||
source "https://rubygems.org"
|
||||
|
||||
gem "fastlane"
|
||||
@@ -1,772 +0,0 @@
|
||||
// !$*UTF8*$!
|
||||
{
|
||||
archiveVersion = 1;
|
||||
classes = {
|
||||
};
|
||||
objectVersion = 77;
|
||||
objects = {
|
||||
|
||||
/* Begin PBXBuildFile section */
|
||||
032E049A4BB3CF0EA990C0CD /* LibNovelApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4F56C8E2BC3614530B81569D /* LibNovelApp.swift */; };
|
||||
07FC69FB9DF3F6073564E489 /* DiscoverViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = AA9111BF29C75E8D60FCEDF6 /* DiscoverViewModel.swift */; };
|
||||
08DFB5F626BA769556C8D145 /* BrowseView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1FA3F0FCA383180EE4C93BBA /* BrowseView.swift */; };
|
||||
0A52BC1CE71BED9E75D20D35 /* Models.swift in Sources */ = {isa = PBXBuildFile; fileRef = 762E378B9BC2161A7AA2CC36 /* Models.swift */; };
|
||||
0B40E3DCE82EBEA7C4ECF148 /* AvatarCropView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 775B5C22D6215D7A7C412E13 /* AvatarCropView.swift */; };
|
||||
192F82518CB8763775E33B38 /* SearchView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 79133D9FA697D1909C8D3973 /* SearchView.swift */; };
|
||||
1945DD2D0DF497FE66FAAF90 /* BookVoicePreferences.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1C0022D98CDAD0B11840AAAC /* BookVoicePreferences.swift */; };
|
||||
1964D61094D4731227384F3A /* VoiceSelectionViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = CB2489CA141D5E19373D0936 /* VoiceSelectionViewModel.swift */; };
|
||||
2790B8C051BE389D83645047 /* BrowseViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9812F5FE30ED657FB40ABD7A /* BrowseViewModel.swift */; };
|
||||
2A15157AD2AE2271675C3485 /* ChapterReaderViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8995E667B3DD9CFCAD8A91D7 /* ChapterReaderViewModel.swift */; };
|
||||
3521DFD5FCBBED7B90368829 /* LibraryViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = FC338B05EA6DB22900712000 /* LibraryViewModel.swift */; };
|
||||
367C88FFC11701D2BAD8CCD0 /* RootTabView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2D5C115992F1CE2326236765 /* RootTabView.swift */; };
|
||||
41FB51553F1F1AEBFEA91C0A /* String+App.swift in Sources */ = {isa = PBXBuildFile; fileRef = FEC6F837FF2E902E334ED72E /* String+App.swift */; };
|
||||
4BB2C76262D5BD5DAD0D5D28 /* LibNovelTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4C918833E173D6B44D06955 /* LibNovelTests.swift */; };
|
||||
58E440CE4360D755401D1672 /* ProfileViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 937A589F84FD412BBB6FBC45 /* ProfileViewModel.swift */; };
|
||||
5D8D783259EF54C773788AAB /* AuthStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = F219788AE5ACBD6F240674F5 /* AuthStore.swift */; };
|
||||
5F7409635F6563E44C836390 /* NetworkMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1FA1B6D9FF31780095F5ACA8 /* NetworkMonitor.swift */; };
|
||||
62B42DB777F53856C57CB6AF /* OfflineBanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = F082F99F2EE05BD98C9EF2AA /* OfflineBanner.swift */; };
|
||||
64D80AACB8E1967B17921EE3 /* ProfileView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0B17D50389C6C98FC78BDBC /* ProfileView.swift */; };
|
||||
65CA672C02F367F72F18F8B8 /* AudioDownloadService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94730324A6BD9D6A772286BB /* AudioDownloadService.swift */; };
|
||||
749292A18C57FA41EC88A30B /* BookDetailView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 39DE056C37FBC5EED8771821 /* BookDetailView.swift */; };
|
||||
774CFCDA8A13311DF85FF051 /* DownloadsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8175390266E8C6CF1437A229 /* DownloadsView.swift */; };
|
||||
7C74C10317D389121922A5E3 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 5A776719B77EDDB5E44743B0 /* Assets.xcassets */; };
|
||||
7D81DEB2EEFF9CA5079AEEF7 /* BookDetailViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 837F83AA12B59924FDF16617 /* BookDetailViewModel.swift */; };
|
||||
880D411C936F7BA92AF83383 /* DownloadQueueButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = 16ECDDD02E6A2F8562111538 /* DownloadQueueButton.swift */; };
|
||||
8B02625CA1B93118B63E9C9D /* VoiceSelectionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A75E148A48D47A5B37CA7FB3 /* VoiceSelectionView.swift */; };
|
||||
9407F80F454D0248D5C779A6 /* UserProfileViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 10777FC4816A7067AF9C4797 /* UserProfileViewModel.swift */; };
|
||||
94D0C4B15734B4056BF3B127 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4B820081FA4817765A39939A /* ContentView.swift */; };
|
||||
9B2D6F241E707312AB80DC31 /* AuthView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CEF6782A2A28B2A485CBD48 /* AuthView.swift */; };
|
||||
9C19B17E746FE6A834E53AF3 /* UserProfileView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F247DE25991F4DB98DF717AA /* UserProfileView.swift */; };
|
||||
A7485E99B9ACBCBCCD1EB7B2 /* CommentsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 16B9AFE90719BDBC718F0621 /* CommentsView.swift */; };
|
||||
A9B95BAD7CE2DCD1DDDABD4C /* AudioPlayerService.swift in Sources */ = {isa = PBXBuildFile; fileRef = DB13E89E50529E3081533A66 /* AudioPlayerService.swift */; };
|
||||
BE7805A4E78037A82B12AE56 /* PlayerViews.swift in Sources */ = {isa = PBXBuildFile; fileRef = DF49C3AEF9D010F9FEDAB1FC /* PlayerViews.swift */; };
|
||||
C807AD8D627CF6BED47D517C /* HomeViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AB2E843D93461074A89A171 /* HomeViewModel.swift */; };
|
||||
CFDAA4776344B075A1E3CD6B /* Kingfisher in Frameworks */ = {isa = PBXBuildFile; productRef = 09584EAB68A07B47F876A062 /* Kingfisher */; };
|
||||
DFA7EB1B0BD53F68FE1335C8 /* DownloadAudioButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = 35942111986E54CC0E83A391 /* DownloadAudioButton.swift */; };
|
||||
E1F564399D1325F6A1B2B84F /* LibraryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C21107BECA55C07416E0CB8B /* LibraryView.swift */; };
|
||||
E2572692178FD17145FDAF77 /* Color+App.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9D83BB88C4306BE7A4F947CB /* Color+App.swift */; };
|
||||
ED54860A709FED5A8CBF4EEB /* AccountMenuSheet.swift in Sources */ = {isa = PBXBuildFile; fileRef = AAD554706F61FE3DC061189F /* AccountMenuSheet.swift */; };
|
||||
EF3C57C400BF05CBEAC1F7FE /* HomeView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D6268D60803940CBD38FB921 /* HomeView.swift */; };
|
||||
F2AF05B9C8C23132A73ACDD3 /* CommonViews.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8E89FD8F46747CA653C5203D /* CommonViews.swift */; };
|
||||
F4FDA3C44752EB979235C042 /* NavDestination.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CAFB96D2500F34F0B0C860C /* NavDestination.swift */; };
|
||||
FB32F3772CA09684F00497F3 /* APIClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = B593F179EC3E9112126B540B /* APIClient.swift */; };
|
||||
FEFB5FDC2424D22914458001 /* ChapterReaderView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 81E3939152E23B4985FAF7E2 /* ChapterReaderView.swift */; };
|
||||
/* End PBXBuildFile section */
|
||||
|
||||
/* Begin PBXContainerItemProxy section */
|
||||
698AC3AA533BC05C985595D0 /* PBXContainerItemProxy */ = {
|
||||
isa = PBXContainerItemProxy;
|
||||
containerPortal = A10A669C0C8B43078C0FEE9F /* Project object */;
|
||||
proxyType = 1;
|
||||
remoteGlobalIDString = D039EDECDE3998D8534BB680;
|
||||
remoteInfo = LibNovel;
|
||||
};
|
||||
/* End PBXContainerItemProxy section */
|
||||
|
||||
/* Begin PBXFileReference section */
|
||||
10777FC4816A7067AF9C4797 /* UserProfileViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserProfileViewModel.swift; sourceTree = "<group>"; };
|
||||
16B9AFE90719BDBC718F0621 /* CommentsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CommentsView.swift; sourceTree = "<group>"; };
|
||||
16ECDDD02E6A2F8562111538 /* DownloadQueueButton.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DownloadQueueButton.swift; sourceTree = "<group>"; };
|
||||
1B8BF3DB582A658386E402C7 /* LibNovel.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = LibNovel.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
1C0022D98CDAD0B11840AAAC /* BookVoicePreferences.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookVoicePreferences.swift; sourceTree = "<group>"; };
|
||||
1FA1B6D9FF31780095F5ACA8 /* NetworkMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NetworkMonitor.swift; sourceTree = "<group>"; };
|
||||
1FA3F0FCA383180EE4C93BBA /* BrowseView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BrowseView.swift; sourceTree = "<group>"; };
|
||||
235967A21B386BE13F56F3F8 /* LibNovelTests.xctest */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.cfbundle; path = LibNovelTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
2D5C115992F1CE2326236765 /* RootTabView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RootTabView.swift; sourceTree = "<group>"; };
|
||||
35942111986E54CC0E83A391 /* DownloadAudioButton.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DownloadAudioButton.swift; sourceTree = "<group>"; };
|
||||
39DE056C37FBC5EED8771821 /* BookDetailView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookDetailView.swift; sourceTree = "<group>"; };
|
||||
3AB2E843D93461074A89A171 /* HomeViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HomeViewModel.swift; sourceTree = "<group>"; };
|
||||
4B820081FA4817765A39939A /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
|
||||
4F56C8E2BC3614530B81569D /* LibNovelApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibNovelApp.swift; sourceTree = "<group>"; };
|
||||
5A776719B77EDDB5E44743B0 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
|
||||
762E378B9BC2161A7AA2CC36 /* Models.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Models.swift; sourceTree = "<group>"; };
|
||||
775B5C22D6215D7A7C412E13 /* AvatarCropView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AvatarCropView.swift; sourceTree = "<group>"; };
|
||||
79133D9FA697D1909C8D3973 /* SearchView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchView.swift; sourceTree = "<group>"; };
|
||||
7CAFB96D2500F34F0B0C860C /* NavDestination.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NavDestination.swift; sourceTree = "<group>"; };
|
||||
7CEF6782A2A28B2A485CBD48 /* AuthView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthView.swift; sourceTree = "<group>"; };
|
||||
8175390266E8C6CF1437A229 /* DownloadsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DownloadsView.swift; sourceTree = "<group>"; };
|
||||
81E3939152E23B4985FAF7E2 /* ChapterReaderView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChapterReaderView.swift; sourceTree = "<group>"; };
|
||||
837F83AA12B59924FDF16617 /* BookDetailViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BookDetailViewModel.swift; sourceTree = "<group>"; };
|
||||
8995E667B3DD9CFCAD8A91D7 /* ChapterReaderViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChapterReaderViewModel.swift; sourceTree = "<group>"; };
|
||||
8E89FD8F46747CA653C5203D /* CommonViews.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CommonViews.swift; sourceTree = "<group>"; };
|
||||
937A589F84FD412BBB6FBC45 /* ProfileViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileViewModel.swift; sourceTree = "<group>"; };
|
||||
94730324A6BD9D6A772286BB /* AudioDownloadService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioDownloadService.swift; sourceTree = "<group>"; };
|
||||
9812F5FE30ED657FB40ABD7A /* BrowseViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BrowseViewModel.swift; sourceTree = "<group>"; };
|
||||
9D83BB88C4306BE7A4F947CB /* Color+App.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Color+App.swift"; sourceTree = "<group>"; };
|
||||
A75E148A48D47A5B37CA7FB3 /* VoiceSelectionView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceSelectionView.swift; sourceTree = "<group>"; };
|
||||
AA9111BF29C75E8D60FCEDF6 /* DiscoverViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiscoverViewModel.swift; sourceTree = "<group>"; };
|
||||
AAD554706F61FE3DC061189F /* AccountMenuSheet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountMenuSheet.swift; sourceTree = "<group>"; };
|
||||
B4C918833E173D6B44D06955 /* LibNovelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibNovelTests.swift; sourceTree = "<group>"; };
|
||||
B593F179EC3E9112126B540B /* APIClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = APIClient.swift; sourceTree = "<group>"; };
|
||||
C0B17D50389C6C98FC78BDBC /* ProfileView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileView.swift; sourceTree = "<group>"; };
|
||||
C21107BECA55C07416E0CB8B /* LibraryView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibraryView.swift; sourceTree = "<group>"; };
|
||||
CB2489CA141D5E19373D0936 /* VoiceSelectionViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceSelectionViewModel.swift; sourceTree = "<group>"; };
|
||||
D6268D60803940CBD38FB921 /* HomeView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HomeView.swift; sourceTree = "<group>"; };
|
||||
DB13E89E50529E3081533A66 /* AudioPlayerService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioPlayerService.swift; sourceTree = "<group>"; };
|
||||
DF49C3AEF9D010F9FEDAB1FC /* PlayerViews.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PlayerViews.swift; sourceTree = "<group>"; };
|
||||
F082F99F2EE05BD98C9EF2AA /* OfflineBanner.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OfflineBanner.swift; sourceTree = "<group>"; };
|
||||
F219788AE5ACBD6F240674F5 /* AuthStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AuthStore.swift; sourceTree = "<group>"; };
|
||||
F247DE25991F4DB98DF717AA /* UserProfileView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserProfileView.swift; sourceTree = "<group>"; };
|
||||
FC338B05EA6DB22900712000 /* LibraryViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibraryViewModel.swift; sourceTree = "<group>"; };
|
||||
FEC6F837FF2E902E334ED72E /* String+App.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "String+App.swift"; sourceTree = "<group>"; };
|
||||
/* End PBXFileReference section */
|
||||
|
||||
/* Begin PBXFrameworksBuildPhase section */
|
||||
EFE3211B202EDF04EB141EFB /* Frameworks */ = {
|
||||
isa = PBXFrameworksBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
CFDAA4776344B075A1E3CD6B /* Kingfisher in Frameworks */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXFrameworksBuildPhase section */
|
||||
|
||||
/* Begin PBXGroup section */
|
||||
2C0FB0EDFF9B3E24B97F4214 /* Resources */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
5A776719B77EDDB5E44743B0 /* Assets.xcassets */,
|
||||
);
|
||||
path = Resources;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
2C57B93EAF19A3B18E7B7E87 /* Views */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
2F18D1275D6022B9847E310E /* Auth */,
|
||||
FB5C0D4925633786D28C6DE3 /* BookDetail */,
|
||||
8E8AAA58A33084ADB8AEA80C /* Browse */,
|
||||
4EAB87A1ED4943A311F26F84 /* ChapterReader */,
|
||||
5D5809803A3D74FAE19DB218 /* Common */,
|
||||
9180FAFE96724B8AACFA9859 /* Components */,
|
||||
3881CBFE9730C6422BE6F03D /* Downloads */,
|
||||
811FC0F6B9C209D6EC8543BD /* Home */,
|
||||
FA994FD601E79EC811D822A4 /* Library */,
|
||||
89F2CB14192E7D7565A588E0 /* Player */,
|
||||
3DB66C5703A4CCAFFA1B7AFE /* Profile */,
|
||||
474BE4FC0353C2DD8D8425D1 /* Search */,
|
||||
);
|
||||
path = Views;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
2F18D1275D6022B9847E310E /* Auth */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
7CEF6782A2A28B2A485CBD48 /* AuthView.swift */,
|
||||
);
|
||||
path = Auth;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
3881CBFE9730C6422BE6F03D /* Downloads */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
16ECDDD02E6A2F8562111538 /* DownloadQueueButton.swift */,
|
||||
8175390266E8C6CF1437A229 /* DownloadsView.swift */,
|
||||
);
|
||||
path = Downloads;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
3DB66C5703A4CCAFFA1B7AFE /* Profile */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
AAD554706F61FE3DC061189F /* AccountMenuSheet.swift */,
|
||||
775B5C22D6215D7A7C412E13 /* AvatarCropView.swift */,
|
||||
C0B17D50389C6C98FC78BDBC /* ProfileView.swift */,
|
||||
F247DE25991F4DB98DF717AA /* UserProfileView.swift */,
|
||||
A75E148A48D47A5B37CA7FB3 /* VoiceSelectionView.swift */,
|
||||
);
|
||||
path = Profile;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
426F7C5465758645B93A1AB1 /* Networking */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
B593F179EC3E9112126B540B /* APIClient.swift */,
|
||||
);
|
||||
path = Networking;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
474BE4FC0353C2DD8D8425D1 /* Search */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
79133D9FA697D1909C8D3973 /* SearchView.swift */,
|
||||
);
|
||||
path = Search;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
4EAB87A1ED4943A311F26F84 /* ChapterReader */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
81E3939152E23B4985FAF7E2 /* ChapterReaderView.swift */,
|
||||
35942111986E54CC0E83A391 /* DownloadAudioButton.swift */,
|
||||
);
|
||||
path = ChapterReader;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
5D5809803A3D74FAE19DB218 /* Common */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
8E89FD8F46747CA653C5203D /* CommonViews.swift */,
|
||||
);
|
||||
path = Common;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
6318D3C6F0DC6C8E2C377103 /* Products */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
1B8BF3DB582A658386E402C7 /* LibNovel.app */,
|
||||
235967A21B386BE13F56F3F8 /* LibNovelTests.xctest */,
|
||||
);
|
||||
name = Products;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
646952B9CE927F8038FF0A13 /* LibNovelTests */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
B4C918833E173D6B44D06955 /* LibNovelTests.swift */,
|
||||
);
|
||||
path = LibNovelTests;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
80148B5E27BD0A3DEDB3ADAA /* Models */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
762E378B9BC2161A7AA2CC36 /* Models.swift */,
|
||||
);
|
||||
path = Models;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
811FC0F6B9C209D6EC8543BD /* Home */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
D6268D60803940CBD38FB921 /* HomeView.swift */,
|
||||
);
|
||||
path = Home;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
89F2CB14192E7D7565A588E0 /* Player */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
DF49C3AEF9D010F9FEDAB1FC /* PlayerViews.swift */,
|
||||
);
|
||||
path = Player;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
8E8AAA58A33084ADB8AEA80C /* Browse */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
1FA3F0FCA383180EE4C93BBA /* BrowseView.swift */,
|
||||
);
|
||||
path = Browse;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
9180FAFE96724B8AACFA9859 /* Components */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
F082F99F2EE05BD98C9EF2AA /* OfflineBanner.swift */,
|
||||
);
|
||||
path = Components;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
9AF55E5D62F980C72431782A = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
A28A184E73B15138A4D13F31 /* LibNovel */,
|
||||
646952B9CE927F8038FF0A13 /* LibNovelTests */,
|
||||
6318D3C6F0DC6C8E2C377103 /* Products */,
|
||||
);
|
||||
indentWidth = 4;
|
||||
sourceTree = "<group>";
|
||||
tabWidth = 4;
|
||||
usesTabs = 0;
|
||||
};
|
||||
A28A184E73B15138A4D13F31 /* LibNovel */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
FE92158CC5DA9AD446062724 /* App */,
|
||||
FD5EDEE9747643D45CA6423E /* Extensions */,
|
||||
80148B5E27BD0A3DEDB3ADAA /* Models */,
|
||||
426F7C5465758645B93A1AB1 /* Networking */,
|
||||
2C0FB0EDFF9B3E24B97F4214 /* Resources */,
|
||||
DA6F6F625578875F3E74F1D3 /* Services */,
|
||||
B6916C5C762A37AB1279DF44 /* ViewModels */,
|
||||
2C57B93EAF19A3B18E7B7E87 /* Views */,
|
||||
);
|
||||
path = LibNovel;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
B6916C5C762A37AB1279DF44 /* ViewModels */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
837F83AA12B59924FDF16617 /* BookDetailViewModel.swift */,
|
||||
9812F5FE30ED657FB40ABD7A /* BrowseViewModel.swift */,
|
||||
8995E667B3DD9CFCAD8A91D7 /* ChapterReaderViewModel.swift */,
|
||||
AA9111BF29C75E8D60FCEDF6 /* DiscoverViewModel.swift */,
|
||||
3AB2E843D93461074A89A171 /* HomeViewModel.swift */,
|
||||
FC338B05EA6DB22900712000 /* LibraryViewModel.swift */,
|
||||
937A589F84FD412BBB6FBC45 /* ProfileViewModel.swift */,
|
||||
10777FC4816A7067AF9C4797 /* UserProfileViewModel.swift */,
|
||||
CB2489CA141D5E19373D0936 /* VoiceSelectionViewModel.swift */,
|
||||
);
|
||||
path = ViewModels;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
DA6F6F625578875F3E74F1D3 /* Services */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
94730324A6BD9D6A772286BB /* AudioDownloadService.swift */,
|
||||
DB13E89E50529E3081533A66 /* AudioPlayerService.swift */,
|
||||
F219788AE5ACBD6F240674F5 /* AuthStore.swift */,
|
||||
1C0022D98CDAD0B11840AAAC /* BookVoicePreferences.swift */,
|
||||
1FA1B6D9FF31780095F5ACA8 /* NetworkMonitor.swift */,
|
||||
);
|
||||
path = Services;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
FA994FD601E79EC811D822A4 /* Library */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
C21107BECA55C07416E0CB8B /* LibraryView.swift */,
|
||||
);
|
||||
path = Library;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
FB5C0D4925633786D28C6DE3 /* BookDetail */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
39DE056C37FBC5EED8771821 /* BookDetailView.swift */,
|
||||
16B9AFE90719BDBC718F0621 /* CommentsView.swift */,
|
||||
);
|
||||
path = BookDetail;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
FD5EDEE9747643D45CA6423E /* Extensions */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
9D83BB88C4306BE7A4F947CB /* Color+App.swift */,
|
||||
7CAFB96D2500F34F0B0C860C /* NavDestination.swift */,
|
||||
FEC6F837FF2E902E334ED72E /* String+App.swift */,
|
||||
);
|
||||
path = Extensions;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
FE92158CC5DA9AD446062724 /* App */ = {
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
4B820081FA4817765A39939A /* ContentView.swift */,
|
||||
4F56C8E2BC3614530B81569D /* LibNovelApp.swift */,
|
||||
2D5C115992F1CE2326236765 /* RootTabView.swift */,
|
||||
);
|
||||
path = App;
|
||||
sourceTree = "<group>";
|
||||
};
|
||||
/* End PBXGroup section */
|
||||
|
||||
/* Begin PBXNativeTarget section */
|
||||
5E6D3E8266BFCF0AAF5EC79D /* LibNovelTests */ = {
|
||||
isa = PBXNativeTarget;
|
||||
buildConfigurationList = 964FF85B62FA35E819BE7661 /* Build configuration list for PBXNativeTarget "LibNovelTests" */;
|
||||
buildPhases = (
|
||||
247D45B3DB26CAC41FA78A0B /* Sources */,
|
||||
);
|
||||
buildRules = (
|
||||
);
|
||||
dependencies = (
|
||||
9FD4A50EB175FC09D6BFD28D /* PBXTargetDependency */,
|
||||
);
|
||||
name = LibNovelTests;
|
||||
packageProductDependencies = (
|
||||
);
|
||||
productName = LibNovelTests;
|
||||
productReference = 235967A21B386BE13F56F3F8 /* LibNovelTests.xctest */;
|
||||
productType = "com.apple.product-type.bundle.unit-test";
|
||||
};
|
||||
D039EDECDE3998D8534BB680 /* LibNovel */ = {
|
||||
isa = PBXNativeTarget;
|
||||
buildConfigurationList = 29B2DE7267A3A4B2D89B32DA /* Build configuration list for PBXNativeTarget "LibNovel" */;
|
||||
buildPhases = (
|
||||
48661ADCA15B54E048CF694C /* Sources */,
|
||||
27446CA4728C022832398376 /* Resources */,
|
||||
EFE3211B202EDF04EB141EFB /* Frameworks */,
|
||||
);
|
||||
buildRules = (
|
||||
);
|
||||
dependencies = (
|
||||
);
|
||||
name = LibNovel;
|
||||
packageProductDependencies = (
|
||||
09584EAB68A07B47F876A062 /* Kingfisher */,
|
||||
);
|
||||
productName = LibNovel;
|
||||
productReference = 1B8BF3DB582A658386E402C7 /* LibNovel.app */;
|
||||
productType = "com.apple.product-type.application";
|
||||
};
|
||||
/* End PBXNativeTarget section */
|
||||
|
||||
/* Begin PBXProject section */
|
||||
A10A669C0C8B43078C0FEE9F /* Project object */ = {
|
||||
isa = PBXProject;
|
||||
attributes = {
|
||||
BuildIndependentTargetsInParallel = YES;
|
||||
LastUpgradeCheck = 1600;
|
||||
};
|
||||
buildConfigurationList = D27899EE96A9AFCBBE62EA3C /* Build configuration list for PBXProject "LibNovel" */;
|
||||
developmentRegion = en;
|
||||
hasScannedForEncodings = 0;
|
||||
knownRegions = (
|
||||
Base,
|
||||
en,
|
||||
);
|
||||
mainGroup = 9AF55E5D62F980C72431782A;
|
||||
minimizedProjectReferenceProxies = 1;
|
||||
packageReferences = (
|
||||
AFEF7128801A76181793EA9C /* XCRemoteSwiftPackageReference "Kingfisher" */,
|
||||
);
|
||||
preferredProjectObjectVersion = 77;
|
||||
productRefGroup = 6318D3C6F0DC6C8E2C377103 /* Products */;
|
||||
projectDirPath = "";
|
||||
projectRoot = "";
|
||||
targets = (
|
||||
D039EDECDE3998D8534BB680 /* LibNovel */,
|
||||
5E6D3E8266BFCF0AAF5EC79D /* LibNovelTests */,
|
||||
);
|
||||
};
|
||||
/* End PBXProject section */
|
||||
|
||||
/* Begin PBXResourcesBuildPhase section */
|
||||
27446CA4728C022832398376 /* Resources */ = {
|
||||
isa = PBXResourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
7C74C10317D389121922A5E3 /* Assets.xcassets in Resources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXResourcesBuildPhase section */
|
||||
|
||||
/* Begin PBXSourcesBuildPhase section */
|
||||
247D45B3DB26CAC41FA78A0B /* Sources */ = {
|
||||
isa = PBXSourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
4BB2C76262D5BD5DAD0D5D28 /* LibNovelTests.swift in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
48661ADCA15B54E048CF694C /* Sources */ = {
|
||||
isa = PBXSourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
FB32F3772CA09684F00497F3 /* APIClient.swift in Sources */,
|
||||
ED54860A709FED5A8CBF4EEB /* AccountMenuSheet.swift in Sources */,
|
||||
65CA672C02F367F72F18F8B8 /* AudioDownloadService.swift in Sources */,
|
||||
A9B95BAD7CE2DCD1DDDABD4C /* AudioPlayerService.swift in Sources */,
|
||||
5D8D783259EF54C773788AAB /* AuthStore.swift in Sources */,
|
||||
9B2D6F241E707312AB80DC31 /* AuthView.swift in Sources */,
|
||||
0B40E3DCE82EBEA7C4ECF148 /* AvatarCropView.swift in Sources */,
|
||||
749292A18C57FA41EC88A30B /* BookDetailView.swift in Sources */,
|
||||
7D81DEB2EEFF9CA5079AEEF7 /* BookDetailViewModel.swift in Sources */,
|
||||
1945DD2D0DF497FE66FAAF90 /* BookVoicePreferences.swift in Sources */,
|
||||
08DFB5F626BA769556C8D145 /* BrowseView.swift in Sources */,
|
||||
2790B8C051BE389D83645047 /* BrowseViewModel.swift in Sources */,
|
||||
FEFB5FDC2424D22914458001 /* ChapterReaderView.swift in Sources */,
|
||||
2A15157AD2AE2271675C3485 /* ChapterReaderViewModel.swift in Sources */,
|
||||
E2572692178FD17145FDAF77 /* Color+App.swift in Sources */,
|
||||
A7485E99B9ACBCBCCD1EB7B2 /* CommentsView.swift in Sources */,
|
||||
F2AF05B9C8C23132A73ACDD3 /* CommonViews.swift in Sources */,
|
||||
94D0C4B15734B4056BF3B127 /* ContentView.swift in Sources */,
|
||||
07FC69FB9DF3F6073564E489 /* DiscoverViewModel.swift in Sources */,
|
||||
DFA7EB1B0BD53F68FE1335C8 /* DownloadAudioButton.swift in Sources */,
|
||||
880D411C936F7BA92AF83383 /* DownloadQueueButton.swift in Sources */,
|
||||
774CFCDA8A13311DF85FF051 /* DownloadsView.swift in Sources */,
|
||||
EF3C57C400BF05CBEAC1F7FE /* HomeView.swift in Sources */,
|
||||
C807AD8D627CF6BED47D517C /* HomeViewModel.swift in Sources */,
|
||||
032E049A4BB3CF0EA990C0CD /* LibNovelApp.swift in Sources */,
|
||||
E1F564399D1325F6A1B2B84F /* LibraryView.swift in Sources */,
|
||||
3521DFD5FCBBED7B90368829 /* LibraryViewModel.swift in Sources */,
|
||||
0A52BC1CE71BED9E75D20D35 /* Models.swift in Sources */,
|
||||
F4FDA3C44752EB979235C042 /* NavDestination.swift in Sources */,
|
||||
5F7409635F6563E44C836390 /* NetworkMonitor.swift in Sources */,
|
||||
62B42DB777F53856C57CB6AF /* OfflineBanner.swift in Sources */,
|
||||
BE7805A4E78037A82B12AE56 /* PlayerViews.swift in Sources */,
|
||||
64D80AACB8E1967B17921EE3 /* ProfileView.swift in Sources */,
|
||||
58E440CE4360D755401D1672 /* ProfileViewModel.swift in Sources */,
|
||||
367C88FFC11701D2BAD8CCD0 /* RootTabView.swift in Sources */,
|
||||
192F82518CB8763775E33B38 /* SearchView.swift in Sources */,
|
||||
41FB51553F1F1AEBFEA91C0A /* String+App.swift in Sources */,
|
||||
9C19B17E746FE6A834E53AF3 /* UserProfileView.swift in Sources */,
|
||||
9407F80F454D0248D5C779A6 /* UserProfileViewModel.swift in Sources */,
|
||||
8B02625CA1B93118B63E9C9D /* VoiceSelectionView.swift in Sources */,
|
||||
1964D61094D4731227384F3A /* VoiceSelectionViewModel.swift in Sources */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXSourcesBuildPhase section */
|
||||
|
||||
/* Begin PBXTargetDependency section */
|
||||
9FD4A50EB175FC09D6BFD28D /* PBXTargetDependency */ = {
|
||||
isa = PBXTargetDependency;
|
||||
target = D039EDECDE3998D8534BB680 /* LibNovel */;
|
||||
targetProxy = 698AC3AA533BC05C985595D0 /* PBXContainerItemProxy */;
|
||||
};
|
||||
/* End PBXTargetDependency section */
|
||||
|
||||
/* Begin XCBuildConfiguration section */
|
||||
428871329DC9E7B31FA1664B /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
BUNDLE_LOADER = "$(TEST_HOST)";
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
"@loader_path/Frameworks",
|
||||
);
|
||||
PRODUCT_BUNDLE_IDENTIFIER = com.kalekber.LibNovel.tests;
|
||||
SDKROOT = iphoneos;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
TEST_HOST = "$(BUILT_PRODUCTS_DIR)/LibNovel.app/LibNovel";
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
49CBF0D367E562629E002A4B /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
BUNDLE_LOADER = "$(TEST_HOST)";
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
"@loader_path/Frameworks",
|
||||
);
|
||||
PRODUCT_BUNDLE_IDENTIFIER = com.kalekber.LibNovel.tests;
|
||||
SDKROOT = iphoneos;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
TEST_HOST = "$(BUILT_PRODUCTS_DIR)/LibNovel.app/LibNovel";
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
8098D4A97F989064EC71E5A1 /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||
CODE_SIGN_IDENTITY = "Apple Development";
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
DEVELOPMENT_TEAM = GHZXC6FVMU;
|
||||
GENERATE_INFOPLIST_FILE = NO;
|
||||
INFOPLIST_FILE = LibNovel/Resources/Info.plist;
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
);
|
||||
PRODUCT_BUNDLE_IDENTIFIER = com.kalekber.LibNovel;
|
||||
PROVISIONING_PROFILE_SPECIFIER = "";
|
||||
SDKROOT = iphoneos;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
9C182367114E72FF84D54A2F /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ALWAYS_SEARCH_USER_PATHS = NO;
|
||||
CLANG_ANALYZER_NONNULL = YES;
|
||||
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
||||
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
||||
CLANG_CXX_LIBRARY = "libc++";
|
||||
CLANG_ENABLE_MODULES = YES;
|
||||
CLANG_ENABLE_OBJC_ARC = YES;
|
||||
CLANG_ENABLE_OBJC_WEAK = YES;
|
||||
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
||||
CLANG_WARN_BOOL_CONVERSION = YES;
|
||||
CLANG_WARN_COMMA = YES;
|
||||
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
||||
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
||||
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
||||
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
||||
CLANG_WARN_EMPTY_BODY = YES;
|
||||
CLANG_WARN_ENUM_CONVERSION = YES;
|
||||
CLANG_WARN_INFINITE_RECURSION = YES;
|
||||
CLANG_WARN_INT_CONVERSION = YES;
|
||||
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
||||
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
||||
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
||||
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
||||
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
||||
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
||||
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
||||
CLANG_WARN_UNREACHABLE_CODE = YES;
|
||||
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
||||
COPY_PHASE_STRIP = NO;
|
||||
CURRENT_PROJECT_VERSION = 1;
|
||||
DEBUG_INFORMATION_FORMAT = dwarf;
|
||||
ENABLE_PREVIEWS = YES;
|
||||
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
||||
ENABLE_TESTABILITY = YES;
|
||||
GCC_C_LANGUAGE_STANDARD = gnu11;
|
||||
GCC_DYNAMIC_NO_PIC = NO;
|
||||
GCC_NO_COMMON_BLOCKS = YES;
|
||||
GCC_OPTIMIZATION_LEVEL = 0;
|
||||
GCC_PREPROCESSOR_DEFINITIONS = (
|
||||
"$(inherited)",
|
||||
"DEBUG=1",
|
||||
);
|
||||
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
||||
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
||||
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
||||
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
||||
GCC_WARN_UNUSED_FUNCTION = YES;
|
||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LIBNOVEL_BASE_URL = "https://v2.libnovel.kalekber.cc";
|
||||
MARKETING_VERSION = 1.0.0;
|
||||
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
|
||||
MTL_FAST_MATH = YES;
|
||||
ONLY_ACTIVE_ARCH = YES;
|
||||
PRODUCT_NAME = "$(TARGET_NAME)";
|
||||
SDKROOT = iphoneos;
|
||||
SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
|
||||
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
|
||||
SWIFT_VERSION = 5.10;
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
D9977A0FA70F052FD0C126D3 /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||
CODE_SIGN_IDENTITY = "Apple Distribution";
|
||||
CODE_SIGN_STYLE = Manual;
|
||||
DEVELOPMENT_TEAM = GHZXC6FVMU;
|
||||
GENERATE_INFOPLIST_FILE = NO;
|
||||
INFOPLIST_FILE = LibNovel/Resources/Info.plist;
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
);
|
||||
PRODUCT_BUNDLE_IDENTIFIER = com.kalekber.LibNovel;
|
||||
PROVISIONING_PROFILE = "af592c3a-f60b-4ac1-a14f-30b8a206017f";
|
||||
SDKROOT = iphoneos;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
F9ED141CFB1E2EC6F5E9F089 /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ALWAYS_SEARCH_USER_PATHS = NO;
|
||||
CLANG_ANALYZER_NONNULL = YES;
|
||||
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
|
||||
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
|
||||
CLANG_CXX_LIBRARY = "libc++";
|
||||
CLANG_ENABLE_MODULES = YES;
|
||||
CLANG_ENABLE_OBJC_ARC = YES;
|
||||
CLANG_ENABLE_OBJC_WEAK = YES;
|
||||
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
|
||||
CLANG_WARN_BOOL_CONVERSION = YES;
|
||||
CLANG_WARN_COMMA = YES;
|
||||
CLANG_WARN_CONSTANT_CONVERSION = YES;
|
||||
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
|
||||
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
|
||||
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
|
||||
CLANG_WARN_EMPTY_BODY = YES;
|
||||
CLANG_WARN_ENUM_CONVERSION = YES;
|
||||
CLANG_WARN_INFINITE_RECURSION = YES;
|
||||
CLANG_WARN_INT_CONVERSION = YES;
|
||||
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
|
||||
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
|
||||
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
|
||||
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
|
||||
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
|
||||
CLANG_WARN_STRICT_PROTOTYPES = YES;
|
||||
CLANG_WARN_SUSPICIOUS_MOVE = YES;
|
||||
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
|
||||
CLANG_WARN_UNREACHABLE_CODE = YES;
|
||||
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
|
||||
COPY_PHASE_STRIP = NO;
|
||||
CURRENT_PROJECT_VERSION = 1;
|
||||
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
|
||||
ENABLE_NS_ASSERTIONS = NO;
|
||||
ENABLE_PREVIEWS = YES;
|
||||
ENABLE_STRICT_OBJC_MSGSEND = YES;
|
||||
GCC_C_LANGUAGE_STANDARD = gnu11;
|
||||
GCC_NO_COMMON_BLOCKS = YES;
|
||||
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
|
||||
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
|
||||
GCC_WARN_UNDECLARED_SELECTOR = YES;
|
||||
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
|
||||
GCC_WARN_UNUSED_FUNCTION = YES;
|
||||
GCC_WARN_UNUSED_VARIABLE = YES;
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LIBNOVEL_BASE_URL = "https://v2.libnovel.kalekber.cc";
|
||||
MARKETING_VERSION = 1.0.0;
|
||||
MTL_ENABLE_DEBUG_INFO = NO;
|
||||
MTL_FAST_MATH = YES;
|
||||
PRODUCT_NAME = "$(TARGET_NAME)";
|
||||
SDKROOT = iphoneos;
|
||||
SWIFT_ACTIVE_COMPILATION_CONDITIONS = "";
|
||||
SWIFT_COMPILATION_MODE = wholemodule;
|
||||
SWIFT_OPTIMIZATION_LEVEL = "-O";
|
||||
SWIFT_VERSION = 5.10;
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
/* End XCBuildConfiguration section */
|
||||
|
||||
/* Begin XCConfigurationList section */
|
||||
29B2DE7267A3A4B2D89B32DA /* Build configuration list for PBXNativeTarget "LibNovel" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
8098D4A97F989064EC71E5A1 /* Debug */,
|
||||
D9977A0FA70F052FD0C126D3 /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Debug;
|
||||
};
|
||||
964FF85B62FA35E819BE7661 /* Build configuration list for PBXNativeTarget "LibNovelTests" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
49CBF0D367E562629E002A4B /* Debug */,
|
||||
428871329DC9E7B31FA1664B /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Debug;
|
||||
};
|
||||
D27899EE96A9AFCBBE62EA3C /* Build configuration list for PBXProject "LibNovel" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
9C182367114E72FF84D54A2F /* Debug */,
|
||||
F9ED141CFB1E2EC6F5E9F089 /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Debug;
|
||||
};
|
||||
/* End XCConfigurationList section */
|
||||
|
||||
/* Begin XCRemoteSwiftPackageReference section */
|
||||
AFEF7128801A76181793EA9C /* XCRemoteSwiftPackageReference "Kingfisher" */ = {
|
||||
isa = XCRemoteSwiftPackageReference;
|
||||
repositoryURL = "https://github.com/onevcat/Kingfisher";
|
||||
requirement = {
|
||||
kind = upToNextMajorVersion;
|
||||
minimumVersion = 8.0.0;
|
||||
};
|
||||
};
|
||||
/* End XCRemoteSwiftPackageReference section */
|
||||
|
||||
/* Begin XCSwiftPackageProductDependency section */
|
||||
09584EAB68A07B47F876A062 /* Kingfisher */ = {
|
||||
isa = XCSwiftPackageProductDependency;
|
||||
package = AFEF7128801A76181793EA9C /* XCRemoteSwiftPackageReference "Kingfisher" */;
|
||||
productName = Kingfisher;
|
||||
};
|
||||
/* End XCSwiftPackageProductDependency section */
|
||||
};
|
||||
rootObject = A10A669C0C8B43078C0FEE9F /* Project object */;
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Workspace
|
||||
version = "1.0">
|
||||
<FileRef
|
||||
location = "self:">
|
||||
</FileRef>
|
||||
</Workspace>
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"originHash" : "ad75ae2d3b8d8b80d99635f65213a3c1092464aa54a86354f850b8317b6fa240",
|
||||
"pins" : [
|
||||
{
|
||||
"identity" : "kingfisher",
|
||||
"kind" : "remoteSourceControl",
|
||||
"location" : "https://github.com/onevcat/Kingfisher",
|
||||
"state" : {
|
||||
"revision" : "c92b84898e34ab46ff0dad86c02a0acbe2d87008",
|
||||
"version" : "8.8.0"
|
||||
}
|
||||
}
|
||||
],
|
||||
"version" : 3
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Scheme
|
||||
LastUpgradeVersion = "1600"
|
||||
version = "1.7">
|
||||
<BuildAction
|
||||
parallelizeBuildables = "YES"
|
||||
buildImplicitDependencies = "YES"
|
||||
runPostActionsOnFailure = "NO">
|
||||
<BuildActionEntries>
|
||||
<BuildActionEntry
|
||||
buildForTesting = "YES"
|
||||
buildForRunning = "YES"
|
||||
buildForProfiling = "YES"
|
||||
buildForArchiving = "YES"
|
||||
buildForAnalyzing = "YES">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "D039EDECDE3998D8534BB680"
|
||||
BuildableName = "LibNovel.app"
|
||||
BlueprintName = "LibNovel"
|
||||
ReferencedContainer = "container:LibNovel.xcodeproj">
|
||||
</BuildableReference>
|
||||
</BuildActionEntry>
|
||||
</BuildActionEntries>
|
||||
</BuildAction>
|
||||
<TestAction
|
||||
buildConfiguration = "Debug"
|
||||
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
||||
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
||||
shouldUseLaunchSchemeArgsEnv = "YES"
|
||||
onlyGenerateCoverageForSpecifiedTargets = "NO">
|
||||
<MacroExpansion>
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "D039EDECDE3998D8534BB680"
|
||||
BuildableName = "LibNovel.app"
|
||||
BlueprintName = "LibNovel"
|
||||
ReferencedContainer = "container:LibNovel.xcodeproj">
|
||||
</BuildableReference>
|
||||
</MacroExpansion>
|
||||
<Testables>
|
||||
<TestableReference
|
||||
skipped = "NO"
|
||||
parallelizable = "NO">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "5E6D3E8266BFCF0AAF5EC79D"
|
||||
BuildableName = "LibNovelTests.xctest"
|
||||
BlueprintName = "LibNovelTests"
|
||||
ReferencedContainer = "container:LibNovel.xcodeproj">
|
||||
</BuildableReference>
|
||||
</TestableReference>
|
||||
</Testables>
|
||||
<CommandLineArguments>
|
||||
</CommandLineArguments>
|
||||
</TestAction>
|
||||
<LaunchAction
|
||||
buildConfiguration = "Debug"
|
||||
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
||||
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
||||
launchStyle = "0"
|
||||
useCustomWorkingDirectory = "NO"
|
||||
ignoresPersistentStateOnLaunch = "NO"
|
||||
debugDocumentVersioning = "YES"
|
||||
debugServiceExtension = "internal"
|
||||
allowLocationSimulation = "YES">
|
||||
<BuildableProductRunnable
|
||||
runnableDebuggingMode = "0">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "D039EDECDE3998D8534BB680"
|
||||
BuildableName = "LibNovel.app"
|
||||
BlueprintName = "LibNovel"
|
||||
ReferencedContainer = "container:LibNovel.xcodeproj">
|
||||
</BuildableReference>
|
||||
</BuildableProductRunnable>
|
||||
<CommandLineArguments>
|
||||
</CommandLineArguments>
|
||||
<EnvironmentVariables>
|
||||
<EnvironmentVariable
|
||||
key = "LIBNOVEL_BASE_URL"
|
||||
value = "["value": "https://v2.libnovel.kalekber.cc", "isEnabled": true]"
|
||||
isEnabled = "YES">
|
||||
</EnvironmentVariable>
|
||||
</EnvironmentVariables>
|
||||
</LaunchAction>
|
||||
<ProfileAction
|
||||
buildConfiguration = "Release"
|
||||
shouldUseLaunchSchemeArgsEnv = "YES"
|
||||
savedToolIdentifier = ""
|
||||
useCustomWorkingDirectory = "NO"
|
||||
debugDocumentVersioning = "YES">
|
||||
<BuildableProductRunnable
|
||||
runnableDebuggingMode = "0">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "D039EDECDE3998D8534BB680"
|
||||
BuildableName = "LibNovel.app"
|
||||
BlueprintName = "LibNovel"
|
||||
ReferencedContainer = "container:LibNovel.xcodeproj">
|
||||
</BuildableReference>
|
||||
</BuildableProductRunnable>
|
||||
<CommandLineArguments>
|
||||
</CommandLineArguments>
|
||||
</ProfileAction>
|
||||
<AnalyzeAction
|
||||
buildConfiguration = "Debug">
|
||||
</AnalyzeAction>
|
||||
<ArchiveAction
|
||||
buildConfiguration = "Release"
|
||||
revealArchiveInOrganizer = "YES">
|
||||
</ArchiveAction>
|
||||
</Scheme>
|
||||
@@ -1,16 +0,0 @@
|
||||
import SwiftUI
|
||||
|
||||
struct ContentView: View {
|
||||
@EnvironmentObject var authStore: AuthStore
|
||||
@EnvironmentObject var audioPlayer: AudioPlayerService
|
||||
|
||||
var body: some View {
|
||||
Group {
|
||||
if authStore.isAuthenticated {
|
||||
RootTabView()
|
||||
} else {
|
||||
AuthView()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
import SwiftUI
|
||||
|
||||
@main
|
||||
struct LibNovelApp: App {
|
||||
@StateObject private var authStore = AuthStore()
|
||||
@StateObject private var audioPlayer = AudioPlayerService()
|
||||
@StateObject private var downloadService = AudioDownloadService.shared
|
||||
@StateObject private var networkMonitor = NetworkMonitor()
|
||||
|
||||
var body: some Scene {
|
||||
WindowGroup {
|
||||
ContentView()
|
||||
.environmentObject(authStore)
|
||||
.environmentObject(audioPlayer)
|
||||
.environmentObject(downloadService)
|
||||
.environmentObject(networkMonitor)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - Root tab container with persistent mini-player overlay
|
||||
|
||||
struct RootTabView: View {
|
||||
@EnvironmentObject var authStore: AuthStore
|
||||
@EnvironmentObject var audioPlayer: AudioPlayerService
|
||||
|
||||
@State private var selectedTab: Tab = .home
|
||||
@State private var showFullPlayer: Bool = false
|
||||
@State private var readerIsActive: Bool = false
|
||||
|
||||
/// Live drag offset while the user is dragging the full player down.
|
||||
@State private var fullPlayerDragOffset: CGFloat = 0
|
||||
|
||||
enum Tab: Hashable {
|
||||
case home, library, browse, search
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
ZStack(alignment: .bottom) {
|
||||
TabView(selection: $selectedTab) {
|
||||
HomeView()
|
||||
.tabItem { Label("Home", systemImage: "house.fill") }
|
||||
.tag(Tab.home)
|
||||
|
||||
LibraryView()
|
||||
.tabItem { Label("Library", systemImage: "book.pages.fill") }
|
||||
.tag(Tab.library)
|
||||
|
||||
BrowseView()
|
||||
.tabItem { Label("Discover", systemImage: "sparkles") }
|
||||
.tag(Tab.browse)
|
||||
|
||||
SearchView()
|
||||
.tabItem { Label("Search", systemImage: "magnifyingglass") }
|
||||
.tag(Tab.search)
|
||||
}
|
||||
|
||||
// Mini player bar — sits above the tab bar, hidden while full player is open
|
||||
// or while the chapter reader is active (it has its own audio chrome).
|
||||
if audioPlayer.isActive && !showFullPlayer && !readerIsActive {
|
||||
MiniPlayerBar(showFullPlayer: $showFullPlayer)
|
||||
// Lift above the tab bar (approx 49 pt on all devices)
|
||||
.padding(.bottom, 49)
|
||||
.transition(.move(edge: .bottom).combined(with: .opacity))
|
||||
.animation(.spring(response: 0.35, dampingFraction: 0.8), value: audioPlayer.isActive)
|
||||
}
|
||||
|
||||
// Full player — slides up from the bottom as a custom overlay.
|
||||
if showFullPlayer {
|
||||
FullPlayerView(onDismiss: {
|
||||
withAnimation(.spring(response: 0.4, dampingFraction: 0.85)) {
|
||||
showFullPlayer = false
|
||||
fullPlayerDragOffset = 0
|
||||
}
|
||||
})
|
||||
.offset(y: max(fullPlayerDragOffset, 0))
|
||||
.gesture(
|
||||
DragGesture(minimumDistance: 10)
|
||||
.onChanged { value in
|
||||
if value.translation.height > 0 {
|
||||
fullPlayerDragOffset = value.translation.height
|
||||
}
|
||||
}
|
||||
.onEnded { value in
|
||||
let velocity = value.predictedEndTranslation.height - value.translation.height
|
||||
if value.translation.height > 120 || velocity > 400 {
|
||||
withAnimation(.spring(response: 0.4, dampingFraction: 0.85)) {
|
||||
showFullPlayer = false
|
||||
fullPlayerDragOffset = 0
|
||||
}
|
||||
} else {
|
||||
withAnimation(.spring(response: 0.35, dampingFraction: 0.8)) {
|
||||
fullPlayerDragOffset = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
.transition(.move(edge: .bottom))
|
||||
.animation(.spring(response: 0.45, dampingFraction: 0.85), value: showFullPlayer)
|
||||
.ignoresSafeArea()
|
||||
}
|
||||
}
|
||||
.animation(.spring(response: 0.45, dampingFraction: 0.85), value: showFullPlayer)
|
||||
.onPreferenceChange(HideMiniPlayerKey.self) { hide in
|
||||
readerIsActive = hide
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - App accent color (amber — mirrors Tailwind amber-500 #f59e0b)
|
||||
extension Color {
|
||||
static let amber = Color(red: 0.96, green: 0.62, blue: 0.04)
|
||||
}
|
||||
|
||||
extension ShapeStyle where Self == Color {
|
||||
static var amber: Color { .amber }
|
||||
}
|
||||
@@ -1,168 +0,0 @@
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - Navigation destination enum used across all tabs
|
||||
|
||||
enum NavDestination: Hashable {
|
||||
case book(String) // slug
|
||||
case chapter(String, Int) // slug + chapter number
|
||||
case userProfile(String) // username
|
||||
case browseCategory(sort: String, genre: String, status: String, title: String) // Browse with filters
|
||||
}
|
||||
|
||||
// MARK: - View extensions for shared navigation + error alert patterns
|
||||
|
||||
extension View {
|
||||
/// Registers the app-wide navigation destinations for NavDestination values.
|
||||
/// Apply once per NavigationStack instead of repeating the switch in every tab.
|
||||
func appNavigationDestination() -> some View {
|
||||
modifier(AppNavigationDestinationModifier())
|
||||
}
|
||||
|
||||
/// Presents a standard "Error" alert driven by an optional String binding.
|
||||
/// Dismissing the alert sets the binding back to nil.
|
||||
/// Silently suppresses network errors when offline (banner shows instead).
|
||||
func errorAlert(_ error: Binding<String?>) -> some View {
|
||||
self.modifier(ErrorAlertModifier(error: error))
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Error Alert Modifier
|
||||
|
||||
private struct ErrorAlertModifier: ViewModifier {
|
||||
@Binding var error: String?
|
||||
@EnvironmentObject var networkMonitor: NetworkMonitor
|
||||
|
||||
private var shouldShowAlert: Bool {
|
||||
guard let errorMessage = error else { return false }
|
||||
|
||||
// If offline, suppress common network error messages
|
||||
if !networkMonitor.isConnected {
|
||||
let networkKeywords = [
|
||||
"internet",
|
||||
"offline",
|
||||
"network",
|
||||
"connection",
|
||||
"unreachable",
|
||||
"timed out",
|
||||
"no data"
|
||||
]
|
||||
|
||||
let lowercased = errorMessage.lowercased()
|
||||
let isNetworkError = networkKeywords.contains { lowercased.contains($0) }
|
||||
|
||||
if isNetworkError {
|
||||
// Clear the error silently
|
||||
DispatchQueue.main.async {
|
||||
self.error = nil
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func body(content: Content) -> some View {
|
||||
content
|
||||
.alert("Error", isPresented: Binding(
|
||||
get: { shouldShowAlert },
|
||||
set: { if !$0 { error = nil } }
|
||||
)) {
|
||||
Button("OK") { error = nil }
|
||||
} message: {
|
||||
Text(error ?? "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Navigation destination modifier
|
||||
|
||||
private struct AppNavigationDestinationModifier: ViewModifier {
|
||||
@Namespace private var zoomNamespace
|
||||
|
||||
func body(content: Content) -> some View {
|
||||
if #available(iOS 18.0, *) {
|
||||
content
|
||||
.navigationDestination(for: NavDestination.self) { dest in
|
||||
switch dest {
|
||||
case .book(let slug):
|
||||
BookDetailView(slug: slug)
|
||||
.navigationTransition(.zoom(sourceID: slug, in: zoomNamespace))
|
||||
case .chapter(let slug, let n):
|
||||
ChapterReaderView(slug: slug, chapterNumber: n)
|
||||
case .userProfile(let username):
|
||||
UserProfileView(username: username)
|
||||
case .browseCategory(let sort, let genre, let status, let title):
|
||||
BrowseCategoryView(sort: sort, genre: genre, status: status, title: title)
|
||||
}
|
||||
}
|
||||
// Expose namespace to child views via environment
|
||||
.environment(\.bookZoomNamespace, zoomNamespace)
|
||||
} else {
|
||||
content
|
||||
.navigationDestination(for: NavDestination.self) { dest in
|
||||
switch dest {
|
||||
case .book(let slug): BookDetailView(slug: slug)
|
||||
case .chapter(let slug, let n): ChapterReaderView(slug: slug, chapterNumber: n)
|
||||
case .userProfile(let username): UserProfileView(username: username)
|
||||
case .browseCategory(let sort, let genre, let status, let title):
|
||||
BrowseCategoryView(sort: sort, genre: genre, status: status, title: title)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Environment key for zoom namespace
|
||||
|
||||
struct BookZoomNamespaceKey: EnvironmentKey {
|
||||
static var defaultValue: Namespace.ID? { nil }
|
||||
}
|
||||
|
||||
extension EnvironmentValues {
|
||||
var bookZoomNamespace: Namespace.ID? {
|
||||
get { self[BookZoomNamespaceKey.self] }
|
||||
set { self[BookZoomNamespaceKey.self] = newValue }
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Preference key: suppress mini player overlay (used by ChapterReaderView)
|
||||
|
||||
struct HideMiniPlayerKey: PreferenceKey {
|
||||
static var defaultValue = false
|
||||
static func reduce(value: inout Bool, nextValue: () -> Bool) {
|
||||
value = value || nextValue()
|
||||
}
|
||||
}
|
||||
|
||||
extension View {
|
||||
/// Signal to the root overlay that the mini player should be hidden.
|
||||
func hideMiniPlayer() -> some View {
|
||||
preference(key: HideMiniPlayerKey.self, value: true)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Cover card zoom source modifier
|
||||
|
||||
/// Apply this to any cover image that should be a zoom source for book navigation.
|
||||
/// Falls back to a no-op on iOS 17 or when no namespace is available.
|
||||
struct BookCoverZoomSource: ViewModifier {
|
||||
let slug: String
|
||||
@Environment(\.bookZoomNamespace) private var namespace
|
||||
|
||||
func body(content: Content) -> some View {
|
||||
if #available(iOS 18.0, *), let ns = namespace {
|
||||
content.matchedTransitionSource(id: slug, in: ns)
|
||||
} else {
|
||||
content
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension View {
|
||||
/// Marks a cover image as the zoom source for a book's navigation transition.
|
||||
func bookCoverZoomSource(slug: String) -> some View {
|
||||
modifier(BookCoverZoomSource(slug: slug))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
import Foundation
|
||||
|
||||
// MARK: - String helpers for display purposes
|
||||
|
||||
extension String {
|
||||
/// Strips trailing relative-date suffixes (e.g. "2 years ago", "3 days ago",
|
||||
/// or "(One)4 years ago" where the number is attached without a preceding space).
|
||||
func strippingTrailingDate() -> String {
|
||||
let units = ["second", "minute", "hour", "day", "week", "month", "year"]
|
||||
let lower = self.lowercased()
|
||||
for unit in units {
|
||||
for suffix in [unit + "s ago", unit + " ago"] {
|
||||
guard let suffixRange = lower.range(of: suffix, options: .backwards) else { continue }
|
||||
// Everything before the suffix
|
||||
let before = String(self[self.startIndex ..< suffixRange.lowerBound])
|
||||
let trimmed = before.trimmingCharacters(in: .whitespaces)
|
||||
// Strip trailing digits (the numeric count, which may be attached without a space)
|
||||
var result = trimmed
|
||||
while let last = result.last, last.isNumber {
|
||||
result.removeLast()
|
||||
}
|
||||
result = result.trimmingCharacters(in: .whitespaces)
|
||||
if result != trimmed {
|
||||
// We actually stripped some digits — return cleaned result
|
||||
return result
|
||||
}
|
||||
// Fallback: number preceded by space
|
||||
if let spaceIdx = trimmed.lastIndex(of: " ") {
|
||||
let potentialNum = String(trimmed[trimmed.index(after: spaceIdx)...])
|
||||
if Int(potentialNum) != nil {
|
||||
return String(trimmed[trimmed.startIndex ..< spaceIdx])
|
||||
.trimmingCharacters(in: .whitespaces)
|
||||
}
|
||||
} else if Int(trimmed) != nil {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
}
|
||||
return self
|
||||
}
|
||||
}
|
||||
@@ -1,395 +0,0 @@
|
||||
import Foundation
|
||||
import SwiftUI
|
||||
|
||||
// MARK: - Book
|
||||
|
||||
struct Book: Identifiable, Codable, Hashable {
|
||||
let id: String
|
||||
let slug: String
|
||||
let title: String
|
||||
let author: String
|
||||
let cover: String
|
||||
let status: String
|
||||
let genres: [String]
|
||||
let summary: String
|
||||
let totalChapters: Int
|
||||
let sourceURL: String
|
||||
let ranking: Int
|
||||
let metaUpdated: String
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id, slug, title, author, cover, status, genres, summary
|
||||
case totalChapters = "total_chapters"
|
||||
case sourceURL = "source_url"
|
||||
case ranking
|
||||
case metaUpdated = "meta_updated"
|
||||
}
|
||||
|
||||
// PocketBase returns genres as either a JSON string array or a real array
|
||||
init(from decoder: Decoder) throws {
|
||||
let container = try decoder.container(keyedBy: CodingKeys.self)
|
||||
id = try container.decode(String.self, forKey: .id)
|
||||
slug = try container.decode(String.self, forKey: .slug)
|
||||
title = try container.decode(String.self, forKey: .title)
|
||||
author = try container.decode(String.self, forKey: .author)
|
||||
cover = try container.decodeIfPresent(String.self, forKey: .cover) ?? ""
|
||||
status = try container.decodeIfPresent(String.self, forKey: .status) ?? ""
|
||||
totalChapters = try container.decodeIfPresent(Int.self, forKey: .totalChapters) ?? 0
|
||||
sourceURL = try container.decodeIfPresent(String.self, forKey: .sourceURL) ?? ""
|
||||
ranking = try container.decodeIfPresent(Int.self, forKey: .ranking) ?? 0
|
||||
metaUpdated = try container.decodeIfPresent(String.self, forKey: .metaUpdated) ?? ""
|
||||
summary = try container.decodeIfPresent(String.self, forKey: .summary) ?? ""
|
||||
|
||||
// genres is sometimes a JSON-encoded string, sometimes a real array
|
||||
if let arr = try? container.decode([String].self, forKey: .genres) {
|
||||
genres = arr
|
||||
} else if let str = try? container.decode(String.self, forKey: .genres),
|
||||
let data = str.data(using: .utf8),
|
||||
let arr = try? JSONDecoder().decode([String].self, from: data) {
|
||||
genres = arr
|
||||
} else {
|
||||
genres = []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - ChapterIndex
|
||||
|
||||
struct ChapterIndex: Identifiable, Codable, Hashable {
|
||||
let id: String
|
||||
let slug: String
|
||||
let number: Int
|
||||
let title: String
|
||||
let dateLabel: String
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id, slug, number, title
|
||||
case dateLabel = "date_label"
|
||||
}
|
||||
}
|
||||
|
||||
struct ChapterIndexBrief: Codable, Hashable {
|
||||
let number: Int
|
||||
let title: String
|
||||
}
|
||||
|
||||
// MARK: - User Settings
|
||||
|
||||
struct UserSettings: Codable {
|
||||
var id: String?
|
||||
var autoNext: Bool
|
||||
var voice: String
|
||||
var speed: Double
|
||||
|
||||
// Server sends/expects camelCase: { autoNext, voice, speed }
|
||||
// (No CodingKeys needed — Swift synthesises the same names by default)
|
||||
|
||||
static let `default` = UserSettings(id: nil, autoNext: false, voice: "af_bella", speed: 1.0)
|
||||
}
|
||||
|
||||
// MARK: - Reading Display Settings (local only — stored in UserDefaults)
|
||||
|
||||
enum ReaderTheme: String, CaseIterable, Codable {
|
||||
case white, sepia, night
|
||||
|
||||
var backgroundColor: Color {
|
||||
switch self {
|
||||
case .white: return Color(.sRGB, white: 1.0, opacity: 1)
|
||||
case .sepia: return Color(red: 0.97, green: 0.93, blue: 0.82)
|
||||
case .night: return Color(red: 0.10, green: 0.10, blue: 0.12)
|
||||
}
|
||||
}
|
||||
|
||||
var textColor: Color {
|
||||
switch self {
|
||||
case .white: return Color(.sRGB, white: 0.1, opacity: 1)
|
||||
case .sepia: return Color(red: 0.25, green: 0.18, blue: 0.08)
|
||||
case .night: return Color(red: 0.85, green: 0.85, blue: 0.87)
|
||||
}
|
||||
}
|
||||
|
||||
var colorScheme: ColorScheme? {
|
||||
switch self {
|
||||
case .white: return nil // follows system
|
||||
case .sepia: return .light
|
||||
case .night: return .dark
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum ReaderFont: String, CaseIterable, Codable {
|
||||
case system = "System"
|
||||
case georgia = "Georgia"
|
||||
case newYork = "New York"
|
||||
|
||||
var fontName: String? {
|
||||
switch self {
|
||||
case .system: return nil
|
||||
case .georgia: return "Georgia"
|
||||
case .newYork: return "NewYorkMedium-Regular"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ReaderSettings: Codable, Equatable {
|
||||
var fontSize: CGFloat
|
||||
var lineSpacing: CGFloat
|
||||
var font: ReaderFont
|
||||
var theme: ReaderTheme
|
||||
var scrollMode: Bool
|
||||
|
||||
static let `default` = ReaderSettings(
|
||||
fontSize: 17,
|
||||
lineSpacing: 1.7,
|
||||
font: .system,
|
||||
theme: .white,
|
||||
scrollMode: false
|
||||
)
|
||||
|
||||
static let userDefaultsKey = "readerSettings"
|
||||
|
||||
static func load() -> ReaderSettings {
|
||||
guard let data = UserDefaults.standard.data(forKey: userDefaultsKey),
|
||||
let decoded = try? JSONDecoder().decode(ReaderSettings.self, from: data)
|
||||
else { return .default }
|
||||
return decoded
|
||||
}
|
||||
|
||||
func save() {
|
||||
if let data = try? JSONEncoder().encode(self) {
|
||||
UserDefaults.standard.set(data, forKey: ReaderSettings.userDefaultsKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - User
|
||||
|
||||
struct AppUser: Codable, Identifiable {
|
||||
let id: String
|
||||
let username: String
|
||||
let role: String
|
||||
let created: String
|
||||
let avatarURL: String?
|
||||
|
||||
var isAdmin: Bool { role == "admin" }
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id, username, role, created
|
||||
case avatarURL = "avatar_url"
|
||||
}
|
||||
|
||||
init(id: String, username: String, role: String, created: String, avatarURL: String?) {
|
||||
self.id = id
|
||||
self.username = username
|
||||
self.role = role
|
||||
self.created = created
|
||||
self.avatarURL = avatarURL
|
||||
}
|
||||
|
||||
init(from decoder: Decoder) throws {
|
||||
let c = try decoder.container(keyedBy: CodingKeys.self)
|
||||
id = try c.decode(String.self, forKey: .id)
|
||||
username = try c.decode(String.self, forKey: .username)
|
||||
role = try c.decodeIfPresent(String.self, forKey: .role) ?? "user"
|
||||
created = try c.decodeIfPresent(String.self, forKey: .created) ?? ""
|
||||
avatarURL = try c.decodeIfPresent(String.self, forKey: .avatarURL)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Ranking
|
||||
|
||||
struct RankingItem: Codable, Identifiable {
|
||||
var id: String { slug }
|
||||
let rank: Int
|
||||
let slug: String
|
||||
let title: String
|
||||
let author: String
|
||||
let cover: String
|
||||
let status: String
|
||||
let genres: [String]
|
||||
let sourceURL: String
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case rank, slug, title, author, cover, status, genres
|
||||
case sourceURL = "source_url"
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Home
|
||||
|
||||
struct ContinueReadingItem: Identifiable {
|
||||
var id: String { book.id }
|
||||
let book: Book
|
||||
let chapter: Int
|
||||
}
|
||||
|
||||
struct HomeStats: Codable {
|
||||
let totalBooks: Int
|
||||
let totalChapters: Int
|
||||
let booksInProgress: Int
|
||||
}
|
||||
|
||||
// MARK: - Session
|
||||
|
||||
struct UserSession: Codable, Identifiable {
|
||||
let id: String
|
||||
let userAgent: String
|
||||
let ip: String
|
||||
let createdAt: String
|
||||
let lastSeen: String
|
||||
var isCurrent: Bool
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id
|
||||
case userAgent = "user_agent"
|
||||
case ip
|
||||
case createdAt = "created_at"
|
||||
case lastSeen = "last_seen"
|
||||
case isCurrent = "is_current"
|
||||
}
|
||||
}
|
||||
|
||||
struct PreviewChapter: Codable, Identifiable {
|
||||
var id: Int { number }
|
||||
let number: Int
|
||||
let title: String
|
||||
let url: String
|
||||
}
|
||||
|
||||
struct BookBrief: Codable {
|
||||
let slug: String
|
||||
let title: String
|
||||
let cover: String
|
||||
}
|
||||
|
||||
// MARK: - Comments
|
||||
|
||||
struct BookComment: Identifiable, Codable, Hashable {
|
||||
let id: String
|
||||
let slug: String
|
||||
let userId: String
|
||||
let username: String
|
||||
let body: String
|
||||
var upvotes: Int
|
||||
var downvotes: Int
|
||||
let created: String
|
||||
let parentId: String // empty = top-level; non-empty = reply
|
||||
var replies: [BookComment]? // populated client-side from the API response
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id, slug, username, body, upvotes, downvotes, created, replies
|
||||
case userId = "user_id"
|
||||
case parentId = "parent_id"
|
||||
}
|
||||
|
||||
init(from decoder: Decoder) throws {
|
||||
let c = try decoder.container(keyedBy: CodingKeys.self)
|
||||
id = try c.decode(String.self, forKey: .id)
|
||||
slug = try c.decodeIfPresent(String.self, forKey: .slug) ?? ""
|
||||
userId = try c.decodeIfPresent(String.self, forKey: .userId) ?? ""
|
||||
username = try c.decodeIfPresent(String.self, forKey: .username) ?? ""
|
||||
body = try c.decodeIfPresent(String.self, forKey: .body) ?? ""
|
||||
upvotes = try c.decodeIfPresent(Int.self, forKey: .upvotes) ?? 0
|
||||
downvotes = try c.decodeIfPresent(Int.self, forKey: .downvotes) ?? 0
|
||||
created = try c.decodeIfPresent(String.self, forKey: .created) ?? ""
|
||||
parentId = try c.decodeIfPresent(String.self, forKey: .parentId) ?? ""
|
||||
replies = try c.decodeIfPresent([BookComment].self, forKey: .replies)
|
||||
}
|
||||
}
|
||||
|
||||
struct CommentsResponse: Decodable {
|
||||
let comments: [BookComment]
|
||||
let myVotes: [String: String]
|
||||
let avatarUrls: [String: String]
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case comments
|
||||
case myVotes = "myVotes"
|
||||
case avatarUrls = "avatarUrls"
|
||||
}
|
||||
|
||||
init(from decoder: Decoder) throws {
|
||||
let c = try decoder.container(keyedBy: CodingKeys.self)
|
||||
comments = try c.decode([BookComment].self, forKey: .comments)
|
||||
myVotes = try c.decodeIfPresent([String: String].self, forKey: .myVotes) ?? [:]
|
||||
avatarUrls = try c.decodeIfPresent([String: String].self, forKey: .avatarUrls) ?? [:]
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - User Profile (public)
|
||||
|
||||
struct PublicUserProfile: Decodable, Identifiable {
|
||||
let id: String
|
||||
let username: String
|
||||
let avatarUrl: String?
|
||||
let created: String
|
||||
let followerCount: Int
|
||||
let followingCount: Int
|
||||
let isSubscribed: Bool
|
||||
let isSelf: Bool
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case id, username, created
|
||||
case avatarUrl = "avatarUrl"
|
||||
case followerCount = "followerCount"
|
||||
case followingCount = "followingCount"
|
||||
case isSubscribed = "isSubscribed"
|
||||
case isSelf = "isSelf"
|
||||
}
|
||||
|
||||
init(from decoder: Decoder) throws {
|
||||
let c = try decoder.container(keyedBy: CodingKeys.self)
|
||||
id = try c.decode(String.self, forKey: .id)
|
||||
username = try c.decode(String.self, forKey: .username)
|
||||
avatarUrl = try c.decodeIfPresent(String.self, forKey: .avatarUrl)
|
||||
created = try c.decodeIfPresent(String.self, forKey: .created) ?? ""
|
||||
followerCount = try c.decodeIfPresent(Int.self, forKey: .followerCount) ?? 0
|
||||
followingCount = try c.decodeIfPresent(Int.self, forKey: .followingCount) ?? 0
|
||||
isSubscribed = try c.decodeIfPresent(Bool.self, forKey: .isSubscribed) ?? false
|
||||
isSelf = try c.decodeIfPresent(Bool.self, forKey: .isSelf) ?? false
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Subscription Feed
|
||||
|
||||
struct SubscriptionFeedItem: Identifiable, Decodable {
|
||||
var id: String { book.id + readerUsername }
|
||||
let book: Book
|
||||
let readerUsername: String
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case book
|
||||
case readerUsername = "readerUsername"
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Public User Library
|
||||
|
||||
struct PublicLibraryItem: Decodable, Identifiable {
|
||||
var id: String { book.id }
|
||||
let book: Book
|
||||
let lastChapter: Int?
|
||||
let saved: Bool
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case book
|
||||
case lastChapter = "last_chapter"
|
||||
case saved
|
||||
}
|
||||
}
|
||||
|
||||
struct PublicUserLibraryResponse: Decodable {
|
||||
let currentlyReading: [PublicLibraryItem]
|
||||
let library: [PublicLibraryItem]
|
||||
|
||||
enum CodingKeys: String, CodingKey {
|
||||
case currentlyReading = "currently_reading"
|
||||
case library
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Audio
|
||||
|
||||
enum NextPrefetchStatus {
|
||||
case none, prefetching, prefetched, failed
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user