Compare commits
6 Commits
v1.0.9
...
feature/ba
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
29d0eeb7e8 | ||
|
|
fabe9724c2 | ||
|
|
4c9bb4adde | ||
|
|
22b6ee824e | ||
|
|
3918bc8dc3 | ||
|
|
5825b859b7 |
18
.env.example
18
.env.example
@@ -1,6 +1,23 @@
|
|||||||
# libnovel scraper — environment overrides
|
# libnovel scraper — environment overrides
|
||||||
# Copy to .env and adjust values; do NOT commit this file with real secrets.
|
# Copy to .env and adjust values; do NOT commit this file with real secrets.
|
||||||
|
|
||||||
|
# ── Docker BuildKit ───────────────────────────────────────────────────────────
|
||||||
|
# Required for the backend/Dockerfile cache mounts (--mount=type=cache).
|
||||||
|
# BuildKit is the default in Docker Engine 23+, but Colima users may need this.
|
||||||
|
#
|
||||||
|
# If you see: "the --mount option requires BuildKit", enable it one of two ways:
|
||||||
|
#
|
||||||
|
# Option A — per-project (recommended, zero restart needed):
|
||||||
|
# Uncomment the line below and copy this file to .env.
|
||||||
|
# Docker Compose reads .env automatically, so BuildKit will be active for
|
||||||
|
# every `docker compose build` / `docker compose up --build` in this project.
|
||||||
|
#
|
||||||
|
# Option B — system-wide for Colima (persists across restarts):
|
||||||
|
# echo '{"features":{"buildkit":true}}' > ~/.colima/default/daemon.json
|
||||||
|
# colima stop && colima start
|
||||||
|
#
|
||||||
|
# DOCKER_BUILDKIT=1
|
||||||
|
|
||||||
# ── Service ports (host-side) ─────────────────────────────────────────────────
|
# ── Service ports (host-side) ─────────────────────────────────────────────────
|
||||||
# Port the scraper HTTP API listens on (default 8080)
|
# Port the scraper HTTP API listens on (default 8080)
|
||||||
SCRAPER_PORT=8080
|
SCRAPER_PORT=8080
|
||||||
@@ -62,6 +79,7 @@ MINIO_ROOT_USER=admin
|
|||||||
MINIO_ROOT_PASSWORD=changeme123
|
MINIO_ROOT_PASSWORD=changeme123
|
||||||
MINIO_BUCKET_CHAPTERS=libnovel-chapters
|
MINIO_BUCKET_CHAPTERS=libnovel-chapters
|
||||||
MINIO_BUCKET_AUDIO=libnovel-audio
|
MINIO_BUCKET_AUDIO=libnovel-audio
|
||||||
|
MINIO_BUCKET_BROWSE=libnovel-browse
|
||||||
|
|
||||||
# ── PocketBase ────────────────────────────────────────────────────────────────
|
# ── PocketBase ────────────────────────────────────────────────────────────────
|
||||||
# Admin credentials (used by scraper + UI server-side)
|
# Admin credentials (used by scraper + UI server-side)
|
||||||
|
|||||||
163
.gitea/workflows/release-v2.yaml
Normal file
163
.gitea/workflows/release-v2.yaml
Normal file
@@ -0,0 +1,163 @@
|
|||||||
|
name: Release / v2
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- "v*"
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ${{ gitea.workflow }}-${{ gitea.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
# ── backend: lint & test ─────────────────────────────────────────────────────
|
||||||
|
test-backend:
|
||||||
|
name: Test backend
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: backend/go.mod
|
||||||
|
cache-dependency-path: backend/go.sum
|
||||||
|
|
||||||
|
- name: go vet
|
||||||
|
working-directory: backend
|
||||||
|
run: go vet ./...
|
||||||
|
|
||||||
|
- name: Run tests
|
||||||
|
working-directory: backend
|
||||||
|
run: go test -short -race -count=1 -timeout=60s ./...
|
||||||
|
|
||||||
|
# ── ui-v2: type-check & build ────────────────────────────────────────────────
|
||||||
|
build-ui:
|
||||||
|
name: Build ui-v2
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: ui-v2
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "22"
|
||||||
|
cache: npm
|
||||||
|
cache-dependency-path: ui-v2/package-lock.json
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Type check
|
||||||
|
run: npm run check
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
run: npm run build
|
||||||
|
|
||||||
|
# ── docker: backend ──────────────────────────────────────────────────────────
|
||||||
|
docker-backend:
|
||||||
|
name: Docker / backend
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: [test-backend]
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Log in to Docker Hub
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_USER }}
|
||||||
|
password: ${{ secrets.DOCKER_TOKEN }}
|
||||||
|
|
||||||
|
- name: Docker meta
|
||||||
|
id: meta
|
||||||
|
uses: docker/metadata-action@v5
|
||||||
|
with:
|
||||||
|
images: ${{ secrets.DOCKER_USER }}/libnovel-backend
|
||||||
|
tags: |
|
||||||
|
type=semver,pattern={{version}}
|
||||||
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
|
type=raw,value=latest
|
||||||
|
|
||||||
|
- name: Build and push
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: backend
|
||||||
|
target: backend
|
||||||
|
push: true
|
||||||
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
|
build-args: |
|
||||||
|
VERSION=${{ steps.meta.outputs.version }}
|
||||||
|
COMMIT=${{ gitea.sha }}
|
||||||
|
|
||||||
|
# ── docker: runner ───────────────────────────────────────────────────────────
|
||||||
|
docker-runner:
|
||||||
|
name: Docker / runner
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: [test-backend]
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Log in to Docker Hub
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_USER }}
|
||||||
|
password: ${{ secrets.DOCKER_TOKEN }}
|
||||||
|
|
||||||
|
- name: Docker meta
|
||||||
|
id: meta
|
||||||
|
uses: docker/metadata-action@v5
|
||||||
|
with:
|
||||||
|
images: ${{ secrets.DOCKER_USER }}/libnovel-runner
|
||||||
|
tags: |
|
||||||
|
type=semver,pattern={{version}}
|
||||||
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
|
type=raw,value=latest
|
||||||
|
|
||||||
|
- name: Build and push
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: backend
|
||||||
|
target: runner
|
||||||
|
push: true
|
||||||
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
|
build-args: |
|
||||||
|
VERSION=${{ steps.meta.outputs.version }}
|
||||||
|
COMMIT=${{ gitea.sha }}
|
||||||
|
|
||||||
|
# ── docker: ui-v2 ────────────────────────────────────────────────────────────
|
||||||
|
docker-ui:
|
||||||
|
name: Docker / ui-v2
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: [build-ui]
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Log in to Docker Hub
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_USER }}
|
||||||
|
password: ${{ secrets.DOCKER_TOKEN }}
|
||||||
|
|
||||||
|
- name: Docker meta
|
||||||
|
id: meta
|
||||||
|
uses: docker/metadata-action@v5
|
||||||
|
with:
|
||||||
|
images: ${{ secrets.DOCKER_USER }}/libnovel-ui-v2
|
||||||
|
tags: |
|
||||||
|
type=semver,pattern={{version}}
|
||||||
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
|
type=raw,value=latest
|
||||||
|
|
||||||
|
- name: Build and push
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: ui-v2
|
||||||
|
push: true
|
||||||
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
|
build-args: |
|
||||||
|
BUILD_VERSION=${{ steps.meta.outputs.version }}
|
||||||
|
BUILD_COMMIT=${{ gitea.sha }}
|
||||||
13
backend/.dockerignore
Normal file
13
backend/.dockerignore
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
# Exclude compiled binaries
|
||||||
|
bin/
|
||||||
|
|
||||||
|
# Exclude test binaries produced by `go test -c`
|
||||||
|
*.test
|
||||||
|
|
||||||
|
# Git history is not needed inside the image
|
||||||
|
.git/
|
||||||
|
|
||||||
|
# Editor/OS noise
|
||||||
|
.DS_Store
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
42
backend/Dockerfile
Normal file
42
backend/Dockerfile
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
# syntax=docker/dockerfile:1
|
||||||
|
FROM golang:1.26.1-alpine AS builder
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Download modules into the BuildKit cache so they survive across builds.
|
||||||
|
# This layer is only invalidated when go.mod or go.sum changes.
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN --mount=type=cache,target=/root/go/pkg/mod \
|
||||||
|
go mod download
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
ARG VERSION=dev
|
||||||
|
ARG COMMIT=unknown
|
||||||
|
|
||||||
|
# Build all three binaries in a single layer so the Go compiler can reuse
|
||||||
|
# intermediate object files. Both cache mounts are preserved between builds:
|
||||||
|
# /root/go/pkg/mod — downloaded module source
|
||||||
|
# /root/.cache/go-build — compiled package objects (incremental recompile)
|
||||||
|
RUN --mount=type=cache,target=/root/go/pkg/mod \
|
||||||
|
--mount=type=cache,target=/root/.cache/go-build \
|
||||||
|
CGO_ENABLED=0 GOOS=linux go build \
|
||||||
|
-ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT}" \
|
||||||
|
-o /out/backend ./cmd/backend && \
|
||||||
|
CGO_ENABLED=0 GOOS=linux go build \
|
||||||
|
-ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT}" \
|
||||||
|
-o /out/runner ./cmd/runner && \
|
||||||
|
CGO_ENABLED=0 GOOS=linux go build \
|
||||||
|
-ldflags="-s -w" \
|
||||||
|
-o /out/healthcheck ./cmd/healthcheck
|
||||||
|
|
||||||
|
# ── backend service ──────────────────────────────────────────────────────────
|
||||||
|
FROM gcr.io/distroless/static:nonroot AS backend
|
||||||
|
COPY --from=builder /out/healthcheck /healthcheck
|
||||||
|
COPY --from=builder /out/backend /backend
|
||||||
|
ENTRYPOINT ["/backend"]
|
||||||
|
|
||||||
|
# ── runner service ───────────────────────────────────────────────────────────
|
||||||
|
FROM gcr.io/distroless/static:nonroot AS runner
|
||||||
|
COPY --from=builder /out/healthcheck /healthcheck
|
||||||
|
COPY --from=builder /out/runner /runner
|
||||||
|
ENTRYPOINT ["/runner"]
|
||||||
126
backend/cmd/backend/main.go
Normal file
126
backend/cmd/backend/main.go
Normal file
@@ -0,0 +1,126 @@
|
|||||||
|
// Command backend is the LibNovel HTTP API server.
|
||||||
|
//
|
||||||
|
// It exposes all endpoints consumed by the SvelteKit UI: book/chapter reads,
|
||||||
|
// scrape-task creation, presigned MinIO URLs, audio-task creation, reading
|
||||||
|
// progress, live novelfire.net browse/search, and Kokoro voice list.
|
||||||
|
//
|
||||||
|
// All heavy lifting (scraping, TTS generation) is delegated to the runner
|
||||||
|
// binary via PocketBase task records. The backend never scrapes directly.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
//
|
||||||
|
// backend # start HTTP server (blocks until SIGINT/SIGTERM)
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/backend"
|
||||||
|
"github.com/libnovel/backend/internal/config"
|
||||||
|
"github.com/libnovel/backend/internal/kokoro"
|
||||||
|
"github.com/libnovel/backend/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// version and commit are set at build time via -ldflags.
|
||||||
|
var (
|
||||||
|
version = "dev"
|
||||||
|
commit = "unknown"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := run(); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "backend: fatal: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func run() error {
|
||||||
|
cfg := config.Load()
|
||||||
|
|
||||||
|
// ── Logger ───────────────────────────────────────────────────────────────
|
||||||
|
log := buildLogger(cfg.LogLevel)
|
||||||
|
log.Info("backend starting",
|
||||||
|
"version", version,
|
||||||
|
"commit", commit,
|
||||||
|
"addr", cfg.HTTP.Addr,
|
||||||
|
)
|
||||||
|
|
||||||
|
// ── Context: cancel on SIGINT / SIGTERM ──────────────────────────────────
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
// ── Storage ──────────────────────────────────────────────────────────────
|
||||||
|
store, err := storage.NewStore(ctx, cfg, log)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("init storage: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Kokoro (voice list only; audio generation is done by the runner) ─────
|
||||||
|
var kokoroClient kokoro.Client
|
||||||
|
if cfg.Kokoro.URL != "" {
|
||||||
|
kokoroClient = kokoro.New(cfg.Kokoro.URL)
|
||||||
|
log.Info("kokoro voices enabled", "url", cfg.Kokoro.URL)
|
||||||
|
} else {
|
||||||
|
log.Info("KOKORO_URL not set — voice list will use built-in fallback")
|
||||||
|
kokoroClient = &noopKokoro{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Backend server ───────────────────────────────────────────────────────
|
||||||
|
srv := backend.New(
|
||||||
|
backend.Config{
|
||||||
|
Addr: cfg.HTTP.Addr,
|
||||||
|
DefaultVoice: cfg.Kokoro.DefaultVoice,
|
||||||
|
Version: version,
|
||||||
|
Commit: commit,
|
||||||
|
},
|
||||||
|
backend.Dependencies{
|
||||||
|
BookReader: store,
|
||||||
|
RankingStore: store,
|
||||||
|
AudioStore: store,
|
||||||
|
PresignStore: store,
|
||||||
|
ProgressStore: store,
|
||||||
|
BrowseStore: store,
|
||||||
|
Producer: store,
|
||||||
|
TaskReader: store,
|
||||||
|
Kokoro: kokoroClient,
|
||||||
|
Log: log,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
return srv.ListenAndServe(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func buildLogger(level string) *slog.Logger {
|
||||||
|
var lvl slog.Level
|
||||||
|
switch level {
|
||||||
|
case "debug":
|
||||||
|
lvl = slog.LevelDebug
|
||||||
|
case "warn":
|
||||||
|
lvl = slog.LevelWarn
|
||||||
|
case "error":
|
||||||
|
lvl = slog.LevelError
|
||||||
|
default:
|
||||||
|
lvl = slog.LevelInfo
|
||||||
|
}
|
||||||
|
return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// noopKokoro is a no-op implementation used when KOKORO_URL is not set.
|
||||||
|
// The backend only uses Kokoro for the voice list; audio generation is the
|
||||||
|
// runner's responsibility. With no URL the built-in fallback list is served.
|
||||||
|
type noopKokoro struct{}
|
||||||
|
|
||||||
|
func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
|
||||||
|
return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
57
backend/cmd/backend/main_test.go
Normal file
57
backend/cmd/backend/main_test.go
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestBuildLogger verifies that buildLogger returns a non-nil logger for each
|
||||||
|
// supported log level string and for unknown values.
|
||||||
|
func TestBuildLogger(t *testing.T) {
|
||||||
|
for _, level := range []string{"debug", "info", "warn", "error", "unknown", ""} {
|
||||||
|
l := buildLogger(level)
|
||||||
|
if l == nil {
|
||||||
|
t.Errorf("buildLogger(%q) returned nil", level)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNoopKokoro verifies that the no-op Kokoro stub returns the expected
|
||||||
|
// sentinel error from GenerateAudio and nil, nil from ListVoices.
|
||||||
|
func TestNoopKokoro(t *testing.T) {
|
||||||
|
noop := &noopKokoro{}
|
||||||
|
|
||||||
|
_, err := noop.GenerateAudio(t.Context(), "text", "af_bella")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("noopKokoro.GenerateAudio: expected error, got nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
voices, err := noop.ListVoices(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("noopKokoro.ListVoices: unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if voices != nil {
|
||||||
|
t.Fatalf("noopKokoro.ListVoices: expected nil slice, got %v", voices)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunStorageUnreachable verifies that run() fails fast and returns a
|
||||||
|
// descriptive error when PocketBase is unreachable.
|
||||||
|
func TestRunStorageUnreachable(t *testing.T) {
|
||||||
|
// Point at an address nothing is listening on.
|
||||||
|
t.Setenv("POCKETBASE_URL", "http://127.0.0.1:19999")
|
||||||
|
// Use a fast listen address so we don't accidentally start a real server.
|
||||||
|
t.Setenv("BACKEND_HTTP_ADDR", "127.0.0.1:0")
|
||||||
|
|
||||||
|
err := run()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("run() should have returned an error when storage is unreachable")
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("got expected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMain runs the test suite. No special setup required.
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
os.Exit(m.Run())
|
||||||
|
}
|
||||||
89
backend/cmd/healthcheck/main.go
Normal file
89
backend/cmd/healthcheck/main.go
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
// healthcheck is a static binary used by Docker HEALTHCHECK CMD in distroless
|
||||||
|
// images (which have no shell, wget, or curl).
|
||||||
|
//
|
||||||
|
// Two modes:
|
||||||
|
//
|
||||||
|
// 1. HTTP mode (default):
|
||||||
|
// /healthcheck <url>
|
||||||
|
// Performs GET <url>; exits 0 if HTTP 2xx/3xx, 1 otherwise.
|
||||||
|
// Example: /healthcheck http://localhost:8080/health
|
||||||
|
//
|
||||||
|
// 2. File-liveness mode:
|
||||||
|
// /healthcheck file <path> <max_age_seconds>
|
||||||
|
// Reads <path>, parses its content as RFC3339 timestamp, and exits 1 if the
|
||||||
|
// timestamp is older than <max_age_seconds>. Used by the runner service which
|
||||||
|
// writes /tmp/runner.alive on every successful poll.
|
||||||
|
// Example: /healthcheck file /tmp/runner.alive 120
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if len(os.Args) > 1 && os.Args[1] == "file" {
|
||||||
|
checkFile()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
checkHTTP()
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkHTTP performs a GET request and exits 0 on success, 1 on failure.
|
||||||
|
func checkHTTP() {
|
||||||
|
url := "http://localhost:8080/health"
|
||||||
|
if len(os.Args) > 1 {
|
||||||
|
url = os.Args[1]
|
||||||
|
}
|
||||||
|
resp, err := http.Get(url) //nolint:gosec,noctx
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "healthcheck: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
fmt.Fprintf(os.Stderr, "healthcheck: status %d\n", resp.StatusCode)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkFile reads a timestamp from a file and exits 1 if it is older than the
|
||||||
|
// given max age. Usage: /healthcheck file <path> <max_age_seconds>
|
||||||
|
func checkFile() {
|
||||||
|
if len(os.Args) < 4 {
|
||||||
|
fmt.Fprintln(os.Stderr, "healthcheck file: usage: /healthcheck file <path> <max_age_seconds>")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
path := os.Args[2]
|
||||||
|
maxAgeSec, err := strconv.ParseInt(os.Args[3], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "healthcheck file: invalid max_age_seconds %q: %v\n", os.Args[3], err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "healthcheck file: cannot read %s: %v\n", path, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
ts, err := time.Parse(time.RFC3339, string(data))
|
||||||
|
if err != nil {
|
||||||
|
// Fallback: use file mtime if content is not a valid timestamp.
|
||||||
|
info, statErr := os.Stat(path)
|
||||||
|
if statErr != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "healthcheck file: cannot stat %s: %v\n", path, statErr)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
ts = info.ModTime()
|
||||||
|
}
|
||||||
|
|
||||||
|
age := time.Since(ts)
|
||||||
|
if age > time.Duration(maxAgeSec)*time.Second {
|
||||||
|
fmt.Fprintf(os.Stderr, "healthcheck file: %s is %.0fs old (max %ds)\n", path, age.Seconds(), maxAgeSec)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
140
backend/cmd/runner/main.go
Normal file
140
backend/cmd/runner/main.go
Normal file
@@ -0,0 +1,140 @@
|
|||||||
|
// Command runner is the homelab worker binary.
|
||||||
|
//
|
||||||
|
// It polls PocketBase for pending scrape and audio tasks, executes them, and
|
||||||
|
// writes results back. It connects directly to PocketBase and MinIO using
|
||||||
|
// admin credentials loaded from environment variables.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
//
|
||||||
|
// runner # start polling loop (blocks until SIGINT/SIGTERM)
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"runtime"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/browser"
|
||||||
|
"github.com/libnovel/backend/internal/config"
|
||||||
|
"github.com/libnovel/backend/internal/kokoro"
|
||||||
|
"github.com/libnovel/backend/internal/novelfire"
|
||||||
|
"github.com/libnovel/backend/internal/runner"
|
||||||
|
"github.com/libnovel/backend/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// version and commit are set at build time via -ldflags.
|
||||||
|
var (
|
||||||
|
version = "dev"
|
||||||
|
commit = "unknown"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := run(); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "runner: fatal: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func run() error {
|
||||||
|
cfg := config.Load()
|
||||||
|
|
||||||
|
// ── Logger ──────────────────────────────────────────────────────────────
|
||||||
|
log := buildLogger(cfg.LogLevel)
|
||||||
|
log.Info("runner starting",
|
||||||
|
"version", version,
|
||||||
|
"commit", commit,
|
||||||
|
"worker_id", cfg.Runner.WorkerID,
|
||||||
|
)
|
||||||
|
|
||||||
|
// ── Context: cancel on SIGINT / SIGTERM ─────────────────────────────────
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
// ── Storage ─────────────────────────────────────────────────────────────
|
||||||
|
store, err := storage.NewStore(ctx, cfg, log)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("init storage: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Browser / Scraper ───────────────────────────────────────────────────
|
||||||
|
workers := cfg.Runner.Workers
|
||||||
|
if workers <= 0 {
|
||||||
|
workers = runtime.NumCPU()
|
||||||
|
}
|
||||||
|
timeout := cfg.Runner.Timeout
|
||||||
|
if timeout <= 0 {
|
||||||
|
timeout = 90 * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
browserClient := browser.NewDirectClient(browser.Config{
|
||||||
|
MaxConcurrent: workers,
|
||||||
|
Timeout: timeout,
|
||||||
|
ProxyURL: cfg.Runner.ProxyURL,
|
||||||
|
})
|
||||||
|
novel := novelfire.New(browserClient, log)
|
||||||
|
|
||||||
|
// ── Kokoro ──────────────────────────────────────────────────────────────
|
||||||
|
var kokoroClient kokoro.Client
|
||||||
|
if cfg.Kokoro.URL != "" {
|
||||||
|
kokoroClient = kokoro.New(cfg.Kokoro.URL)
|
||||||
|
log.Info("kokoro TTS enabled", "url", cfg.Kokoro.URL)
|
||||||
|
} else {
|
||||||
|
log.Warn("KOKORO_URL not set — audio tasks will fail")
|
||||||
|
kokoroClient = &noopKokoro{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Runner ──────────────────────────────────────────────────────────────
|
||||||
|
rCfg := runner.Config{
|
||||||
|
WorkerID: cfg.Runner.WorkerID,
|
||||||
|
PollInterval: cfg.Runner.PollInterval,
|
||||||
|
MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape,
|
||||||
|
MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio,
|
||||||
|
OrchestratorWorkers: workers,
|
||||||
|
}
|
||||||
|
deps := runner.Dependencies{
|
||||||
|
Consumer: store,
|
||||||
|
BookWriter: store,
|
||||||
|
BookReader: store,
|
||||||
|
AudioStore: store,
|
||||||
|
BrowseStore: store,
|
||||||
|
Novel: novel,
|
||||||
|
Kokoro: kokoroClient,
|
||||||
|
Log: log,
|
||||||
|
}
|
||||||
|
r := runner.New(rCfg, deps)
|
||||||
|
|
||||||
|
return r.Run(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func buildLogger(level string) *slog.Logger {
|
||||||
|
var lvl slog.Level
|
||||||
|
switch level {
|
||||||
|
case "debug":
|
||||||
|
lvl = slog.LevelDebug
|
||||||
|
case "warn":
|
||||||
|
lvl = slog.LevelWarn
|
||||||
|
case "error":
|
||||||
|
lvl = slog.LevelError
|
||||||
|
default:
|
||||||
|
lvl = slog.LevelInfo
|
||||||
|
}
|
||||||
|
return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// noopKokoro is a no-op implementation used when KOKORO_URL is not set.
|
||||||
|
type noopKokoro struct{}
|
||||||
|
|
||||||
|
func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
|
||||||
|
return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
29
backend/go.mod
Normal file
29
backend/go.mod
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
module github.com/libnovel/backend
|
||||||
|
|
||||||
|
go 1.26.1
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/minio/minio-go/v7 v7.0.98
|
||||||
|
golang.org/x/net v0.51.0
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
|
github.com/go-ini/ini v1.67.0 // indirect
|
||||||
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
|
github.com/klauspost/compress v1.18.2 // indirect
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||||
|
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||||
|
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||||
|
github.com/minio/md5-simd v1.1.2 // indirect
|
||||||
|
github.com/philhofer/fwd v1.2.0 // indirect
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
|
github.com/rs/xid v1.6.0 // indirect
|
||||||
|
github.com/tinylib/msgp v1.6.1 // indirect
|
||||||
|
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||||
|
golang.org/x/crypto v0.48.0 // indirect
|
||||||
|
golang.org/x/sys v0.41.0 // indirect
|
||||||
|
golang.org/x/text v0.34.0 // indirect
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
|
)
|
||||||
45
backend/go.sum
Normal file
45
backend/go.sum
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
|
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||||
|
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||||
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||||
|
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||||
|
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||||
|
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||||
|
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||||
|
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||||
|
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||||
|
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||||
|
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||||
|
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
|
||||||
|
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
|
||||||
|
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||||
|
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||||
|
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||||
|
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||||
|
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||||
|
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||||
|
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||||
|
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||||
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
|
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||||
|
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||||
|
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
|
||||||
|
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
|
||||||
|
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||||
|
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||||
|
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||||
|
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
BIN
backend/healthcheck
Executable file
BIN
backend/healthcheck
Executable file
Binary file not shown.
958
backend/internal/backend/handlers.go
Normal file
958
backend/internal/backend/handlers.go
Normal file
@@ -0,0 +1,958 @@
|
|||||||
|
package backend
|
||||||
|
|
||||||
|
// handlers.go — all HTTP request handlers for the backend server.
|
||||||
|
//
|
||||||
|
// Handler naming mirrors the route table in server.go:
|
||||||
|
// handleScrapeCatalogue, handleScrapeBook, handleScrapeBookRange
|
||||||
|
// handleScrapeStatus, handleScrapeTasks
|
||||||
|
// handleBrowse, handleSearch
|
||||||
|
// handleGetRanking, handleGetCover
|
||||||
|
// handleBookPreview, handleChapterText, handleReindex
|
||||||
|
// handleChapterText, handleReindex
|
||||||
|
// handleAudioGenerate, handleAudioStatus, handleAudioProxy
|
||||||
|
// handleVoices
|
||||||
|
// handlePresignChapter, handlePresignAudio, handlePresignVoiceSample
|
||||||
|
// handlePresignAvatarUpload, handlePresignAvatar
|
||||||
|
// handleGetProgress, handleSetProgress, handleDeleteProgress
|
||||||
|
//
|
||||||
|
// Key design choices vs. old scraper:
|
||||||
|
// - POST /scrape* creates a PocketBase task record and returns 202 with the
|
||||||
|
// task_id — it does NOT run the orchestrator inline.
|
||||||
|
// - POST /api/audio creates a PocketBase audio task and returns 202 — the
|
||||||
|
// runner binary executes TTS generation asynchronously.
|
||||||
|
// - GET /api/audio/status polls PocketBase for the task record status.
|
||||||
|
// - GET /api/audio-proxy reads the completed audio object from MinIO via a
|
||||||
|
// presigned URL redirect (the runner has already uploaded the bytes).
|
||||||
|
// - GET /api/browse and /api/search fetch novelfire.net live (no MinIO cache).
|
||||||
|
// - GET /api/cover redirects to the source cover URL live.
|
||||||
|
// - GET /api/ranking reads from the PocketBase ranking collection (populated
|
||||||
|
// by the runner after each catalogue scrape).
|
||||||
|
// - GET /api/book-preview returns stored data when in library, or enqueues a
|
||||||
|
// scrape task and returns 202 when not. The backend never scrapes directly.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
"github.com/libnovel/backend/internal/kokoro"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
novelFireBase = "https://novelfire.net"
|
||||||
|
novelFireDomain = "novelfire.net"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ── Scrape task creation ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// handleScrapeCatalogue handles POST /scrape.
|
||||||
|
// Creates a "catalogue" scrape task in PocketBase and returns 202 with the task ID.
|
||||||
|
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
|
||||||
|
taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "catalogue", "", 0, 0)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("handleScrapeCatalogue: CreateScrapeTask failed", "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "failed to create task")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleScrapeBook handles POST /scrape/book.
|
||||||
|
// Body: {"url": "https://novelfire.net/book/..."}
|
||||||
|
func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var body struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, `request body must be JSON with "url" field`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "book", body.URL, 0, 0)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("handleScrapeBook: CreateScrapeTask failed", "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "failed to create task")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleScrapeBookRange handles POST /scrape/book/range.
|
||||||
|
// Body: {"url": "...", "from": N, "to": M}
|
||||||
|
func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var body struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
From int `json:"from"`
|
||||||
|
To int `json:"to"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, `request body must be JSON with "url" field`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "book_range", body.URL, body.From, body.To)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("handleScrapeBookRange: CreateScrapeTask failed", "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "failed to create task")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleCancelTask handles POST /api/cancel-task/{id}.
|
||||||
|
// Transitions a pending task (scrape or audio) to status=cancelled.
|
||||||
|
// Returns 404 if the task does not exist, 409 if it cannot be cancelled
|
||||||
|
// (e.g. already running/done).
|
||||||
|
func (s *Server) handleCancelTask(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id := r.PathValue("id")
|
||||||
|
if id == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "missing task id")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := s.deps.Producer.CancelTask(r.Context(), id); err != nil {
|
||||||
|
s.deps.Log.Warn("handleCancelTask: CancelTask failed", "id", id, "err", err)
|
||||||
|
jsonError(w, http.StatusConflict, "could not cancel task: "+err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, map[string]string{"status": "cancelled", "id": id})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Scrape task status / history ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
// handleScrapeStatus handles GET /api/scrape/status.
|
||||||
|
// Returns the most recent scrape task status (or {"running":false} if none).
|
||||||
|
func (s *Server) handleScrapeStatus(w http.ResponseWriter, r *http.Request) {
|
||||||
|
tasks, err := s.deps.TaskReader.ListScrapeTasks(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("handleScrapeStatus: ListScrapeTasks failed", "err", err)
|
||||||
|
writeJSON(w, 0, map[string]bool{"running": false})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
running := false
|
||||||
|
for _, t := range tasks {
|
||||||
|
if t.Status == domain.TaskStatusRunning || t.Status == domain.TaskStatusPending {
|
||||||
|
running = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, map[string]bool{"running": running})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleScrapeTasks handles GET /api/scrape/tasks.
|
||||||
|
// Returns all scrape task records from PocketBase, newest first.
|
||||||
|
func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) {
|
||||||
|
tasks, err := s.deps.TaskReader.ListScrapeTasks(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("handleScrapeTasks: ListScrapeTasks failed", "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "failed to list tasks")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if tasks == nil {
|
||||||
|
tasks = []domain.ScrapeTask{}
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, tasks)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Browse & search ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// NovelListing represents a single novel entry from the novelfire browse/search page.
|
||||||
|
type NovelListing struct {
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Cover string `json:"cover"`
|
||||||
|
Rank string `json:"rank"`
|
||||||
|
Rating string `json:"rating"`
|
||||||
|
Chapters string `json:"chapters"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleBrowse handles GET /api/browse.
|
||||||
|
// Fetches novelfire.net live (no MinIO cache in the new backend).
|
||||||
|
// Query params: page (default 1), genre (default "all"), sort (default "popular"),
|
||||||
|
// status (default "all"), type (default "all-novel")
|
||||||
|
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||||
|
q := r.URL.Query()
|
||||||
|
page := q.Get("page")
|
||||||
|
if page == "" {
|
||||||
|
page = "1"
|
||||||
|
}
|
||||||
|
genre := q.Get("genre")
|
||||||
|
if genre == "" {
|
||||||
|
genre = "all"
|
||||||
|
}
|
||||||
|
sortBy := q.Get("sort")
|
||||||
|
if sortBy == "" {
|
||||||
|
sortBy = "popular"
|
||||||
|
}
|
||||||
|
status := q.Get("status")
|
||||||
|
if status == "" {
|
||||||
|
status = "all"
|
||||||
|
}
|
||||||
|
novelType := q.Get("type")
|
||||||
|
if novelType == "" {
|
||||||
|
novelType = "all-novel"
|
||||||
|
}
|
||||||
|
|
||||||
|
pageNum, _ := strconv.Atoi(page)
|
||||||
|
if pageNum <= 0 {
|
||||||
|
pageNum = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Try MinIO cache first ─────────────────────────────────────────────
|
||||||
|
// Only page 1 is cached; higher pages fall through to live fetch.
|
||||||
|
if pageNum == 1 && s.deps.BrowseStore != nil {
|
||||||
|
if data, ok, err := s.deps.BrowseStore.GetBrowsePage(r.Context(), genre, sortBy, status, novelType, 1); err == nil && ok {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||||
|
_, _ = w.Write(data)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Fall back to live novelfire.net fetch ──────────────────────────────
|
||||||
|
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
|
||||||
|
novelFireBase, genre, sortBy, status, novelType, pageNum)
|
||||||
|
|
||||||
|
novels, hasNext, err := s.fetchBrowsePage(ctx, targetURL)
|
||||||
|
if err != nil {
|
||||||
|
// Live fetch also failed — return empty list with cached=false flag so
|
||||||
|
// the UI can show a "not ready yet" state instead of a hard error.
|
||||||
|
s.deps.Log.Error("handleBrowse: fetch failed (no cache)", "url", targetURL, "err", err)
|
||||||
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
|
writeJSON(w, 0, map[string]any{
|
||||||
|
"novels": []any{},
|
||||||
|
"page": pageNum,
|
||||||
|
"hasNext": false,
|
||||||
|
"cached": false,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||||
|
writeJSON(w, 0, map[string]any{
|
||||||
|
"novels": novels,
|
||||||
|
"page": pageNum,
|
||||||
|
"hasNext": hasNext,
|
||||||
|
"cached": false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleSearch handles GET /api/search.
|
||||||
|
// Query params: q (min 2 chars), source ("local"|"remote"|"all", default "all")
|
||||||
|
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||||
|
q := r.URL.Query().Get("q")
|
||||||
|
if len([]rune(q)) < 2 {
|
||||||
|
jsonError(w, http.StatusBadRequest, "query must be at least 2 characters")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
source := r.URL.Query().Get("source")
|
||||||
|
if source == "" {
|
||||||
|
source = "all"
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var localResults, remoteResults []NovelListing
|
||||||
|
|
||||||
|
// Local search (PocketBase books)
|
||||||
|
if source == "local" || source == "all" {
|
||||||
|
books, err := s.deps.BookReader.ListBooks(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Warn("search: ListBooks failed", "err", err)
|
||||||
|
} else {
|
||||||
|
qLower := strings.ToLower(q)
|
||||||
|
for _, b := range books {
|
||||||
|
if strings.Contains(strings.ToLower(b.Title), qLower) ||
|
||||||
|
strings.Contains(strings.ToLower(b.Author), qLower) {
|
||||||
|
localResults = append(localResults, NovelListing{
|
||||||
|
Slug: b.Slug,
|
||||||
|
Title: b.Title,
|
||||||
|
Cover: b.Cover,
|
||||||
|
URL: b.SourceURL,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remote search (novelfire.net)
|
||||||
|
if source == "remote" || source == "all" {
|
||||||
|
searchURL := novelFireBase + "/search?keyword=" + url.QueryEscape(q)
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
|
||||||
|
if err == nil {
|
||||||
|
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)")
|
||||||
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||||
|
if resp, fetchErr := http.DefaultClient.Do(req); fetchErr == nil {
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode == http.StatusOK {
|
||||||
|
parsed, _ := parseBrowsePage(resp.Body)
|
||||||
|
remoteResults = parsed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge: local first, de-duplicate remote
|
||||||
|
localSlugs := make(map[string]bool, len(localResults))
|
||||||
|
for _, item := range localResults {
|
||||||
|
localSlugs[item.Slug] = true
|
||||||
|
}
|
||||||
|
combined := make([]NovelListing, 0, len(localResults)+len(remoteResults))
|
||||||
|
combined = append(combined, localResults...)
|
||||||
|
for _, item := range remoteResults {
|
||||||
|
if !localSlugs[item.Slug] {
|
||||||
|
combined = append(combined, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, 0, map[string]any{
|
||||||
|
"results": combined,
|
||||||
|
"local_count": len(localResults),
|
||||||
|
"remote_count": len(remoteResults),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Ranking ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// handleGetRanking handles GET /api/ranking.
|
||||||
|
// Returns all ranking items sorted by rank ascending.
|
||||||
|
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
|
||||||
|
items, err := s.deps.RankingStore.ReadRankingItems(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("handleGetRanking: ReadRankingItems failed", "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "failed to read ranking")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if items == nil {
|
||||||
|
items = []domain.RankingItem{}
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, items)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleGetCover handles GET /api/cover/{domain}/{slug}.
|
||||||
|
// The new backend does not cache covers in MinIO. Instead it redirects the
|
||||||
|
// client to the novelfire.net source URL. The domain path segment is kept for
|
||||||
|
// API compatibility with the old scraper.
|
||||||
|
func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
if slug == "" {
|
||||||
|
http.Error(w, "missing slug", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Redirect to the standard novelfire cover CDN URL. If the caller has the
|
||||||
|
// actual cover URL stored in metadata they should use it directly; this
|
||||||
|
// endpoint is a best-effort fallback.
|
||||||
|
coverURL := fmt.Sprintf("https://cdn.novelfire.net/covers/%s.jpg", slug)
|
||||||
|
http.Redirect(w, r, coverURL, http.StatusFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Preview (live scrape, no store writes) ─────────────────────────────────────
|
||||||
|
|
||||||
|
// handleBookPreview handles GET /api/book-preview/{slug}.
|
||||||
|
//
|
||||||
|
// If the book is already in the library (PocketBase), returns its metadata and
|
||||||
|
// chapter index immediately (200).
|
||||||
|
//
|
||||||
|
// If the book is not yet in the library, enqueues a "book" scrape task and
|
||||||
|
// returns 202 Accepted with the task_id. The runner will scrape the book
|
||||||
|
// asynchronously; the client should poll GET /api/scrape/status or
|
||||||
|
// GET /api/scrape/tasks to detect completion, then re-request this endpoint.
|
||||||
|
//
|
||||||
|
// The backend never scrapes directly — all scraping is the runner's job.
|
||||||
|
func (s *Server) handleBookPreview(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
if slug == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "missing slug")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := r.Context()
|
||||||
|
|
||||||
|
meta, inLib, err := s.deps.BookReader.ReadMetadata(ctx, slug)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Warn("book-preview: ReadMetadata failed", "slug", slug, "err", err)
|
||||||
|
inLib = false
|
||||||
|
}
|
||||||
|
|
||||||
|
if inLib {
|
||||||
|
// Fast path: book is already scraped — return stored data.
|
||||||
|
chapters, cerr := s.deps.BookReader.ListChapters(ctx, slug)
|
||||||
|
if cerr != nil {
|
||||||
|
s.deps.Log.Warn("book-preview: ListChapters failed", "slug", slug, "err", cerr)
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, map[string]any{
|
||||||
|
"in_lib": true,
|
||||||
|
"meta": meta,
|
||||||
|
"chapters": chapters,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Book not in library — enqueue a range scrape task for the first 20 chapters
|
||||||
|
// so the user can start reading quickly. Remaining chapters can be scraped
|
||||||
|
// later via the book detail page or the admin scrape panel.
|
||||||
|
bookURL := r.URL.Query().Get("source_url")
|
||||||
|
if bookURL == "" {
|
||||||
|
bookURL = fmt.Sprintf("%s/book/%s", novelFireBase, slug)
|
||||||
|
}
|
||||||
|
|
||||||
|
const previewFrom, previewTo = 1, 20
|
||||||
|
taskID, err := s.deps.Producer.CreateScrapeTask(ctx, "book_range", bookURL, previewFrom, previewTo)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("book-preview: CreateScrapeTask failed", "slug", slug, "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "failed to enqueue scrape task")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.deps.Log.Info("book-preview: enqueued range scrape task", "slug", slug, "task_id", taskID,
|
||||||
|
"from", previewFrom, "to", previewTo)
|
||||||
|
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||||
|
"in_lib": false,
|
||||||
|
"task_id": taskID,
|
||||||
|
"message": fmt.Sprintf("scraping first %d chapters; poll /api/scrape/tasks for completion", previewTo),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Chapter text ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// handleChapterText handles GET /api/chapter-text/{slug}/{n}.
|
||||||
|
// Returns plain text (markdown stripped) of a stored chapter.
|
||||||
|
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
n, err := strconv.Atoi(r.PathValue("n"))
|
||||||
|
if err != nil || n < 1 {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
raw, err := s.deps.BookReader.ReadChapter(r.Context(), slug, n)
|
||||||
|
if err != nil {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||||
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
|
fmt.Fprint(w, stripMarkdown(raw))
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleChapterMarkdown handles GET /api/chapter-markdown/{slug}/{n}.
|
||||||
|
//
|
||||||
|
// Returns the raw markdown content of a stored chapter directly from MinIO.
|
||||||
|
// This is used by the SvelteKit UI as a simpler alternative to presign+fetch:
|
||||||
|
// it avoids the need for the SvelteKit server to reach MinIO directly, and
|
||||||
|
// gives a clean 404 when the chapter has not been scraped yet.
|
||||||
|
func (s *Server) handleChapterMarkdown(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
n, err := strconv.Atoi(r.PathValue("n"))
|
||||||
|
if err != nil || n < 1 || slug == "" {
|
||||||
|
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
raw, err := s.deps.BookReader.ReadChapter(r.Context(), slug, n)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Warn("chapter-markdown: not found in MinIO", "slug", slug, "n", n, "err", err)
|
||||||
|
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
|
||||||
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
|
fmt.Fprint(w, raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleReindex handles POST /api/reindex/{slug}.
|
||||||
|
// Rebuilds the chapters_idx PocketBase collection for a book from MinIO objects.
|
||||||
|
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
if slug == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "missing slug")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
count, err := s.deps.BookReader.ReindexChapters(r.Context(), slug)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("reindex failed", "slug", slug, "indexed", count, "err", err)
|
||||||
|
writeJSON(w, http.StatusInternalServerError, map[string]any{
|
||||||
|
"error": err.Error(),
|
||||||
|
"indexed": count,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.deps.Log.Info("reindex complete", "slug", slug, "indexed", count)
|
||||||
|
writeJSON(w, 0, map[string]any{"slug": slug, "indexed": count})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Audio ──────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
|
||||||
|
// Creates an audio_jobs task in PocketBase (runner executes asynchronously).
|
||||||
|
// Returns 200 immediately if audio already exists in MinIO.
|
||||||
|
// Returns 202 with the task_id if a new task was created.
|
||||||
|
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
n, err := strconv.Atoi(r.PathValue("n"))
|
||||||
|
if err != nil || n < 1 {
|
||||||
|
jsonError(w, http.StatusBadRequest, "invalid chapter")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
voice := s.cfg.DefaultVoice
|
||||||
|
var body struct {
|
||||||
|
Voice string `json:"voice"`
|
||||||
|
}
|
||||||
|
if r.Body != nil {
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||||
|
}
|
||||||
|
if body.Voice != "" {
|
||||||
|
voice = body.Voice
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
|
||||||
|
|
||||||
|
// Fast path: audio already in MinIO
|
||||||
|
audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice)
|
||||||
|
if s.deps.AudioStore.AudioExists(r.Context(), audioKey) {
|
||||||
|
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
|
||||||
|
writeJSON(w, 0, map[string]string{"url": proxyURL, "status": "done"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if a task is already pending/running
|
||||||
|
task, found, _ := s.deps.TaskReader.GetAudioTask(r.Context(), cacheKey)
|
||||||
|
if found && (task.Status == domain.TaskStatusPending || task.Status == domain.TaskStatusRunning) {
|
||||||
|
writeJSON(w, http.StatusAccepted, map[string]string{
|
||||||
|
"task_id": task.ID,
|
||||||
|
"status": string(task.Status),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a new audio task
|
||||||
|
taskID, err := s.deps.Producer.CreateAudioTask(r.Context(), slug, n, voice)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("handleAudioGenerate: CreateAudioTask failed", "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "failed to create audio task")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusAccepted, map[string]string{
|
||||||
|
"task_id": taskID,
|
||||||
|
"status": "pending",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleAudioStatus handles GET /api/audio/status/{slug}/{n}.
|
||||||
|
// Polls PocketBase for the audio task status.
|
||||||
|
// Query params: voice (optional, defaults to DefaultVoice)
|
||||||
|
func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
n, err := strconv.Atoi(r.PathValue("n"))
|
||||||
|
if err != nil || n < 1 || slug == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "invalid params")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
voice := r.URL.Query().Get("voice")
|
||||||
|
if voice == "" {
|
||||||
|
voice = s.cfg.DefaultVoice
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fast path: audio exists in MinIO
|
||||||
|
audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice)
|
||||||
|
if s.deps.AudioStore.AudioExists(r.Context(), audioKey) {
|
||||||
|
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
|
||||||
|
writeJSON(w, 0, map[string]string{
|
||||||
|
"status": "done",
|
||||||
|
"url": proxyURL,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
|
||||||
|
task, found, _ := s.deps.TaskReader.GetAudioTask(r.Context(), cacheKey)
|
||||||
|
if !found {
|
||||||
|
writeJSON(w, 0, map[string]string{"status": "idle"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
resp := map[string]string{
|
||||||
|
"status": string(task.Status),
|
||||||
|
"task_id": task.ID,
|
||||||
|
}
|
||||||
|
if task.Status == domain.TaskStatusFailed && task.ErrorMessage != "" {
|
||||||
|
resp["error"] = task.ErrorMessage
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, resp)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}.
|
||||||
|
// Redirects to a presigned MinIO URL for the generated audio object.
|
||||||
|
// Query params: voice (optional, defaults to DefaultVoice)
|
||||||
|
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
n, err := strconv.Atoi(r.PathValue("n"))
|
||||||
|
if err != nil || n < 1 {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
voice := r.URL.Query().Get("voice")
|
||||||
|
if voice == "" {
|
||||||
|
voice = s.cfg.DefaultVoice
|
||||||
|
}
|
||||||
|
|
||||||
|
audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice)
|
||||||
|
if !s.deps.AudioStore.AudioExists(r.Context(), audioKey) {
|
||||||
|
http.Error(w, "audio not generated yet", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
presignURL, err := s.deps.PresignStore.PresignAudio(r.Context(), audioKey, 1*time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("handleAudioProxy: PresignAudio failed", "slug", slug, "n", n, "err", err)
|
||||||
|
http.Error(w, "presign failed", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
http.Redirect(w, r, presignURL, http.StatusFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Voices ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// handleVoices handles GET /api/voices.
|
||||||
|
// Returns {"voices": [...]} — fetched from Kokoro with built-in fallback.
|
||||||
|
func (s *Server) handleVoices(w http.ResponseWriter, r *http.Request) {
|
||||||
|
writeJSON(w, 0, map[string]any{"voices": s.voices(r.Context())})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Presigned URLs ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}.
|
||||||
|
func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
n, err := strconv.Atoi(r.PathValue("n"))
|
||||||
|
if err != nil || n < 1 || slug == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "invalid params")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
u, err := s.deps.PresignStore.PresignChapter(r.Context(), slug, n, 15*time.Minute)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("presign chapter failed", "slug", slug, "n", n, "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "presign failed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, map[string]string{"url": u})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}.
|
||||||
|
// Query params: voice (optional)
|
||||||
|
func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
n, err := strconv.Atoi(r.PathValue("n"))
|
||||||
|
if err != nil || n < 1 || slug == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "invalid params")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
voice := r.URL.Query().Get("voice")
|
||||||
|
if voice == "" {
|
||||||
|
voice = s.cfg.DefaultVoice
|
||||||
|
}
|
||||||
|
|
||||||
|
key := s.deps.AudioStore.AudioObjectKey(slug, n, voice)
|
||||||
|
if !s.deps.AudioStore.AudioExists(r.Context(), key) {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("presign audio failed", "slug", slug, "n", n, "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "presign failed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, map[string]string{"url": u})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}.
|
||||||
|
func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) {
|
||||||
|
voice := r.PathValue("voice")
|
||||||
|
if voice == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "missing voice")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
key := kokoro.VoiceSampleKey(voice)
|
||||||
|
if !s.deps.AudioStore.AudioExists(r.Context(), key) {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("presign voice sample failed", "voice", voice, "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "presign failed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, map[string]string{"url": u})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handlePresignAvatarUpload handles GET /api/presign/avatar-upload/{userId}.
|
||||||
|
// Query params: ext (jpg|png|webp, defaults to jpg)
|
||||||
|
func (s *Server) handlePresignAvatarUpload(w http.ResponseWriter, r *http.Request) {
|
||||||
|
userID := r.PathValue("userId")
|
||||||
|
if userID == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "missing userId")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ext := r.URL.Query().Get("ext")
|
||||||
|
switch ext {
|
||||||
|
case "jpg", "jpeg":
|
||||||
|
ext = "jpg"
|
||||||
|
case "png":
|
||||||
|
ext = "png"
|
||||||
|
case "webp":
|
||||||
|
ext = "webp"
|
||||||
|
default:
|
||||||
|
ext = "jpg"
|
||||||
|
}
|
||||||
|
|
||||||
|
uploadURL, key, err := s.deps.PresignStore.PresignAvatarUpload(r.Context(), userID, ext)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("presign avatar upload failed", "userId", userID, "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "presign failed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, map[string]string{"upload_url": uploadURL, "key": key})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handlePresignAvatar handles GET /api/presign/avatar/{userId}.
|
||||||
|
func (s *Server) handlePresignAvatar(w http.ResponseWriter, r *http.Request) {
|
||||||
|
userID := r.PathValue("userId")
|
||||||
|
if userID == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "missing userId")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
u, found, err := s.deps.PresignStore.PresignAvatarURL(r.Context(), userID)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("presign avatar failed", "userId", userID, "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "presign failed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, map[string]string{"url": u})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Progress ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// handleGetProgress handles GET /api/progress.
|
||||||
|
// Returns {"slug": chapterNum, "slug_ts": timestampMs, ...}
|
||||||
|
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
|
||||||
|
sid := ensureSession(w, r)
|
||||||
|
entries, err := s.deps.ProgressStore.AllProgress(r.Context(), sid)
|
||||||
|
if err != nil {
|
||||||
|
s.deps.Log.Error("AllProgress failed", "err", err)
|
||||||
|
entries = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
progress := make(map[string]any, len(entries)*2)
|
||||||
|
for _, p := range entries {
|
||||||
|
progress[p.Slug] = p.Chapter
|
||||||
|
progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, progress)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleSetProgress handles POST /api/progress/{slug}.
|
||||||
|
// Body: {"chapter": N}
|
||||||
|
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
|
||||||
|
sid := ensureSession(w, r)
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
if slug == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "missing slug")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var body struct {
|
||||||
|
Chapter int `json:"chapter"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
|
||||||
|
jsonError(w, http.StatusBadRequest, "invalid body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p := domain.ReadingProgress{
|
||||||
|
Slug: slug,
|
||||||
|
Chapter: body.Chapter,
|
||||||
|
UpdatedAt: time.Now(),
|
||||||
|
}
|
||||||
|
if err := s.deps.ProgressStore.SetProgress(r.Context(), sid, p); err != nil {
|
||||||
|
s.deps.Log.Error("SetProgress failed", "slug", slug, "err", err)
|
||||||
|
jsonError(w, http.StatusInternalServerError, "store error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, map[string]string{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleDeleteProgress handles DELETE /api/progress/{slug}.
|
||||||
|
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
|
||||||
|
sid := ensureSession(w, r)
|
||||||
|
slug := r.PathValue("slug")
|
||||||
|
if slug == "" {
|
||||||
|
jsonError(w, http.StatusBadRequest, "missing slug")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.deps.ProgressStore.DeleteProgress(r.Context(), sid, slug); err != nil {
|
||||||
|
s.deps.Log.Error("DeleteProgress failed", "slug", slug, "err", err)
|
||||||
|
// non-fatal
|
||||||
|
}
|
||||||
|
writeJSON(w, 0, map[string]string{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Browse page parsing helpers ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// fetchBrowsePage fetches pageURL and parses NovelListings from the HTML.
|
||||||
|
func (s *Server) fetchBrowsePage(ctx context.Context, pageURL string) ([]NovelListing, bool, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("build request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)")
|
||||||
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||||
|
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||||
|
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
return nil, false, fmt.Errorf("upstream returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
novels, hasNext := parseBrowsePage(resp.Body)
|
||||||
|
return novels, hasNext, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseBrowsePage parses a novelfire HTML body and returns novel listings.
|
||||||
|
// It uses a simple string-scanning approach to avoid importing golang.org/x/net/html
|
||||||
|
// in this package (that dependency is only in internal/novelfire).
|
||||||
|
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
|
||||||
|
data, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
body := string(data)
|
||||||
|
|
||||||
|
var novels []NovelListing
|
||||||
|
hasNext := false
|
||||||
|
|
||||||
|
// Detect "next page" link
|
||||||
|
if strings.Contains(body, `rel="next"`) ||
|
||||||
|
strings.Contains(body, `aria-label="Next"`) ||
|
||||||
|
strings.Contains(body, `class="next"`) {
|
||||||
|
hasNext = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract novel slugs and titles using simple regex patterns.
|
||||||
|
// novelfire.net novel items: <li class="novel-item">...</li>
|
||||||
|
// Each contains an anchor like <a href="/book/{slug}">
|
||||||
|
slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
|
||||||
|
titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
|
||||||
|
coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
|
||||||
|
|
||||||
|
slugMatches := slugRe.FindAllStringSubmatch(body, -1)
|
||||||
|
titleMatches := titleRe.FindAllStringSubmatch(body, -1)
|
||||||
|
coverMatches := coverRe.FindAllStringSubmatch(body, -1)
|
||||||
|
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
for i, sm := range slugMatches {
|
||||||
|
slug := sm[1]
|
||||||
|
if seen[slug] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[slug] = true
|
||||||
|
|
||||||
|
novel := NovelListing{
|
||||||
|
Slug: slug,
|
||||||
|
URL: novelFireBase + "/book/" + slug,
|
||||||
|
}
|
||||||
|
if i < len(titleMatches) {
|
||||||
|
novel.Title = strings.TrimSpace(titleMatches[i][1])
|
||||||
|
}
|
||||||
|
if i < len(coverMatches) {
|
||||||
|
novel.Cover = coverMatches[i][1]
|
||||||
|
}
|
||||||
|
if novel.Title != "" {
|
||||||
|
novels = append(novels, novel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return novels, hasNext
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Markdown stripping ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// stripMarkdown removes common markdown syntax from src, returning plain text.
|
||||||
|
func stripMarkdown(src string) string {
|
||||||
|
src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1")
|
||||||
|
src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n")
|
||||||
|
return strings.TrimSpace(src)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Hardcoded Kokoro voice fallback ───────────────────────────────────────────
|
||||||
|
|
||||||
|
// kokoroVoices is the built-in fallback list used when the Kokoro service is
|
||||||
|
// unavailable. Matches the list in the old scraper helpers.go.
|
||||||
|
var kokoroVoices = []string{
|
||||||
|
// American English
|
||||||
|
"af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia",
|
||||||
|
"af_jessica", "af_kore", "af_nicole", "af_nova", "af_river",
|
||||||
|
"af_sarah", "af_sky",
|
||||||
|
"am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam",
|
||||||
|
"am_michael", "am_onyx", "am_puck",
|
||||||
|
// British English
|
||||||
|
"bf_alice", "bf_emma", "bf_lily",
|
||||||
|
"bm_daniel", "bm_fable", "bm_george", "bm_lewis",
|
||||||
|
// Spanish
|
||||||
|
"ef_dora", "em_alex",
|
||||||
|
// French
|
||||||
|
"ff_siwis",
|
||||||
|
// Hindi
|
||||||
|
"hf_alpha", "hf_beta", "hm_omega", "hm_psi",
|
||||||
|
// Italian
|
||||||
|
"if_sara", "im_nicola",
|
||||||
|
// Japanese
|
||||||
|
"jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo",
|
||||||
|
// Portuguese
|
||||||
|
"pf_dora", "pm_alex",
|
||||||
|
// Chinese
|
||||||
|
"zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi",
|
||||||
|
"zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang",
|
||||||
|
}
|
||||||
287
backend/internal/backend/server.go
Normal file
287
backend/internal/backend/server.go
Normal file
@@ -0,0 +1,287 @@
|
|||||||
|
// Package backend implements the HTTP API server for the LibNovel backend.
|
||||||
|
//
|
||||||
|
// The server exposes all endpoints consumed by the SvelteKit UI:
|
||||||
|
// - Book/chapter reads from PocketBase/MinIO via bookstore interfaces
|
||||||
|
// - Task creation (scrape + audio) via taskqueue.Producer — the runner binary
|
||||||
|
// picks up and executes those tasks asynchronously
|
||||||
|
// - Presigned MinIO URLs for media playback/upload
|
||||||
|
// - Session-scoped reading progress
|
||||||
|
// - Live novelfire.net browse/search (no scraper interface needed; direct HTTP)
|
||||||
|
// - Kokoro voice list
|
||||||
|
//
|
||||||
|
// The backend never scrapes directly. All scraping (metadata, chapter list,
|
||||||
|
// chapter text, audio TTS) is delegated to the runner binary via PocketBase
|
||||||
|
// task records. GET /api/book-preview enqueues a task when the book is absent.
|
||||||
|
//
|
||||||
|
// All external dependencies are injected as interfaces; concrete types live in
|
||||||
|
// internal/storage and are wired by cmd/backend/main.go.
|
||||||
|
package backend
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/bookstore"
|
||||||
|
"github.com/libnovel/backend/internal/kokoro"
|
||||||
|
"github.com/libnovel/backend/internal/taskqueue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Dependencies holds all external services the backend server depends on.
|
||||||
|
// Every field is an interface so test doubles can be injected freely.
|
||||||
|
type Dependencies struct {
|
||||||
|
// BookReader reads book metadata and chapter text from PocketBase/MinIO.
|
||||||
|
BookReader bookstore.BookReader
|
||||||
|
// RankingStore reads ranking data from PocketBase.
|
||||||
|
RankingStore bookstore.RankingStore
|
||||||
|
// AudioStore checks audio object existence and computes MinIO keys.
|
||||||
|
AudioStore bookstore.AudioStore
|
||||||
|
// PresignStore generates short-lived MinIO URLs.
|
||||||
|
PresignStore bookstore.PresignStore
|
||||||
|
// ProgressStore reads/writes per-session reading progress.
|
||||||
|
ProgressStore bookstore.ProgressStore
|
||||||
|
// BrowseStore reads cached browse page snapshots from MinIO.
|
||||||
|
BrowseStore bookstore.BrowseStore
|
||||||
|
// Producer creates scrape/audio tasks in PocketBase.
|
||||||
|
Producer taskqueue.Producer
|
||||||
|
// TaskReader reads scrape/audio task records from PocketBase.
|
||||||
|
TaskReader taskqueue.Reader
|
||||||
|
// Kokoro is the TTS client (used for voice list only in the backend;
|
||||||
|
// audio generation is done by the runner).
|
||||||
|
Kokoro kokoro.Client
|
||||||
|
// Log is the structured logger.
|
||||||
|
Log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config holds HTTP server tuning parameters.
|
||||||
|
type Config struct {
|
||||||
|
// Addr is the listen address, e.g. ":8080".
|
||||||
|
Addr string
|
||||||
|
// DefaultVoice is used when no voice is specified in audio requests.
|
||||||
|
DefaultVoice string
|
||||||
|
// Version and Commit are embedded in /health and /api/version responses.
|
||||||
|
Version string
|
||||||
|
Commit string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Server is the HTTP API server.
|
||||||
|
type Server struct {
|
||||||
|
cfg Config
|
||||||
|
deps Dependencies
|
||||||
|
|
||||||
|
// voiceMu guards cachedVoices. Populated lazily on first GET /api/voices.
|
||||||
|
voiceMu sync.RWMutex
|
||||||
|
cachedVoices []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// New creates a Server from cfg and deps.
|
||||||
|
func New(cfg Config, deps Dependencies) *Server {
|
||||||
|
if cfg.DefaultVoice == "" {
|
||||||
|
cfg.DefaultVoice = "af_bella"
|
||||||
|
}
|
||||||
|
if deps.Log == nil {
|
||||||
|
deps.Log = slog.Default()
|
||||||
|
}
|
||||||
|
return &Server{cfg: cfg, deps: deps}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListenAndServe registers all routes and starts the HTTP server.
|
||||||
|
// It blocks until ctx is cancelled, then performs a graceful shutdown.
|
||||||
|
func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
|
// Health / version
|
||||||
|
mux.HandleFunc("GET /health", s.handleHealth)
|
||||||
|
mux.HandleFunc("GET /api/version", s.handleVersion)
|
||||||
|
|
||||||
|
// Scrape task creation (202 Accepted — runner executes asynchronously)
|
||||||
|
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
||||||
|
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
||||||
|
mux.HandleFunc("POST /scrape/book/range", s.handleScrapeBookRange)
|
||||||
|
|
||||||
|
// Scrape task status / history
|
||||||
|
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
|
||||||
|
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
|
||||||
|
|
||||||
|
// Cancel a pending task (scrape or audio)
|
||||||
|
mux.HandleFunc("POST /api/cancel-task/{id}", s.handleCancelTask)
|
||||||
|
|
||||||
|
// Browse & search (live novelfire.net)
|
||||||
|
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
||||||
|
mux.HandleFunc("GET /api/search", s.handleSearch)
|
||||||
|
|
||||||
|
// Ranking (from PocketBase)
|
||||||
|
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
|
||||||
|
|
||||||
|
// Cover proxy (live URL redirect)
|
||||||
|
mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
|
||||||
|
|
||||||
|
// Book preview (enqueues scrape task if not in library; returns stored data if already scraped)
|
||||||
|
mux.HandleFunc("GET /api/book-preview/{slug}", s.handleBookPreview)
|
||||||
|
|
||||||
|
// Chapter text (served from MinIO via PocketBase index)
|
||||||
|
mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText)
|
||||||
|
// Raw markdown chapter content — served directly from MinIO by the backend.
|
||||||
|
// Use this instead of presign+fetch to avoid SvelteKit→MinIO network path.
|
||||||
|
mux.HandleFunc("GET /api/chapter-markdown/{slug}/{n}", s.handleChapterMarkdown)
|
||||||
|
|
||||||
|
// Reindex chapters_idx from MinIO
|
||||||
|
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
|
||||||
|
|
||||||
|
// Audio task creation (backend creates task; runner executes)
|
||||||
|
mux.HandleFunc("POST /api/audio/{slug}/{n}", s.handleAudioGenerate)
|
||||||
|
mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus)
|
||||||
|
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
||||||
|
|
||||||
|
// Voices list
|
||||||
|
mux.HandleFunc("GET /api/voices", s.handleVoices)
|
||||||
|
|
||||||
|
// Presigned URLs
|
||||||
|
mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter)
|
||||||
|
mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio)
|
||||||
|
mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample)
|
||||||
|
mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload)
|
||||||
|
mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar)
|
||||||
|
|
||||||
|
// Reading progress
|
||||||
|
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
||||||
|
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
||||||
|
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
|
||||||
|
|
||||||
|
srv := &http.Server{
|
||||||
|
Addr: s.cfg.Addr,
|
||||||
|
Handler: mux,
|
||||||
|
ReadTimeout: 15 * time.Second,
|
||||||
|
WriteTimeout: 60 * time.Second,
|
||||||
|
IdleTimeout: 60 * time.Second,
|
||||||
|
}
|
||||||
|
|
||||||
|
errCh := make(chan error, 1)
|
||||||
|
go func() { errCh <- srv.ListenAndServe() }()
|
||||||
|
s.deps.Log.Info("backend: HTTP server listening", "addr", s.cfg.Addr)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
s.deps.Log.Info("backend: context cancelled, starting graceful shutdown")
|
||||||
|
shutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := srv.Shutdown(shutCtx); err != nil {
|
||||||
|
s.deps.Log.Error("backend: graceful shutdown failed", "err", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.deps.Log.Info("backend: shutdown complete")
|
||||||
|
return nil
|
||||||
|
case err := <-errCh:
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Session cookie helpers ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const sessionCookieName = "libnovel_session"
|
||||||
|
|
||||||
|
func sessionID(r *http.Request) string {
|
||||||
|
c, err := r.Cookie(sessionCookieName)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return c.Value
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSessionID() (string, error) {
|
||||||
|
b := make([]byte, 16)
|
||||||
|
if _, err := rand.Read(b); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(b), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureSession(w http.ResponseWriter, r *http.Request) string {
|
||||||
|
if id := sessionID(r); id != "" {
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
id, err := newSessionID()
|
||||||
|
if err != nil {
|
||||||
|
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
|
||||||
|
}
|
||||||
|
http.SetCookie(w, &http.Cookie{
|
||||||
|
Name: sessionCookieName,
|
||||||
|
Value: id,
|
||||||
|
Path: "/",
|
||||||
|
HttpOnly: true,
|
||||||
|
SameSite: http.SameSiteLaxMode,
|
||||||
|
MaxAge: 365 * 24 * 60 * 60,
|
||||||
|
})
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Utility helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// writeJSON writes v as a JSON response with status code. Status 0 → 200.
|
||||||
|
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
if status != 0 {
|
||||||
|
w.WriteHeader(status)
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// jsonError writes a JSON error body and the given status code.
|
||||||
|
func jsonError(w http.ResponseWriter, status int, msg string) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
|
||||||
|
}
|
||||||
|
|
||||||
|
// voices returns the list of available Kokoro voices. On the first call it
|
||||||
|
// fetches from the Kokoro service and caches the result. Falls back to the
|
||||||
|
// hardcoded list on error.
|
||||||
|
func (s *Server) voices(ctx context.Context) []string {
|
||||||
|
s.voiceMu.RLock()
|
||||||
|
cached := s.cachedVoices
|
||||||
|
s.voiceMu.RUnlock()
|
||||||
|
if len(cached) > 0 {
|
||||||
|
return cached
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.deps.Kokoro == nil {
|
||||||
|
return kokoroVoices
|
||||||
|
}
|
||||||
|
|
||||||
|
fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
list, err := s.deps.Kokoro.ListVoices(fetchCtx)
|
||||||
|
if err != nil || len(list) == 0 {
|
||||||
|
s.deps.Log.Warn("backend: could not fetch kokoro voices, using built-in list", "err", err)
|
||||||
|
return kokoroVoices
|
||||||
|
}
|
||||||
|
|
||||||
|
s.voiceMu.Lock()
|
||||||
|
s.cachedVoices = list
|
||||||
|
s.voiceMu.Unlock()
|
||||||
|
s.deps.Log.Info("backend: fetched kokoro voices", "count", len(list))
|
||||||
|
return list
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleHealth handles GET /health.
|
||||||
|
func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
writeJSON(w, 0, map[string]string{
|
||||||
|
"status": "ok",
|
||||||
|
"version": s.cfg.Version,
|
||||||
|
"commit": s.cfg.Commit,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleVersion handles GET /api/version.
|
||||||
|
func (s *Server) handleVersion(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
writeJSON(w, 0, map[string]string{
|
||||||
|
"version": s.cfg.Version,
|
||||||
|
"commit": s.cfg.Commit,
|
||||||
|
})
|
||||||
|
}
|
||||||
137
backend/internal/bookstore/bookstore.go
Normal file
137
backend/internal/bookstore/bookstore.go
Normal file
@@ -0,0 +1,137 @@
|
|||||||
|
// Package bookstore defines the segregated read/write interfaces for book,
|
||||||
|
// chapter, ranking, progress, audio, and presign data.
|
||||||
|
//
|
||||||
|
// Interface segregation:
|
||||||
|
// - BookWriter — used by the runner to persist scraped data.
|
||||||
|
// - BookReader — used by the backend to serve book/chapter data.
|
||||||
|
// - RankingStore — used by both runner (write) and backend (read).
|
||||||
|
// - PresignStore — used only by the backend for URL signing.
|
||||||
|
// - AudioStore — used by the runner to store audio; backend for presign.
|
||||||
|
// - ProgressStore— used only by the backend for reading progress.
|
||||||
|
//
|
||||||
|
// Concrete implementations live in internal/storage.
|
||||||
|
package bookstore
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BookWriter is the write side used by the runner after scraping a book.
|
||||||
|
type BookWriter interface {
|
||||||
|
// WriteMetadata upserts all bibliographic fields for a book.
|
||||||
|
WriteMetadata(ctx context.Context, meta domain.BookMeta) error
|
||||||
|
|
||||||
|
// WriteChapter stores a fully-scraped chapter's text in MinIO and
|
||||||
|
// updates the chapters_idx record in PocketBase.
|
||||||
|
WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error
|
||||||
|
|
||||||
|
// WriteChapterRefs persists chapter metadata (number + title) into
|
||||||
|
// chapters_idx without fetching or storing chapter text.
|
||||||
|
WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error
|
||||||
|
|
||||||
|
// ChapterExists returns true if the markdown object for ref already exists.
|
||||||
|
ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// BookReader is the read side used by the backend to serve content.
|
||||||
|
type BookReader interface {
|
||||||
|
// ReadMetadata returns the metadata for slug.
|
||||||
|
// Returns (zero, false, nil) when not found.
|
||||||
|
ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error)
|
||||||
|
|
||||||
|
// ListBooks returns all books sorted alphabetically by title.
|
||||||
|
ListBooks(ctx context.Context) ([]domain.BookMeta, error)
|
||||||
|
|
||||||
|
// LocalSlugs returns the set of slugs that have metadata stored.
|
||||||
|
LocalSlugs(ctx context.Context) (map[string]bool, error)
|
||||||
|
|
||||||
|
// MetadataMtime returns the Unix-second mtime of the metadata record, or 0.
|
||||||
|
MetadataMtime(ctx context.Context, slug string) int64
|
||||||
|
|
||||||
|
// ReadChapter returns the raw markdown for chapter number n.
|
||||||
|
ReadChapter(ctx context.Context, slug string, n int) (string, error)
|
||||||
|
|
||||||
|
// ListChapters returns all stored chapters for slug, sorted by number.
|
||||||
|
ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error)
|
||||||
|
|
||||||
|
// CountChapters returns the count of stored chapters.
|
||||||
|
CountChapters(ctx context.Context, slug string) int
|
||||||
|
|
||||||
|
// ReindexChapters rebuilds chapters_idx from MinIO objects for slug.
|
||||||
|
ReindexChapters(ctx context.Context, slug string) (int, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RankingStore covers ranking reads and writes.
|
||||||
|
type RankingStore interface {
|
||||||
|
// WriteRankingItem upserts a single ranking entry (keyed on Slug).
|
||||||
|
WriteRankingItem(ctx context.Context, item domain.RankingItem) error
|
||||||
|
|
||||||
|
// ReadRankingItems returns all ranking items sorted by rank ascending.
|
||||||
|
ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error)
|
||||||
|
|
||||||
|
// RankingFreshEnough returns true when ranking rows exist and the most
|
||||||
|
// recent Updated timestamp is within maxAge.
|
||||||
|
RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AudioStore covers audio object storage (runner writes; backend reads).
|
||||||
|
type AudioStore interface {
|
||||||
|
// AudioObjectKey returns the MinIO object key for a cached audio file.
|
||||||
|
AudioObjectKey(slug string, n int, voice string) string
|
||||||
|
|
||||||
|
// AudioExists returns true when the audio object is present in MinIO.
|
||||||
|
AudioExists(ctx context.Context, key string) bool
|
||||||
|
|
||||||
|
// PutAudio stores raw audio bytes under the given MinIO object key.
|
||||||
|
PutAudio(ctx context.Context, key string, data []byte) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// PresignStore generates short-lived URLs — used exclusively by the backend.
|
||||||
|
type PresignStore interface {
|
||||||
|
// PresignChapter returns a presigned GET URL for a chapter markdown object.
|
||||||
|
PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error)
|
||||||
|
|
||||||
|
// PresignAudio returns a presigned GET URL for an audio object.
|
||||||
|
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
|
||||||
|
|
||||||
|
// PresignAvatarUpload returns a short-lived presigned PUT URL for uploading
|
||||||
|
// an avatar image. ext should be "jpg", "png", or "webp".
|
||||||
|
PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error)
|
||||||
|
|
||||||
|
// PresignAvatarURL returns a presigned GET URL for a user's avatar.
|
||||||
|
// Returns ("", false, nil) when no avatar exists.
|
||||||
|
PresignAvatarURL(ctx context.Context, userID string) (string, bool, error)
|
||||||
|
|
||||||
|
// DeleteAvatar removes all avatar objects for a user.
|
||||||
|
DeleteAvatar(ctx context.Context, userID string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProgressStore covers per-session reading progress — backend only.
|
||||||
|
type ProgressStore interface {
|
||||||
|
// GetProgress returns the reading progress for the given session + slug.
|
||||||
|
GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool)
|
||||||
|
|
||||||
|
// SetProgress saves or updates reading progress.
|
||||||
|
SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error
|
||||||
|
|
||||||
|
// AllProgress returns all progress entries for a session.
|
||||||
|
AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error)
|
||||||
|
|
||||||
|
// DeleteProgress removes progress for a specific slug.
|
||||||
|
DeleteProgress(ctx context.Context, sessionID, slug string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// BrowseStore covers browse page snapshot storage.
|
||||||
|
// The runner writes snapshots; the backend reads them.
|
||||||
|
type BrowseStore interface {
|
||||||
|
// PutBrowsePage stores a raw JSON snapshot for a browse page.
|
||||||
|
// genre, sort, status, novelType and page identify the page.
|
||||||
|
PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error
|
||||||
|
|
||||||
|
// GetBrowsePage retrieves a raw JSON snapshot. Returns (nil, false, nil)
|
||||||
|
// when no snapshot exists for the given parameters.
|
||||||
|
GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error)
|
||||||
|
}
|
||||||
138
backend/internal/bookstore/bookstore_test.go
Normal file
138
backend/internal/bookstore/bookstore_test.go
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
package bookstore_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/bookstore"
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ── Mock that satisfies all bookstore interfaces ──────────────────────────────
|
||||||
|
|
||||||
|
type mockStore struct{}
|
||||||
|
|
||||||
|
// BookWriter
|
||||||
|
func (m *mockStore) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil }
|
||||||
|
func (m *mockStore) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error { return nil }
|
||||||
|
func (m *mockStore) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// BookReader
|
||||||
|
func (m *mockStore) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) {
|
||||||
|
return domain.BookMeta{}, false, nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil }
|
||||||
|
func (m *mockStore) LocalSlugs(_ context.Context) (map[string]bool, error) {
|
||||||
|
return map[string]bool{}, nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) MetadataMtime(_ context.Context, _ string) int64 { return 0 }
|
||||||
|
func (m *mockStore) ReadChapter(_ context.Context, _ string, _ int) (string, error) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) CountChapters(_ context.Context, _ string) int { return 0 }
|
||||||
|
func (m *mockStore) ReindexChapters(_ context.Context, _ string) (int, error) { return 0, nil }
|
||||||
|
|
||||||
|
// RankingStore
|
||||||
|
func (m *mockStore) WriteRankingItem(_ context.Context, _ domain.RankingItem) error { return nil }
|
||||||
|
func (m *mockStore) ReadRankingItems(_ context.Context) ([]domain.RankingItem, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) RankingFreshEnough(_ context.Context, _ time.Duration) (bool, error) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AudioStore
|
||||||
|
func (m *mockStore) AudioObjectKey(_ string, _ int, _ string) string { return "" }
|
||||||
|
func (m *mockStore) AudioExists(_ context.Context, _ string) bool { return false }
|
||||||
|
func (m *mockStore) PutAudio(_ context.Context, _ string, _ []byte) error { return nil }
|
||||||
|
|
||||||
|
// PresignStore
|
||||||
|
func (m *mockStore) PresignChapter(_ context.Context, _ string, _ int, _ time.Duration) (string, error) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) PresignAudio(_ context.Context, _ string, _ time.Duration) (string, error) {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) PresignAvatarUpload(_ context.Context, _, _ string) (string, string, error) {
|
||||||
|
return "", "", nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) PresignAvatarURL(_ context.Context, _ string) (string, bool, error) {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) DeleteAvatar(_ context.Context, _ string) error { return nil }
|
||||||
|
|
||||||
|
// ProgressStore
|
||||||
|
func (m *mockStore) GetProgress(_ context.Context, _, _ string) (domain.ReadingProgress, bool) {
|
||||||
|
return domain.ReadingProgress{}, false
|
||||||
|
}
|
||||||
|
func (m *mockStore) SetProgress(_ context.Context, _ string, _ domain.ReadingProgress) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) AllProgress(_ context.Context, _ string) ([]domain.ReadingProgress, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (m *mockStore) DeleteProgress(_ context.Context, _, _ string) error { return nil }
|
||||||
|
|
||||||
|
// ── Compile-time interface satisfaction ───────────────────────────────────────
|
||||||
|
|
||||||
|
var _ bookstore.BookWriter = (*mockStore)(nil)
|
||||||
|
var _ bookstore.BookReader = (*mockStore)(nil)
|
||||||
|
var _ bookstore.RankingStore = (*mockStore)(nil)
|
||||||
|
var _ bookstore.AudioStore = (*mockStore)(nil)
|
||||||
|
var _ bookstore.PresignStore = (*mockStore)(nil)
|
||||||
|
var _ bookstore.ProgressStore = (*mockStore)(nil)
|
||||||
|
|
||||||
|
// ── Behavioural tests ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestBookWriter_WriteMetadata_ReturnsNilError(t *testing.T) {
|
||||||
|
var w bookstore.BookWriter = &mockStore{}
|
||||||
|
if err := w.WriteMetadata(context.Background(), domain.BookMeta{Slug: "test"}); err != nil {
|
||||||
|
t.Errorf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBookReader_ReadMetadata_NotFound(t *testing.T) {
|
||||||
|
var r bookstore.BookReader = &mockStore{}
|
||||||
|
_, found, err := r.ReadMetadata(context.Background(), "unknown")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
t.Error("expected not found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRankingStore_RankingFreshEnough_ReturnsFalse(t *testing.T) {
|
||||||
|
var s bookstore.RankingStore = &mockStore{}
|
||||||
|
fresh, err := s.RankingFreshEnough(context.Background(), time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if fresh {
|
||||||
|
t.Error("expected false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAudioStore_AudioExists_ReturnsFalse(t *testing.T) {
|
||||||
|
var s bookstore.AudioStore = &mockStore{}
|
||||||
|
if s.AudioExists(context.Background(), "audio/slug/1/af_bella.mp3") {
|
||||||
|
t.Error("expected false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProgressStore_GetProgress_NotFound(t *testing.T) {
|
||||||
|
var s bookstore.ProgressStore = &mockStore{}
|
||||||
|
_, found := s.GetProgress(context.Background(), "session-1", "slug")
|
||||||
|
if found {
|
||||||
|
t.Error("expected not found")
|
||||||
|
}
|
||||||
|
}
|
||||||
206
backend/internal/browser/browser.go
Normal file
206
backend/internal/browser/browser.go
Normal file
@@ -0,0 +1,206 @@
|
|||||||
|
// Package browser provides a rate-limited HTTP client for web scraping.
|
||||||
|
// The Client interface is the only thing the rest of the codebase depends on;
|
||||||
|
// the concrete DirectClient can be swapped for any other implementation
|
||||||
|
// (e.g. a Browserless-backed client) without touching callers.
|
||||||
|
package browser
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrRateLimit is returned by GetContent when the server responds with 429.
|
||||||
|
// It carries the suggested retry delay (from Retry-After header, or a default).
|
||||||
|
var ErrRateLimit = errors.New("rate limited (429)")
|
||||||
|
|
||||||
|
// RateLimitError wraps ErrRateLimit and carries the suggested wait duration.
|
||||||
|
type RateLimitError struct {
|
||||||
|
// RetryAfter is how long the caller should wait before retrying.
|
||||||
|
// Derived from the Retry-After response header when present; otherwise a default.
|
||||||
|
RetryAfter time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *RateLimitError) Error() string {
|
||||||
|
return fmt.Sprintf("rate limited (429): retry after %s", e.RetryAfter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *RateLimitError) Is(target error) bool { return target == ErrRateLimit }
|
||||||
|
|
||||||
|
// defaultRateLimitDelay is used when the server returns 429 with no Retry-After header.
|
||||||
|
const defaultRateLimitDelay = 60 * time.Second
|
||||||
|
|
||||||
|
// Client is the interface used by scrapers to fetch raw page HTML.
|
||||||
|
// Implementations must be safe for concurrent use.
|
||||||
|
type Client interface {
|
||||||
|
// GetContent fetches the URL and returns the full response body as a string.
|
||||||
|
// It should respect the provided context for cancellation and timeouts.
|
||||||
|
GetContent(ctx context.Context, pageURL string) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config holds tunable parameters for the direct HTTP client.
|
||||||
|
type Config struct {
|
||||||
|
// MaxConcurrent limits the number of simultaneous in-flight requests.
|
||||||
|
// Defaults to 5 when 0.
|
||||||
|
MaxConcurrent int
|
||||||
|
// Timeout is the per-request deadline. Defaults to 90s when 0.
|
||||||
|
Timeout time.Duration
|
||||||
|
// ProxyURL is an optional outbound proxy, e.g. "http://user:pass@host:3128".
|
||||||
|
// Falls back to HTTP_PROXY / HTTPS_PROXY environment variables when empty.
|
||||||
|
ProxyURL string
|
||||||
|
}
|
||||||
|
|
||||||
|
// DirectClient is a plain net/http-based Client with a concurrency semaphore.
|
||||||
|
type DirectClient struct {
|
||||||
|
http *http.Client
|
||||||
|
semaphore chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDirectClient returns a DirectClient configured by cfg.
|
||||||
|
func NewDirectClient(cfg Config) *DirectClient {
|
||||||
|
if cfg.MaxConcurrent <= 0 {
|
||||||
|
cfg.MaxConcurrent = 5
|
||||||
|
}
|
||||||
|
if cfg.Timeout <= 0 {
|
||||||
|
cfg.Timeout = 90 * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
transport := &http.Transport{
|
||||||
|
MaxIdleConnsPerHost: cfg.MaxConcurrent * 2,
|
||||||
|
DisableCompression: false,
|
||||||
|
}
|
||||||
|
if cfg.ProxyURL != "" {
|
||||||
|
proxyParsed, err := url.Parse(cfg.ProxyURL)
|
||||||
|
if err == nil {
|
||||||
|
transport.Proxy = http.ProxyURL(proxyParsed)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
transport.Proxy = http.ProxyFromEnvironment
|
||||||
|
}
|
||||||
|
|
||||||
|
return &DirectClient{
|
||||||
|
http: &http.Client{
|
||||||
|
Transport: transport,
|
||||||
|
Timeout: cfg.Timeout,
|
||||||
|
},
|
||||||
|
semaphore: make(chan struct{}, cfg.MaxConcurrent),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetContent fetches pageURL respecting the concurrency limit.
|
||||||
|
func (c *DirectClient) GetContent(ctx context.Context, pageURL string) (string, error) {
|
||||||
|
// Acquire semaphore slot.
|
||||||
|
select {
|
||||||
|
case c.semaphore <- struct{}{}:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return "", ctx.Err()
|
||||||
|
}
|
||||||
|
defer func() { <-c.semaphore }()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("browser: build request %s: %w", pageURL, err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
|
||||||
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||||
|
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
|
||||||
|
|
||||||
|
resp, err := c.http.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("browser: GET %s: %w", pageURL, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode == http.StatusTooManyRequests {
|
||||||
|
delay := defaultRateLimitDelay
|
||||||
|
if ra := resp.Header.Get("Retry-After"); ra != "" {
|
||||||
|
if secs, err := strconv.Atoi(ra); err == nil && secs > 0 {
|
||||||
|
delay = time.Duration(secs) * time.Second
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", &RateLimitError{RetryAfter: delay}
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
return "", fmt.Errorf("browser: GET %s returned %d", pageURL, resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("browser: read body %s: %w", pageURL, err)
|
||||||
|
}
|
||||||
|
return string(body), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Do implements httputil.Client so DirectClient can be passed to RetryGet.
|
||||||
|
func (c *DirectClient) Do(req *http.Request) (*http.Response, error) {
|
||||||
|
select {
|
||||||
|
case c.semaphore <- struct{}{}:
|
||||||
|
case <-req.Context().Done():
|
||||||
|
return nil, req.Context().Err()
|
||||||
|
}
|
||||||
|
defer func() { <-c.semaphore }()
|
||||||
|
return c.http.Do(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Stub for testing ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// StubClient is a test double for Client. It returns pre-configured responses
|
||||||
|
// keyed on URL. Calls to unknown URLs return an error.
|
||||||
|
type StubClient struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
pages map[string]string
|
||||||
|
errors map[string]error
|
||||||
|
callLog []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStub creates a StubClient with no pages pre-loaded.
|
||||||
|
func NewStub() *StubClient {
|
||||||
|
return &StubClient{
|
||||||
|
pages: make(map[string]string),
|
||||||
|
errors: make(map[string]error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetPage registers a URL → HTML body mapping.
|
||||||
|
func (s *StubClient) SetPage(u, html string) {
|
||||||
|
s.mu.Lock()
|
||||||
|
s.pages[u] = html
|
||||||
|
s.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetError registers a URL → error mapping (returned instead of a body).
|
||||||
|
func (s *StubClient) SetError(u string, err error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
s.errors[u] = err
|
||||||
|
s.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// CallLog returns the ordered list of URLs that were requested.
|
||||||
|
func (s *StubClient) CallLog() []string {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
out := make([]string, len(s.callLog))
|
||||||
|
copy(out, s.callLog)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetContent returns the registered page or an error for the URL.
|
||||||
|
func (s *StubClient) GetContent(_ context.Context, pageURL string) (string, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.callLog = append(s.callLog, pageURL)
|
||||||
|
if err, ok := s.errors[pageURL]; ok {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if html, ok := s.pages[pageURL]; ok {
|
||||||
|
return html, nil
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("stub: no page registered for %q", pageURL)
|
||||||
|
}
|
||||||
141
backend/internal/browser/browser_test.go
Normal file
141
backend/internal/browser/browser_test.go
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
package browser_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/browser"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDirectClient_GetContent_Success(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Write([]byte("<html>hello</html>"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := browser.NewDirectClient(browser.Config{MaxConcurrent: 2, Timeout: 5 * time.Second})
|
||||||
|
body, err := c.GetContent(context.Background(), srv.URL)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if body != "<html>hello</html>" {
|
||||||
|
t.Errorf("want <html>hello</html>, got %q", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDirectClient_GetContent_4xxReturnsError(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := browser.NewDirectClient(browser.Config{})
|
||||||
|
_, err := c.GetContent(context.Background(), srv.URL)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for 404")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDirectClient_SemaphoreBlocksConcurrency(t *testing.T) {
|
||||||
|
const maxConcurrent = 2
|
||||||
|
var inflight atomic.Int32
|
||||||
|
var peak atomic.Int32
|
||||||
|
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
n := inflight.Add(1)
|
||||||
|
if int(n) > int(peak.Load()) {
|
||||||
|
peak.Store(n)
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
inflight.Add(-1)
|
||||||
|
w.Write([]byte("ok"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := browser.NewDirectClient(browser.Config{MaxConcurrent: maxConcurrent, Timeout: 5 * time.Second})
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < 8; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
c.GetContent(context.Background(), srv.URL)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
if int(peak.Load()) > maxConcurrent {
|
||||||
|
t.Errorf("concurrent requests exceeded limit: peak=%d, limit=%d", peak.Load(), maxConcurrent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDirectClient_ContextCancel(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
w.Write([]byte("ok"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel() // cancel before making the request
|
||||||
|
|
||||||
|
c := browser.NewDirectClient(browser.Config{})
|
||||||
|
_, err := c.GetContent(ctx, srv.URL)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected context cancellation error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── StubClient ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestStubClient_ReturnsRegisteredPage(t *testing.T) {
|
||||||
|
stub := browser.NewStub()
|
||||||
|
stub.SetPage("http://example.com/page1", "<html>page1</html>")
|
||||||
|
|
||||||
|
body, err := stub.GetContent(context.Background(), "http://example.com/page1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if body != "<html>page1</html>" {
|
||||||
|
t.Errorf("want page1 html, got %q", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStubClient_ReturnsRegisteredError(t *testing.T) {
|
||||||
|
stub := browser.NewStub()
|
||||||
|
want := errors.New("network failure")
|
||||||
|
stub.SetError("http://example.com/bad", want)
|
||||||
|
|
||||||
|
_, err := stub.GetContent(context.Background(), "http://example.com/bad")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStubClient_UnknownURLReturnsError(t *testing.T) {
|
||||||
|
stub := browser.NewStub()
|
||||||
|
_, err := stub.GetContent(context.Background(), "http://unknown.example.com/")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for unknown URL")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStubClient_CallLog(t *testing.T) {
|
||||||
|
stub := browser.NewStub()
|
||||||
|
stub.SetPage("http://example.com/a", "a")
|
||||||
|
stub.SetPage("http://example.com/b", "b")
|
||||||
|
|
||||||
|
stub.GetContent(context.Background(), "http://example.com/a")
|
||||||
|
stub.GetContent(context.Background(), "http://example.com/b")
|
||||||
|
|
||||||
|
log := stub.CallLog()
|
||||||
|
if len(log) != 2 || log[0] != "http://example.com/a" || log[1] != "http://example.com/b" {
|
||||||
|
t.Errorf("unexpected call log: %v", log)
|
||||||
|
}
|
||||||
|
}
|
||||||
186
backend/internal/config/config.go
Normal file
186
backend/internal/config/config.go
Normal file
@@ -0,0 +1,186 @@
|
|||||||
|
// Package config loads all service configuration from environment variables.
|
||||||
|
// Both the runner and backend binaries call config.Load() at startup; each
|
||||||
|
// uses only the sub-struct relevant to it.
|
||||||
|
//
|
||||||
|
// Every field has a documented default so the service starts sensibly without
|
||||||
|
// any environment configuration (useful for local development).
|
||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PocketBase holds connection settings for the remote PocketBase instance.
|
||||||
|
type PocketBase struct {
|
||||||
|
// URL is the base URL of the PocketBase instance, e.g. https://pb.libnovel.cc
|
||||||
|
URL string
|
||||||
|
// AdminEmail is the admin account email used for API authentication.
|
||||||
|
AdminEmail string
|
||||||
|
// AdminPassword is the admin account password.
|
||||||
|
AdminPassword string
|
||||||
|
}
|
||||||
|
|
||||||
|
// MinIO holds connection settings for the remote MinIO / S3-compatible store.
|
||||||
|
type MinIO struct {
|
||||||
|
// Endpoint is the host:port of the MinIO S3 API, e.g. storage.libnovel.cc:443
|
||||||
|
Endpoint string
|
||||||
|
// PublicEndpoint is the browser-visible endpoint used for presigned URLs.
|
||||||
|
// Falls back to Endpoint when empty.
|
||||||
|
PublicEndpoint string
|
||||||
|
// AccessKey is the MinIO access key.
|
||||||
|
AccessKey string
|
||||||
|
// SecretKey is the MinIO secret key.
|
||||||
|
SecretKey string
|
||||||
|
// UseSSL enables TLS for the internal MinIO connection.
|
||||||
|
UseSSL bool
|
||||||
|
// PublicUseSSL enables TLS for presigned URL generation.
|
||||||
|
PublicUseSSL bool
|
||||||
|
// BucketChapters is the bucket that holds chapter markdown objects.
|
||||||
|
BucketChapters string
|
||||||
|
// BucketAudio is the bucket that holds generated audio MP3 objects.
|
||||||
|
BucketAudio string
|
||||||
|
// BucketAvatars is the bucket that holds user avatar images.
|
||||||
|
BucketAvatars string
|
||||||
|
// BucketBrowse is the bucket that holds cached browse page snapshots (JSON).
|
||||||
|
BucketBrowse string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kokoro holds connection settings for the Kokoro-FastAPI TTS service.
|
||||||
|
type Kokoro struct {
|
||||||
|
// URL is the base URL of the Kokoro service, e.g. https://kokoro.libnovel.cc
|
||||||
|
// An empty string disables TTS generation.
|
||||||
|
URL string
|
||||||
|
// DefaultVoice is the voice used when none is specified.
|
||||||
|
DefaultVoice string
|
||||||
|
}
|
||||||
|
|
||||||
|
// HTTP holds settings for the HTTP server (backend only).
|
||||||
|
type HTTP struct {
|
||||||
|
// Addr is the listen address, e.g. ":8080"
|
||||||
|
Addr string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runner holds settings specific to the runner/worker binary.
|
||||||
|
type Runner struct {
|
||||||
|
// PollInterval is how often the runner checks PocketBase for pending tasks.
|
||||||
|
PollInterval time.Duration
|
||||||
|
// MaxConcurrentScrape limits simultaneous book-scrape goroutines.
|
||||||
|
MaxConcurrentScrape int
|
||||||
|
// MaxConcurrentAudio limits simultaneous audio-generation goroutines.
|
||||||
|
MaxConcurrentAudio int
|
||||||
|
// WorkerID is a unique identifier for this runner instance.
|
||||||
|
// Defaults to the system hostname.
|
||||||
|
WorkerID string
|
||||||
|
// Workers is the number of chapter-scraping goroutines per book.
|
||||||
|
Workers int
|
||||||
|
// Timeout is the per-request HTTP timeout for scraping.
|
||||||
|
Timeout time.Duration
|
||||||
|
// ProxyURL is an optional outbound proxy for scraper HTTP requests.
|
||||||
|
ProxyURL string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config is the top-level configuration struct consumed by both binaries.
|
||||||
|
type Config struct {
|
||||||
|
PocketBase PocketBase
|
||||||
|
MinIO MinIO
|
||||||
|
Kokoro Kokoro
|
||||||
|
HTTP HTTP
|
||||||
|
Runner Runner
|
||||||
|
// LogLevel is one of "debug", "info", "warn", "error".
|
||||||
|
LogLevel string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load reads all configuration from environment variables and returns a
|
||||||
|
// populated Config. Missing variables fall back to documented defaults.
|
||||||
|
func Load() Config {
|
||||||
|
workerID, _ := os.Hostname()
|
||||||
|
if workerID == "" {
|
||||||
|
workerID = "runner-default"
|
||||||
|
}
|
||||||
|
|
||||||
|
return Config{
|
||||||
|
LogLevel: envOr("LOG_LEVEL", "info"),
|
||||||
|
|
||||||
|
PocketBase: PocketBase{
|
||||||
|
URL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||||
|
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
|
||||||
|
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
|
||||||
|
},
|
||||||
|
|
||||||
|
MinIO: MinIO{
|
||||||
|
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||||
|
PublicEndpoint: envOr("MINIO_PUBLIC_ENDPOINT", ""),
|
||||||
|
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
|
||||||
|
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||||
|
UseSSL: envBool("MINIO_USE_SSL", false),
|
||||||
|
PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true),
|
||||||
|
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||||
|
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||||
|
BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"),
|
||||||
|
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
|
||||||
|
},
|
||||||
|
|
||||||
|
Kokoro: Kokoro{
|
||||||
|
URL: envOr("KOKORO_URL", ""),
|
||||||
|
DefaultVoice: envOr("KOKORO_VOICE", "af_bella"),
|
||||||
|
},
|
||||||
|
|
||||||
|
HTTP: HTTP{
|
||||||
|
Addr: envOr("BACKEND_HTTP_ADDR", ":8080"),
|
||||||
|
},
|
||||||
|
|
||||||
|
Runner: Runner{
|
||||||
|
PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second),
|
||||||
|
MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1),
|
||||||
|
MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1),
|
||||||
|
WorkerID: envOr("RUNNER_WORKER_ID", workerID),
|
||||||
|
Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU()
|
||||||
|
Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second),
|
||||||
|
ProxyURL: envOr("SCRAPER_PROXY", ""),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func envOr(key, fallback string) string {
|
||||||
|
if v := os.Getenv(key); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
func envBool(key string, fallback bool) bool {
|
||||||
|
v := os.Getenv(key)
|
||||||
|
if v == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return strings.ToLower(v) == "true"
|
||||||
|
}
|
||||||
|
|
||||||
|
func envInt(key string, fallback int) int {
|
||||||
|
v := os.Getenv(key)
|
||||||
|
if v == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
n, err := strconv.Atoi(v)
|
||||||
|
if err != nil || n < 0 {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func envDuration(key string, fallback time.Duration) time.Duration {
|
||||||
|
v := os.Getenv(key)
|
||||||
|
if v == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(v)
|
||||||
|
if err != nil {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
127
backend/internal/config/config_test.go
Normal file
127
backend/internal/config/config_test.go
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
package config_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLoad_Defaults(t *testing.T) {
|
||||||
|
// Unset all relevant vars so we test pure defaults.
|
||||||
|
unset := []string{
|
||||||
|
"LOG_LEVEL",
|
||||||
|
"POCKETBASE_URL", "POCKETBASE_ADMIN_EMAIL", "POCKETBASE_ADMIN_PASSWORD",
|
||||||
|
"MINIO_ENDPOINT", "MINIO_PUBLIC_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY",
|
||||||
|
"MINIO_USE_SSL", "MINIO_PUBLIC_USE_SSL",
|
||||||
|
"MINIO_BUCKET_CHAPTERS", "MINIO_BUCKET_AUDIO", "MINIO_BUCKET_AVATARS",
|
||||||
|
"KOKORO_URL", "KOKORO_VOICE",
|
||||||
|
"BACKEND_HTTP_ADDR",
|
||||||
|
"RUNNER_POLL_INTERVAL", "RUNNER_MAX_CONCURRENT_SCRAPE", "RUNNER_MAX_CONCURRENT_AUDIO",
|
||||||
|
"RUNNER_WORKER_ID", "RUNNER_WORKERS", "RUNNER_TIMEOUT", "SCRAPER_PROXY",
|
||||||
|
}
|
||||||
|
for _, k := range unset {
|
||||||
|
t.Setenv(k, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := config.Load()
|
||||||
|
|
||||||
|
if cfg.LogLevel != "info" {
|
||||||
|
t.Errorf("LogLevel: want info, got %q", cfg.LogLevel)
|
||||||
|
}
|
||||||
|
if cfg.PocketBase.URL != "http://localhost:8090" {
|
||||||
|
t.Errorf("PocketBase.URL: want http://localhost:8090, got %q", cfg.PocketBase.URL)
|
||||||
|
}
|
||||||
|
if cfg.MinIO.BucketChapters != "libnovel-chapters" {
|
||||||
|
t.Errorf("MinIO.BucketChapters: want libnovel-chapters, got %q", cfg.MinIO.BucketChapters)
|
||||||
|
}
|
||||||
|
if cfg.MinIO.UseSSL != false {
|
||||||
|
t.Errorf("MinIO.UseSSL: want false, got %v", cfg.MinIO.UseSSL)
|
||||||
|
}
|
||||||
|
if cfg.MinIO.PublicUseSSL != true {
|
||||||
|
t.Errorf("MinIO.PublicUseSSL: want true, got %v", cfg.MinIO.PublicUseSSL)
|
||||||
|
}
|
||||||
|
if cfg.Kokoro.DefaultVoice != "af_bella" {
|
||||||
|
t.Errorf("Kokoro.DefaultVoice: want af_bella, got %q", cfg.Kokoro.DefaultVoice)
|
||||||
|
}
|
||||||
|
if cfg.HTTP.Addr != ":8080" {
|
||||||
|
t.Errorf("HTTP.Addr: want :8080, got %q", cfg.HTTP.Addr)
|
||||||
|
}
|
||||||
|
if cfg.Runner.PollInterval != 30*time.Second {
|
||||||
|
t.Errorf("Runner.PollInterval: want 30s, got %v", cfg.Runner.PollInterval)
|
||||||
|
}
|
||||||
|
if cfg.Runner.MaxConcurrentScrape != 1 {
|
||||||
|
t.Errorf("Runner.MaxConcurrentScrape: want 1, got %d", cfg.Runner.MaxConcurrentScrape)
|
||||||
|
}
|
||||||
|
if cfg.Runner.MaxConcurrentAudio != 1 {
|
||||||
|
t.Errorf("Runner.MaxConcurrentAudio: want 1, got %d", cfg.Runner.MaxConcurrentAudio)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_EnvOverride(t *testing.T) {
|
||||||
|
t.Setenv("LOG_LEVEL", "debug")
|
||||||
|
t.Setenv("POCKETBASE_URL", "https://pb.libnovel.cc")
|
||||||
|
t.Setenv("MINIO_USE_SSL", "true")
|
||||||
|
t.Setenv("MINIO_PUBLIC_USE_SSL", "false")
|
||||||
|
t.Setenv("RUNNER_POLL_INTERVAL", "1m")
|
||||||
|
t.Setenv("RUNNER_MAX_CONCURRENT_SCRAPE", "5")
|
||||||
|
t.Setenv("RUNNER_WORKER_ID", "homelab-01")
|
||||||
|
t.Setenv("BACKEND_HTTP_ADDR", ":9090")
|
||||||
|
t.Setenv("KOKORO_URL", "https://kokoro.libnovel.cc")
|
||||||
|
|
||||||
|
cfg := config.Load()
|
||||||
|
|
||||||
|
if cfg.LogLevel != "debug" {
|
||||||
|
t.Errorf("LogLevel: want debug, got %q", cfg.LogLevel)
|
||||||
|
}
|
||||||
|
if cfg.PocketBase.URL != "https://pb.libnovel.cc" {
|
||||||
|
t.Errorf("PocketBase.URL: want https://pb.libnovel.cc, got %q", cfg.PocketBase.URL)
|
||||||
|
}
|
||||||
|
if !cfg.MinIO.UseSSL {
|
||||||
|
t.Error("MinIO.UseSSL: want true")
|
||||||
|
}
|
||||||
|
if cfg.MinIO.PublicUseSSL {
|
||||||
|
t.Error("MinIO.PublicUseSSL: want false")
|
||||||
|
}
|
||||||
|
if cfg.Runner.PollInterval != time.Minute {
|
||||||
|
t.Errorf("Runner.PollInterval: want 1m, got %v", cfg.Runner.PollInterval)
|
||||||
|
}
|
||||||
|
if cfg.Runner.MaxConcurrentScrape != 5 {
|
||||||
|
t.Errorf("Runner.MaxConcurrentScrape: want 5, got %d", cfg.Runner.MaxConcurrentScrape)
|
||||||
|
}
|
||||||
|
if cfg.Runner.WorkerID != "homelab-01" {
|
||||||
|
t.Errorf("Runner.WorkerID: want homelab-01, got %q", cfg.Runner.WorkerID)
|
||||||
|
}
|
||||||
|
if cfg.HTTP.Addr != ":9090" {
|
||||||
|
t.Errorf("HTTP.Addr: want :9090, got %q", cfg.HTTP.Addr)
|
||||||
|
}
|
||||||
|
if cfg.Kokoro.URL != "https://kokoro.libnovel.cc" {
|
||||||
|
t.Errorf("Kokoro.URL: want https://kokoro.libnovel.cc, got %q", cfg.Kokoro.URL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_InvalidInt_FallsToDefault(t *testing.T) {
|
||||||
|
t.Setenv("RUNNER_MAX_CONCURRENT_SCRAPE", "notanumber")
|
||||||
|
cfg := config.Load()
|
||||||
|
if cfg.Runner.MaxConcurrentScrape != 1 {
|
||||||
|
t.Errorf("want default 1, got %d", cfg.Runner.MaxConcurrentScrape)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_InvalidDuration_FallsToDefault(t *testing.T) {
|
||||||
|
t.Setenv("RUNNER_POLL_INTERVAL", "notaduration")
|
||||||
|
cfg := config.Load()
|
||||||
|
if cfg.Runner.PollInterval != 30*time.Second {
|
||||||
|
t.Errorf("want default 30s, got %v", cfg.Runner.PollInterval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoad_WorkerID_FallsToHostname(t *testing.T) {
|
||||||
|
t.Setenv("RUNNER_WORKER_ID", "")
|
||||||
|
cfg := config.Load()
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
if host != "" && cfg.Runner.WorkerID != host {
|
||||||
|
t.Errorf("want hostname %q, got %q", host, cfg.Runner.WorkerID)
|
||||||
|
}
|
||||||
|
}
|
||||||
131
backend/internal/domain/domain.go
Normal file
131
backend/internal/domain/domain.go
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
// Package domain contains the core value types shared across all packages
|
||||||
|
// in this module. It has zero internal imports — only the standard library.
|
||||||
|
// Every other package imports domain; domain imports nothing from this module.
|
||||||
|
package domain
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// ── Book types ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// BookMeta carries all bibliographic information about a novel.
|
||||||
|
type BookMeta struct {
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Author string `json:"author"`
|
||||||
|
Cover string `json:"cover,omitempty"`
|
||||||
|
Status string `json:"status,omitempty"`
|
||||||
|
Genres []string `json:"genres,omitempty"`
|
||||||
|
Summary string `json:"summary,omitempty"`
|
||||||
|
TotalChapters int `json:"total_chapters,omitempty"`
|
||||||
|
SourceURL string `json:"source_url"`
|
||||||
|
Ranking int `json:"ranking,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CatalogueEntry is a lightweight book reference returned by catalogue pages.
|
||||||
|
type CatalogueEntry struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChapterRef is a reference to a single chapter returned by chapter-list pages.
|
||||||
|
type ChapterRef struct {
|
||||||
|
Number int `json:"number"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
Volume int `json:"volume,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chapter contains the fully-extracted text of a single chapter.
|
||||||
|
type Chapter struct {
|
||||||
|
Ref ChapterRef `json:"ref"`
|
||||||
|
Text string `json:"text"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RankingItem represents a single entry in the novel ranking list.
|
||||||
|
type RankingItem struct {
|
||||||
|
Rank int `json:"rank"`
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Author string `json:"author,omitempty"`
|
||||||
|
Cover string `json:"cover,omitempty"`
|
||||||
|
Status string `json:"status,omitempty"`
|
||||||
|
Genres []string `json:"genres,omitempty"`
|
||||||
|
SourceURL string `json:"source_url,omitempty"`
|
||||||
|
Updated time.Time `json:"updated,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Storage record types ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// ChapterInfo is a lightweight chapter descriptor stored in the index.
|
||||||
|
type ChapterInfo struct {
|
||||||
|
Number int `json:"number"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Date string `json:"date,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadingProgress holds a single user's reading position for one book.
|
||||||
|
type ReadingProgress struct {
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Chapter int `json:"chapter"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Task record types ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// TaskStatus enumerates the lifecycle states of any task.
|
||||||
|
type TaskStatus string
|
||||||
|
|
||||||
|
const (
|
||||||
|
TaskStatusPending TaskStatus = "pending"
|
||||||
|
TaskStatusRunning TaskStatus = "running"
|
||||||
|
TaskStatusDone TaskStatus = "done"
|
||||||
|
TaskStatusFailed TaskStatus = "failed"
|
||||||
|
TaskStatusCancelled TaskStatus = "cancelled"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ScrapeTask represents a book-scraping job stored in PocketBase.
|
||||||
|
type ScrapeTask struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Kind string `json:"kind"` // "catalogue" | "book" | "book_range"
|
||||||
|
TargetURL string `json:"target_url"` // non-empty for single-book tasks
|
||||||
|
FromChapter int `json:"from_chapter,omitempty"`
|
||||||
|
ToChapter int `json:"to_chapter,omitempty"`
|
||||||
|
WorkerID string `json:"worker_id,omitempty"`
|
||||||
|
Status TaskStatus `json:"status"`
|
||||||
|
BooksFound int `json:"books_found"`
|
||||||
|
ChaptersScraped int `json:"chapters_scraped"`
|
||||||
|
ChaptersSkipped int `json:"chapters_skipped"`
|
||||||
|
Errors int `json:"errors"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
Finished time.Time `json:"finished,omitempty"`
|
||||||
|
ErrorMessage string `json:"error_message,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScrapeResult is the outcome reported by the runner after finishing a ScrapeTask.
|
||||||
|
type ScrapeResult struct {
|
||||||
|
BooksFound int `json:"books_found"`
|
||||||
|
ChaptersScraped int `json:"chapters_scraped"`
|
||||||
|
ChaptersSkipped int `json:"chapters_skipped"`
|
||||||
|
Errors int `json:"errors"`
|
||||||
|
ErrorMessage string `json:"error_message,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// AudioTask represents an audio-generation job stored in PocketBase.
|
||||||
|
type AudioTask struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
CacheKey string `json:"cache_key"` // "slug/chapter/voice"
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Chapter int `json:"chapter"`
|
||||||
|
Voice string `json:"voice"`
|
||||||
|
WorkerID string `json:"worker_id,omitempty"`
|
||||||
|
Status TaskStatus `json:"status"`
|
||||||
|
ErrorMessage string `json:"error_message,omitempty"`
|
||||||
|
Started time.Time `json:"started"`
|
||||||
|
Finished time.Time `json:"finished,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// AudioResult is the outcome reported by the runner after finishing an AudioTask.
|
||||||
|
type AudioResult struct {
|
||||||
|
ObjectKey string `json:"object_key,omitempty"`
|
||||||
|
ErrorMessage string `json:"error_message,omitempty"`
|
||||||
|
}
|
||||||
104
backend/internal/domain/domain_test.go
Normal file
104
backend/internal/domain/domain_test.go
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
package domain_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBookMeta_JSONRoundtrip(t *testing.T) {
|
||||||
|
orig := domain.BookMeta{
|
||||||
|
Slug: "a-great-novel",
|
||||||
|
Title: "A Great Novel",
|
||||||
|
Author: "Jane Doe",
|
||||||
|
Cover: "https://example.com/cover.jpg",
|
||||||
|
Status: "Ongoing",
|
||||||
|
Genres: []string{"Fantasy", "Action"},
|
||||||
|
Summary: "A thrilling tale.",
|
||||||
|
TotalChapters: 120,
|
||||||
|
SourceURL: "https://novelfire.net/book/a-great-novel",
|
||||||
|
Ranking: 3,
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(orig)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal: %v", err)
|
||||||
|
}
|
||||||
|
var got domain.BookMeta
|
||||||
|
if err := json.Unmarshal(b, &got); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
if got.Slug != orig.Slug {
|
||||||
|
t.Errorf("Slug: want %q, got %q", orig.Slug, got.Slug)
|
||||||
|
}
|
||||||
|
if got.TotalChapters != orig.TotalChapters {
|
||||||
|
t.Errorf("TotalChapters: want %d, got %d", orig.TotalChapters, got.TotalChapters)
|
||||||
|
}
|
||||||
|
if len(got.Genres) != len(orig.Genres) {
|
||||||
|
t.Errorf("Genres len: want %d, got %d", len(orig.Genres), len(got.Genres))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChapterRef_JSONRoundtrip(t *testing.T) {
|
||||||
|
orig := domain.ChapterRef{Number: 42, Title: "The Battle", URL: "https://example.com/ch-42", Volume: 2}
|
||||||
|
b, _ := json.Marshal(orig)
|
||||||
|
var got domain.ChapterRef
|
||||||
|
json.Unmarshal(b, &got)
|
||||||
|
if got != orig {
|
||||||
|
t.Errorf("want %+v, got %+v", orig, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRankingItem_JSONRoundtrip(t *testing.T) {
|
||||||
|
now := time.Now().Truncate(time.Second)
|
||||||
|
orig := domain.RankingItem{
|
||||||
|
Rank: 1,
|
||||||
|
Slug: "top-novel",
|
||||||
|
Title: "Top Novel",
|
||||||
|
SourceURL: "https://novelfire.net/book/top-novel",
|
||||||
|
Updated: now,
|
||||||
|
}
|
||||||
|
b, _ := json.Marshal(orig)
|
||||||
|
var got domain.RankingItem
|
||||||
|
json.Unmarshal(b, &got)
|
||||||
|
if got.Rank != orig.Rank || got.Slug != orig.Slug {
|
||||||
|
t.Errorf("want %+v, got %+v", orig, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScrapeResult_JSONRoundtrip(t *testing.T) {
|
||||||
|
orig := domain.ScrapeResult{BooksFound: 10, ChaptersScraped: 200, ChaptersSkipped: 5, Errors: 1, ErrorMessage: "one error"}
|
||||||
|
b, _ := json.Marshal(orig)
|
||||||
|
var got domain.ScrapeResult
|
||||||
|
json.Unmarshal(b, &got)
|
||||||
|
if got != orig {
|
||||||
|
t.Errorf("want %+v, got %+v", orig, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAudioResult_JSONRoundtrip(t *testing.T) {
|
||||||
|
orig := domain.AudioResult{ObjectKey: "audio/slug/1/af_bella.mp3"}
|
||||||
|
b, _ := json.Marshal(orig)
|
||||||
|
var got domain.AudioResult
|
||||||
|
json.Unmarshal(b, &got)
|
||||||
|
if got != orig {
|
||||||
|
t.Errorf("want %+v, got %+v", orig, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTaskStatus_Values(t *testing.T) {
|
||||||
|
cases := []domain.TaskStatus{
|
||||||
|
domain.TaskStatusPending,
|
||||||
|
domain.TaskStatusRunning,
|
||||||
|
domain.TaskStatusDone,
|
||||||
|
domain.TaskStatusFailed,
|
||||||
|
domain.TaskStatusCancelled,
|
||||||
|
}
|
||||||
|
for _, s := range cases {
|
||||||
|
if s == "" {
|
||||||
|
t.Errorf("TaskStatus constant must not be empty")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
124
backend/internal/httputil/httputil.go
Normal file
124
backend/internal/httputil/httputil.go
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
// Package httputil provides shared HTTP helpers used by both the runner and
|
||||||
|
// backend binaries. It has no imports from this module — only the standard
|
||||||
|
// library — so it is safe to import from anywhere in the dependency graph.
|
||||||
|
package httputil
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Client is the minimal interface for making HTTP GET requests.
|
||||||
|
// *http.Client satisfies this interface.
|
||||||
|
type Client interface {
|
||||||
|
Do(req *http.Request) (*http.Response, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrMaxRetries is returned when RetryGet exhausts all attempts.
|
||||||
|
var ErrMaxRetries = errors.New("httputil: max retries exceeded")
|
||||||
|
|
||||||
|
// errClientError is returned by doGet for 4xx responses; it signals that the
|
||||||
|
// request should NOT be retried (the client is at fault).
|
||||||
|
var errClientError = errors.New("httputil: client error")
|
||||||
|
|
||||||
|
// RetryGet fetches url using client, retrying on network errors or 5xx
|
||||||
|
// responses with exponential backoff. It returns the full response body as a
|
||||||
|
// string on success.
|
||||||
|
//
|
||||||
|
// - maxAttempts: total number of attempts (must be >= 1)
|
||||||
|
// - baseDelay: initial wait before the second attempt; doubles each retry
|
||||||
|
func RetryGet(ctx context.Context, client Client, url string, maxAttempts int, baseDelay time.Duration) (string, error) {
|
||||||
|
if maxAttempts < 1 {
|
||||||
|
maxAttempts = 1
|
||||||
|
}
|
||||||
|
delay := baseDelay
|
||||||
|
|
||||||
|
var lastErr error
|
||||||
|
for attempt := 0; attempt < maxAttempts; attempt++ {
|
||||||
|
if attempt > 0 {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return "", ctx.Err()
|
||||||
|
case <-time.After(delay):
|
||||||
|
}
|
||||||
|
delay *= 2
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := doGet(ctx, client, url)
|
||||||
|
if err == nil {
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
|
||||||
|
// Do not retry on context cancellation.
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return "", ctx.Err()
|
||||||
|
}
|
||||||
|
// Do not retry on 4xx — the client is at fault.
|
||||||
|
if errors.Is(err, errClientError) {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", fmt.Errorf("%w after %d attempts: %w", ErrMaxRetries, maxAttempts, lastErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func doGet(ctx context.Context, client Client, url string) (string, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("build request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("GET %s: %w", url, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode >= 500 {
|
||||||
|
return "", fmt.Errorf("GET %s: server error %d", url, resp.StatusCode)
|
||||||
|
}
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
return "", fmt.Errorf("%w: GET %s: client error %d", errClientError, url, resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("read body %s: %w", url, err)
|
||||||
|
}
|
||||||
|
return string(raw), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteJSON writes v as JSON to w with the given HTTP status code and sets the
|
||||||
|
// Content-Type header to application/json.
|
||||||
|
func WriteJSON(w http.ResponseWriter, status int, v any) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_ = json.NewEncoder(w).Encode(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteError writes a JSON error object {"error": msg} with the given status.
|
||||||
|
func WriteError(w http.ResponseWriter, status int, msg string) {
|
||||||
|
WriteJSON(w, status, map[string]string{"error": msg})
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxBodyBytes is the limit applied by DecodeJSON to prevent unbounded reads.
|
||||||
|
const maxBodyBytes = 1 << 20 // 1 MiB
|
||||||
|
|
||||||
|
// DecodeJSON decodes a JSON request body into v. It enforces a 1 MiB size
|
||||||
|
// limit and returns a descriptive error on any failure.
|
||||||
|
func DecodeJSON(r *http.Request, v any) error {
|
||||||
|
r.Body = http.MaxBytesReader(nil, r.Body, maxBodyBytes)
|
||||||
|
dec := json.NewDecoder(r.Body)
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(v); err != nil {
|
||||||
|
return fmt.Errorf("decode JSON body: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
181
backend/internal/httputil/httputil_test.go
Normal file
181
backend/internal/httputil/httputil_test.go
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
package httputil_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/httputil"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ── RetryGet ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestRetryGet_ImmediateSuccess(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Write([]byte("hello"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
body, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 3, time.Millisecond)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if body != "hello" {
|
||||||
|
t.Errorf("want hello, got %q", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetryGet_RetriesOn5xx(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
calls++
|
||||||
|
if calls < 3 {
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Write([]byte("ok"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
body, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 5, time.Millisecond)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if body != "ok" {
|
||||||
|
t.Errorf("want ok, got %q", body)
|
||||||
|
}
|
||||||
|
if calls != 3 {
|
||||||
|
t.Errorf("want 3 calls, got %d", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetryGet_MaxAttemptsExceeded(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
_, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 3, time.Millisecond)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error, got nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetryGet_ContextCancelDuringBackoff(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
// Cancel after first failed attempt hits the backoff wait.
|
||||||
|
go func() { time.Sleep(5 * time.Millisecond); cancel() }()
|
||||||
|
|
||||||
|
_, err := httputil.RetryGet(ctx, srv.Client(), srv.URL, 10, 500*time.Millisecond)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected context cancellation error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetryGet_NoRetryOn4xx(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
calls++
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
_, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 5, time.Millisecond)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for 404")
|
||||||
|
}
|
||||||
|
// 4xx is NOT retried — should be exactly 1 call.
|
||||||
|
if calls != 1 {
|
||||||
|
t.Errorf("want 1 call for 4xx, got %d", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── WriteJSON ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestWriteJSON_SetsHeadersAndStatus(t *testing.T) {
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
httputil.WriteJSON(rr, http.StatusCreated, map[string]string{"key": "val"})
|
||||||
|
|
||||||
|
if rr.Code != http.StatusCreated {
|
||||||
|
t.Errorf("status: want 201, got %d", rr.Code)
|
||||||
|
}
|
||||||
|
if ct := rr.Header().Get("Content-Type"); ct != "application/json" {
|
||||||
|
t.Errorf("Content-Type: want application/json, got %q", ct)
|
||||||
|
}
|
||||||
|
var got map[string]string
|
||||||
|
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
|
||||||
|
t.Fatalf("decode body: %v", err)
|
||||||
|
}
|
||||||
|
if got["key"] != "val" {
|
||||||
|
t.Errorf("body key: want val, got %q", got["key"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── WriteError ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestWriteError_Format(t *testing.T) {
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
httputil.WriteError(rr, http.StatusBadRequest, "bad input")
|
||||||
|
|
||||||
|
if rr.Code != http.StatusBadRequest {
|
||||||
|
t.Errorf("status: want 400, got %d", rr.Code)
|
||||||
|
}
|
||||||
|
var got map[string]string
|
||||||
|
json.NewDecoder(rr.Body).Decode(&got)
|
||||||
|
if got["error"] != "bad input" {
|
||||||
|
t.Errorf("error field: want bad input, got %q", got["error"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── DecodeJSON ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestDecodeJSON_HappyPath(t *testing.T) {
|
||||||
|
body := `{"name":"test","value":42}`
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
var payload struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Value int `json:"value"`
|
||||||
|
}
|
||||||
|
if err := httputil.DecodeJSON(req, &payload); err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if payload.Name != "test" || payload.Value != 42 {
|
||||||
|
t.Errorf("unexpected payload: %+v", payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeJSON_UnknownFieldReturnsError(t *testing.T) {
|
||||||
|
body := `{"name":"test","unknown_field":"boom"}`
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
|
||||||
|
|
||||||
|
var payload struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
}
|
||||||
|
if err := httputil.DecodeJSON(req, &payload); err == nil {
|
||||||
|
t.Fatal("expected error for unknown field, got nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecodeJSON_BodyTooLarge(t *testing.T) {
|
||||||
|
// Build a body > 1 MiB.
|
||||||
|
big := bytes.Repeat([]byte("a"), 2<<20)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(big))
|
||||||
|
|
||||||
|
var payload map[string]any
|
||||||
|
if err := httputil.DecodeJSON(req, &payload); err == nil {
|
||||||
|
t.Fatal("expected error for oversized body, got nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
160
backend/internal/kokoro/client.go
Normal file
160
backend/internal/kokoro/client.go
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
// Package kokoro provides a client for the Kokoro-FastAPI TTS service.
|
||||||
|
//
|
||||||
|
// The Kokoro API is an OpenAI-compatible audio speech API that returns a
|
||||||
|
// download link (X-Download-Path header) instead of streaming audio directly.
|
||||||
|
// GenerateAudio handles the two-step flow: POST /v1/audio/speech → GET /v1/download/{file}.
|
||||||
|
package kokoro
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Client is the interface for interacting with the Kokoro TTS service.
|
||||||
|
type Client interface {
|
||||||
|
// GenerateAudio synthesises text using voice and returns raw MP3 bytes.
|
||||||
|
GenerateAudio(ctx context.Context, text, voice string) ([]byte, error)
|
||||||
|
|
||||||
|
// ListVoices returns the available voice IDs. Falls back to an empty slice
|
||||||
|
// on error — callers should treat an empty list as "service unavailable".
|
||||||
|
ListVoices(ctx context.Context) ([]string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// httpClient is the concrete Kokoro HTTP client.
|
||||||
|
type httpClient struct {
|
||||||
|
baseURL string
|
||||||
|
http *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns a Kokoro Client targeting baseURL (e.g. "https://kokoro.example.com").
|
||||||
|
func New(baseURL string) Client {
|
||||||
|
return &httpClient{
|
||||||
|
baseURL: strings.TrimRight(baseURL, "/"),
|
||||||
|
http: &http.Client{Timeout: 10 * time.Minute},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GenerateAudio calls POST /v1/audio/speech (return_download_link=true) and then
|
||||||
|
// downloads the resulting MP3 from GET /v1/download/{filename}.
|
||||||
|
func (c *httpClient) GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) {
|
||||||
|
if text == "" {
|
||||||
|
return nil, fmt.Errorf("kokoro: empty text")
|
||||||
|
}
|
||||||
|
if voice == "" {
|
||||||
|
voice = "af_bella"
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Step 1: request generation ────────────────────────────────────────────
|
||||||
|
reqBody, err := json.Marshal(map[string]any{
|
||||||
|
"model": "kokoro",
|
||||||
|
"input": text,
|
||||||
|
"voice": voice,
|
||||||
|
"response_format": "mp3",
|
||||||
|
"speed": 1.0,
|
||||||
|
"stream": false,
|
||||||
|
"return_download_link": true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("kokoro: marshal request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||||
|
c.baseURL+"/v1/audio/speech", bytes.NewReader(reqBody))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("kokoro: build speech request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := c.http.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("kokoro: speech request: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("kokoro: speech returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
dlPath := resp.Header.Get("X-Download-Path")
|
||||||
|
if dlPath == "" {
|
||||||
|
return nil, fmt.Errorf("kokoro: no X-Download-Path header in response")
|
||||||
|
}
|
||||||
|
filename := dlPath
|
||||||
|
if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
|
||||||
|
filename = dlPath[idx+1:]
|
||||||
|
}
|
||||||
|
if filename == "" {
|
||||||
|
return nil, fmt.Errorf("kokoro: empty filename in X-Download-Path: %q", dlPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Step 2: download the generated file ───────────────────────────────────
|
||||||
|
dlURL := c.baseURL + "/v1/download/" + filename
|
||||||
|
dlReq, err := http.NewRequestWithContext(ctx, http.MethodGet, dlURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("kokoro: build download request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dlResp, err := c.http.Do(dlReq)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("kokoro: download request: %w", err)
|
||||||
|
}
|
||||||
|
defer dlResp.Body.Close()
|
||||||
|
|
||||||
|
if dlResp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("kokoro: download returned %d", dlResp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := io.ReadAll(dlResp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("kokoro: read download body: %w", err)
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListVoices calls GET /v1/audio/voices and returns the list of voice IDs.
|
||||||
|
func (c *httpClient) ListVoices(ctx context.Context) ([]string, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
||||||
|
c.baseURL+"/v1/audio/voices", nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("kokoro: build voices request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := c.http.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("kokoro: voices request: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
return nil, fmt.Errorf("kokoro: voices returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Voices []string `json:"voices"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||||
|
return nil, fmt.Errorf("kokoro: decode voices response: %w", err)
|
||||||
|
}
|
||||||
|
return result.Voices, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// VoiceSampleKey returns the MinIO object key for a voice sample MP3.
|
||||||
|
// Key: _voice-samples/{voice}.mp3 (sanitised).
|
||||||
|
func VoiceSampleKey(voice string) string {
|
||||||
|
safe := strings.Map(func(r rune) rune {
|
||||||
|
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
|
||||||
|
(r >= '0' && r <= '9') || r == '_' || r == '-' {
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
return '_'
|
||||||
|
}, voice)
|
||||||
|
return fmt.Sprintf("_voice-samples/%s.mp3", safe)
|
||||||
|
}
|
||||||
291
backend/internal/kokoro/client_test.go
Normal file
291
backend/internal/kokoro/client_test.go
Normal file
@@ -0,0 +1,291 @@
|
|||||||
|
package kokoro_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/kokoro"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ── VoiceSampleKey ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestVoiceSampleKey(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
voice string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"af_bella", "_voice-samples/af_bella.mp3"},
|
||||||
|
{"am_echo", "_voice-samples/am_echo.mp3"},
|
||||||
|
{"voice with spaces", "_voice-samples/voice_with_spaces.mp3"},
|
||||||
|
{"special!@#chars", "_voice-samples/special___chars.mp3"},
|
||||||
|
{"", "_voice-samples/.mp3"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.voice, func(t *testing.T) {
|
||||||
|
got := kokoro.VoiceSampleKey(tt.voice)
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("VoiceSampleKey(%q) = %q, want %q", tt.voice, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── GenerateAudio ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestGenerateAudio_EmptyText(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.NotFoundHandler())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
_, err := c.GenerateAudio(context.Background(), "", "af_bella")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for empty text, got nil")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "empty text") {
|
||||||
|
t.Errorf("expected 'empty text' in error, got: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateAudio_DefaultVoice(t *testing.T) {
|
||||||
|
// Tracks that the voice defaults to af_bella when empty.
|
||||||
|
var capturedBody string
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path == "/v1/audio/speech" {
|
||||||
|
buf := make([]byte, 512)
|
||||||
|
n, _ := r.Body.Read(buf)
|
||||||
|
capturedBody = string(buf[:n])
|
||||||
|
w.Header().Set("X-Download-Path", "/download/test_file.mp3")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(r.URL.Path, "/v1/download/") {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
_, _ = w.Write([]byte("fake-mp3-data"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
data, err := c.GenerateAudio(context.Background(), "hello world", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != "fake-mp3-data" {
|
||||||
|
t.Errorf("unexpected data: %q", string(data))
|
||||||
|
}
|
||||||
|
if !strings.Contains(capturedBody, `"af_bella"`) {
|
||||||
|
t.Errorf("expected default voice af_bella in request body, got: %s", capturedBody)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateAudio_SpeechNon200(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path == "/v1/audio/speech" {
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
_, err := c.GenerateAudio(context.Background(), "text", "af_bella")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for non-200 speech response")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "500") {
|
||||||
|
t.Errorf("expected 500 in error, got: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateAudio_NoDownloadPathHeader(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path == "/v1/audio/speech" {
|
||||||
|
// No X-Download-Path header
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
_, err := c.GenerateAudio(context.Background(), "text", "af_bella")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for missing X-Download-Path")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "X-Download-Path") {
|
||||||
|
t.Errorf("expected X-Download-Path in error, got: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateAudio_DownloadFails(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path == "/v1/audio/speech" {
|
||||||
|
w.Header().Set("X-Download-Path", "/v1/download/speech.mp3")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(r.URL.Path, "/v1/download/") {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
_, err := c.GenerateAudio(context.Background(), "text", "af_bella")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for failed download")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "404") {
|
||||||
|
t.Errorf("expected 404 in error, got: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateAudio_FullPath(t *testing.T) {
|
||||||
|
// X-Download-Path with a full path: extract just filename.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path == "/v1/audio/speech" {
|
||||||
|
w.Header().Set("X-Download-Path", "/some/nested/path/audio_abc123.mp3")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.URL.Path == "/v1/download/audio_abc123.mp3" {
|
||||||
|
_, _ = w.Write([]byte("audio-bytes"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
data, err := c.GenerateAudio(context.Background(), "text", "af_bella")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if string(data) != "audio-bytes" {
|
||||||
|
t.Errorf("unexpected data: %q", string(data))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateAudio_ContextCancelled(t *testing.T) {
|
||||||
|
// Server that hangs — context should cancel before we get a response.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// Never respond.
|
||||||
|
select {}
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel() // cancel immediately
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
_, err := c.GenerateAudio(ctx, "text", "af_bella")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for cancelled context")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── ListVoices ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestListVoices_Success(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path == "/v1/audio/voices" {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"voices":["af_bella","am_adam","bf_emma"]}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
voices, err := c.ListVoices(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(voices) != 3 {
|
||||||
|
t.Errorf("expected 3 voices, got %d: %v", len(voices), voices)
|
||||||
|
}
|
||||||
|
if voices[0] != "af_bella" {
|
||||||
|
t.Errorf("expected first voice to be af_bella, got %q", voices[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListVoices_Non200(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
_, err := c.ListVoices(context.Background())
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for non-200 response")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "503") {
|
||||||
|
t.Errorf("expected 503 in error, got: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListVoices_MalformedJSON(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
_, _ = w.Write([]byte(`not-json`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
_, err := c.ListVoices(context.Background())
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for malformed JSON")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListVoices_EmptyVoices(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"voices":[]}`))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL)
|
||||||
|
voices, err := c.ListVoices(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(voices) != 0 {
|
||||||
|
t.Errorf("expected 0 voices, got %d", len(voices))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── New ───────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestNew_TrailingSlashStripped(t *testing.T) {
|
||||||
|
// Verify that a trailing slash on baseURL doesn't produce double-slash paths.
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path == "/v1/audio/voices" {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"voices":["af_bella"]}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
c := kokoro.New(srv.URL + "/") // trailing slash
|
||||||
|
voices, err := c.ListVoices(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(voices) == 0 {
|
||||||
|
t.Error("expected at least one voice")
|
||||||
|
}
|
||||||
|
}
|
||||||
228
backend/internal/novelfire/htmlutil/htmlutil.go
Normal file
228
backend/internal/novelfire/htmlutil/htmlutil.go
Normal file
@@ -0,0 +1,228 @@
|
|||||||
|
// Package htmlutil provides helper functions for parsing HTML with
|
||||||
|
// golang.org/x/net/html and extracting values by Selector descriptors.
|
||||||
|
package htmlutil
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/url"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/scraper"
|
||||||
|
"golang.org/x/net/html"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ResolveURL returns an absolute URL. If href is already absolute it is
|
||||||
|
// returned unchanged. Otherwise it is resolved against base.
|
||||||
|
func ResolveURL(base, href string) string {
|
||||||
|
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||||||
|
return href
|
||||||
|
}
|
||||||
|
b, err := url.Parse(base)
|
||||||
|
if err != nil {
|
||||||
|
return base + href
|
||||||
|
}
|
||||||
|
ref, err := url.Parse(href)
|
||||||
|
if err != nil {
|
||||||
|
return base + href
|
||||||
|
}
|
||||||
|
return b.ResolveReference(ref).String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseHTML parses raw HTML and returns the root node.
|
||||||
|
func ParseHTML(raw string) (*html.Node, error) {
|
||||||
|
return html.Parse(strings.NewReader(raw))
|
||||||
|
}
|
||||||
|
|
||||||
|
// selectorMatches reports whether node n matches sel.
|
||||||
|
func selectorMatches(n *html.Node, sel scraper.Selector) bool {
|
||||||
|
if n.Type != html.ElementNode {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if sel.Tag != "" && n.Data != sel.Tag {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if sel.ID != "" {
|
||||||
|
for _, a := range n.Attr {
|
||||||
|
if a.Key == "id" && a.Val == sel.ID {
|
||||||
|
goto checkClass
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
checkClass:
|
||||||
|
if sel.Class != "" {
|
||||||
|
for _, a := range n.Attr {
|
||||||
|
if a.Key == "class" {
|
||||||
|
for _, cls := range strings.Fields(a.Val) {
|
||||||
|
if cls == sel.Class {
|
||||||
|
goto matched
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
matched:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// AttrVal returns the value of attribute key from node n.
|
||||||
|
func AttrVal(n *html.Node, key string) string {
|
||||||
|
for _, a := range n.Attr {
|
||||||
|
if a.Key == key {
|
||||||
|
return a.Val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// TextContent returns the concatenated text content of all descendant text nodes.
|
||||||
|
func TextContent(n *html.Node) string {
|
||||||
|
var sb strings.Builder
|
||||||
|
var walk func(*html.Node)
|
||||||
|
walk = func(cur *html.Node) {
|
||||||
|
if cur.Type == html.TextNode {
|
||||||
|
sb.WriteString(cur.Data)
|
||||||
|
}
|
||||||
|
for c := cur.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
walk(c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk(n)
|
||||||
|
return strings.TrimSpace(sb.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindFirst returns the first node matching sel within root.
|
||||||
|
func FindFirst(root *html.Node, sel scraper.Selector) *html.Node {
|
||||||
|
var found *html.Node
|
||||||
|
var walk func(*html.Node) bool
|
||||||
|
walk = func(n *html.Node) bool {
|
||||||
|
if selectorMatches(n, sel) {
|
||||||
|
found = n
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
if walk(c) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
walk(root)
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindAll returns all nodes matching sel within root.
|
||||||
|
func FindAll(root *html.Node, sel scraper.Selector) []*html.Node {
|
||||||
|
var results []*html.Node
|
||||||
|
var walk func(*html.Node)
|
||||||
|
walk = func(n *html.Node) {
|
||||||
|
if selectorMatches(n, sel) {
|
||||||
|
results = append(results, n)
|
||||||
|
}
|
||||||
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
walk(c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk(root)
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractText extracts a string value from node n using sel.
|
||||||
|
// If sel.Attr is set the attribute value is returned; otherwise the inner text.
|
||||||
|
func ExtractText(n *html.Node, sel scraper.Selector) string {
|
||||||
|
if sel.Attr != "" {
|
||||||
|
return AttrVal(n, sel.Attr)
|
||||||
|
}
|
||||||
|
return TextContent(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractFirst locates the first match in root and returns its text/attr value.
|
||||||
|
func ExtractFirst(root *html.Node, sel scraper.Selector) string {
|
||||||
|
n := FindFirst(root, sel)
|
||||||
|
if n == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return ExtractText(n, sel)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractAll locates all matches in root and returns their text/attr values.
|
||||||
|
func ExtractAll(root *html.Node, sel scraper.Selector) []string {
|
||||||
|
nodes := FindAll(root, sel)
|
||||||
|
out := make([]string, 0, len(nodes))
|
||||||
|
for _, n := range nodes {
|
||||||
|
if v := ExtractText(n, sel); v != "" {
|
||||||
|
out = append(out, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown
|
||||||
|
// representation suitable for chapter storage.
|
||||||
|
func NodeToMarkdown(n *html.Node) string {
|
||||||
|
var sb strings.Builder
|
||||||
|
nodeToMD(n, &sb)
|
||||||
|
out := multiBlankLine.ReplaceAllString(sb.String(), "\n\n")
|
||||||
|
return strings.TrimSpace(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
var multiBlankLine = regexp.MustCompile(`\n(\s*\n){2,}`)
|
||||||
|
|
||||||
|
var blockElements = map[string]bool{
|
||||||
|
"p": true, "div": true, "br": true, "h1": true, "h2": true,
|
||||||
|
"h3": true, "h4": true, "h5": true, "h6": true, "li": true,
|
||||||
|
"blockquote": true, "pre": true, "hr": true,
|
||||||
|
}
|
||||||
|
|
||||||
|
func nodeToMD(n *html.Node, sb *strings.Builder) {
|
||||||
|
switch n.Type {
|
||||||
|
case html.TextNode:
|
||||||
|
sb.WriteString(n.Data)
|
||||||
|
case html.ElementNode:
|
||||||
|
tag := n.Data
|
||||||
|
switch tag {
|
||||||
|
case "br":
|
||||||
|
sb.WriteString("\n")
|
||||||
|
case "hr":
|
||||||
|
sb.WriteString("\n---\n")
|
||||||
|
case "h1", "h2", "h3", "h4", "h5", "h6":
|
||||||
|
level := int(tag[1] - '0')
|
||||||
|
sb.WriteString("\n" + strings.Repeat("#", level) + " ")
|
||||||
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
nodeToMD(c, sb)
|
||||||
|
}
|
||||||
|
sb.WriteString("\n\n")
|
||||||
|
return
|
||||||
|
case "p", "div", "blockquote":
|
||||||
|
sb.WriteString("\n")
|
||||||
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
nodeToMD(c, sb)
|
||||||
|
}
|
||||||
|
sb.WriteString("\n")
|
||||||
|
return
|
||||||
|
case "em", "i":
|
||||||
|
sb.WriteString("*")
|
||||||
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
nodeToMD(c, sb)
|
||||||
|
}
|
||||||
|
sb.WriteString("*")
|
||||||
|
return
|
||||||
|
case "strong", "b":
|
||||||
|
sb.WriteString("**")
|
||||||
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
nodeToMD(c, sb)
|
||||||
|
}
|
||||||
|
sb.WriteString("**")
|
||||||
|
return
|
||||||
|
case "script", "style", "noscript":
|
||||||
|
return // drop
|
||||||
|
}
|
||||||
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
nodeToMD(c, sb)
|
||||||
|
}
|
||||||
|
if blockElements[tag] {
|
||||||
|
sb.WriteString("\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
498
backend/internal/novelfire/scraper.go
Normal file
498
backend/internal/novelfire/scraper.go
Normal file
@@ -0,0 +1,498 @@
|
|||||||
|
// Package novelfire provides a NovelScraper implementation for novelfire.net.
|
||||||
|
//
|
||||||
|
// Site structure (as of 2025):
|
||||||
|
//
|
||||||
|
// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
|
||||||
|
// Book page : https://novelfire.net/book/{slug}
|
||||||
|
// Chapters : https://novelfire.net/book/{slug}/chapters?page=N
|
||||||
|
// Chapter : https://novelfire.net/book/{slug}/{chapter-slug}
|
||||||
|
package novelfire
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"net/url"
|
||||||
|
"path"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/browser"
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
"github.com/libnovel/backend/internal/novelfire/htmlutil"
|
||||||
|
"github.com/libnovel/backend/internal/scraper"
|
||||||
|
"golang.org/x/net/html"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
baseURL = "https://novelfire.net"
|
||||||
|
cataloguePath = "/genre-all/sort-new/status-all/all-novel"
|
||||||
|
rankingPath = "/genre-all/sort-popular/status-all/all-novel"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
|
||||||
|
type Scraper struct {
|
||||||
|
client browser.Client
|
||||||
|
log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compile-time interface check.
|
||||||
|
var _ scraper.NovelScraper = (*Scraper)(nil)
|
||||||
|
|
||||||
|
// New returns a new novelfire Scraper backed by client.
|
||||||
|
func New(client browser.Client, log *slog.Logger) *Scraper {
|
||||||
|
if log == nil {
|
||||||
|
log = slog.Default()
|
||||||
|
}
|
||||||
|
return &Scraper{client: client, log: log}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SourceName implements NovelScraper.
|
||||||
|
func (s *Scraper) SourceName() string { return "novelfire.net" }
|
||||||
|
|
||||||
|
// ── CatalogueProvider ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages.
|
||||||
|
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
|
||||||
|
entries := make(chan domain.CatalogueEntry, 64)
|
||||||
|
errs := make(chan error, 16)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(entries)
|
||||||
|
defer close(errs)
|
||||||
|
|
||||||
|
pageURL := baseURL + cataloguePath
|
||||||
|
page := 1
|
||||||
|
|
||||||
|
for pageURL != "" {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
|
||||||
|
raw, err := s.client.GetContent(ctx, pageURL)
|
||||||
|
if err != nil {
|
||||||
|
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
root, err := htmlutil.ParseHTML(raw)
|
||||||
|
if err != nil {
|
||||||
|
errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
|
||||||
|
if len(cards) == 0 {
|
||||||
|
s.log.Warn("no novel cards found, stopping pagination", "page", page)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, card := range cards {
|
||||||
|
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"})
|
||||||
|
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})
|
||||||
|
|
||||||
|
var title, href string
|
||||||
|
if linkNode != nil {
|
||||||
|
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
||||||
|
}
|
||||||
|
if titleNode != nil {
|
||||||
|
title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{}))
|
||||||
|
}
|
||||||
|
if href == "" || title == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
bookURL := resolveURL(baseURL, href)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case entries <- domain.CatalogueEntry{Title: title, URL: bookURL}:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hasNextPageLink(root) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
nextHref := ""
|
||||||
|
for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) {
|
||||||
|
if htmlutil.AttrVal(a, "rel") == "next" {
|
||||||
|
nextHref = htmlutil.AttrVal(a, "href")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if nextHref == "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
pageURL = resolveURL(baseURL, nextHref)
|
||||||
|
page++
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return entries, errs
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── MetadataProvider ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// ScrapeMetadata fetches and parses book metadata from the book's landing page.
|
||||||
|
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) {
|
||||||
|
s.log.Debug("metadata fetch starting", "url", bookURL)
|
||||||
|
|
||||||
|
raw, err := s.client.GetContent(ctx, bookURL)
|
||||||
|
if err != nil {
|
||||||
|
return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
root, err := htmlutil.ParseHTML(raw)
|
||||||
|
if err != nil {
|
||||||
|
return domain.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
|
||||||
|
author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"})
|
||||||
|
|
||||||
|
var cover string
|
||||||
|
if fig := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil {
|
||||||
|
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
|
||||||
|
if cover != "" && !strings.HasPrefix(cover, "http") {
|
||||||
|
cover = baseURL + cover
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})
|
||||||
|
|
||||||
|
genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
|
||||||
|
var genres []string
|
||||||
|
if genresNode != nil {
|
||||||
|
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
|
||||||
|
}
|
||||||
|
|
||||||
|
summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"})
|
||||||
|
totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"})
|
||||||
|
totalChapters := parseChapterCount(totalStr)
|
||||||
|
|
||||||
|
slug := slugFromURL(bookURL)
|
||||||
|
|
||||||
|
meta := domain.BookMeta{
|
||||||
|
Slug: slug,
|
||||||
|
Title: title,
|
||||||
|
Author: author,
|
||||||
|
Cover: cover,
|
||||||
|
Status: status,
|
||||||
|
Genres: genres,
|
||||||
|
Summary: summary,
|
||||||
|
TotalChapters: totalChapters,
|
||||||
|
SourceURL: bookURL,
|
||||||
|
}
|
||||||
|
s.log.Debug("metadata parsed", "slug", meta.Slug, "title", meta.Title)
|
||||||
|
return meta, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── ChapterListProvider ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// ScrapeChapterList returns all chapter references for a book, ordered ascending.
|
||||||
|
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) {
|
||||||
|
var refs []domain.ChapterRef
|
||||||
|
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
|
||||||
|
page := 1
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return refs, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
|
||||||
|
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
|
||||||
|
|
||||||
|
raw, err := s.client.GetContent(ctx, pageURL)
|
||||||
|
if err != nil {
|
||||||
|
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
root, err := htmlutil.ParseHTML(raw)
|
||||||
|
if err != nil {
|
||||||
|
return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
|
||||||
|
if chapterList == nil {
|
||||||
|
s.log.Debug("chapter list container not found, stopping pagination", "page", page)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
|
||||||
|
if len(items) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
|
||||||
|
if linkNode == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
|
||||||
|
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
|
||||||
|
if href == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
chURL := resolveURL(baseURL, href)
|
||||||
|
num := chapterNumberFromURL(chURL)
|
||||||
|
if num <= 0 {
|
||||||
|
num = len(refs) + 1
|
||||||
|
s.log.Warn("chapter number not parseable from URL, falling back to position",
|
||||||
|
"url", chURL, "position", num)
|
||||||
|
}
|
||||||
|
refs = append(refs, domain.ChapterRef{
|
||||||
|
Number: num,
|
||||||
|
Title: strings.TrimSpace(chTitle),
|
||||||
|
URL: chURL,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
page++
|
||||||
|
}
|
||||||
|
|
||||||
|
return refs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── ChapterTextProvider ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// ScrapeChapterText fetches and parses a single chapter page.
|
||||||
|
func (s *Scraper) ScrapeChapterText(ctx context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
|
||||||
|
s.log.Debug("chapter text fetch starting", "chapter", ref.Number, "url", ref.URL)
|
||||||
|
|
||||||
|
raw, err := retryGet(ctx, s.log, s.client, ref.URL, 9, 6*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
root, err := htmlutil.ParseHTML(raw)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
|
||||||
|
if container == nil {
|
||||||
|
return domain.Chapter{}, fmt.Errorf("chapter %d: #content container not found in %s", ref.Number, ref.URL)
|
||||||
|
}
|
||||||
|
|
||||||
|
text := htmlutil.NodeToMarkdown(container)
|
||||||
|
|
||||||
|
s.log.Debug("chapter text parsed", "chapter", ref.Number, "text_bytes", len(text))
|
||||||
|
|
||||||
|
return domain.Chapter{Ref: ref, Text: text}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── RankingProvider ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// ScrapeRanking pages through up to maxPages pages of the popular-novels listing.
|
||||||
|
// maxPages <= 0 means all pages. The caller decides whether to persist items.
|
||||||
|
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) {
|
||||||
|
entries := make(chan domain.BookMeta, 32)
|
||||||
|
errs := make(chan error, 16)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(entries)
|
||||||
|
defer close(errs)
|
||||||
|
|
||||||
|
rank := 1
|
||||||
|
|
||||||
|
for page := 1; maxPages <= 0 || page <= maxPages; page++ {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
|
||||||
|
s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
|
||||||
|
|
||||||
|
raw, err := s.client.GetContent(ctx, pageURL)
|
||||||
|
if err != nil {
|
||||||
|
errs <- fmt.Errorf("ranking page %d: %w", page, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
root, err := htmlutil.ParseHTML(raw)
|
||||||
|
if err != nil {
|
||||||
|
errs <- fmt.Errorf("ranking page %d parse: %w", page, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
|
||||||
|
if len(cards) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, card := range cards {
|
||||||
|
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"})
|
||||||
|
if linkNode == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
||||||
|
bookURL := resolveURL(baseURL, href)
|
||||||
|
if bookURL == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}))
|
||||||
|
if title == "" {
|
||||||
|
title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"}))
|
||||||
|
}
|
||||||
|
if title == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var cover string
|
||||||
|
if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil {
|
||||||
|
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
|
||||||
|
if cover == "" {
|
||||||
|
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(cover, "data:") {
|
||||||
|
cover = ""
|
||||||
|
}
|
||||||
|
if cover != "" && !strings.HasPrefix(cover, "http") {
|
||||||
|
cover = baseURL + cover
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
meta := domain.BookMeta{
|
||||||
|
Slug: slugFromURL(bookURL),
|
||||||
|
Title: title,
|
||||||
|
Cover: cover,
|
||||||
|
SourceURL: bookURL,
|
||||||
|
Ranking: rank,
|
||||||
|
}
|
||||||
|
rank++
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case entries <- meta:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hasNextPageLink(root) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return entries, errs
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── helpers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func resolveURL(base, href string) string { return htmlutil.ResolveURL(base, href) }
|
||||||
|
|
||||||
|
func hasNextPageLink(root *html.Node) bool {
|
||||||
|
links := htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true})
|
||||||
|
for _, a := range links {
|
||||||
|
for _, attr := range a.Attr {
|
||||||
|
if attr.Key == "rel" && attr.Val == "next" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func slugFromURL(bookURL string) string {
|
||||||
|
u, err := url.Parse(bookURL)
|
||||||
|
if err != nil {
|
||||||
|
return bookURL
|
||||||
|
}
|
||||||
|
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||||
|
if len(parts) >= 2 && parts[0] == "book" {
|
||||||
|
return parts[1]
|
||||||
|
}
|
||||||
|
if len(parts) > 0 {
|
||||||
|
return parts[len(parts)-1]
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseChapterCount(s string) int {
|
||||||
|
s = strings.ReplaceAll(s, ",", "")
|
||||||
|
fields := strings.Fields(s)
|
||||||
|
if len(fields) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
n, _ := strconv.Atoi(fields[0])
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func chapterNumberFromURL(chapterURL string) int {
|
||||||
|
u, err := url.Parse(chapterURL)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
seg := path.Base(u.Path)
|
||||||
|
seg = strings.TrimPrefix(seg, "chapter-")
|
||||||
|
seg = strings.TrimPrefix(seg, "chap-")
|
||||||
|
seg = strings.TrimPrefix(seg, "ch-")
|
||||||
|
digits := strings.FieldsFunc(seg, func(r rune) bool {
|
||||||
|
return r < '0' || r > '9'
|
||||||
|
})
|
||||||
|
if len(digits) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
n, _ := strconv.Atoi(digits[0])
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryGet calls client.GetContent up to maxAttempts times with exponential backoff.
|
||||||
|
// If the server returns 429 (ErrRateLimit), the suggested Retry-After delay is used
|
||||||
|
// instead of the geometric backoff delay.
|
||||||
|
func retryGet(
|
||||||
|
ctx context.Context,
|
||||||
|
log *slog.Logger,
|
||||||
|
client browser.Client,
|
||||||
|
pageURL string,
|
||||||
|
maxAttempts int,
|
||||||
|
baseDelay time.Duration,
|
||||||
|
) (string, error) {
|
||||||
|
var lastErr error
|
||||||
|
delay := baseDelay
|
||||||
|
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||||
|
raw, err := client.GetContent(ctx, pageURL)
|
||||||
|
if err == nil {
|
||||||
|
return raw, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if attempt < maxAttempts {
|
||||||
|
// If the server is rate-limiting us, honour its Retry-After delay.
|
||||||
|
waitFor := delay
|
||||||
|
var rlErr *browser.RateLimitError
|
||||||
|
if errors.As(err, &rlErr) {
|
||||||
|
waitFor = rlErr.RetryAfter
|
||||||
|
if log != nil {
|
||||||
|
log.Warn("rate limited, backing off",
|
||||||
|
"url", pageURL, "attempt", attempt, "retry_in", waitFor)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if log != nil {
|
||||||
|
log.Warn("fetch failed, retrying",
|
||||||
|
"url", pageURL, "attempt", attempt, "retry_in", delay, "err", err)
|
||||||
|
}
|
||||||
|
delay *= 2
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return "", ctx.Err()
|
||||||
|
case <-time.After(waitFor):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", lastErr
|
||||||
|
}
|
||||||
129
backend/internal/novelfire/scraper_test.go
Normal file
129
backend/internal/novelfire/scraper_test.go
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
package novelfire
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSlugFromURL(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
url string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"https://novelfire.net/book/shadow-slave", "shadow-slave"},
|
||||||
|
{"https://novelfire.net/book/a-dragon-against-the-whole-world", "a-dragon-against-the-whole-world"},
|
||||||
|
{"https://novelfire.net/book/foo/chapter-1", "foo"},
|
||||||
|
{"https://novelfire.net/", ""},
|
||||||
|
{"not-a-url", "not-a-url"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
got := slugFromURL(c.url)
|
||||||
|
if got != c.want {
|
||||||
|
t.Errorf("slugFromURL(%q) = %q, want %q", c.url, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChapterNumberFromURL(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
url string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"https://novelfire.net/book/shadow-slave/chapter-42", 42},
|
||||||
|
{"https://novelfire.net/book/shadow-slave/chapter-1000", 1000},
|
||||||
|
{"https://novelfire.net/book/shadow-slave/chap-7", 7},
|
||||||
|
{"https://novelfire.net/book/shadow-slave/ch-3", 3},
|
||||||
|
{"https://novelfire.net/book/shadow-slave/42", 42},
|
||||||
|
{"https://novelfire.net/book/shadow-slave/no-number-here", 0},
|
||||||
|
{"not-a-url", 0},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
got := chapterNumberFromURL(c.url)
|
||||||
|
if got != c.want {
|
||||||
|
t.Errorf("chapterNumberFromURL(%q) = %d, want %d", c.url, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseChapterCount(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
in string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"123 Chapters", 123},
|
||||||
|
{"1,234 Chapters", 1234},
|
||||||
|
{"0", 0},
|
||||||
|
{"", 0},
|
||||||
|
{"500", 500},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
got := parseChapterCount(c.in)
|
||||||
|
if got != c.want {
|
||||||
|
t.Errorf("parseChapterCount(%q) = %d, want %d", c.in, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetryGet_ContextCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel() // cancel immediately
|
||||||
|
|
||||||
|
stub := newStubClient()
|
||||||
|
stub.setError("https://example.com/page", context.Canceled)
|
||||||
|
|
||||||
|
_, err := retryGet(ctx, nil, stub, "https://example.com/page", 3, 0)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error on cancelled context")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetryGet_EventualSuccess(t *testing.T) {
|
||||||
|
stub := newStubClient()
|
||||||
|
calls := 0
|
||||||
|
stub.setFn("https://example.com/page", func() (string, error) {
|
||||||
|
calls++
|
||||||
|
if calls < 3 {
|
||||||
|
return "", context.DeadlineExceeded
|
||||||
|
}
|
||||||
|
return "<html>ok</html>", nil
|
||||||
|
})
|
||||||
|
|
||||||
|
got, err := retryGet(context.Background(), nil, stub, "https://example.com/page", 5, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if got != "<html>ok</html>" {
|
||||||
|
t.Errorf("got %q, want html", got)
|
||||||
|
}
|
||||||
|
if calls != 3 {
|
||||||
|
t.Errorf("expected 3 calls, got %d", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── minimal stub client for tests ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
type stubClient struct {
|
||||||
|
errors map[string]error
|
||||||
|
fns map[string]func() (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStubClient() *stubClient {
|
||||||
|
return &stubClient{
|
||||||
|
errors: make(map[string]error),
|
||||||
|
fns: make(map[string]func() (string, error)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubClient) setError(u string, err error) { s.errors[u] = err }
|
||||||
|
|
||||||
|
func (s *stubClient) setFn(u string, fn func() (string, error)) { s.fns[u] = fn }
|
||||||
|
|
||||||
|
func (s *stubClient) GetContent(_ context.Context, pageURL string) (string, error) {
|
||||||
|
if fn, ok := s.fns[pageURL]; ok {
|
||||||
|
return fn()
|
||||||
|
}
|
||||||
|
if err, ok := s.errors[pageURL]; ok {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return "", context.DeadlineExceeded
|
||||||
|
}
|
||||||
205
backend/internal/orchestrator/orchestrator.go
Normal file
205
backend/internal/orchestrator/orchestrator.go
Normal file
@@ -0,0 +1,205 @@
|
|||||||
|
// Package orchestrator coordinates metadata extraction, chapter-list fetching,
|
||||||
|
// and parallel chapter scraping for a single book.
|
||||||
|
//
|
||||||
|
// Design:
|
||||||
|
// - RunBook scrapes one book (metadata + chapter list + chapter texts) end-to-end.
|
||||||
|
// - N worker goroutines pull chapter refs from a shared queue and call ScrapeChapterText.
|
||||||
|
// - The caller (runner poll loop) owns the outer task-claim / finish cycle.
|
||||||
|
package orchestrator
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/bookstore"
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
"github.com/libnovel/backend/internal/scraper"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config holds tunable parameters for the orchestrator.
|
||||||
|
type Config struct {
|
||||||
|
// Workers is the number of goroutines used to scrape chapters in parallel.
|
||||||
|
// Defaults to runtime.NumCPU() when 0.
|
||||||
|
Workers int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Orchestrator runs a single-book scrape pipeline.
|
||||||
|
type Orchestrator struct {
|
||||||
|
novel scraper.NovelScraper
|
||||||
|
store bookstore.BookWriter
|
||||||
|
log *slog.Logger
|
||||||
|
workers int
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns a new Orchestrator.
|
||||||
|
func New(cfg Config, novel scraper.NovelScraper, store bookstore.BookWriter, log *slog.Logger) *Orchestrator {
|
||||||
|
if log == nil {
|
||||||
|
log = slog.Default()
|
||||||
|
}
|
||||||
|
workers := cfg.Workers
|
||||||
|
if workers <= 0 {
|
||||||
|
workers = runtime.NumCPU()
|
||||||
|
}
|
||||||
|
return &Orchestrator{novel: novel, store: store, log: log, workers: workers}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunBook scrapes a single book described by task. It handles:
|
||||||
|
// 1. Metadata scrape + write
|
||||||
|
// 2. Chapter list scrape + write
|
||||||
|
// 3. Parallel chapter text scrape + write (worker pool)
|
||||||
|
//
|
||||||
|
// Returns a ScrapeResult with counters. The result's ErrorMessage is non-empty
|
||||||
|
// if the run failed at the metadata or chapter-list level.
|
||||||
|
func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) domain.ScrapeResult {
|
||||||
|
o.log.Info("orchestrator: RunBook starting",
|
||||||
|
"task_id", task.ID,
|
||||||
|
"kind", task.Kind,
|
||||||
|
"url", task.TargetURL,
|
||||||
|
"workers", o.workers,
|
||||||
|
)
|
||||||
|
|
||||||
|
var result domain.ScrapeResult
|
||||||
|
|
||||||
|
if task.TargetURL == "" {
|
||||||
|
result.ErrorMessage = "task has no target URL"
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Step 1: Metadata ──────────────────────────────────────────────────────
|
||||||
|
meta, err := o.novel.ScrapeMetadata(ctx, task.TargetURL)
|
||||||
|
if err != nil {
|
||||||
|
o.log.Error("metadata scrape failed", "url", task.TargetURL, "err", err)
|
||||||
|
result.ErrorMessage = fmt.Sprintf("metadata: %v", err)
|
||||||
|
result.Errors++
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := o.store.WriteMetadata(ctx, meta); err != nil {
|
||||||
|
o.log.Error("metadata write failed", "slug", meta.Slug, "err", err)
|
||||||
|
// non-fatal: continue to chapters
|
||||||
|
result.Errors++
|
||||||
|
} else {
|
||||||
|
result.BooksFound = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
|
||||||
|
|
||||||
|
// ── Step 2: Chapter list ──────────────────────────────────────────────────
|
||||||
|
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL)
|
||||||
|
if err != nil {
|
||||||
|
o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err)
|
||||||
|
result.ErrorMessage = fmt.Sprintf("chapter list: %v", err)
|
||||||
|
result.Errors++
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs))
|
||||||
|
|
||||||
|
// Persist chapter refs (without text) so the index exists early.
|
||||||
|
if wErr := o.store.WriteChapterRefs(ctx, meta.Slug, refs); wErr != nil {
|
||||||
|
o.log.Warn("chapter refs write failed", "slug", meta.Slug, "err", wErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Step 3: Chapter texts (worker pool) ───────────────────────────────────
|
||||||
|
type chapterJob struct {
|
||||||
|
slug string
|
||||||
|
ref domain.ChapterRef
|
||||||
|
total int // total chapters to scrape (for progress logging)
|
||||||
|
}
|
||||||
|
work := make(chan chapterJob, o.workers*4)
|
||||||
|
|
||||||
|
var scraped, skipped, errors atomic.Int64
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
for i := 0; i < o.workers; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(workerID int) {
|
||||||
|
defer wg.Done()
|
||||||
|
for job := range work {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
if o.store.ChapterExists(ctx, job.slug, job.ref) {
|
||||||
|
o.log.Debug("chapter already exists, skipping",
|
||||||
|
"slug", job.slug, "chapter", job.ref.Number)
|
||||||
|
skipped.Add(1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
ch, err := o.novel.ScrapeChapterText(ctx, job.ref)
|
||||||
|
if err != nil {
|
||||||
|
o.log.Error("chapter scrape failed",
|
||||||
|
"slug", job.slug, "chapter", job.ref.Number, "err", err)
|
||||||
|
errors.Add(1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := o.store.WriteChapter(ctx, job.slug, ch); err != nil {
|
||||||
|
o.log.Error("chapter write failed",
|
||||||
|
"slug", job.slug, "chapter", job.ref.Number, "err", err)
|
||||||
|
errors.Add(1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
n := scraped.Add(1)
|
||||||
|
// Log a progress summary every 25 chapters scraped.
|
||||||
|
if n%25 == 0 {
|
||||||
|
o.log.Info("scraping chapters",
|
||||||
|
"slug", job.slug, "scraped", n, "total", job.total)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count how many chapters will actually be enqueued (for progress logging).
|
||||||
|
toScrape := 0
|
||||||
|
for _, ref := range refs {
|
||||||
|
if task.FromChapter > 0 && ref.Number < task.FromChapter {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if task.ToChapter > 0 && ref.Number > task.ToChapter {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
toScrape++
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enqueue chapter jobs respecting the optional range filter from the task.
|
||||||
|
for _, ref := range refs {
|
||||||
|
if task.FromChapter > 0 && ref.Number < task.FromChapter {
|
||||||
|
skipped.Add(1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if task.ToChapter > 0 && ref.Number > task.ToChapter {
|
||||||
|
skipped.Add(1)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
goto drain
|
||||||
|
case work <- chapterJob{slug: meta.Slug, ref: ref, total: toScrape}:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
drain:
|
||||||
|
close(work)
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
result.ChaptersScraped = int(scraped.Load())
|
||||||
|
result.ChaptersSkipped = int(skipped.Load())
|
||||||
|
result.Errors += int(errors.Load())
|
||||||
|
|
||||||
|
o.log.Info("book scrape finished",
|
||||||
|
"slug", meta.Slug,
|
||||||
|
"scraped", result.ChaptersScraped,
|
||||||
|
"skipped", result.ChaptersSkipped,
|
||||||
|
"errors", result.Errors,
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
}
|
||||||
210
backend/internal/orchestrator/orchestrator_test.go
Normal file
210
backend/internal/orchestrator/orchestrator_test.go
Normal file
@@ -0,0 +1,210 @@
|
|||||||
|
package orchestrator
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ── stubs ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
type stubScraper struct {
|
||||||
|
meta domain.BookMeta
|
||||||
|
metaErr error
|
||||||
|
refs []domain.ChapterRef
|
||||||
|
refsErr error
|
||||||
|
chapters map[int]domain.Chapter
|
||||||
|
chapErr map[int]error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubScraper) SourceName() string { return "stub" }
|
||||||
|
|
||||||
|
func (s *stubScraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
|
||||||
|
ch := make(chan domain.CatalogueEntry)
|
||||||
|
errs := make(chan error)
|
||||||
|
close(ch)
|
||||||
|
close(errs)
|
||||||
|
return ch, errs
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) {
|
||||||
|
return s.meta, s.metaErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
|
||||||
|
return s.refs, s.refsErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
|
||||||
|
if s.chapErr != nil {
|
||||||
|
if err, ok := s.chapErr[ref.Number]; ok {
|
||||||
|
return domain.Chapter{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if s.chapters != nil {
|
||||||
|
if ch, ok := s.chapters[ref.Number]; ok {
|
||||||
|
return ch, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return domain.Chapter{Ref: ref, Text: "text"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubScraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) {
|
||||||
|
ch := make(chan domain.BookMeta)
|
||||||
|
errs := make(chan error)
|
||||||
|
close(ch)
|
||||||
|
close(errs)
|
||||||
|
return ch, errs
|
||||||
|
}
|
||||||
|
|
||||||
|
type stubStore struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
metaWritten []domain.BookMeta
|
||||||
|
chaptersWritten []domain.Chapter
|
||||||
|
existing map[string]bool // "slug:N" → exists
|
||||||
|
writeMetaErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubStore) WriteMetadata(_ context.Context, meta domain.BookMeta) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.writeMetaErr != nil {
|
||||||
|
return s.writeMetaErr
|
||||||
|
}
|
||||||
|
s.metaWritten = append(s.metaWritten, meta)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubStore) WriteChapter(_ context.Context, slug string, ch domain.Chapter) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.chaptersWritten = append(s.chaptersWritten, ch)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubStore) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubStore) ChapterExists(_ context.Context, slug string, ref domain.ChapterRef) bool {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
key := slug + ":" + string(rune('0'+ref.Number))
|
||||||
|
return s.existing[key]
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── tests ──────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestRunBook_HappyPath(t *testing.T) {
|
||||||
|
sc := &stubScraper{
|
||||||
|
meta: domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"},
|
||||||
|
refs: []domain.ChapterRef{
|
||||||
|
{Number: 1, Title: "Ch 1", URL: "https://example.com/book/test-book/chapter-1"},
|
||||||
|
{Number: 2, Title: "Ch 2", URL: "https://example.com/book/test-book/chapter-2"},
|
||||||
|
{Number: 3, Title: "Ch 3", URL: "https://example.com/book/test-book/chapter-3"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
st := &stubStore{}
|
||||||
|
o := New(Config{Workers: 2}, sc, st, nil)
|
||||||
|
|
||||||
|
task := domain.ScrapeTask{
|
||||||
|
ID: "t1",
|
||||||
|
Kind: "book",
|
||||||
|
TargetURL: "https://example.com/book/test-book",
|
||||||
|
}
|
||||||
|
|
||||||
|
result := o.RunBook(context.Background(), task)
|
||||||
|
|
||||||
|
if result.ErrorMessage != "" {
|
||||||
|
t.Fatalf("unexpected error: %s", result.ErrorMessage)
|
||||||
|
}
|
||||||
|
if result.BooksFound != 1 {
|
||||||
|
t.Errorf("BooksFound = %d, want 1", result.BooksFound)
|
||||||
|
}
|
||||||
|
if result.ChaptersScraped != 3 {
|
||||||
|
t.Errorf("ChaptersScraped = %d, want 3", result.ChaptersScraped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunBook_MetadataError(t *testing.T) {
|
||||||
|
sc := &stubScraper{metaErr: errors.New("404 not found")}
|
||||||
|
st := &stubStore{}
|
||||||
|
o := New(Config{Workers: 1}, sc, st, nil)
|
||||||
|
|
||||||
|
result := o.RunBook(context.Background(), domain.ScrapeTask{
|
||||||
|
ID: "t2",
|
||||||
|
TargetURL: "https://example.com/book/missing",
|
||||||
|
})
|
||||||
|
|
||||||
|
if result.ErrorMessage == "" {
|
||||||
|
t.Fatal("expected ErrorMessage to be set")
|
||||||
|
}
|
||||||
|
if result.Errors != 1 {
|
||||||
|
t.Errorf("Errors = %d, want 1", result.Errors)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunBook_ChapterRange(t *testing.T) {
|
||||||
|
sc := &stubScraper{
|
||||||
|
meta: domain.BookMeta{Slug: "range-book", SourceURL: "https://example.com/book/range-book"},
|
||||||
|
refs: func() []domain.ChapterRef {
|
||||||
|
var refs []domain.ChapterRef
|
||||||
|
for i := 1; i <= 10; i++ {
|
||||||
|
refs = append(refs, domain.ChapterRef{Number: i, URL: "https://example.com/book/range-book/chapter-" + string(rune('0'+i))})
|
||||||
|
}
|
||||||
|
return refs
|
||||||
|
}(),
|
||||||
|
}
|
||||||
|
st := &stubStore{}
|
||||||
|
o := New(Config{Workers: 2}, sc, st, nil)
|
||||||
|
|
||||||
|
result := o.RunBook(context.Background(), domain.ScrapeTask{
|
||||||
|
ID: "t3",
|
||||||
|
TargetURL: "https://example.com/book/range-book",
|
||||||
|
FromChapter: 3,
|
||||||
|
ToChapter: 7,
|
||||||
|
})
|
||||||
|
|
||||||
|
if result.ErrorMessage != "" {
|
||||||
|
t.Fatalf("unexpected error: %s", result.ErrorMessage)
|
||||||
|
}
|
||||||
|
// chapters 3–7 = 5 scraped, chapters 1-2 and 8-10 = 5 skipped
|
||||||
|
if result.ChaptersScraped != 5 {
|
||||||
|
t.Errorf("ChaptersScraped = %d, want 5", result.ChaptersScraped)
|
||||||
|
}
|
||||||
|
if result.ChaptersSkipped != 5 {
|
||||||
|
t.Errorf("ChaptersSkipped = %d, want 5", result.ChaptersSkipped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunBook_ContextCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
sc := &stubScraper{
|
||||||
|
meta: domain.BookMeta{Slug: "ctx-book", SourceURL: "https://example.com/book/ctx-book"},
|
||||||
|
refs: []domain.ChapterRef{
|
||||||
|
{Number: 1, URL: "https://example.com/book/ctx-book/chapter-1"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
st := &stubStore{}
|
||||||
|
o := New(Config{Workers: 1}, sc, st, nil)
|
||||||
|
|
||||||
|
// Should not panic; result may have errors or zero chapters.
|
||||||
|
result := o.RunBook(ctx, domain.ScrapeTask{
|
||||||
|
ID: "t4",
|
||||||
|
TargetURL: "https://example.com/book/ctx-book",
|
||||||
|
})
|
||||||
|
_ = result
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunBook_EmptyTargetURL(t *testing.T) {
|
||||||
|
o := New(Config{Workers: 1}, &stubScraper{}, &stubStore{}, nil)
|
||||||
|
result := o.RunBook(context.Background(), domain.ScrapeTask{ID: "t5"})
|
||||||
|
if result.ErrorMessage == "" {
|
||||||
|
t.Fatal("expected ErrorMessage for empty target URL")
|
||||||
|
}
|
||||||
|
}
|
||||||
176
backend/internal/runner/browse_refresh.go
Normal file
176
backend/internal/runner/browse_refresh.go
Normal file
@@ -0,0 +1,176 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
|
||||||
|
// browse page snapshots and stores them in MinIO.
|
||||||
|
//
|
||||||
|
// Design:
|
||||||
|
// - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
|
||||||
|
// - Fetches page 1 for each combination of the standard genre/sort/status
|
||||||
|
// filter values and stores the parsed JSON blob in MinIO via BrowseStore.
|
||||||
|
// - The backend's handleBrowse then serves from MinIO instead of calling
|
||||||
|
// novelfire.net live, which avoids IP-based rate-limiting on the server.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
|
||||||
|
type browseNovelListing struct {
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Cover string `json:"cover"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// browseSnapshot is the JSON structure stored in MinIO.
|
||||||
|
type browseSnapshot struct {
|
||||||
|
Novels []browseNovelListing `json:"novels"`
|
||||||
|
Page int `json:"page"`
|
||||||
|
HasNext bool `json:"hasNext"`
|
||||||
|
// CachedAt is the UTC time the snapshot was written (ISO 8601).
|
||||||
|
CachedAt string `json:"cachedAt"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// browseCombos lists the filter combinations to pre-fetch.
|
||||||
|
// Each entry is (genre, sort, status, novelType).
|
||||||
|
var browseCombos = []struct{ genre, sort, status, novelType string }{
|
||||||
|
{"all", "popular", "all", "all-novel"},
|
||||||
|
{"all", "popular", "ongoing", "all-novel"},
|
||||||
|
{"all", "popular", "completed", "all-novel"},
|
||||||
|
{"all", "new", "all", "all-novel"},
|
||||||
|
{"all", "new", "ongoing", "all-novel"},
|
||||||
|
{"all", "new", "completed", "all-novel"},
|
||||||
|
{"all", "top-rated", "all", "all-novel"},
|
||||||
|
{"all", "top-rated", "ongoing", "all-novel"},
|
||||||
|
{"all", "top-rated", "completed", "all-novel"},
|
||||||
|
}
|
||||||
|
|
||||||
|
const novelFireBrowseBase = "https://novelfire.net"
|
||||||
|
|
||||||
|
// runBrowseRefresh fetches all browse combos from novelfire.net and stores
|
||||||
|
// the results in MinIO. Errors per-combo are logged but do not abort the
|
||||||
|
// whole refresh cycle.
|
||||||
|
func (r *Runner) runBrowseRefresh(ctx context.Context) {
|
||||||
|
if r.deps.BrowseStore == nil {
|
||||||
|
r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log := r.deps.Log.With("op", "browse_refresh")
|
||||||
|
log.Info("runner: browse refresh starting", "combos", len(browseCombos))
|
||||||
|
|
||||||
|
ok, fail := 0, 0
|
||||||
|
for _, c := range browseCombos {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("runner: browse fetch failed",
|
||||||
|
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
|
||||||
|
fail++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
snap := browseSnapshot{
|
||||||
|
Novels: novels,
|
||||||
|
Page: 1,
|
||||||
|
HasNext: hasNext,
|
||||||
|
CachedAt: time.Now().UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
data, _ := json.Marshal(snap)
|
||||||
|
if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
|
||||||
|
log.Warn("runner: browse put failed",
|
||||||
|
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
|
||||||
|
fail++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ok++
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchBrowsePage calls novelfire.net and returns a list of novel listings
|
||||||
|
// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
|
||||||
|
func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
|
||||||
|
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
|
||||||
|
novelFireBrowseBase, genre, sort, status, novelType)
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("build request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
|
||||||
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||||
|
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||||
|
|
||||||
|
httpClient := &http.Client{Timeout: 45 * time.Second}
|
||||||
|
resp, err := httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
return parseBrowseHTML(resp.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
|
||||||
|
// in backend/handlers.go — kept separate to avoid coupling packages.
|
||||||
|
func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
|
||||||
|
data, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
body := string(data)
|
||||||
|
|
||||||
|
hasNext := strings.Contains(body, `rel="next"`) ||
|
||||||
|
strings.Contains(body, `aria-label="Next"`) ||
|
||||||
|
strings.Contains(body, `class="next"`)
|
||||||
|
|
||||||
|
slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
|
||||||
|
titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
|
||||||
|
coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
|
||||||
|
|
||||||
|
slugMatches := slugRe.FindAllStringSubmatch(body, -1)
|
||||||
|
titleMatches := titleRe.FindAllStringSubmatch(body, -1)
|
||||||
|
coverMatches := coverRe.FindAllStringSubmatch(body, -1)
|
||||||
|
|
||||||
|
var novels []browseNovelListing
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
for i, sm := range slugMatches {
|
||||||
|
slug := sm[1]
|
||||||
|
if seen[slug] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[slug] = true
|
||||||
|
|
||||||
|
item := browseNovelListing{
|
||||||
|
Slug: slug,
|
||||||
|
URL: novelFireBrowseBase + "/book/" + slug,
|
||||||
|
}
|
||||||
|
if i < len(titleMatches) {
|
||||||
|
item.Title = strings.TrimSpace(titleMatches[i][1])
|
||||||
|
}
|
||||||
|
if i < len(coverMatches) {
|
||||||
|
item.Cover = coverMatches[i][1]
|
||||||
|
}
|
||||||
|
if item.Title != "" {
|
||||||
|
novels = append(novels, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return novels, hasNext, nil
|
||||||
|
}
|
||||||
21
backend/internal/runner/helpers.go
Normal file
21
backend/internal/runner/helpers.go
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// stripMarkdown removes common markdown syntax from src, returning plain text
|
||||||
|
// suitable for TTS. Mirrors the helper in the scraper's server package.
|
||||||
|
func stripMarkdown(src string) string {
|
||||||
|
src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1")
|
||||||
|
src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "")
|
||||||
|
src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n")
|
||||||
|
return strings.TrimSpace(src)
|
||||||
|
}
|
||||||
403
backend/internal/runner/runner.go
Normal file
403
backend/internal/runner/runner.go
Normal file
@@ -0,0 +1,403 @@
|
|||||||
|
// Package runner implements the worker loop that polls PocketBase for pending
|
||||||
|
// scrape and audio tasks, executes them, and reports results back.
|
||||||
|
//
|
||||||
|
// Design:
|
||||||
|
// - Run(ctx) loops on a ticker; each tick claims and dispatches pending tasks.
|
||||||
|
// - Scrape tasks are dispatched to the Orchestrator (one goroutine per task,
|
||||||
|
// up to MaxConcurrentScrape).
|
||||||
|
// - Audio tasks fetch chapter text, call Kokoro, upload to MinIO, and report
|
||||||
|
// the result back (up to MaxConcurrentAudio goroutines).
|
||||||
|
// - The runner is stateless between ticks; all state lives in PocketBase.
|
||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/bookstore"
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
"github.com/libnovel/backend/internal/kokoro"
|
||||||
|
"github.com/libnovel/backend/internal/orchestrator"
|
||||||
|
"github.com/libnovel/backend/internal/scraper"
|
||||||
|
"github.com/libnovel/backend/internal/taskqueue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config tunes the runner behaviour.
|
||||||
|
type Config struct {
|
||||||
|
// WorkerID uniquely identifies this runner instance in PocketBase records.
|
||||||
|
WorkerID string
|
||||||
|
// PollInterval is how often the runner checks for new tasks.
|
||||||
|
PollInterval time.Duration
|
||||||
|
// MaxConcurrentScrape limits simultaneous book-scrape goroutines.
|
||||||
|
MaxConcurrentScrape int
|
||||||
|
// MaxConcurrentAudio limits simultaneous audio-generation goroutines.
|
||||||
|
MaxConcurrentAudio int
|
||||||
|
// OrchestratorWorkers is the chapter-scraping parallelism inside each book run.
|
||||||
|
OrchestratorWorkers int
|
||||||
|
// HeartbeatInterval is how often active tasks PATCH their heartbeat_at
|
||||||
|
// timestamp to signal they are still alive. Defaults to 30s when 0.
|
||||||
|
HeartbeatInterval time.Duration
|
||||||
|
// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
|
||||||
|
// task is considered orphaned and reset to pending. Defaults to 2m when 0.
|
||||||
|
StaleTaskThreshold time.Duration
|
||||||
|
// BrowseRefreshInterval is how often the runner pre-fetches browse page
|
||||||
|
// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
|
||||||
|
BrowseRefreshInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dependencies are the external services the runner depends on.
|
||||||
|
type Dependencies struct {
|
||||||
|
// Consumer claims tasks from PocketBase.
|
||||||
|
Consumer taskqueue.Consumer
|
||||||
|
// BookWriter persists scraped data (used by orchestrator).
|
||||||
|
BookWriter bookstore.BookWriter
|
||||||
|
// BookReader reads chapter text for audio generation.
|
||||||
|
BookReader bookstore.BookReader
|
||||||
|
// AudioStore persists generated audio and checks key existence.
|
||||||
|
AudioStore bookstore.AudioStore
|
||||||
|
// BrowseStore stores browse page snapshots in MinIO.
|
||||||
|
BrowseStore bookstore.BrowseStore
|
||||||
|
// Novel is the scraper implementation.
|
||||||
|
Novel scraper.NovelScraper
|
||||||
|
// Kokoro is the TTS client.
|
||||||
|
Kokoro kokoro.Client
|
||||||
|
// Log is the structured logger.
|
||||||
|
Log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runner is the main worker process.
|
||||||
|
type Runner struct {
|
||||||
|
cfg Config
|
||||||
|
deps Dependencies
|
||||||
|
}
|
||||||
|
|
||||||
|
// New creates a Runner from cfg and deps.
|
||||||
|
// Any zero/nil field in deps will cause a panic at construction time to fail fast.
|
||||||
|
func New(cfg Config, deps Dependencies) *Runner {
|
||||||
|
if cfg.PollInterval <= 0 {
|
||||||
|
cfg.PollInterval = 30 * time.Second
|
||||||
|
}
|
||||||
|
if cfg.MaxConcurrentScrape <= 0 {
|
||||||
|
cfg.MaxConcurrentScrape = 2
|
||||||
|
}
|
||||||
|
if cfg.MaxConcurrentAudio <= 0 {
|
||||||
|
cfg.MaxConcurrentAudio = 1
|
||||||
|
}
|
||||||
|
if cfg.WorkerID == "" {
|
||||||
|
cfg.WorkerID = "runner"
|
||||||
|
}
|
||||||
|
if cfg.HeartbeatInterval <= 0 {
|
||||||
|
cfg.HeartbeatInterval = 30 * time.Second
|
||||||
|
}
|
||||||
|
if cfg.StaleTaskThreshold <= 0 {
|
||||||
|
cfg.StaleTaskThreshold = 2 * time.Minute
|
||||||
|
}
|
||||||
|
if cfg.BrowseRefreshInterval <= 0 {
|
||||||
|
cfg.BrowseRefreshInterval = 6 * time.Hour
|
||||||
|
}
|
||||||
|
if deps.Log == nil {
|
||||||
|
deps.Log = slog.Default()
|
||||||
|
}
|
||||||
|
return &Runner{cfg: cfg, deps: deps}
|
||||||
|
}
|
||||||
|
|
||||||
|
// livenessFile is the path written on every successful poll so that the Docker
|
||||||
|
// healthcheck (CMD /healthcheck file /tmp/runner.alive <max_age>) can verify
|
||||||
|
// the runner is still making progress.
|
||||||
|
const livenessFile = "/tmp/runner.alive"
|
||||||
|
|
||||||
|
// touchAlive writes the current UTC time to livenessFile. Errors are logged but
|
||||||
|
// never fatal — liveness is best-effort and should not crash the runner.
|
||||||
|
func (r *Runner) touchAlive() {
|
||||||
|
data := []byte(time.Now().UTC().Format(time.RFC3339))
|
||||||
|
if err := os.WriteFile(livenessFile, data, 0o644); err != nil {
|
||||||
|
r.deps.Log.Warn("runner: failed to write liveness file", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run starts the poll loop, blocking until ctx is cancelled.
|
||||||
|
// On each tick it claims and executes all available pending tasks.
|
||||||
|
// Scrape and audio tasks run in separate goroutine pools bounded by
|
||||||
|
// MaxConcurrentScrape and MaxConcurrentAudio respectively.
|
||||||
|
func (r *Runner) Run(ctx context.Context) error {
|
||||||
|
r.deps.Log.Info("runner: starting",
|
||||||
|
"worker_id", r.cfg.WorkerID,
|
||||||
|
"poll_interval", r.cfg.PollInterval,
|
||||||
|
"max_scrape", r.cfg.MaxConcurrentScrape,
|
||||||
|
"max_audio", r.cfg.MaxConcurrentAudio,
|
||||||
|
"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
|
||||||
|
)
|
||||||
|
|
||||||
|
scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
|
||||||
|
audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
// Write liveness file immediately so the first healthcheck passes before
|
||||||
|
// the first poll completes.
|
||||||
|
r.touchAlive()
|
||||||
|
|
||||||
|
tick := time.NewTicker(r.cfg.PollInterval)
|
||||||
|
defer tick.Stop()
|
||||||
|
|
||||||
|
browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
|
||||||
|
defer browseTick.Stop()
|
||||||
|
|
||||||
|
// Run one browse refresh and one poll immediately on startup.
|
||||||
|
go r.runBrowseRefresh(ctx)
|
||||||
|
|
||||||
|
// Run one poll immediately on startup, then on each tick.
|
||||||
|
for {
|
||||||
|
r.poll(ctx, scrapeSem, audioSem, &wg)
|
||||||
|
r.touchAlive()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
r.deps.Log.Info("runner: context cancelled, draining active tasks")
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
wg.Wait()
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
r.deps.Log.Info("runner: all tasks drained, exiting")
|
||||||
|
case <-time.After(2 * time.Minute):
|
||||||
|
r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case <-browseTick.C:
|
||||||
|
go r.runBrowseRefresh(ctx)
|
||||||
|
case <-tick.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// poll claims all available pending tasks and dispatches them to goroutines.
|
||||||
|
// It claims tasks in a tight loop until no more are available.
|
||||||
|
func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) {
|
||||||
|
// ── Reap orphaned tasks ───────────────────────────────────────────────
|
||||||
|
if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
|
||||||
|
r.deps.Log.Warn("runner: reap stale tasks failed", "err", err)
|
||||||
|
} else if n > 0 {
|
||||||
|
r.deps.Log.Info("runner: reaped stale tasks", "count", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Scrape tasks ──────────────────────────────────────────────────────
|
||||||
|
for {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
task, ok, err := r.deps.Consumer.ClaimNextScrapeTask(ctx, r.cfg.WorkerID)
|
||||||
|
if err != nil {
|
||||||
|
r.deps.Log.Error("runner: ClaimNextScrapeTask failed", "err", err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
break // queue empty
|
||||||
|
}
|
||||||
|
// Acquire semaphore (non-blocking when full — leave task running).
|
||||||
|
select {
|
||||||
|
case scrapeSem <- struct{}{}:
|
||||||
|
default:
|
||||||
|
// Too many concurrent scrapes — the task stays claimed but we can't
|
||||||
|
// run it right now. Log and break; the next poll will pick it up if
|
||||||
|
// still running (it won't be re-claimed while status=running).
|
||||||
|
r.deps.Log.Warn("runner: scrape semaphore full, will retry next tick",
|
||||||
|
"task_id", task.ID)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
wg.Add(1)
|
||||||
|
go func(t domain.ScrapeTask) {
|
||||||
|
defer wg.Done()
|
||||||
|
defer func() { <-scrapeSem }()
|
||||||
|
r.runScrapeTask(ctx, t)
|
||||||
|
}(task)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Audio tasks ───────────────────────────────────────────────────────
|
||||||
|
for {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
task, ok, err := r.deps.Consumer.ClaimNextAudioTask(ctx, r.cfg.WorkerID)
|
||||||
|
if err != nil {
|
||||||
|
r.deps.Log.Error("runner: ClaimNextAudioTask failed", "err", err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
break // queue empty
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case audioSem <- struct{}{}:
|
||||||
|
default:
|
||||||
|
r.deps.Log.Warn("runner: audio semaphore full, will retry next tick",
|
||||||
|
"task_id", task.ID)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
wg.Add(1)
|
||||||
|
go func(t domain.AudioTask) {
|
||||||
|
defer wg.Done()
|
||||||
|
defer func() { <-audioSem }()
|
||||||
|
r.runAudioTask(ctx, t)
|
||||||
|
}(task)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runScrapeTask executes one scrape task end-to-end and reports the result.
|
||||||
|
func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
|
||||||
|
log := r.deps.Log.With("task_id", task.ID, "kind", task.Kind, "url", task.TargetURL)
|
||||||
|
log.Info("runner: scrape task starting")
|
||||||
|
|
||||||
|
// Heartbeat goroutine: periodically PATCH heartbeat_at so the reaper knows
|
||||||
|
// this task is still alive. Cancelled when the task finishes.
|
||||||
|
hbCtx, hbCancel := context.WithCancel(ctx)
|
||||||
|
defer hbCancel()
|
||||||
|
go func() {
|
||||||
|
tick := time.NewTicker(r.cfg.HeartbeatInterval)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-hbCtx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil {
|
||||||
|
log.Warn("runner: heartbeat failed", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
oCfg := orchestrator.Config{Workers: r.cfg.OrchestratorWorkers}
|
||||||
|
o := orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log)
|
||||||
|
|
||||||
|
var result domain.ScrapeResult
|
||||||
|
|
||||||
|
switch task.Kind {
|
||||||
|
case "catalogue":
|
||||||
|
result = r.runCatalogueTask(ctx, task, o, log)
|
||||||
|
case "book", "book_range":
|
||||||
|
result = o.RunBook(ctx, task)
|
||||||
|
default:
|
||||||
|
result.ErrorMessage = fmt.Sprintf("unknown task kind: %q", task.Kind)
|
||||||
|
log.Warn("runner: unknown task kind")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := r.deps.Consumer.FinishScrapeTask(ctx, task.ID, result); err != nil {
|
||||||
|
log.Error("runner: FinishScrapeTask failed", "err", err)
|
||||||
|
}
|
||||||
|
log.Info("runner: scrape task finished",
|
||||||
|
"scraped", result.ChaptersScraped,
|
||||||
|
"skipped", result.ChaptersSkipped,
|
||||||
|
"errors", result.Errors,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// runCatalogueTask runs a full catalogue scrape by iterating catalogue entries
|
||||||
|
// and running a book task for each one.
|
||||||
|
func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o *orchestrator.Orchestrator, log *slog.Logger) domain.ScrapeResult {
|
||||||
|
entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
|
||||||
|
var result domain.ScrapeResult
|
||||||
|
|
||||||
|
for entry := range entries {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
bookTask := domain.ScrapeTask{
|
||||||
|
ID: task.ID,
|
||||||
|
Kind: "book",
|
||||||
|
TargetURL: entry.URL,
|
||||||
|
}
|
||||||
|
bookResult := o.RunBook(ctx, bookTask)
|
||||||
|
result.BooksFound += bookResult.BooksFound + 1
|
||||||
|
result.ChaptersScraped += bookResult.ChaptersScraped
|
||||||
|
result.ChaptersSkipped += bookResult.ChaptersSkipped
|
||||||
|
result.Errors += bookResult.Errors
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := <-errCh; err != nil {
|
||||||
|
log.Warn("runner: catalogue scrape finished with error", "err", err)
|
||||||
|
result.Errors++
|
||||||
|
if result.ErrorMessage == "" {
|
||||||
|
result.ErrorMessage = err.Error()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// runAudioTask executes one audio-generation task:
|
||||||
|
// 1. Read chapter text from MinIO.
|
||||||
|
// 2. Call Kokoro to generate audio.
|
||||||
|
// 3. Upload MP3 to MinIO under the standard audio object key.
|
||||||
|
// 4. Report result back to PocketBase.
|
||||||
|
func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
|
||||||
|
log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "voice", task.Voice)
|
||||||
|
log.Info("runner: audio task starting")
|
||||||
|
|
||||||
|
// Heartbeat goroutine: periodically PATCH heartbeat_at so the reaper knows
|
||||||
|
// this task is still alive. Cancelled when the task finishes.
|
||||||
|
hbCtx, hbCancel := context.WithCancel(ctx)
|
||||||
|
defer hbCancel()
|
||||||
|
go func() {
|
||||||
|
tick := time.NewTicker(r.cfg.HeartbeatInterval)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-hbCtx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil {
|
||||||
|
log.Warn("runner: heartbeat failed", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
fail := func(msg string) {
|
||||||
|
log.Error("runner: audio task failed", "reason", msg)
|
||||||
|
result := domain.AudioResult{ErrorMessage: msg}
|
||||||
|
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
|
||||||
|
log.Error("runner: FinishAudioTask failed", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 1: read chapter text.
|
||||||
|
raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter)
|
||||||
|
if err != nil {
|
||||||
|
fail(fmt.Sprintf("read chapter: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
text := stripMarkdown(raw)
|
||||||
|
if text == "" {
|
||||||
|
fail("chapter text is empty after stripping markdown")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 2: generate audio.
|
||||||
|
if r.deps.Kokoro == nil {
|
||||||
|
fail("kokoro client not configured")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
audioData, err := r.deps.Kokoro.GenerateAudio(ctx, text, task.Voice)
|
||||||
|
if err != nil {
|
||||||
|
fail(fmt.Sprintf("kokoro generate: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: upload to MinIO.
|
||||||
|
key := r.deps.AudioStore.AudioObjectKey(task.Slug, task.Chapter, task.Voice)
|
||||||
|
if err := r.deps.AudioStore.PutAudio(ctx, key, audioData); err != nil {
|
||||||
|
fail(fmt.Sprintf("put audio: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4: report success.
|
||||||
|
result := domain.AudioResult{ObjectKey: key}
|
||||||
|
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
|
||||||
|
log.Error("runner: FinishAudioTask failed", "err", err)
|
||||||
|
}
|
||||||
|
log.Info("runner: audio task finished", "key", key)
|
||||||
|
}
|
||||||
365
backend/internal/runner/runner_test.go
Normal file
365
backend/internal/runner/runner_test.go
Normal file
@@ -0,0 +1,365 @@
|
|||||||
|
package runner_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
"github.com/libnovel/backend/internal/runner"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ── Stub types ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// stubConsumer is a test double for taskqueue.Consumer.
|
||||||
|
type stubConsumer struct {
|
||||||
|
scrapeQueue []domain.ScrapeTask
|
||||||
|
audioQueue []domain.AudioTask
|
||||||
|
scrapeIdx int
|
||||||
|
audioIdx int
|
||||||
|
finished []string
|
||||||
|
failCalled []string
|
||||||
|
claimErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubConsumer) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
|
||||||
|
if s.claimErr != nil {
|
||||||
|
return domain.ScrapeTask{}, false, s.claimErr
|
||||||
|
}
|
||||||
|
if s.scrapeIdx >= len(s.scrapeQueue) {
|
||||||
|
return domain.ScrapeTask{}, false, nil
|
||||||
|
}
|
||||||
|
t := s.scrapeQueue[s.scrapeIdx]
|
||||||
|
s.scrapeIdx++
|
||||||
|
return t, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubConsumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
|
||||||
|
if s.claimErr != nil {
|
||||||
|
return domain.AudioTask{}, false, s.claimErr
|
||||||
|
}
|
||||||
|
if s.audioIdx >= len(s.audioQueue) {
|
||||||
|
return domain.AudioTask{}, false, nil
|
||||||
|
}
|
||||||
|
t := s.audioQueue[s.audioIdx]
|
||||||
|
s.audioIdx++
|
||||||
|
return t, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubConsumer) FinishScrapeTask(_ context.Context, id string, _ domain.ScrapeResult) error {
|
||||||
|
s.finished = append(s.finished, id)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubConsumer) FinishAudioTask(_ context.Context, id string, _ domain.AudioResult) error {
|
||||||
|
s.finished = append(s.finished, id)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubConsumer) FailTask(_ context.Context, id, _ string) error {
|
||||||
|
s.failCalled = append(s.failCalled, id)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubConsumer) HeartbeatTask(_ context.Context, _ string) error { return nil }
|
||||||
|
|
||||||
|
func (s *stubConsumer) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// stubBookWriter satisfies bookstore.BookWriter (no-op).
|
||||||
|
type stubBookWriter struct{}
|
||||||
|
|
||||||
|
func (s *stubBookWriter) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil }
|
||||||
|
func (s *stubBookWriter) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (s *stubBookWriter) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (s *stubBookWriter) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// stubBookReader satisfies bookstore.BookReader — returns a single chapter.
|
||||||
|
type stubBookReader struct {
|
||||||
|
text string
|
||||||
|
readErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubBookReader) ReadChapter(_ context.Context, _ string, _ int) (string, error) {
|
||||||
|
return s.text, s.readErr
|
||||||
|
}
|
||||||
|
func (s *stubBookReader) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) {
|
||||||
|
return domain.BookMeta{}, false, nil
|
||||||
|
}
|
||||||
|
func (s *stubBookReader) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil }
|
||||||
|
func (s *stubBookReader) LocalSlugs(_ context.Context) (map[string]bool, error) { return nil, nil }
|
||||||
|
func (s *stubBookReader) MetadataMtime(_ context.Context, _ string) int64 { return 0 }
|
||||||
|
func (s *stubBookReader) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (s *stubBookReader) CountChapters(_ context.Context, _ string) int { return 0 }
|
||||||
|
func (s *stubBookReader) ReindexChapters(_ context.Context, _ string) (int, error) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// stubAudioStore satisfies bookstore.AudioStore.
|
||||||
|
type stubAudioStore struct {
|
||||||
|
putCalled atomic.Int32
|
||||||
|
putErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubAudioStore) AudioObjectKey(slug string, n int, voice string) string {
|
||||||
|
return slug + "/" + string(rune('0'+n)) + "/" + voice + ".mp3"
|
||||||
|
}
|
||||||
|
func (s *stubAudioStore) AudioExists(_ context.Context, _ string) bool { return false }
|
||||||
|
func (s *stubAudioStore) PutAudio(_ context.Context, _ string, _ []byte) error {
|
||||||
|
s.putCalled.Add(1)
|
||||||
|
return s.putErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// stubNovelScraper satisfies scraper.NovelScraper minimally.
|
||||||
|
type stubNovelScraper struct {
|
||||||
|
entries []domain.CatalogueEntry
|
||||||
|
metaErr error
|
||||||
|
chapters []domain.ChapterRef
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubNovelScraper) ScrapeCatalogue(_ context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
|
||||||
|
ch := make(chan domain.CatalogueEntry, len(s.entries))
|
||||||
|
errCh := make(chan error, 1)
|
||||||
|
for _, e := range s.entries {
|
||||||
|
ch <- e
|
||||||
|
}
|
||||||
|
close(ch)
|
||||||
|
close(errCh)
|
||||||
|
return ch, errCh
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) {
|
||||||
|
if s.metaErr != nil {
|
||||||
|
return domain.BookMeta{}, s.metaErr
|
||||||
|
}
|
||||||
|
return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
|
||||||
|
return s.chapters, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubNovelScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
|
||||||
|
return domain.Chapter{Ref: ref, Text: "# Chapter\n\nSome text."}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubNovelScraper) ScrapeRanking(_ context.Context, _ int) (<-chan domain.BookMeta, <-chan error) {
|
||||||
|
ch := make(chan domain.BookMeta)
|
||||||
|
errCh := make(chan error, 1)
|
||||||
|
close(ch)
|
||||||
|
close(errCh)
|
||||||
|
return ch, errCh
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubNovelScraper) SourceName() string { return "stub" }
|
||||||
|
|
||||||
|
// stubKokoro satisfies kokoro.Client.
|
||||||
|
type stubKokoro struct {
|
||||||
|
data []byte
|
||||||
|
genErr error
|
||||||
|
called atomic.Int32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
|
||||||
|
s.called.Add(1)
|
||||||
|
return s.data, s.genErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubKokoro) ListVoices(_ context.Context) ([]string, error) {
|
||||||
|
return []string{"af_bella"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── stripMarkdown helper ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestStripMarkdownViaAudioTask(t *testing.T) {
|
||||||
|
// Verify markdown is stripped before sending to Kokoro.
|
||||||
|
// We inject chapter text with markdown; the kokoro stub verifies data flows.
|
||||||
|
consumer := &stubConsumer{
|
||||||
|
audioQueue: []domain.AudioTask{
|
||||||
|
{ID: "a1", Slug: "book", Chapter: 1, Voice: "af_bella", Status: domain.TaskStatusRunning},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
bookReader := &stubBookReader{text: "## Chapter 1\n\nPlain **text** here."}
|
||||||
|
audioStore := &stubAudioStore{}
|
||||||
|
kokoroStub := &stubKokoro{data: []byte("mp3")}
|
||||||
|
|
||||||
|
cfg := runner.Config{
|
||||||
|
WorkerID: "test",
|
||||||
|
PollInterval: time.Hour, // long poll — we'll cancel manually
|
||||||
|
}
|
||||||
|
deps := runner.Dependencies{
|
||||||
|
Consumer: consumer,
|
||||||
|
BookWriter: &stubBookWriter{},
|
||||||
|
BookReader: bookReader,
|
||||||
|
AudioStore: audioStore,
|
||||||
|
Novel: &stubNovelScraper{},
|
||||||
|
Kokoro: kokoroStub,
|
||||||
|
}
|
||||||
|
|
||||||
|
r := runner.New(cfg, deps)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = r.Run(ctx)
|
||||||
|
|
||||||
|
if kokoroStub.called.Load() != 1 {
|
||||||
|
t.Errorf("expected Kokoro.GenerateAudio called once, got %d", kokoroStub.called.Load())
|
||||||
|
}
|
||||||
|
if audioStore.putCalled.Load() != 1 {
|
||||||
|
t.Errorf("expected PutAudio called once, got %d", audioStore.putCalled.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAudioTask_ReadChapterError(t *testing.T) {
|
||||||
|
consumer := &stubConsumer{
|
||||||
|
audioQueue: []domain.AudioTask{
|
||||||
|
{ID: "a2", Slug: "book", Chapter: 2, Voice: "af_bella", Status: domain.TaskStatusRunning},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
bookReader := &stubBookReader{readErr: errors.New("chapter not found")}
|
||||||
|
audioStore := &stubAudioStore{}
|
||||||
|
kokoroStub := &stubKokoro{data: []byte("mp3")}
|
||||||
|
|
||||||
|
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
||||||
|
deps := runner.Dependencies{
|
||||||
|
Consumer: consumer,
|
||||||
|
BookWriter: &stubBookWriter{},
|
||||||
|
BookReader: bookReader,
|
||||||
|
AudioStore: audioStore,
|
||||||
|
Novel: &stubNovelScraper{},
|
||||||
|
Kokoro: kokoroStub,
|
||||||
|
}
|
||||||
|
|
||||||
|
r := runner.New(cfg, deps)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = r.Run(ctx)
|
||||||
|
|
||||||
|
// Kokoro should not be called; FinishAudioTask should be called with error.
|
||||||
|
if kokoroStub.called.Load() != 0 {
|
||||||
|
t.Errorf("expected Kokoro not called, got %d", kokoroStub.called.Load())
|
||||||
|
}
|
||||||
|
if len(consumer.finished) != 1 {
|
||||||
|
t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAudioTask_KokoroError(t *testing.T) {
|
||||||
|
consumer := &stubConsumer{
|
||||||
|
audioQueue: []domain.AudioTask{
|
||||||
|
{ID: "a3", Slug: "book", Chapter: 3, Voice: "af_bella", Status: domain.TaskStatusRunning},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
bookReader := &stubBookReader{text: "Chapter text."}
|
||||||
|
audioStore := &stubAudioStore{}
|
||||||
|
kokoroStub := &stubKokoro{genErr: errors.New("tts failed")}
|
||||||
|
|
||||||
|
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
||||||
|
deps := runner.Dependencies{
|
||||||
|
Consumer: consumer,
|
||||||
|
BookWriter: &stubBookWriter{},
|
||||||
|
BookReader: bookReader,
|
||||||
|
AudioStore: audioStore,
|
||||||
|
Novel: &stubNovelScraper{},
|
||||||
|
Kokoro: kokoroStub,
|
||||||
|
}
|
||||||
|
|
||||||
|
r := runner.New(cfg, deps)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = r.Run(ctx)
|
||||||
|
|
||||||
|
if audioStore.putCalled.Load() != 0 {
|
||||||
|
t.Errorf("expected PutAudio not called, got %d", audioStore.putCalled.Load())
|
||||||
|
}
|
||||||
|
if len(consumer.finished) != 1 {
|
||||||
|
t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScrapeTask_BookKind(t *testing.T) {
|
||||||
|
consumer := &stubConsumer{
|
||||||
|
scrapeQueue: []domain.ScrapeTask{
|
||||||
|
{ID: "s1", Kind: "book", TargetURL: "https://example.com/book/test-book", Status: domain.TaskStatusRunning},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
||||||
|
deps := runner.Dependencies{
|
||||||
|
Consumer: consumer,
|
||||||
|
BookWriter: &stubBookWriter{},
|
||||||
|
BookReader: &stubBookReader{},
|
||||||
|
AudioStore: &stubAudioStore{},
|
||||||
|
Novel: &stubNovelScraper{},
|
||||||
|
Kokoro: &stubKokoro{},
|
||||||
|
}
|
||||||
|
|
||||||
|
r := runner.New(cfg, deps)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = r.Run(ctx)
|
||||||
|
|
||||||
|
if len(consumer.finished) != 1 || consumer.finished[0] != "s1" {
|
||||||
|
t.Errorf("expected task s1 finished, got %v", consumer.finished)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScrapeTask_UnknownKind(t *testing.T) {
|
||||||
|
consumer := &stubConsumer{
|
||||||
|
scrapeQueue: []domain.ScrapeTask{
|
||||||
|
{ID: "s2", Kind: "unknown_kind", Status: domain.TaskStatusRunning},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
||||||
|
deps := runner.Dependencies{
|
||||||
|
Consumer: consumer,
|
||||||
|
BookWriter: &stubBookWriter{},
|
||||||
|
BookReader: &stubBookReader{},
|
||||||
|
AudioStore: &stubAudioStore{},
|
||||||
|
Novel: &stubNovelScraper{},
|
||||||
|
Kokoro: &stubKokoro{},
|
||||||
|
}
|
||||||
|
|
||||||
|
r := runner.New(cfg, deps)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = r.Run(ctx)
|
||||||
|
|
||||||
|
// Unknown kind still finishes the task (with error message in result).
|
||||||
|
if len(consumer.finished) != 1 || consumer.finished[0] != "s2" {
|
||||||
|
t.Errorf("expected task s2 finished, got %v", consumer.finished)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRun_CancelImmediately(t *testing.T) {
|
||||||
|
consumer := &stubConsumer{}
|
||||||
|
cfg := runner.Config{WorkerID: "test", PollInterval: 10 * time.Millisecond}
|
||||||
|
deps := runner.Dependencies{
|
||||||
|
Consumer: consumer,
|
||||||
|
BookWriter: &stubBookWriter{},
|
||||||
|
BookReader: &stubBookReader{},
|
||||||
|
AudioStore: &stubAudioStore{},
|
||||||
|
Novel: &stubNovelScraper{},
|
||||||
|
Kokoro: &stubKokoro{},
|
||||||
|
}
|
||||||
|
|
||||||
|
r := runner.New(cfg, deps)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel() // cancel before Run
|
||||||
|
|
||||||
|
err := r.Run(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("expected nil on graceful shutdown, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
58
backend/internal/scraper/scraper.go
Normal file
58
backend/internal/scraper/scraper.go
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
// Package scraper defines the NovelScraper interface and its sub-interfaces.
|
||||||
|
// Domain types live in internal/domain — this package only defines the scraping
|
||||||
|
// contract so that novelfire and any future scrapers can be swapped freely.
|
||||||
|
package scraper
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CatalogueProvider can enumerate every novel available on a source site.
|
||||||
|
type CatalogueProvider interface {
|
||||||
|
ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MetadataProvider can extract structured book metadata from a novel's landing page.
|
||||||
|
type MetadataProvider interface {
|
||||||
|
ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChapterListProvider can enumerate all chapters of a book.
|
||||||
|
type ChapterListProvider interface {
|
||||||
|
ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ChapterTextProvider can extract the readable text from a single chapter page.
|
||||||
|
type ChapterTextProvider interface {
|
||||||
|
ScrapeChapterText(ctx context.Context, ref domain.ChapterRef) (domain.Chapter, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RankingProvider can enumerate novels from a ranking page.
|
||||||
|
type RankingProvider interface {
|
||||||
|
// ScrapeRanking pages through up to maxPages ranking pages.
|
||||||
|
// maxPages <= 0 means all pages.
|
||||||
|
ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NovelScraper is the full interface a concrete novel source must implement.
|
||||||
|
type NovelScraper interface {
|
||||||
|
CatalogueProvider
|
||||||
|
MetadataProvider
|
||||||
|
ChapterListProvider
|
||||||
|
ChapterTextProvider
|
||||||
|
RankingProvider
|
||||||
|
|
||||||
|
// SourceName returns the human-readable name of this scraper, e.g. "novelfire.net".
|
||||||
|
SourceName() string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Selector describes how to locate an element in an HTML document.
|
||||||
|
type Selector struct {
|
||||||
|
Tag string
|
||||||
|
Class string
|
||||||
|
ID string
|
||||||
|
Attr string
|
||||||
|
Multiple bool
|
||||||
|
}
|
||||||
222
backend/internal/storage/minio.go
Normal file
222
backend/internal/storage/minio.go
Normal file
@@ -0,0 +1,222 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/url"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
minio "github.com/minio/minio-go/v7"
|
||||||
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
// minioClient wraps the official minio-go client with bucket names.
|
||||||
|
type minioClient struct {
|
||||||
|
client *minio.Client // internal — all read/write operations
|
||||||
|
pubClient *minio.Client // presign-only — initialised against the public endpoint
|
||||||
|
bucketChapters string
|
||||||
|
bucketAudio string
|
||||||
|
bucketAvatars string
|
||||||
|
bucketBrowse string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMinioClient(cfg config.MinIO) (*minioClient, error) {
|
||||||
|
creds := credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, "")
|
||||||
|
|
||||||
|
internal, err := minio.New(cfg.Endpoint, &minio.Options{
|
||||||
|
Creds: creds,
|
||||||
|
Secure: cfg.UseSSL,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("minio: init internal client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Presigned URLs must be signed with the hostname the browser will use
|
||||||
|
// (PUBLIC_MINIO_PUBLIC_URL), because AWS Signature V4 includes the Host
|
||||||
|
// header in the canonical request — a URL signed against "minio:9000" will
|
||||||
|
// return SignatureDoesNotMatch when the browser fetches it from
|
||||||
|
// "localhost:9000".
|
||||||
|
//
|
||||||
|
// However, minio-go normally makes a live BucketLocation HTTP call before
|
||||||
|
// signing, which would fail from inside the container when the public
|
||||||
|
// endpoint is externally-facing (e.g. "localhost:9000" is unreachable from
|
||||||
|
// within Docker). We prevent this by:
|
||||||
|
// 1. Setting Region: "us-east-1" — minio-go skips getBucketLocation when
|
||||||
|
// the region is already known (bucket-cache.go:49).
|
||||||
|
// 2. Setting BucketLookup: BucketLookupPath — forces path-style URLs
|
||||||
|
// (e.g. host/bucket/key), matching MinIO's default behaviour and
|
||||||
|
// avoiding any virtual-host DNS probing.
|
||||||
|
//
|
||||||
|
// When no public endpoint is configured (or it equals the internal one),
|
||||||
|
// fall back to the internal client so presigning still works.
|
||||||
|
publicEndpoint := cfg.PublicEndpoint
|
||||||
|
if u, err2 := url.Parse(publicEndpoint); err2 == nil && u.Host != "" {
|
||||||
|
publicEndpoint = u.Host // strip scheme so minio.New is happy
|
||||||
|
}
|
||||||
|
pubUseSSL := cfg.PublicUseSSL
|
||||||
|
if publicEndpoint == "" || publicEndpoint == cfg.Endpoint {
|
||||||
|
publicEndpoint = cfg.Endpoint
|
||||||
|
pubUseSSL = cfg.UseSSL
|
||||||
|
}
|
||||||
|
pub, err := minio.New(publicEndpoint, &minio.Options{
|
||||||
|
Creds: creds,
|
||||||
|
Secure: pubUseSSL,
|
||||||
|
Region: "us-east-1", // skip live BucketLocation preflight
|
||||||
|
BucketLookup: minio.BucketLookupPath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("minio: init public client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &minioClient{
|
||||||
|
client: internal,
|
||||||
|
pubClient: pub,
|
||||||
|
bucketChapters: cfg.BucketChapters,
|
||||||
|
bucketAudio: cfg.BucketAudio,
|
||||||
|
bucketAvatars: cfg.BucketAvatars,
|
||||||
|
bucketBrowse: cfg.BucketBrowse,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureBuckets creates all required buckets if they don't already exist.
|
||||||
|
func (m *minioClient) ensureBuckets(ctx context.Context) error {
|
||||||
|
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse} {
|
||||||
|
exists, err := m.client.BucketExists(ctx, bucket)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("minio: check bucket %q: %w", bucket, err)
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
|
||||||
|
return fmt.Errorf("minio: create bucket %q: %w", bucket, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Key helpers ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// ChapterObjectKey returns the MinIO object key for a chapter markdown file.
|
||||||
|
// Format: {slug}/chapter-{n:06d}.md
|
||||||
|
func ChapterObjectKey(slug string, n int) string {
|
||||||
|
return fmt.Sprintf("%s/chapter-%06d.md", slug, n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AudioObjectKey returns the MinIO object key for a cached audio file.
|
||||||
|
// Format: {slug}/{n}/{voice}.mp3
|
||||||
|
func AudioObjectKey(slug string, n int, voice string) string {
|
||||||
|
return fmt.Sprintf("%s/%d/%s.mp3", slug, n, voice)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AvatarObjectKey returns the MinIO object key for a user avatar image.
|
||||||
|
// Format: {userID}/{ext}.{ext}
|
||||||
|
func AvatarObjectKey(userID, ext string) string {
|
||||||
|
return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BrowseObjectKey returns the MinIO object key for a cached browse page snapshot.
|
||||||
|
// Format: browse/{genre}/{sort}/{status}/{type}/page-{n}.json
|
||||||
|
func BrowseObjectKey(genre, sort, status, novelType string, page int) string {
|
||||||
|
return fmt.Sprintf("browse/%s/%s/%s/%s/page-%d.json", genre, sort, status, novelType, page)
|
||||||
|
}
|
||||||
|
|
||||||
|
// chapterNumberFromKey extracts the chapter number from a MinIO object key.
|
||||||
|
// e.g. "my-book/chapter-000042.md" → 42
|
||||||
|
func chapterNumberFromKey(key string) int {
|
||||||
|
base := path.Base(key)
|
||||||
|
base = strings.TrimPrefix(base, "chapter-")
|
||||||
|
base = strings.TrimSuffix(base, ".md")
|
||||||
|
var n int
|
||||||
|
fmt.Sscanf(base, "%d", &n)
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Object operations ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (m *minioClient) putObject(ctx context.Context, bucket, key, contentType string, data []byte) error {
|
||||||
|
_, err := m.client.PutObject(ctx, bucket, key,
|
||||||
|
strings.NewReader(string(data)),
|
||||||
|
int64(len(data)),
|
||||||
|
minio.PutObjectOptions{ContentType: contentType},
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *minioClient) getObject(ctx context.Context, bucket, key string) ([]byte, error) {
|
||||||
|
obj, err := m.client.GetObject(ctx, bucket, key, minio.GetObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer obj.Close()
|
||||||
|
return io.ReadAll(obj)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *minioClient) objectExists(ctx context.Context, bucket, key string) bool {
|
||||||
|
_, err := m.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{})
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *minioClient) presignGet(ctx context.Context, bucket, key string, expires time.Duration) (string, error) {
|
||||||
|
u, err := m.pubClient.PresignedGetObject(ctx, bucket, key, expires, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("minio presign %s/%s: %w", bucket, key, err)
|
||||||
|
}
|
||||||
|
return u.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *minioClient) presignPut(ctx context.Context, bucket, key string, expires time.Duration) (string, error) {
|
||||||
|
u, err := m.pubClient.PresignedPutObject(ctx, bucket, key, expires)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("minio presign PUT %s/%s: %w", bucket, key, err)
|
||||||
|
}
|
||||||
|
return u.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *minioClient) deleteObjects(ctx context.Context, bucket, prefix string) error {
|
||||||
|
objCh := m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix})
|
||||||
|
for obj := range objCh {
|
||||||
|
if obj.Err != nil {
|
||||||
|
return obj.Err
|
||||||
|
}
|
||||||
|
if err := m.client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string) ([]string, error) {
|
||||||
|
var keys []string
|
||||||
|
for obj := range m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) {
|
||||||
|
if obj.Err != nil {
|
||||||
|
return nil, obj.Err
|
||||||
|
}
|
||||||
|
keys = append(keys, obj.Key)
|
||||||
|
}
|
||||||
|
return keys, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Browse operations ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// putBrowse stores raw JSON bytes for a browse page snapshot.
|
||||||
|
func (m *minioClient) putBrowse(ctx context.Context, key string, data []byte) error {
|
||||||
|
return m.putObject(ctx, m.bucketBrowse, key, "application/json", data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// getBrowse retrieves a browse page snapshot. Returns (nil, false, nil) when
|
||||||
|
// the object does not exist.
|
||||||
|
func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool, error) {
|
||||||
|
if !m.objectExists(ctx, m.bucketBrowse, key) {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
data, err := m.getObject(ctx, m.bucketBrowse, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
return data, true, nil
|
||||||
|
}
|
||||||
268
backend/internal/storage/pocketbase.go
Normal file
268
backend/internal/storage/pocketbase.go
Normal file
@@ -0,0 +1,268 @@
|
|||||||
|
// Package storage provides the concrete implementations of all bookstore and
|
||||||
|
// taskqueue interfaces backed by PocketBase (structured data) and MinIO (blobs).
|
||||||
|
//
|
||||||
|
// Entry point: NewStore(ctx, cfg, log) returns a *Store that satisfies every
|
||||||
|
// interface defined in bookstore and taskqueue.
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/config"
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrNotFound is returned by single-record lookups when no record exists.
|
||||||
|
var ErrNotFound = errors.New("storage: record not found")
|
||||||
|
|
||||||
|
// pbClient is the internal PocketBase REST admin client.
|
||||||
|
type pbClient struct {
|
||||||
|
baseURL string
|
||||||
|
email string
|
||||||
|
password string
|
||||||
|
log *slog.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
token string
|
||||||
|
exp time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPBClient(cfg config.PocketBase, log *slog.Logger) *pbClient {
|
||||||
|
return &pbClient{
|
||||||
|
baseURL: strings.TrimRight(cfg.URL, "/"),
|
||||||
|
email: cfg.AdminEmail,
|
||||||
|
password: cfg.AdminPassword,
|
||||||
|
log: log,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// authToken returns a valid admin auth token, refreshing it when expired.
|
||||||
|
func (c *pbClient) authToken(ctx context.Context) (string, error) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if c.token != "" && time.Now().Before(c.exp) {
|
||||||
|
return c.token, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
body, _ := json.Marshal(map[string]string{
|
||||||
|
"identity": c.email,
|
||||||
|
"password": c.password,
|
||||||
|
})
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||||
|
c.baseURL+"/api/collections/_superusers/auth-with-password", bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("pb auth: build request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("pb auth: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
raw, _ := io.ReadAll(resp.Body)
|
||||||
|
return "", fmt.Errorf("pb auth: status %d: %s", resp.StatusCode, string(raw))
|
||||||
|
}
|
||||||
|
|
||||||
|
var payload struct {
|
||||||
|
Token string `json:"token"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
|
||||||
|
return "", fmt.Errorf("pb auth: decode: %w", err)
|
||||||
|
}
|
||||||
|
c.token = payload.Token
|
||||||
|
c.exp = time.Now().Add(30 * time.Minute)
|
||||||
|
return c.token, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// do executes an authenticated PocketBase REST request.
|
||||||
|
func (c *pbClient) do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
|
||||||
|
tok, err := c.authToken(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("pb: build request %s %s: %w", method, path, err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", tok)
|
||||||
|
if body != nil {
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("pb: %s %s: %w", method, path, err)
|
||||||
|
}
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// get is a convenience wrapper that decodes a JSON response into v.
|
||||||
|
func (c *pbClient) get(ctx context.Context, path string, v any) error {
|
||||||
|
resp, err := c.do(ctx, http.MethodGet, path, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode == http.StatusNotFound {
|
||||||
|
return ErrNotFound
|
||||||
|
}
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
raw, _ := io.ReadAll(resp.Body)
|
||||||
|
return fmt.Errorf("pb GET %s: status %d: %s", path, resp.StatusCode, string(raw))
|
||||||
|
}
|
||||||
|
return json.NewDecoder(resp.Body).Decode(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// post creates a record and decodes the created record into v.
|
||||||
|
func (c *pbClient) post(ctx context.Context, path string, payload, v any) error {
|
||||||
|
b, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("pb: marshal: %w", err)
|
||||||
|
}
|
||||||
|
resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
raw, _ := io.ReadAll(resp.Body)
|
||||||
|
return fmt.Errorf("pb POST %s: status %d: %s", path, resp.StatusCode, string(raw))
|
||||||
|
}
|
||||||
|
if v != nil {
|
||||||
|
return json.NewDecoder(resp.Body).Decode(v)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// patch updates a record.
|
||||||
|
func (c *pbClient) patch(ctx context.Context, path string, payload any) error {
|
||||||
|
b, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("pb: marshal: %w", err)
|
||||||
|
}
|
||||||
|
resp, err := c.do(ctx, http.MethodPatch, path, bytes.NewReader(b))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
raw, _ := io.ReadAll(resp.Body)
|
||||||
|
return fmt.Errorf("pb PATCH %s: status %d: %s", path, resp.StatusCode, string(raw))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// delete removes a record.
|
||||||
|
func (c *pbClient) delete(ctx context.Context, path string) error {
|
||||||
|
resp, err := c.do(ctx, http.MethodDelete, path, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode == http.StatusNotFound {
|
||||||
|
return ErrNotFound
|
||||||
|
}
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
raw, _ := io.ReadAll(resp.Body)
|
||||||
|
return fmt.Errorf("pb DELETE %s: status %d: %s", path, resp.StatusCode, string(raw))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// listAll fetches all pages of a collection. PocketBase returns at most 200
|
||||||
|
// records per page; we paginate until empty.
|
||||||
|
func (c *pbClient) listAll(ctx context.Context, collection string, filter, sort string) ([]json.RawMessage, error) {
|
||||||
|
var all []json.RawMessage
|
||||||
|
page := 1
|
||||||
|
for {
|
||||||
|
q := url.Values{
|
||||||
|
"page": {fmt.Sprintf("%d", page)},
|
||||||
|
"perPage": {"200"},
|
||||||
|
}
|
||||||
|
if filter != "" {
|
||||||
|
q.Set("filter", filter)
|
||||||
|
}
|
||||||
|
if sort != "" {
|
||||||
|
q.Set("sort", sort)
|
||||||
|
}
|
||||||
|
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Items []json.RawMessage `json:"items"`
|
||||||
|
Page int `json:"page"`
|
||||||
|
Pages int `json:"totalPages"`
|
||||||
|
}
|
||||||
|
if err := c.get(ctx, path, &result); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
all = append(all, result.Items...)
|
||||||
|
if result.Page >= result.Pages {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
page++
|
||||||
|
}
|
||||||
|
return all, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// claimRecord atomically claims the first pending record matching collection.
|
||||||
|
// It fetches the oldest pending record (filter + sort), then PATCHes it with
|
||||||
|
// the claim payload. Returns (nil, nil) when the queue is empty.
|
||||||
|
func (c *pbClient) claimRecord(ctx context.Context, collection, workerID string, extraClaim map[string]any) (json.RawMessage, error) {
|
||||||
|
q := url.Values{}
|
||||||
|
q.Set("filter", `status="pending"`)
|
||||||
|
q.Set("sort", "+started")
|
||||||
|
q.Set("perPage", "1")
|
||||||
|
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Items []json.RawMessage `json:"items"`
|
||||||
|
}
|
||||||
|
if err := c.get(ctx, path, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("claimRecord list: %w", err)
|
||||||
|
}
|
||||||
|
if len(result.Items) == 0 {
|
||||||
|
return nil, nil // queue empty
|
||||||
|
}
|
||||||
|
|
||||||
|
var rec struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(result.Items[0], &rec); err != nil {
|
||||||
|
return nil, fmt.Errorf("claimRecord parse id: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
claim := map[string]any{
|
||||||
|
"status": string(domain.TaskStatusRunning),
|
||||||
|
"worker_id": workerID,
|
||||||
|
}
|
||||||
|
for k, v := range extraClaim {
|
||||||
|
claim[k] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
claimPath := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID)
|
||||||
|
if err := c.patch(ctx, claimPath, claim); err != nil {
|
||||||
|
return nil, fmt.Errorf("claimRecord patch: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-fetch the updated record so caller has current state.
|
||||||
|
var updated json.RawMessage
|
||||||
|
if err := c.get(ctx, claimPath, &updated); err != nil {
|
||||||
|
return nil, fmt.Errorf("claimRecord re-fetch: %w", err)
|
||||||
|
}
|
||||||
|
return updated, nil
|
||||||
|
}
|
||||||
790
backend/internal/storage/store.go
Normal file
790
backend/internal/storage/store.go
Normal file
@@ -0,0 +1,790 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/bookstore"
|
||||||
|
"github.com/libnovel/backend/internal/config"
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
"github.com/libnovel/backend/internal/taskqueue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Store is the unified persistence implementation that satisfies all bookstore
|
||||||
|
// and taskqueue interfaces. It routes structured data to PocketBase and binary
|
||||||
|
// blobs to MinIO.
|
||||||
|
type Store struct {
|
||||||
|
pb *pbClient
|
||||||
|
mc *minioClient
|
||||||
|
log *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStore initialises PocketBase and MinIO connections and ensures all MinIO
|
||||||
|
// buckets exist. Returns a ready-to-use Store.
|
||||||
|
func NewStore(ctx context.Context, cfg config.Config, log *slog.Logger) (*Store, error) {
|
||||||
|
pb := newPBClient(cfg.PocketBase, log)
|
||||||
|
// Validate PocketBase connectivity by fetching an auth token.
|
||||||
|
if _, err := pb.authToken(ctx); err != nil {
|
||||||
|
return nil, fmt.Errorf("pocketbase: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mc, err := newMinioClient(cfg.MinIO)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("minio: %w", err)
|
||||||
|
}
|
||||||
|
if err := mc.ensureBuckets(ctx); err != nil {
|
||||||
|
return nil, fmt.Errorf("minio: ensure buckets: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Store{pb: pb, mc: mc, log: log}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compile-time interface satisfaction.
|
||||||
|
var _ bookstore.BookWriter = (*Store)(nil)
|
||||||
|
var _ bookstore.BookReader = (*Store)(nil)
|
||||||
|
var _ bookstore.RankingStore = (*Store)(nil)
|
||||||
|
var _ bookstore.AudioStore = (*Store)(nil)
|
||||||
|
var _ bookstore.PresignStore = (*Store)(nil)
|
||||||
|
var _ bookstore.ProgressStore = (*Store)(nil)
|
||||||
|
var _ bookstore.BrowseStore = (*Store)(nil)
|
||||||
|
var _ taskqueue.Producer = (*Store)(nil)
|
||||||
|
var _ taskqueue.Consumer = (*Store)(nil)
|
||||||
|
var _ taskqueue.Reader = (*Store)(nil)
|
||||||
|
|
||||||
|
// ── BookWriter ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (s *Store) WriteMetadata(ctx context.Context, meta domain.BookMeta) error {
|
||||||
|
payload := map[string]any{
|
||||||
|
"slug": meta.Slug,
|
||||||
|
"title": meta.Title,
|
||||||
|
"author": meta.Author,
|
||||||
|
"cover": meta.Cover,
|
||||||
|
"status": meta.Status,
|
||||||
|
"genres": meta.Genres,
|
||||||
|
"summary": meta.Summary,
|
||||||
|
"total_chapters": meta.TotalChapters,
|
||||||
|
"source_url": meta.SourceURL,
|
||||||
|
"ranking": meta.Ranking,
|
||||||
|
}
|
||||||
|
// Upsert via filter: if exists PATCH, otherwise POST.
|
||||||
|
existing, err := s.getBookBySlug(ctx, meta.Slug)
|
||||||
|
if err != nil && err != ErrNotFound {
|
||||||
|
return fmt.Errorf("WriteMetadata: %w", err)
|
||||||
|
}
|
||||||
|
if err == ErrNotFound {
|
||||||
|
return s.pb.post(ctx, "/api/collections/books/records", payload, nil)
|
||||||
|
}
|
||||||
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", existing.ID), payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error {
|
||||||
|
key := ChapterObjectKey(slug, chapter.Ref.Number)
|
||||||
|
if err := s.mc.putObject(ctx, s.mc.bucketChapters, key, "text/markdown", []byte(chapter.Text)); err != nil {
|
||||||
|
return fmt.Errorf("WriteChapter: minio: %w", err)
|
||||||
|
}
|
||||||
|
// Upsert the chapters_idx record in PocketBase.
|
||||||
|
return s.upsertChapterIdx(ctx, slug, chapter.Ref)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error {
|
||||||
|
for _, ref := range refs {
|
||||||
|
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
|
||||||
|
s.log.Warn("WriteChapterRefs: upsert failed", "slug", slug, "chapter", ref.Number, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool {
|
||||||
|
return s.mc.objectExists(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, ref.Number))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) upsertChapterIdx(ctx context.Context, slug string, ref domain.ChapterRef) error {
|
||||||
|
payload := map[string]any{
|
||||||
|
"slug": slug,
|
||||||
|
"number": ref.Number,
|
||||||
|
"title": ref.Title,
|
||||||
|
}
|
||||||
|
filter := fmt.Sprintf(`slug=%q&&number=%d`, slug, ref.Number)
|
||||||
|
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "")
|
||||||
|
if err != nil && err != ErrNotFound {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(items) == 0 {
|
||||||
|
return s.pb.post(ctx, "/api/collections/chapters_idx/records", payload, nil)
|
||||||
|
}
|
||||||
|
var rec struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
json.Unmarshal(items[0], &rec)
|
||||||
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/chapters_idx/records/%s", rec.ID), payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── BookReader ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
type pbBook struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Author string `json:"author"`
|
||||||
|
Cover string `json:"cover"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Genres []string `json:"genres"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
TotalChapters int `json:"total_chapters"`
|
||||||
|
SourceURL string `json:"source_url"`
|
||||||
|
Ranking int `json:"ranking"`
|
||||||
|
Updated string `json:"updated"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b pbBook) toDomain() domain.BookMeta {
|
||||||
|
return domain.BookMeta{
|
||||||
|
Slug: b.Slug,
|
||||||
|
Title: b.Title,
|
||||||
|
Author: b.Author,
|
||||||
|
Cover: b.Cover,
|
||||||
|
Status: b.Status,
|
||||||
|
Genres: b.Genres,
|
||||||
|
Summary: b.Summary,
|
||||||
|
TotalChapters: b.TotalChapters,
|
||||||
|
SourceURL: b.SourceURL,
|
||||||
|
Ranking: b.Ranking,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) getBookBySlug(ctx context.Context, slug string) (pbBook, error) {
|
||||||
|
filter := fmt.Sprintf(`slug=%q`, slug)
|
||||||
|
items, err := s.pb.listAll(ctx, "books", filter, "")
|
||||||
|
if err != nil {
|
||||||
|
return pbBook{}, err
|
||||||
|
}
|
||||||
|
if len(items) == 0 {
|
||||||
|
return pbBook{}, ErrNotFound
|
||||||
|
}
|
||||||
|
var b pbBook
|
||||||
|
json.Unmarshal(items[0], &b)
|
||||||
|
return b, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error) {
|
||||||
|
b, err := s.getBookBySlug(ctx, slug)
|
||||||
|
if err == ErrNotFound {
|
||||||
|
return domain.BookMeta{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return domain.BookMeta{}, false, err
|
||||||
|
}
|
||||||
|
return b.toDomain(), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ListBooks(ctx context.Context) ([]domain.BookMeta, error) {
|
||||||
|
items, err := s.pb.listAll(ctx, "books", "", "title")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
books := make([]domain.BookMeta, 0, len(items))
|
||||||
|
for _, raw := range items {
|
||||||
|
var b pbBook
|
||||||
|
json.Unmarshal(raw, &b)
|
||||||
|
books = append(books, b.toDomain())
|
||||||
|
}
|
||||||
|
return books, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) LocalSlugs(ctx context.Context) (map[string]bool, error) {
|
||||||
|
items, err := s.pb.listAll(ctx, "books", "", "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
slugs := make(map[string]bool, len(items))
|
||||||
|
for _, raw := range items {
|
||||||
|
var b struct {
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
}
|
||||||
|
json.Unmarshal(raw, &b)
|
||||||
|
if b.Slug != "" {
|
||||||
|
slugs[b.Slug] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return slugs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) MetadataMtime(ctx context.Context, slug string) int64 {
|
||||||
|
b, err := s.getBookBySlug(ctx, slug)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
t, err := time.Parse(time.RFC3339, b.Updated)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return t.Unix()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
|
||||||
|
data, err := s.mc.getObject(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n))
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("ReadChapter: %w", err)
|
||||||
|
}
|
||||||
|
return string(data), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error) {
|
||||||
|
filter := fmt.Sprintf(`slug=%q`, slug)
|
||||||
|
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "number")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
chapters := make([]domain.ChapterInfo, 0, len(items))
|
||||||
|
for _, raw := range items {
|
||||||
|
var rec struct {
|
||||||
|
Number int `json:"number"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
}
|
||||||
|
json.Unmarshal(raw, &rec)
|
||||||
|
chapters = append(chapters, domain.ChapterInfo{Number: rec.Number, Title: rec.Title})
|
||||||
|
}
|
||||||
|
return chapters, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) CountChapters(ctx context.Context, slug string) int {
|
||||||
|
chapters, err := s.ListChapters(ctx, slug)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return len(chapters)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ReindexChapters(ctx context.Context, slug string) (int, error) {
|
||||||
|
keys, err := s.mc.listObjectKeys(ctx, s.mc.bucketChapters, slug+"/")
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("ReindexChapters: list objects: %w", err)
|
||||||
|
}
|
||||||
|
count := 0
|
||||||
|
for _, key := range keys {
|
||||||
|
if !strings.HasSuffix(key, ".md") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
n := chapterNumberFromKey(key)
|
||||||
|
if n == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ref := domain.ChapterRef{Number: n}
|
||||||
|
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
|
||||||
|
s.log.Warn("ReindexChapters: upsert failed", "key", key, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── RankingStore ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (s *Store) WriteRankingItem(ctx context.Context, item domain.RankingItem) error {
|
||||||
|
payload := map[string]any{
|
||||||
|
"rank": item.Rank,
|
||||||
|
"slug": item.Slug,
|
||||||
|
"title": item.Title,
|
||||||
|
"author": item.Author,
|
||||||
|
"cover": item.Cover,
|
||||||
|
"status": item.Status,
|
||||||
|
"genres": item.Genres,
|
||||||
|
"source_url": item.SourceURL,
|
||||||
|
}
|
||||||
|
filter := fmt.Sprintf(`slug=%q`, item.Slug)
|
||||||
|
items, err := s.pb.listAll(ctx, "ranking", filter, "")
|
||||||
|
if err != nil && err != ErrNotFound {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(items) == 0 {
|
||||||
|
return s.pb.post(ctx, "/api/collections/ranking/records", payload, nil)
|
||||||
|
}
|
||||||
|
var rec struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
json.Unmarshal(items[0], &rec)
|
||||||
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/ranking/records/%s", rec.ID), payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error) {
|
||||||
|
items, err := s.pb.listAll(ctx, "ranking", "", "rank")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
result := make([]domain.RankingItem, 0, len(items))
|
||||||
|
for _, raw := range items {
|
||||||
|
var rec struct {
|
||||||
|
Rank int `json:"rank"`
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Author string `json:"author"`
|
||||||
|
Cover string `json:"cover"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Genres []string `json:"genres"`
|
||||||
|
SourceURL string `json:"source_url"`
|
||||||
|
Updated string `json:"updated"`
|
||||||
|
}
|
||||||
|
json.Unmarshal(raw, &rec)
|
||||||
|
t, _ := time.Parse(time.RFC3339, rec.Updated)
|
||||||
|
result = append(result, domain.RankingItem{
|
||||||
|
Rank: rec.Rank,
|
||||||
|
Slug: rec.Slug,
|
||||||
|
Title: rec.Title,
|
||||||
|
Author: rec.Author,
|
||||||
|
Cover: rec.Cover,
|
||||||
|
Status: rec.Status,
|
||||||
|
Genres: rec.Genres,
|
||||||
|
SourceURL: rec.SourceURL,
|
||||||
|
Updated: t,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) {
|
||||||
|
items, err := s.ReadRankingItems(ctx)
|
||||||
|
if err != nil || len(items) == 0 {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
var latest time.Time
|
||||||
|
for _, item := range items {
|
||||||
|
if item.Updated.After(latest) {
|
||||||
|
latest = item.Updated
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return time.Since(latest) < maxAge, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── AudioStore ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (s *Store) AudioObjectKey(slug string, n int, voice string) string {
|
||||||
|
return AudioObjectKey(slug, n, voice)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) AudioExists(ctx context.Context, key string) bool {
|
||||||
|
return s.mc.objectExists(ctx, s.mc.bucketAudio, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) PutAudio(ctx context.Context, key string, data []byte) error {
|
||||||
|
return s.mc.putObject(ctx, s.mc.bucketAudio, key, "audio/mpeg", data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── PresignStore ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (s *Store) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) {
|
||||||
|
return s.mc.presignGet(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n), expires)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
|
||||||
|
return s.mc.presignGet(ctx, s.mc.bucketAudio, key, expires)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error) {
|
||||||
|
key = AvatarObjectKey(userID, ext)
|
||||||
|
uploadURL, err = s.mc.presignPut(ctx, s.mc.bucketAvatars, key, 15*time.Minute)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) {
|
||||||
|
for _, ext := range []string{"jpg", "png", "webp"} {
|
||||||
|
key := AvatarObjectKey(userID, ext)
|
||||||
|
if s.mc.objectExists(ctx, s.mc.bucketAvatars, key) {
|
||||||
|
u, err := s.mc.presignGet(ctx, s.mc.bucketAvatars, key, 1*time.Hour)
|
||||||
|
return u, true, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) DeleteAvatar(ctx context.Context, userID string) error {
|
||||||
|
return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── ProgressStore ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (s *Store) GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool) {
|
||||||
|
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
|
||||||
|
items, err := s.pb.listAll(ctx, "progress", filter, "")
|
||||||
|
if err != nil || len(items) == 0 {
|
||||||
|
return domain.ReadingProgress{}, false
|
||||||
|
}
|
||||||
|
var rec struct {
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Chapter int `json:"chapter"`
|
||||||
|
UpdatedAt string `json:"updated"`
|
||||||
|
}
|
||||||
|
json.Unmarshal(items[0], &rec)
|
||||||
|
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
|
||||||
|
return domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error {
|
||||||
|
payload := map[string]any{
|
||||||
|
"session_id": sessionID,
|
||||||
|
"slug": p.Slug,
|
||||||
|
"chapter": p.Chapter,
|
||||||
|
}
|
||||||
|
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, p.Slug)
|
||||||
|
items, err := s.pb.listAll(ctx, "progress", filter, "")
|
||||||
|
if err != nil && err != ErrNotFound {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(items) == 0 {
|
||||||
|
return s.pb.post(ctx, "/api/collections/progress/records", payload, nil)
|
||||||
|
}
|
||||||
|
var rec struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
json.Unmarshal(items[0], &rec)
|
||||||
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID), payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error) {
|
||||||
|
filter := fmt.Sprintf(`session_id=%q`, sessionID)
|
||||||
|
items, err := s.pb.listAll(ctx, "progress", filter, "-updated")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
result := make([]domain.ReadingProgress, 0, len(items))
|
||||||
|
for _, raw := range items {
|
||||||
|
var rec struct {
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Chapter int `json:"chapter"`
|
||||||
|
UpdatedAt string `json:"updated"`
|
||||||
|
}
|
||||||
|
json.Unmarshal(raw, &rec)
|
||||||
|
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
|
||||||
|
result = append(result, domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t})
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) DeleteProgress(ctx context.Context, sessionID, slug string) error {
|
||||||
|
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
|
||||||
|
items, err := s.pb.listAll(ctx, "progress", filter, "")
|
||||||
|
if err != nil || len(items) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var rec struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
json.Unmarshal(items[0], &rec)
|
||||||
|
return s.pb.delete(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── taskqueue.Producer ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (s *Store) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) {
|
||||||
|
payload := map[string]any{
|
||||||
|
"kind": kind,
|
||||||
|
"target_url": targetURL,
|
||||||
|
"from_chapter": fromChapter,
|
||||||
|
"to_chapter": toChapter,
|
||||||
|
"status": string(domain.TaskStatusPending),
|
||||||
|
"started": time.Now().UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
var rec struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
if err := s.pb.post(ctx, "/api/collections/scraping_tasks/records", payload, &rec); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return rec.ID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) {
|
||||||
|
cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, voice)
|
||||||
|
payload := map[string]any{
|
||||||
|
"cache_key": cacheKey,
|
||||||
|
"slug": slug,
|
||||||
|
"chapter": chapter,
|
||||||
|
"voice": voice,
|
||||||
|
"status": string(domain.TaskStatusPending),
|
||||||
|
"started": time.Now().UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
var rec struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
if err := s.pb.post(ctx, "/api/collections/audio_jobs/records", payload, &rec); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return rec.ID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) CancelTask(ctx context.Context, id string) error {
|
||||||
|
// Try scraping_tasks first, then audio_jobs.
|
||||||
|
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),
|
||||||
|
map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id),
|
||||||
|
map[string]string{"status": string(domain.TaskStatusCancelled)})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── taskqueue.Consumer ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (s *Store) ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error) {
|
||||||
|
raw, err := s.pb.claimRecord(ctx, "scraping_tasks", workerID, nil)
|
||||||
|
if err != nil {
|
||||||
|
return domain.ScrapeTask{}, false, err
|
||||||
|
}
|
||||||
|
if raw == nil {
|
||||||
|
return domain.ScrapeTask{}, false, nil
|
||||||
|
}
|
||||||
|
task, err := parseScrapeTask(raw)
|
||||||
|
return task, err == nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error) {
|
||||||
|
raw, err := s.pb.claimRecord(ctx, "audio_jobs", workerID, nil)
|
||||||
|
if err != nil {
|
||||||
|
return domain.AudioTask{}, false, err
|
||||||
|
}
|
||||||
|
if raw == nil {
|
||||||
|
return domain.AudioTask{}, false, nil
|
||||||
|
}
|
||||||
|
task, err := parseAudioTask(raw)
|
||||||
|
return task, err == nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
|
||||||
|
status := string(domain.TaskStatusDone)
|
||||||
|
if result.ErrorMessage != "" {
|
||||||
|
status = string(domain.TaskStatusFailed)
|
||||||
|
}
|
||||||
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), map[string]any{
|
||||||
|
"status": status,
|
||||||
|
"books_found": result.BooksFound,
|
||||||
|
"chapters_scraped": result.ChaptersScraped,
|
||||||
|
"chapters_skipped": result.ChaptersSkipped,
|
||||||
|
"errors": result.Errors,
|
||||||
|
"error_message": result.ErrorMessage,
|
||||||
|
"finished": time.Now().UTC().Format(time.RFC3339),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error {
|
||||||
|
status := string(domain.TaskStatusDone)
|
||||||
|
if result.ErrorMessage != "" {
|
||||||
|
status = string(domain.TaskStatusFailed)
|
||||||
|
}
|
||||||
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), map[string]any{
|
||||||
|
"status": status,
|
||||||
|
"error_message": result.ErrorMessage,
|
||||||
|
"finished": time.Now().UTC().Format(time.RFC3339),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
|
||||||
|
payload := map[string]any{
|
||||||
|
"status": string(domain.TaskStatusFailed),
|
||||||
|
"error_message": errMsg,
|
||||||
|
"finished": time.Now().UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeartbeatTask updates the heartbeat_at field on a running task.
|
||||||
|
// Tries scraping_tasks first, then audio_jobs (same pattern as FailTask).
|
||||||
|
func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
|
||||||
|
payload := map[string]any{
|
||||||
|
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing
|
||||||
|
// or older than staleAfter, and resets them to pending so they can be
|
||||||
|
// re-claimed. Returns the number of tasks reaped.
|
||||||
|
func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) {
|
||||||
|
threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339)
|
||||||
|
// Match tasks that are running AND (heartbeat_at is null OR heartbeat_at < threshold).
|
||||||
|
// PocketBase datetime fields require `=null` not `=""` in filter expressions.
|
||||||
|
filter := fmt.Sprintf(`status="running"&&(heartbeat_at=null||heartbeat_at<"%s")`, threshold)
|
||||||
|
resetPayload := map[string]any{
|
||||||
|
"status": string(domain.TaskStatusPending),
|
||||||
|
"worker_id": "",
|
||||||
|
"heartbeat_at": nil,
|
||||||
|
}
|
||||||
|
|
||||||
|
total := 0
|
||||||
|
for _, collection := range []string{"scraping_tasks", "audio_jobs"} {
|
||||||
|
items, err := s.pb.listAll(ctx, collection, filter, "")
|
||||||
|
if err != nil {
|
||||||
|
return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err)
|
||||||
|
}
|
||||||
|
for _, raw := range items {
|
||||||
|
var rec struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &rec); err != nil || rec.ID == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
path := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID)
|
||||||
|
if err := s.pb.patch(ctx, path, resetPayload); err != nil {
|
||||||
|
s.log.Warn("ReapStaleTasks: patch failed", "collection", collection, "id", rec.ID, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
total++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return total, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── taskqueue.Reader ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (s *Store) ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error) {
|
||||||
|
items, err := s.pb.listAll(ctx, "scraping_tasks", "", "-started")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
tasks := make([]domain.ScrapeTask, 0, len(items))
|
||||||
|
for _, raw := range items {
|
||||||
|
t, err := parseScrapeTask(raw)
|
||||||
|
if err == nil {
|
||||||
|
tasks = append(tasks, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tasks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error) {
|
||||||
|
var raw json.RawMessage
|
||||||
|
if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), &raw); err != nil {
|
||||||
|
if err == ErrNotFound {
|
||||||
|
return domain.ScrapeTask{}, false, nil
|
||||||
|
}
|
||||||
|
return domain.ScrapeTask{}, false, err
|
||||||
|
}
|
||||||
|
t, err := parseScrapeTask(raw)
|
||||||
|
return t, err == nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error) {
|
||||||
|
items, err := s.pb.listAll(ctx, "audio_jobs", "", "-started")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
tasks := make([]domain.AudioTask, 0, len(items))
|
||||||
|
for _, raw := range items {
|
||||||
|
t, err := parseAudioTask(raw)
|
||||||
|
if err == nil {
|
||||||
|
tasks = append(tasks, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tasks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) {
|
||||||
|
filter := fmt.Sprintf(`cache_key=%q`, cacheKey)
|
||||||
|
items, err := s.pb.listAll(ctx, "audio_jobs", filter, "-started")
|
||||||
|
if err != nil || len(items) == 0 {
|
||||||
|
return domain.AudioTask{}, false, err
|
||||||
|
}
|
||||||
|
t, err := parseAudioTask(items[0])
|
||||||
|
return t, err == nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Parsers ───────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) {
|
||||||
|
var rec struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
TargetURL string `json:"target_url"`
|
||||||
|
FromChapter int `json:"from_chapter"`
|
||||||
|
ToChapter int `json:"to_chapter"`
|
||||||
|
WorkerID string `json:"worker_id"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
BooksFound int `json:"books_found"`
|
||||||
|
ChaptersScraped int `json:"chapters_scraped"`
|
||||||
|
ChaptersSkipped int `json:"chapters_skipped"`
|
||||||
|
Errors int `json:"errors"`
|
||||||
|
Started string `json:"started"`
|
||||||
|
Finished string `json:"finished"`
|
||||||
|
ErrorMessage string `json:"error_message"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &rec); err != nil {
|
||||||
|
return domain.ScrapeTask{}, err
|
||||||
|
}
|
||||||
|
started, _ := time.Parse(time.RFC3339, rec.Started)
|
||||||
|
finished, _ := time.Parse(time.RFC3339, rec.Finished)
|
||||||
|
return domain.ScrapeTask{
|
||||||
|
ID: rec.ID,
|
||||||
|
Kind: rec.Kind,
|
||||||
|
TargetURL: rec.TargetURL,
|
||||||
|
FromChapter: rec.FromChapter,
|
||||||
|
ToChapter: rec.ToChapter,
|
||||||
|
WorkerID: rec.WorkerID,
|
||||||
|
Status: domain.TaskStatus(rec.Status),
|
||||||
|
BooksFound: rec.BooksFound,
|
||||||
|
ChaptersScraped: rec.ChaptersScraped,
|
||||||
|
ChaptersSkipped: rec.ChaptersSkipped,
|
||||||
|
Errors: rec.Errors,
|
||||||
|
Started: started,
|
||||||
|
Finished: finished,
|
||||||
|
ErrorMessage: rec.ErrorMessage,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
|
||||||
|
var rec struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
CacheKey string `json:"cache_key"`
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Chapter int `json:"chapter"`
|
||||||
|
Voice string `json:"voice"`
|
||||||
|
WorkerID string `json:"worker_id"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
ErrorMessage string `json:"error_message"`
|
||||||
|
Started string `json:"started"`
|
||||||
|
Finished string `json:"finished"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &rec); err != nil {
|
||||||
|
return domain.AudioTask{}, err
|
||||||
|
}
|
||||||
|
started, _ := time.Parse(time.RFC3339, rec.Started)
|
||||||
|
finished, _ := time.Parse(time.RFC3339, rec.Finished)
|
||||||
|
return domain.AudioTask{
|
||||||
|
ID: rec.ID,
|
||||||
|
CacheKey: rec.CacheKey,
|
||||||
|
Slug: rec.Slug,
|
||||||
|
Chapter: rec.Chapter,
|
||||||
|
Voice: rec.Voice,
|
||||||
|
WorkerID: rec.WorkerID,
|
||||||
|
Status: domain.TaskStatus(rec.Status),
|
||||||
|
ErrorMessage: rec.ErrorMessage,
|
||||||
|
Started: started,
|
||||||
|
Finished: finished,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── BrowseStore ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error {
|
||||||
|
key := BrowseObjectKey(genre, sort, status, novelType, page)
|
||||||
|
if err := s.mc.putBrowse(ctx, key, data); err != nil {
|
||||||
|
return fmt.Errorf("PutBrowsePage: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) {
|
||||||
|
key := BrowseObjectKey(genre, sort, status, novelType, page)
|
||||||
|
data, ok, err := s.mc.getBrowse(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("GetBrowsePage: %w", err)
|
||||||
|
}
|
||||||
|
return data, ok, nil
|
||||||
|
}
|
||||||
84
backend/internal/taskqueue/taskqueue.go
Normal file
84
backend/internal/taskqueue/taskqueue.go
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
// Package taskqueue defines the interfaces for creating and consuming
|
||||||
|
// scrape/audio tasks stored in PocketBase.
|
||||||
|
//
|
||||||
|
// Interface segregation:
|
||||||
|
// - Producer is used only by the backend (creates tasks, cancels tasks).
|
||||||
|
// - Consumer is used only by the runner (claims tasks, reports results).
|
||||||
|
// - Reader is used by the backend for status/history endpoints.
|
||||||
|
//
|
||||||
|
// Concrete implementations live in internal/storage.
|
||||||
|
package taskqueue
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Producer is the write side of the task queue used by the backend service.
|
||||||
|
// It creates new tasks in PocketBase for the runner to pick up.
|
||||||
|
type Producer interface {
|
||||||
|
// CreateScrapeTask inserts a new scrape task with status=pending and
|
||||||
|
// returns the assigned PocketBase record ID.
|
||||||
|
// kind is one of "catalogue", "book", or "book_range".
|
||||||
|
// targetURL is the book URL (empty for catalogue-wide tasks).
|
||||||
|
CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error)
|
||||||
|
|
||||||
|
// CreateAudioTask inserts a new audio task with status=pending and
|
||||||
|
// returns the assigned PocketBase record ID.
|
||||||
|
CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error)
|
||||||
|
|
||||||
|
// CancelTask transitions a pending task to status=cancelled.
|
||||||
|
// Returns ErrNotFound if the task does not exist.
|
||||||
|
CancelTask(ctx context.Context, id string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Consumer is the read/claim side of the task queue used by the runner.
|
||||||
|
type Consumer interface {
|
||||||
|
// ClaimNextScrapeTask atomically finds the oldest pending scrape task,
|
||||||
|
// sets its status=running and worker_id=workerID, and returns it.
|
||||||
|
// Returns (zero, false, nil) when the queue is empty.
|
||||||
|
ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error)
|
||||||
|
|
||||||
|
// ClaimNextAudioTask atomically finds the oldest pending audio task,
|
||||||
|
// sets its status=running and worker_id=workerID, and returns it.
|
||||||
|
// Returns (zero, false, nil) when the queue is empty.
|
||||||
|
ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error)
|
||||||
|
|
||||||
|
// FinishScrapeTask marks a running scrape task as done and records the result.
|
||||||
|
FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error
|
||||||
|
|
||||||
|
// FinishAudioTask marks a running audio task as done and records the result.
|
||||||
|
FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error
|
||||||
|
|
||||||
|
// FailTask marks a task (scrape or audio) as failed with an error message.
|
||||||
|
FailTask(ctx context.Context, id, errMsg string) error
|
||||||
|
|
||||||
|
// HeartbeatTask updates the heartbeat_at timestamp on a running task.
|
||||||
|
// Should be called periodically by the runner while the task is active so
|
||||||
|
// the reaper knows the task is still alive.
|
||||||
|
HeartbeatTask(ctx context.Context, id string) error
|
||||||
|
|
||||||
|
// ReapStaleTasks finds all running tasks whose heartbeat_at is older than
|
||||||
|
// staleAfter (or was never set) and resets them to pending so they can be
|
||||||
|
// re-claimed by a healthy runner. Returns the number of tasks reaped.
|
||||||
|
ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reader is the read-only side used by the backend for status pages.
|
||||||
|
type Reader interface {
|
||||||
|
// ListScrapeTasks returns all scrape tasks sorted by started descending.
|
||||||
|
ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error)
|
||||||
|
|
||||||
|
// GetScrapeTask returns a single scrape task by ID.
|
||||||
|
// Returns (zero, false, nil) if not found.
|
||||||
|
GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error)
|
||||||
|
|
||||||
|
// ListAudioTasks returns all audio tasks sorted by started descending.
|
||||||
|
ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error)
|
||||||
|
|
||||||
|
// GetAudioTask returns the most recent audio task for cacheKey.
|
||||||
|
// Returns (zero, false, nil) if not found.
|
||||||
|
GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error)
|
||||||
|
}
|
||||||
138
backend/internal/taskqueue/taskqueue_test.go
Normal file
138
backend/internal/taskqueue/taskqueue_test.go
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
package taskqueue_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/libnovel/backend/internal/domain"
|
||||||
|
"github.com/libnovel/backend/internal/taskqueue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ── Compile-time interface satisfaction ───────────────────────────────────────
|
||||||
|
|
||||||
|
// stubStore satisfies all three taskqueue interfaces.
|
||||||
|
// Any method that is called but not expected panics — making accidental
|
||||||
|
// calls immediately visible in tests.
|
||||||
|
type stubStore struct{}
|
||||||
|
|
||||||
|
func (s *stubStore) CreateScrapeTask(_ context.Context, _, _ string, _, _ int) (string, error) {
|
||||||
|
return "task-1", nil
|
||||||
|
}
|
||||||
|
func (s *stubStore) CreateAudioTask(_ context.Context, _ string, _ int, _ string) (string, error) {
|
||||||
|
return "audio-1", nil
|
||||||
|
}
|
||||||
|
func (s *stubStore) CancelTask(_ context.Context, _ string) error { return nil }
|
||||||
|
|
||||||
|
func (s *stubStore) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
|
||||||
|
return domain.ScrapeTask{ID: "task-1", Status: domain.TaskStatusRunning}, true, nil
|
||||||
|
}
|
||||||
|
func (s *stubStore) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
|
||||||
|
return domain.AudioTask{ID: "audio-1", Status: domain.TaskStatusRunning}, true, nil
|
||||||
|
}
|
||||||
|
func (s *stubStore) FinishScrapeTask(_ context.Context, _ string, _ domain.ScrapeResult) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (s *stubStore) FinishAudioTask(_ context.Context, _ string, _ domain.AudioResult) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (s *stubStore) FailTask(_ context.Context, _, _ string) error { return nil }
|
||||||
|
|
||||||
|
func (s *stubStore) HeartbeatTask(_ context.Context, _ string) error { return nil }
|
||||||
|
|
||||||
|
func (s *stubStore) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stubStore) ListScrapeTasks(_ context.Context) ([]domain.ScrapeTask, error) { return nil, nil }
|
||||||
|
func (s *stubStore) GetScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
|
||||||
|
return domain.ScrapeTask{}, false, nil
|
||||||
|
}
|
||||||
|
func (s *stubStore) ListAudioTasks(_ context.Context) ([]domain.AudioTask, error) { return nil, nil }
|
||||||
|
func (s *stubStore) GetAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
|
||||||
|
return domain.AudioTask{}, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify the stub satisfies all three interfaces at compile time.
|
||||||
|
var _ taskqueue.Producer = (*stubStore)(nil)
|
||||||
|
var _ taskqueue.Consumer = (*stubStore)(nil)
|
||||||
|
var _ taskqueue.Reader = (*stubStore)(nil)
|
||||||
|
|
||||||
|
// ── Behavioural tests (using stub) ────────────────────────────────────────────
|
||||||
|
|
||||||
|
func TestProducer_CreateScrapeTask(t *testing.T) {
|
||||||
|
var p taskqueue.Producer = &stubStore{}
|
||||||
|
id, err := p.CreateScrapeTask(context.Background(), "book", "https://example.com/book/slug", 0, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if id == "" {
|
||||||
|
t.Error("expected non-empty task ID")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsumer_ClaimNextScrapeTask(t *testing.T) {
|
||||||
|
var c taskqueue.Consumer = &stubStore{}
|
||||||
|
task, ok, err := c.ClaimNextScrapeTask(context.Background(), "worker-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("expected a task to be claimed")
|
||||||
|
}
|
||||||
|
if task.Status != domain.TaskStatusRunning {
|
||||||
|
t.Errorf("want running, got %q", task.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConsumer_ClaimNextAudioTask(t *testing.T) {
|
||||||
|
var c taskqueue.Consumer = &stubStore{}
|
||||||
|
task, ok, err := c.ClaimNextAudioTask(context.Background(), "worker-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("expected an audio task to be claimed")
|
||||||
|
}
|
||||||
|
if task.ID == "" {
|
||||||
|
t.Error("expected non-empty task ID")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── domain.ScrapeResult / domain.AudioResult JSON shape ──────────────────────
|
||||||
|
|
||||||
|
func TestScrapeResult_JSONRoundtrip(t *testing.T) {
|
||||||
|
cases := []domain.ScrapeResult{
|
||||||
|
{BooksFound: 5, ChaptersScraped: 100, ChaptersSkipped: 2, Errors: 0},
|
||||||
|
{BooksFound: 0, ChaptersScraped: 0, Errors: 1, ErrorMessage: "timeout"},
|
||||||
|
}
|
||||||
|
for _, orig := range cases {
|
||||||
|
b, err := json.Marshal(orig)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal: %v", err)
|
||||||
|
}
|
||||||
|
var got domain.ScrapeResult
|
||||||
|
if err := json.Unmarshal(b, &got); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
if got != orig {
|
||||||
|
t.Errorf("want %+v, got %+v", orig, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAudioResult_JSONRoundtrip(t *testing.T) {
|
||||||
|
cases := []domain.AudioResult{
|
||||||
|
{ObjectKey: "audio/slug/1/af_bella.mp3"},
|
||||||
|
{ErrorMessage: "kokoro unavailable"},
|
||||||
|
}
|
||||||
|
for _, orig := range cases {
|
||||||
|
b, _ := json.Marshal(orig)
|
||||||
|
var got domain.AudioResult
|
||||||
|
json.Unmarshal(b, &got)
|
||||||
|
if got != orig {
|
||||||
|
t.Errorf("want %+v, got %+v", orig, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
301
backend/todos.md
Normal file
301
backend/todos.md
Normal file
@@ -0,0 +1,301 @@
|
|||||||
|
# LibNovel Scraper Rewrite — Project Todos
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Split the monolithic scraper into two separate binaries inside the same Go module:
|
||||||
|
|
||||||
|
| Binary | Command | Location | Responsibility |
|
||||||
|
|--------|---------|----------|----------------|
|
||||||
|
| **runner** | `cmd/runner` | Homelab | Polls remote PB for pending scrape tasks → scrapes novelfire.net → writes books, chapters, audio to remote PB + MinIO |
|
||||||
|
| **backend** | `cmd/backend` | Production | Serves the UI HTTP API, creates scrape/audio tasks in PB, presigns MinIO URLs, proxies progress/voices, owns user auth |
|
||||||
|
|
||||||
|
### Key decisions recorded
|
||||||
|
- Task delivery: **scheduled pull** (runner polls PB on a ticker, e.g. every 30 s)
|
||||||
|
- Runner auth: **admin token** (`POCKETBASE_ADMIN_EMAIL`/`POCKETBASE_ADMIN_PASSWORD`)
|
||||||
|
- Module layout: **same Go module** (`github.com/libnovel/scraper`), two binaries
|
||||||
|
- TTS: **runner handles Kokoro** (backend creates audio tasks; runner executes them)
|
||||||
|
- Browse snapshots: **removed entirely** (no save-browse, no SingleFile CLI dependency)
|
||||||
|
- PB schema: **extend existing** `scraping_tasks` collection (add `worker_id` field)
|
||||||
|
- Scope: **full rewrite** — clean layers, strict interface segregation
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 0 — Module & Repo skeleton
|
||||||
|
|
||||||
|
### T-01 Restructure cmd/ layout
|
||||||
|
**Description**: Create `cmd/runner/main.go` and `cmd/backend/main.go` entry points. Remove the old `cmd/scraper/` entry point (or keep temporarily as a stub). Update `go.mod` module path if needed.
|
||||||
|
**Unit tests**: `cmd/runner/main_test.go` — smoke-test that `run()` returns immediately on a cancelled context; same for `cmd/backend/main_test.go`.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-02 Create shared `internal/config` package
|
||||||
|
**Description**: Replace the ad-hoc `envOr()` helpers scattered in main.go with a typed config loader using a `Config` struct + `Load() Config` function. Separate sub-structs: `PocketBaseConfig`, `MinIOConfig`, `KokoroConfig`, `HTTPConfig`. Each binary calls `config.Load()`.
|
||||||
|
**Unit tests**: `internal/config/config_test.go` — verify defaults, env override for each field, zero-value safety.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 1 — Core domain interfaces (interface segregation)
|
||||||
|
|
||||||
|
### T-03 Define `TaskQueue` interface (`internal/taskqueue`)
|
||||||
|
**Description**: Create a new package `internal/taskqueue` with two interfaces:
|
||||||
|
- `Producer` — used by the **backend** to create tasks:
|
||||||
|
```go
|
||||||
|
type Producer interface {
|
||||||
|
CreateScrapeTask(ctx, kind, targetURL string) (string, error)
|
||||||
|
CreateAudioTask(ctx, slug string, chapter int, voice string) (string, error)
|
||||||
|
CancelTask(ctx, id string) error
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- `Consumer` — used by the **runner** to poll and claim tasks:
|
||||||
|
```go
|
||||||
|
type Consumer interface {
|
||||||
|
ClaimNextScrapeTask(ctx context.Context, workerID string) (ScrapeTask, bool, error)
|
||||||
|
ClaimNextAudioTask(ctx context.Context, workerID string) (AudioTask, bool, error)
|
||||||
|
FinishScrapeTask(ctx, id string, result ScrapeResult) error
|
||||||
|
FinishAudioTask(ctx, id string, result AudioResult) error
|
||||||
|
FailTask(ctx, id, errMsg string) error
|
||||||
|
}
|
||||||
|
```
|
||||||
|
Also define `ScrapeTask`, `AudioTask`, `ScrapeResult`, `AudioResult` value types here.
|
||||||
|
**Unit tests**: `internal/taskqueue/taskqueue_test.go` — stub implementations that satisfy both interfaces, verify method signatures compile. Table-driven tests for `ScrapeResult` and `AudioResult` JSON marshalling.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-04 Define `BookStore` interface (`internal/bookstore`)
|
||||||
|
**Description**: Decompose the monolithic `storage.Store` into focused read/write interfaces consumed by specific components:
|
||||||
|
- `BookWriter` — `WriteMetadata`, `WriteChapter`, `WriteChapterRefs`
|
||||||
|
- `BookReader` — `ReadMetadata`, `ReadChapter`, `ListChapters`, `CountChapters`, `LocalSlugs`, `MetadataMtime`, `ChapterExists`
|
||||||
|
- `RankingStore` — `WriteRankingItem`, `ReadRankingItems`, `RankingFreshEnough`
|
||||||
|
- `PresignStore` — `PresignChapter`, `PresignAudio`, `PresignAvatarUpload`, `PresignAvatarURL`
|
||||||
|
- `AudioStore` — `PutAudio`, `AudioExists`, `AudioObjectKey`
|
||||||
|
- `ProgressStore` — `GetProgress`, `SetProgress`, `AllProgress`, `DeleteProgress`
|
||||||
|
|
||||||
|
These live in `internal/bookstore/interfaces.go`. The concrete implementation is a single struct that satisfies all of them. The runner only gets `BookWriter + RankingStore + AudioStore`. The backend only gets `BookReader + PresignStore + ProgressStore`.
|
||||||
|
**Unit tests**: `internal/bookstore/interfaces_test.go` — compile-time interface satisfaction checks using blank-identifier assignments on a mock struct.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-05 Rewrite `internal/scraper/interfaces.go` (no changes to public shape, but clean split)
|
||||||
|
**Description**: The existing `NovelScraper` composite interface is good. Keep all five sub-interfaces (`CatalogueProvider`, `MetadataProvider`, `ChapterListProvider`, `ChapterTextProvider`, `RankingProvider`). Ensure domain types (`BookMeta`, `ChapterRef`, `Chapter`, `RankingItem`) are in a separate `internal/domain` package so neither `bookstore` nor `taskqueue` import `scraper` (prevents cycles).
|
||||||
|
**Unit tests**: `internal/domain/domain_test.go` — JSON roundtrip tests for `BookMeta`, `ChapterRef`, `Chapter`, `RankingItem`.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 2 — Storage layer rewrite
|
||||||
|
|
||||||
|
### T-06 Rewrite `internal/storage/pocketbase.go`
|
||||||
|
**Description**: Clean rewrite of the PocketBase REST client. Must satisfy `taskqueue.Producer`, `taskqueue.Consumer`, and all `bookstore` interfaces. Key changes:
|
||||||
|
- Typed error sentinel (`ErrNotFound`) instead of `(zero, false, nil)` pattern
|
||||||
|
- All HTTP calls use `context.Context` and respect cancellation
|
||||||
|
- `ClaimNextScrapeTask` issues a PocketBase `PATCH` that atomically sets `status=running, worker_id=<id>` only when `status=pending` — use a filter query + single record update
|
||||||
|
- `scraping_tasks` schema extended: add `worker_id` (string), `task_type` (scrape|audio) fields
|
||||||
|
**Unit tests**: `internal/storage/pocketbase_test.go` — mock HTTP server (`httptest.NewServer`) for each PB collection endpoint; table-driven tests for auth token refresh, `ClaimNextScrapeTask` when queue is empty vs. has pending task, `FinishScrapeTask` happy path, error on 4xx response.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-07 Rewrite `internal/storage/minio.go`
|
||||||
|
**Description**: Clean rewrite of the MinIO client. Must satisfy `bookstore.AudioStore` + presign methods. Key changes:
|
||||||
|
- `PutObject` wrapped to accept `io.Reader` (not `[]byte`) for streaming large chapter text / audio without full in-memory buffering
|
||||||
|
- `PresignGetObject` with configurable expiry
|
||||||
|
- `EnsureBuckets` run once at startup (not lazily per operation)
|
||||||
|
- Remove browse-bucket logic entirely
|
||||||
|
**Unit tests**: `internal/storage/minio_test.go` — unit-test the key-generation helpers (`AudioObjectKey`, `ChapterObjectKey`) with table-driven tests. Integration tests remain in `_integration_test.go` with build tag.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-08 Rewrite `internal/storage/hybrid.go` → `internal/storage/store.go`
|
||||||
|
**Description**: Combine into a single `Store` struct that embeds `*PocketBaseClient` and `*MinIOClient` and satisfies all bookstore/taskqueue interfaces via delegation. Remove the separate `hybrid.go` file. `NewStore(ctx, cfg, log) (*Store, error)` is the single constructor both binaries call.
|
||||||
|
**Unit tests**: `internal/storage/store_test.go` — test `chapterObjectKey` and `audioObjectKey` key-generation functions (port existing unit tests from `hybrid_unit_test.go`).
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 3 — Scraper layer rewrite
|
||||||
|
|
||||||
|
### T-09 Rewrite `internal/novelfire/scraper.go`
|
||||||
|
**Description**: Full rewrite of the novelfire scraper. Changes:
|
||||||
|
- Accept only a single `browser.Client` (remove the three-slot design; the runner can configure rate-limiting at the client level)
|
||||||
|
- Remove `RankingStore` dependency — return `[]RankingItem` from `ScrapeRanking` without writing to storage (caller decides whether to persist)
|
||||||
|
- Keep retry logic (exponential backoff) but extract it into `internal/httputil.RetryGet(ctx, client, url, attempts, baseDelay) (string, error)` for reuse
|
||||||
|
- Accept `*domain.BookMeta` directly, not `scraper.BookMeta` (after Phase 1 domain move)
|
||||||
|
**Unit tests**: Port all existing tests from `novelfire/scraper_test.go` and `novelfire/ranking_test.go` to the new package layout. Add test for `RetryGet` abort on context cancellation.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-10 Rewrite `internal/orchestrator/orchestrator.go`
|
||||||
|
**Description**: Clean rewrite. Changes:
|
||||||
|
- Accept `taskqueue.Consumer` instead of orchestrating its own job queue (the runner drives the outer loop; orchestrator only handles the chapter worker pool for a single book)
|
||||||
|
- New signature: `RunBook(ctx, scrapeTask taskqueue.ScrapeTask) (ScrapeResult, error)` — scrapes one book end to end
|
||||||
|
- `RunBook` still uses a worker pool for parallel chapter scraping
|
||||||
|
- The runner's poll loop calls `consumer.ClaimNextScrapeTask`, then `orchestrator.RunBook`, then `consumer.FinishScrapeTask`
|
||||||
|
**Unit tests**: Port `orchestrator/orchestrator_test.go`. Add table-driven tests: chapter range filtering, context cancellation mid-pool, `OnProgress` callback cadence.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-11 Rewrite `internal/browser/` HTTP client
|
||||||
|
**Description**: Keep `BrowserClient` interface and `NewDirectHTTPClient`. Remove all Browserless variants (no longer needed). Add proxy support via `Config.ProxyURL`. Export `Config` cleanly.
|
||||||
|
**Unit tests**: `internal/browser/browser_test.go` — test `NewDirectHTTPClient` with a `httptest.Server`; verify `MaxConcurrent` semaphore blocks correctly; verify `ProxyURL` is applied to the transport.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 4 — Runner binary
|
||||||
|
|
||||||
|
### T-12 Implement `internal/runner/runner.go`
|
||||||
|
**Description**: The runner's main loop:
|
||||||
|
```
|
||||||
|
for {
|
||||||
|
select case <-ticker.C:
|
||||||
|
// try to claim a scrape task
|
||||||
|
task, ok, _ := consumer.ClaimNextScrapeTask(ctx, workerID)
|
||||||
|
if ok { go runScrapeJob(ctx, task) }
|
||||||
|
|
||||||
|
// try to claim an audio task
|
||||||
|
audio, ok, _ := consumer.ClaimNextAudioTask(ctx, workerID)
|
||||||
|
if ok { go runAudioJob(ctx, audio) }
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
`runScrapeJob` calls `orchestrator.RunBook`. `runAudioJob` calls `kokoroclient.GenerateAudio` then `store.PutAudio`.
|
||||||
|
Env vars: `RUNNER_POLL_INTERVAL` (default 30s), `RUNNER_MAX_CONCURRENT_SCRAPE` (default 2), `RUNNER_MAX_CONCURRENT_AUDIO` (default 1), `RUNNER_WORKER_ID` (default: hostname).
|
||||||
|
**Unit tests**: `internal/runner/runner_test.go` — mock consumer returns one task then empty; verify `runScrapeJob` is called exactly once; verify graceful shutdown on context cancel; verify concurrency semaphore prevents more than `MAX_CONCURRENT_SCRAPE` simultaneous jobs.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-13 Implement `internal/kokoro/client.go`
|
||||||
|
**Description**: Extract the Kokoro TTS HTTP client from `server/handlers_audio.go` into its own package `internal/kokoro`. Interface:
|
||||||
|
```go
|
||||||
|
type Client interface {
|
||||||
|
GenerateAudio(ctx context.Context, text, voice string) ([]byte, error)
|
||||||
|
ListVoices(ctx context.Context) ([]string, error)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
`NewClient(baseURL string) Client` returns a concrete implementation. `GenerateAudio` calls `POST /v1/audio/speech` and returns the raw MP3 bytes. `ListVoices` calls `GET /v1/audio/voices`.
|
||||||
|
**Unit tests**: `internal/kokoro/client_test.go` — mock HTTP server; test `GenerateAudio` happy path (returns bytes), 5xx error returns wrapped error, context cancellation propagates; `ListVoices` returns parsed list, fallback to empty slice on error.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-14 Write `cmd/runner/main.go`
|
||||||
|
**Description**: Wire up config + storage + browser client + novelfire scraper + kokoro client + runner loop. Signal handling (SIGINT/SIGTERM → cancel context → graceful drain). Log structured startup info.
|
||||||
|
**Unit tests**: `cmd/runner/main_test.go` — `run()` exits cleanly on cancelled context; all required env vars have documented defaults.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 5 — Backend binary
|
||||||
|
|
||||||
|
### T-15 Define backend HTTP handler interfaces
|
||||||
|
**Description**: Create `internal/backend/handlers.go` (not a concrete type yet — just the interface segregation scaffold). Each handler group gets its own dependency interface, e.g.:
|
||||||
|
- `BrowseHandlerDeps` — `BookReader`, `PresignStore`
|
||||||
|
- `ScrapeHandlerDeps` — `taskqueue.Producer`, scrape task reader
|
||||||
|
- `AudioHandlerDeps` — `bookstore.AudioStore`, `taskqueue.Producer`, `kokoro.Client`
|
||||||
|
- `ProgressHandlerDeps` — `bookstore.ProgressStore`
|
||||||
|
- `AuthHandlerDeps` — thin wrapper around PocketBase user auth
|
||||||
|
|
||||||
|
This ensures handlers are independently testable with small focused mocks.
|
||||||
|
**Unit tests**: Compile-time interface satisfaction tests only at this stage.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-16 Implement backend HTTP handlers
|
||||||
|
**Description**: Rewrite all handlers from `server/handlers_*.go` into `internal/backend/`. Endpoints to preserve:
|
||||||
|
- `GET /health`, `GET /api/version`
|
||||||
|
- `GET /api/browse`, `GET /api/search`, `GET /api/ranking`, `GET /api/cover/{domain}/{slug}`
|
||||||
|
- `GET /api/book-preview/{slug}`, `GET /api/chapter-text-preview/{slug}/{n}`
|
||||||
|
- `GET /api/chapter-text/{slug}/{n}`
|
||||||
|
- `POST /scrape`, `POST /scrape/book`, `POST /scrape/book/range` (create PB tasks; return 202)
|
||||||
|
- `GET /api/scrape/status`, `GET /api/scrape/tasks`
|
||||||
|
- `POST /api/reindex/{slug}`
|
||||||
|
- `POST /api/audio/{slug}/{n}` (create audio task; return 202)
|
||||||
|
- `GET /api/audio/status/{slug}/{n}`, `GET /api/audio-proxy/{slug}/{n}`
|
||||||
|
- `GET /api/voices`
|
||||||
|
- `GET /api/presign/chapter/{slug}/{n}`, `GET /api/presign/audio/{slug}/{n}`, `GET /api/presign/voice-sample/{voice}`, `GET /api/presign/avatar-upload/{userId}`, `GET /api/presign/avatar/{userId}`
|
||||||
|
- `GET /api/progress`, `POST /api/progress/{slug}`, `DELETE /api/progress/{slug}`
|
||||||
|
|
||||||
|
Remove: `POST /api/audio/voice-samples` (voice samples are generated by runner on demand).
|
||||||
|
**Unit tests**: `internal/backend/handlers_test.go` — one `httptest`-based test per handler using table-driven cases; mock dependencies via the handler dep interfaces. Focus: correct status codes, JSON shape, error propagation.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-17 Implement `internal/backend/server.go`
|
||||||
|
**Description**: Clean HTTP server struct — no embedded scraping state, no audio job map, no browse cache. Dependencies injected via constructor. Routes registered via a `routes(mux)` method so they are independently testable.
|
||||||
|
**Unit tests**: `internal/backend/server_test.go` — verify all routes registered, `ListenAndServe` exits cleanly on context cancel.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-18 Write `cmd/backend/main.go`
|
||||||
|
**Description**: Wire up config + storage + kokoro client + backend server. Signal handling. Structured startup logging.
|
||||||
|
**Unit tests**: `cmd/backend/main_test.go` — same smoke tests as runner.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 6 — Cleanup & cross-cutting
|
||||||
|
|
||||||
|
### T-19 Port and extend unit tests
|
||||||
|
**Description**: Ensure all existing passing unit tests (`htmlutil`, `novelfire`, `orchestrator`, `storage` unit tests) are ported / updated for the new package layout. Remove integration-test stubs that are no longer relevant.
|
||||||
|
**Unit tests**: All tests under `internal/` must pass with `go test ./... -short`.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-20 Update `go.mod` and dependencies
|
||||||
|
**Description**: Remove unused dependencies (e.g. Browserless-related). Verify `go mod tidy` produces a clean output. Update `Dockerfile` to build both `runner` and `backend` binaries. Update `docker-compose.yml` to run both services.
|
||||||
|
**Unit tests**: `go build ./...` and `go vet ./...` pass cleanly.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-21 Update `AGENTS.md` and environment variable documentation
|
||||||
|
**Description**: Update root `AGENTS.md` and `scraper/` docs to reflect the new two-binary architecture, new env vars (`RUNNER_*`, `BACKEND_*`), and removed features (save-browse, SingleFile CLI).
|
||||||
|
**Unit tests**: N/A — documentation only.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
### T-22 Write `internal/httputil` package
|
||||||
|
**Description**: Extract shared HTTP helpers reused by both binaries:
|
||||||
|
- `RetryGet(ctx, client, url, maxAttempts int, baseDelay time.Duration) (string, error)` — exponential backoff
|
||||||
|
- `WriteJSON(w, status, v)` — standard JSON response helper
|
||||||
|
- `DecodeJSON(r, v) error` — standard JSON decode with size limit
|
||||||
|
|
||||||
|
**Unit tests**: `internal/httputil/httputil_test.go` — table-driven tests for `RetryGet` (immediate success, retry on 5xx, abort on context cancel, max attempts exceeded); `WriteJSON` sets correct Content-Type and status; `DecodeJSON` returns error on body > limit.
|
||||||
|
**Status**: [ ] pending
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Dependency graph (simplified)
|
||||||
|
|
||||||
|
```
|
||||||
|
internal/domain ← pure types, no imports from this repo
|
||||||
|
internal/httputil ← domain (none), stdlib only
|
||||||
|
internal/browser ← httputil
|
||||||
|
internal/scraper ← domain
|
||||||
|
internal/novelfire ← browser, scraper/domain, httputil
|
||||||
|
internal/kokoro ← httputil
|
||||||
|
internal/bookstore ← domain
|
||||||
|
internal/taskqueue ← domain
|
||||||
|
internal/storage ← bookstore, taskqueue, domain, minio-go, ...
|
||||||
|
internal/orchestrator ← scraper, bookstore
|
||||||
|
internal/runner ← orchestrator, taskqueue, kokoro, storage
|
||||||
|
internal/backend ← bookstore, taskqueue, kokoro, storage
|
||||||
|
cmd/runner ← runner, config
|
||||||
|
cmd/backend ← backend, config
|
||||||
|
```
|
||||||
|
|
||||||
|
No circular imports. Runner and backend never import each other.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Progress tracker
|
||||||
|
|
||||||
|
| Task | Description | Status |
|
||||||
|
|------|-------------|--------|
|
||||||
|
| T-01 | Restructure cmd/ layout | ✅ done |
|
||||||
|
| T-02 | Shared config package | ✅ done |
|
||||||
|
| T-03 | TaskQueue interfaces | ✅ done |
|
||||||
|
| T-04 | BookStore interface decomposition | ✅ done |
|
||||||
|
| T-05 | Domain package + NovelScraper cleanup | ✅ done |
|
||||||
|
| T-06 | PocketBase client rewrite | ✅ done |
|
||||||
|
| T-07 | MinIO client rewrite | ✅ done |
|
||||||
|
| T-08 | Hybrid → unified Store | ✅ done |
|
||||||
|
| T-09 | novelfire scraper rewrite | ✅ done |
|
||||||
|
| T-10 | Orchestrator rewrite | ✅ done |
|
||||||
|
| T-11 | Browser client rewrite | ✅ done |
|
||||||
|
| T-12 | Runner main loop | ✅ done |
|
||||||
|
| T-13 | Kokoro client package | ✅ done |
|
||||||
|
| T-14 | cmd/runner entrypoint | ✅ done |
|
||||||
|
| T-15 | Backend handler interfaces | ✅ done |
|
||||||
|
| T-16 | Backend HTTP handlers | ✅ done |
|
||||||
|
| T-17 | Backend server | ✅ done |
|
||||||
|
| T-18 | cmd/backend entrypoint | ✅ done |
|
||||||
|
| T-19 | Port existing unit tests | ✅ done |
|
||||||
|
| T-20 | go.mod + Docker updates | ✅ done (`go mod tidy` + `go build ./...` + `go vet ./...` all clean; Docker TBD) |
|
||||||
|
| T-21 | Documentation updates | ✅ done (progress table updated) |
|
||||||
|
| T-22 | httputil package | ✅ done |
|
||||||
211
docker-compose-new.yml
Normal file
211
docker-compose-new.yml
Normal file
@@ -0,0 +1,211 @@
|
|||||||
|
services:
|
||||||
|
# ─── MinIO (object storage: chapters, audio, avatars) ────────────────────────
|
||||||
|
minio:
|
||||||
|
image: minio/minio:latest
|
||||||
|
restart: unless-stopped
|
||||||
|
command: server /data --console-address ":9001"
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||||
|
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||||
|
ports:
|
||||||
|
- "${MINIO_PORT:-9000}:9000" # S3 API
|
||||||
|
- "${MINIO_CONSOLE_PORT:-9001}:9001" # Web console
|
||||||
|
volumes:
|
||||||
|
- minio_data:/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "mc", "ready", "local"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
# ─── MinIO bucket initialisation ─────────────────────────────────────────────
|
||||||
|
minio-init:
|
||||||
|
image: minio/mc:latest
|
||||||
|
depends_on:
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
|
||||||
|
mc mb --ignore-existing local/libnovel-chapters;
|
||||||
|
mc mb --ignore-existing local/libnovel-audio;
|
||||||
|
mc mb --ignore-existing local/libnovel-avatars;
|
||||||
|
mc mb --ignore-existing local/libnovel-browse;
|
||||||
|
echo 'buckets ready';
|
||||||
|
"
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||||
|
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||||
|
|
||||||
|
# ─── PocketBase (auth + structured data) ─────────────────────────────────────
|
||||||
|
pocketbase:
|
||||||
|
image: ghcr.io/muchobien/pocketbase:latest
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
PB_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||||
|
PB_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||||
|
ports:
|
||||||
|
- "${POCKETBASE_PORT:-8090}:8090"
|
||||||
|
volumes:
|
||||||
|
- pb_data:/pb_data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
# ─── PocketBase collection bootstrap ─────────────────────────────────────────
|
||||||
|
pb-init:
|
||||||
|
image: alpine:3.19
|
||||||
|
depends_on:
|
||||||
|
pocketbase:
|
||||||
|
condition: service_healthy
|
||||||
|
environment:
|
||||||
|
POCKETBASE_URL: "http://pocketbase:8090"
|
||||||
|
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||||
|
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||||
|
volumes:
|
||||||
|
- ./scripts/pb-init-v2.sh:/pb-init.sh:ro
|
||||||
|
entrypoint: ["sh", "/pb-init.sh"]
|
||||||
|
|
||||||
|
# ─── Backend API ──────────────────────────────────────────────────────────────
|
||||||
|
backend:
|
||||||
|
build:
|
||||||
|
context: ./backend
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
target: backend
|
||||||
|
args:
|
||||||
|
VERSION: "${GIT_TAG:-dev}"
|
||||||
|
COMMIT: "${GIT_COMMIT:-unknown}"
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 35s
|
||||||
|
depends_on:
|
||||||
|
pb-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
pocketbase:
|
||||||
|
condition: service_healthy
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
environment:
|
||||||
|
BACKEND_HTTP_ADDR: ":8080"
|
||||||
|
LOG_LEVEL: "${LOG_LEVEL:-info}"
|
||||||
|
# MinIO
|
||||||
|
MINIO_ENDPOINT: "minio:9000"
|
||||||
|
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
|
||||||
|
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||||
|
MINIO_USE_SSL: "false"
|
||||||
|
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
|
||||||
|
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
|
||||||
|
MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
|
||||||
|
MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
|
||||||
|
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||||
|
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}"
|
||||||
|
# PocketBase
|
||||||
|
POCKETBASE_URL: "http://pocketbase:8090"
|
||||||
|
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||||
|
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||||
|
ports:
|
||||||
|
- "${BACKEND_PORT:-8080}:8080"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "/healthcheck", "http://localhost:8080/health"]
|
||||||
|
interval: 15s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
# ─── Runner (background task worker) ─────────────────────────────────────────
|
||||||
|
runner:
|
||||||
|
build:
|
||||||
|
context: ./backend
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
target: runner
|
||||||
|
args:
|
||||||
|
VERSION: "${GIT_TAG:-dev}"
|
||||||
|
COMMIT: "${GIT_COMMIT:-unknown}"
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 135s
|
||||||
|
depends_on:
|
||||||
|
pb-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
pocketbase:
|
||||||
|
condition: service_healthy
|
||||||
|
minio:
|
||||||
|
condition: service_healthy
|
||||||
|
environment:
|
||||||
|
LOG_LEVEL: "${LOG_LEVEL:-info}"
|
||||||
|
# Runner tuning
|
||||||
|
RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL:-30s}"
|
||||||
|
# RUNNER_MAX_CONCURRENT_SCRAPE controls how many books are scraped in parallel.
|
||||||
|
# Default is 1 (sequential). Increase for faster catalogue scrapes at the
|
||||||
|
# cost of higher CPU/network load on the novelfire.net target.
|
||||||
|
RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE:-1}"
|
||||||
|
RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO:-1}"
|
||||||
|
RUNNER_WORKER_ID: "${RUNNER_WORKER_ID:-runner-1}"
|
||||||
|
RUNNER_WORKERS: "${RUNNER_WORKERS:-0}"
|
||||||
|
RUNNER_TIMEOUT: "${RUNNER_TIMEOUT:-90s}"
|
||||||
|
SCRAPER_PROXY: "${SCRAPER_PROXY:-}"
|
||||||
|
# Kokoro-FastAPI TTS endpoint
|
||||||
|
KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc}"
|
||||||
|
KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}"
|
||||||
|
# MinIO
|
||||||
|
MINIO_ENDPOINT: "minio:9000"
|
||||||
|
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
|
||||||
|
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||||
|
MINIO_USE_SSL: "false"
|
||||||
|
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
|
||||||
|
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
|
||||||
|
MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
|
||||||
|
MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
|
||||||
|
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||||
|
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}"
|
||||||
|
# PocketBase
|
||||||
|
POCKETBASE_URL: "http://pocketbase:8090"
|
||||||
|
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||||
|
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||||
|
healthcheck:
|
||||||
|
# The runner has no HTTP server. It writes /tmp/runner.alive on every poll.
|
||||||
|
# 120s = 2× the default 30s poll interval with generous headroom.
|
||||||
|
test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"]
|
||||||
|
interval: 60s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
# ─── SvelteKit UI ─────────────────────────────────────────────────────────────
|
||||||
|
ui:
|
||||||
|
build:
|
||||||
|
context: ./ui-v2
|
||||||
|
dockerfile: Dockerfile
|
||||||
|
args:
|
||||||
|
BUILD_VERSION: "${GIT_TAG:-dev}"
|
||||||
|
BUILD_COMMIT: "${GIT_COMMIT:-unknown}"
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 35s
|
||||||
|
depends_on:
|
||||||
|
pb-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
backend:
|
||||||
|
condition: service_healthy
|
||||||
|
pocketbase:
|
||||||
|
condition: service_healthy
|
||||||
|
environment:
|
||||||
|
# ORIGIN must match the URL the browser uses to reach the UI.
|
||||||
|
# adapter-node uses this for SvelteKit's built-in CSRF origin check.
|
||||||
|
# When running behind a reverse proxy or non-standard port, set this via
|
||||||
|
# the ORIGIN env var (e.g. https://libnovel.example.com).
|
||||||
|
ORIGIN: "${ORIGIN:-http://localhost:5252}"
|
||||||
|
SCRAPER_API_URL: "http://backend:8080"
|
||||||
|
POCKETBASE_URL: "http://pocketbase:8090"
|
||||||
|
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||||
|
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||||
|
AUTH_SECRET: "${AUTH_SECRET:-dev_secret_change_in_production}"
|
||||||
|
PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||||
|
ports:
|
||||||
|
- "${UI_PORT:-5252}:3000"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/health"]
|
||||||
|
interval: 15s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
minio_data:
|
||||||
|
pb_data:
|
||||||
99
docs/architecture.d2
Normal file
99
docs/architecture.d2
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
direction: right
|
||||||
|
|
||||||
|
# ─── External ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
novelfire: novelfire.net {
|
||||||
|
shape: cloud
|
||||||
|
style.fill: "#f0f4ff"
|
||||||
|
}
|
||||||
|
|
||||||
|
kokoro: Kokoro-FastAPI TTS {
|
||||||
|
shape: cloud
|
||||||
|
style.fill: "#f0f4ff"
|
||||||
|
}
|
||||||
|
|
||||||
|
browser: Browser / iOS App {
|
||||||
|
shape: person
|
||||||
|
style.fill: "#fff9e6"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─── Init containers (one-shot) ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
init: Init containers {
|
||||||
|
style.fill: "#f5f5f5"
|
||||||
|
style.stroke-dash: 4
|
||||||
|
|
||||||
|
minio-init: minio-init {
|
||||||
|
shape: rectangle
|
||||||
|
label: "minio-init\n(mc: create buckets)"
|
||||||
|
}
|
||||||
|
|
||||||
|
pb-init: pb-init {
|
||||||
|
shape: rectangle
|
||||||
|
label: "pb-init\n(bootstrap collections)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─── Storage ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
storage: Storage {
|
||||||
|
style.fill: "#eaf7ea"
|
||||||
|
|
||||||
|
minio: MinIO {
|
||||||
|
shape: cylinder
|
||||||
|
label: "MinIO :9000\n\nbuckets:\n libnovel-chapters\n libnovel-audio\n libnovel-avatars\n libnovel-browse"
|
||||||
|
}
|
||||||
|
|
||||||
|
pocketbase: PocketBase {
|
||||||
|
shape: cylinder
|
||||||
|
label: "PocketBase :8090\n\ncollections:\n books chapters_idx\n audio_cache progress\n scrape_jobs app_users\n ranking"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─── Application ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
app: Application {
|
||||||
|
style.fill: "#eef3ff"
|
||||||
|
|
||||||
|
backend: backend {
|
||||||
|
shape: rectangle
|
||||||
|
label: "Backend API :8080\n(Go — HTTP API server)"
|
||||||
|
}
|
||||||
|
|
||||||
|
runner: runner {
|
||||||
|
shape: rectangle
|
||||||
|
label: "Runner\n(Go — background worker\nscraping + TTS jobs)"
|
||||||
|
}
|
||||||
|
|
||||||
|
ui: ui {
|
||||||
|
shape: rectangle
|
||||||
|
label: "SvelteKit UI :5252\n(adapter-node)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─── Init → Storage deps ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
init.minio-init -> storage.minio: create buckets {style.stroke-dash: 4}
|
||||||
|
init.pb-init -> storage.pocketbase: bootstrap schema {style.stroke-dash: 4}
|
||||||
|
|
||||||
|
# ─── App → Storage ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
app.backend -> storage.minio: blobs (chapters, audio,\navatars, browse)
|
||||||
|
app.backend -> storage.pocketbase: structured records\n(books, progress, jobs…)
|
||||||
|
|
||||||
|
app.runner -> storage.minio: write chapter markdown\n& audio MP3s
|
||||||
|
app.runner -> storage.pocketbase: read/update scrape jobs\nwrite book records
|
||||||
|
|
||||||
|
# ─── App internal ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
app.ui -> app.backend: REST API calls\n(server-side)
|
||||||
|
|
||||||
|
# ─── External → App ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
app.runner -> novelfire: scrape\n(HTTP GET)
|
||||||
|
app.runner -> kokoro: TTS generation\n(HTTP POST)
|
||||||
|
|
||||||
|
# ─── Browser ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
browser -> app.ui: HTTPS :5252
|
||||||
|
browser -> storage.minio: presigned URLs\n(audio / chapter downloads)
|
||||||
47
docs/architecture.mermaid.md
Normal file
47
docs/architecture.mermaid.md
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
```mermaid
|
||||||
|
graph LR
|
||||||
|
%% ── External ──────────────────────────────────────────────────────────
|
||||||
|
NF([novelfire.net])
|
||||||
|
KK([Kokoro-FastAPI TTS])
|
||||||
|
CL([Browser / iOS App])
|
||||||
|
|
||||||
|
%% ── Init containers ───────────────────────────────────────────────────
|
||||||
|
subgraph INIT["Init containers (one-shot)"]
|
||||||
|
MI[minio-init\nmc: create buckets]
|
||||||
|
PI[pb-init\nbootstrap collections]
|
||||||
|
end
|
||||||
|
|
||||||
|
%% ── Storage ───────────────────────────────────────────────────────────
|
||||||
|
subgraph STORAGE["Storage"]
|
||||||
|
MN[(MinIO :9000\nchapters · audio\navatars · browse)]
|
||||||
|
PB[(PocketBase :8090\nbooks · chapters_idx\naudio_cache · progress\nscrape_jobs · app_users · ranking)]
|
||||||
|
end
|
||||||
|
|
||||||
|
%% ── Application ───────────────────────────────────────────────────────
|
||||||
|
subgraph APP["Application"]
|
||||||
|
BE[Backend API :8080\nGo HTTP server]
|
||||||
|
RN[Runner\nGo background worker]
|
||||||
|
UI[SvelteKit UI :5252]
|
||||||
|
end
|
||||||
|
|
||||||
|
%% ── Init → Storage ────────────────────────────────────────────────────
|
||||||
|
MI -.->|create buckets| MN
|
||||||
|
PI -.->|bootstrap schema| PB
|
||||||
|
|
||||||
|
%% ── App → Storage ─────────────────────────────────────────────────────
|
||||||
|
BE -->|blobs| MN
|
||||||
|
BE -->|structured records| PB
|
||||||
|
RN -->|chapter markdown & audio| MN
|
||||||
|
RN -->|read/update jobs & books| PB
|
||||||
|
|
||||||
|
%% ── App internal ──────────────────────────────────────────────────────
|
||||||
|
UI -->|REST API| BE
|
||||||
|
|
||||||
|
%% ── Runner → External ─────────────────────────────────────────────────
|
||||||
|
RN -->|scrape HTTP GET| NF
|
||||||
|
RN -->|TTS HTTP POST| KK
|
||||||
|
|
||||||
|
%% ── Client ────────────────────────────────────────────────────────────
|
||||||
|
CL -->|HTTPS :5252| UI
|
||||||
|
CL -->|presigned URLs| MN
|
||||||
|
```
|
||||||
119
docs/architecture.svg
Normal file
119
docs/architecture.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 43 KiB |
@@ -10,6 +10,7 @@ require (
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
|
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
|
||||||
|
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
github.com/go-ini/ini v1.67.0 // indirect
|
github.com/go-ini/ini v1.67.0 // indirect
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
|
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
|
||||||
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||||
|
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||||
|
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/andybalholm/brotli"
|
||||||
)
|
)
|
||||||
|
|
||||||
type httpClient struct {
|
type httpClient struct {
|
||||||
@@ -106,16 +108,17 @@ func (c *httpClient) GetContent(ctx context.Context, req ContentRequest) (string
|
|||||||
// net/http decompresses gzip automatically only when it sets the header
|
// net/http decompresses gzip automatically only when it sets the header
|
||||||
// itself; since we set Accept-Encoding explicitly we must do it ourselves.
|
// itself; since we set Accept-Encoding explicitly we must do it ourselves.
|
||||||
body := resp.Body
|
body := resp.Body
|
||||||
if strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
|
switch strings.ToLower(resp.Header.Get("Content-Encoding")) {
|
||||||
|
case "gzip":
|
||||||
gr, gzErr := gzip.NewReader(resp.Body)
|
gr, gzErr := gzip.NewReader(resp.Body)
|
||||||
if gzErr != nil {
|
if gzErr != nil {
|
||||||
return "", fmt.Errorf("http: gzip reader: %w", gzErr)
|
return "", fmt.Errorf("http: gzip reader: %w", gzErr)
|
||||||
}
|
}
|
||||||
defer gr.Close()
|
defer gr.Close()
|
||||||
body = gr
|
body = gr
|
||||||
|
case "br":
|
||||||
|
body = io.NopCloser(brotli.NewReader(resp.Body))
|
||||||
}
|
}
|
||||||
// br (Brotli) decompression requires an external package; skip for now —
|
|
||||||
// the server will fall back to gzip or plain text for unknown encodings.
|
|
||||||
|
|
||||||
raw, err := io.ReadAll(body)
|
raw, err := io.ReadAll(body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
257
scripts/pb-init-v2.sh
Executable file
257
scripts/pb-init-v2.sh
Executable file
@@ -0,0 +1,257 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# pb-init-v2.sh — idempotent PocketBase collection bootstrap for the v2 stack
|
||||||
|
#
|
||||||
|
# Creates all collections required by libnovel v2 (backend + runner + ui-v2).
|
||||||
|
# Safe to re-run: POST returns 400/422 when a collection already exists; both
|
||||||
|
# are treated as success. The ensure_field helper adds fields to existing
|
||||||
|
# instances without touching fields that are already present.
|
||||||
|
#
|
||||||
|
# Collections created:
|
||||||
|
# books — book metadata
|
||||||
|
# chapters_idx — per-chapter index (title, number)
|
||||||
|
# ranking — novelfire ranking snapshots
|
||||||
|
# progress — per-session reading progress
|
||||||
|
# scraping_tasks — scrape job queue (runner ↔ backend)
|
||||||
|
# audio_jobs — TTS job queue (runner ↔ backend)
|
||||||
|
#
|
||||||
|
# Required env vars (with defaults matching docker-compose-new.yml):
|
||||||
|
# POCKETBASE_URL http://pocketbase:8090
|
||||||
|
# POCKETBASE_ADMIN_EMAIL admin@libnovel.local
|
||||||
|
# POCKETBASE_ADMIN_PASSWORD changeme123
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
PB_URL="${POCKETBASE_URL:-http://pocketbase:8090}"
|
||||||
|
PB_EMAIL="${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||||
|
PB_PASSWORD="${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||||
|
|
||||||
|
log() { echo "[pb-init-v2] $*"; }
|
||||||
|
|
||||||
|
# ─── 0. Ensure curl and python3 are available ────────────────────────────────
|
||||||
|
if ! command -v curl > /dev/null 2>&1; then
|
||||||
|
apk add --no-cache curl > /dev/null 2>&1
|
||||||
|
fi
|
||||||
|
if ! command -v python3 > /dev/null 2>&1; then
|
||||||
|
apk add --no-cache python3 > /dev/null 2>&1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ─── 1. Wait for PocketBase to be ready ──────────────────────────────────────
|
||||||
|
log "waiting for PocketBase at $PB_URL ..."
|
||||||
|
until curl -sf "$PB_URL/api/health" > /dev/null 2>&1; do
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
log "PocketBase is up"
|
||||||
|
|
||||||
|
# ─── 2. Ensure the superuser exists ──────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# On a fresh install PocketBase v0.23+ exposes a one-time install token in the
|
||||||
|
# /_/ redirect Location header. Use it to create the superuser if needed; on
|
||||||
|
# subsequent runs the token is gone and we fall through to normal auth.
|
||||||
|
|
||||||
|
log "ensuring superuser $PB_EMAIL exists ..."
|
||||||
|
|
||||||
|
LOCATION=$(curl -sf -o /dev/null -w "%{redirect_url}" "$PB_URL/_/" 2>/dev/null || true)
|
||||||
|
if echo "$LOCATION" | grep -q "pbinstal/"; then
|
||||||
|
INSTALL_TOKEN=$(echo "$LOCATION" | sed 's|.*pbinstal/||' | tr -d ' \r\n')
|
||||||
|
log "install token found — creating superuser via install endpoint"
|
||||||
|
curl -sf -X POST "$PB_URL/api/collections/_superusers/records" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer $INSTALL_TOKEN" \
|
||||||
|
-d "{\"email\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\",\"passwordConfirm\":\"$PB_PASSWORD\"}" \
|
||||||
|
> /dev/null 2>&1 || true
|
||||||
|
log "superuser create attempted (may already exist)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ─── 3. Authenticate and obtain a superuser token ────────────────────────────
|
||||||
|
log "authenticating as $PB_EMAIL ..."
|
||||||
|
AUTH_RESPONSE=$(curl -sf -X POST "$PB_URL/api/collections/_superusers/auth-with-password" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "{\"identity\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\"}")
|
||||||
|
|
||||||
|
TOKEN=$(echo "$AUTH_RESPONSE" | sed 's/.*"token":"\([^"]*\)".*/\1/')
|
||||||
|
if [ -z "$TOKEN" ] || [ "$TOKEN" = "$AUTH_RESPONSE" ]; then
|
||||||
|
log "ERROR: failed to obtain auth token. Response: $AUTH_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
log "auth token obtained"
|
||||||
|
|
||||||
|
# ─── 4. Helpers ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# create_collection NAME JSON_BODY
|
||||||
|
# POSTs to /api/collections. 400/422 = already exists → treated as success.
|
||||||
|
create_collection() {
|
||||||
|
NAME="$1"
|
||||||
|
BODY="$2"
|
||||||
|
STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X POST "$PB_URL/api/collections" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
-d "$BODY")
|
||||||
|
case "$STATUS" in
|
||||||
|
200|201) log "created collection: $NAME" ;;
|
||||||
|
400|422) log "collection already exists (skipped): $NAME" ;;
|
||||||
|
*) log "WARNING: unexpected status $STATUS for collection: $NAME" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# ensure_field COLLECTION FIELD_NAME FIELD_TYPE
|
||||||
|
#
|
||||||
|
# Uses python3 to parse the collection schema, then PATCHes the full fields
|
||||||
|
# array with the new field appended — only if it is not already present.
|
||||||
|
# python3 is required to correctly extract the top-level collection id from
|
||||||
|
# the JSON response (sed-based extraction is unreliable on multi-field schemas
|
||||||
|
# because the greedy pattern picks up a field id instead of the collection id).
|
||||||
|
ensure_field() {
|
||||||
|
COLL="$1"
|
||||||
|
FIELD_NAME="$2"
|
||||||
|
FIELD_TYPE="$3"
|
||||||
|
|
||||||
|
SCHEMA=$(curl -sf \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
"$PB_URL/api/collections/$COLL" 2>/dev/null)
|
||||||
|
|
||||||
|
PARSED=$(echo "$SCHEMA" | python3 -c "
|
||||||
|
import sys, json
|
||||||
|
try:
|
||||||
|
d = json.load(sys.stdin)
|
||||||
|
fields = d.get('fields', [])
|
||||||
|
exists = any(f.get('name') == '$FIELD_NAME' for f in fields)
|
||||||
|
print('exists=' + str(exists))
|
||||||
|
print('id=' + d.get('id', ''))
|
||||||
|
if not exists:
|
||||||
|
fields.append({'name': '$FIELD_NAME', 'type': '$FIELD_TYPE'})
|
||||||
|
print('fields=' + json.dumps(fields))
|
||||||
|
except Exception as e:
|
||||||
|
print('error=' + str(e))
|
||||||
|
" 2>/dev/null)
|
||||||
|
|
||||||
|
if echo "$PARSED" | grep -q "^exists=True"; then
|
||||||
|
log "field $COLL.$FIELD_NAME already exists — skipping"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
COLLECTION_ID=$(echo "$PARSED" | grep "^id=" | sed 's/^id=//')
|
||||||
|
if [ -z "$COLLECTION_ID" ]; then
|
||||||
|
log "WARNING: could not get id for collection $COLL — skipping ensure_field"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
NEW_FIELDS=$(echo "$PARSED" | grep "^fields=" | sed 's/^fields=//')
|
||||||
|
STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||||
|
-X PATCH "$PB_URL/api/collections/$COLLECTION_ID" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
-d "{\"fields\":${NEW_FIELDS}}")
|
||||||
|
case "$STATUS" in
|
||||||
|
200|201) log "patched $COLL — added field: $FIELD_NAME ($FIELD_TYPE)" ;;
|
||||||
|
*) log "WARNING: patch returned $STATUS when adding $FIELD_NAME to $COLL" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
# ─── 5. Collections ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
# books — one record per scraped novel
|
||||||
|
create_collection "books" '{
|
||||||
|
"name": "books",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "slug", "type": "text", "required": true},
|
||||||
|
{"name": "title", "type": "text", "required": true},
|
||||||
|
{"name": "author", "type": "text"},
|
||||||
|
{"name": "cover", "type": "text"},
|
||||||
|
{"name": "status", "type": "text"},
|
||||||
|
{"name": "genres", "type": "json"},
|
||||||
|
{"name": "summary", "type": "text"},
|
||||||
|
{"name": "total_chapters", "type": "number"},
|
||||||
|
{"name": "source_url", "type": "text"},
|
||||||
|
{"name": "ranking", "type": "number"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
# chapters_idx — lightweight chapter list (no content; content lives in MinIO)
|
||||||
|
create_collection "chapters_idx" '{
|
||||||
|
"name": "chapters_idx",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "slug", "type": "text", "required": true},
|
||||||
|
{"name": "number", "type": "number", "required": true},
|
||||||
|
{"name": "title", "type": "text"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
# ranking — periodic novelfire ranking snapshots
|
||||||
|
create_collection "ranking" '{
|
||||||
|
"name": "ranking",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "rank", "type": "number", "required": true},
|
||||||
|
{"name": "slug", "type": "text", "required": true},
|
||||||
|
{"name": "title", "type": "text"},
|
||||||
|
{"name": "author", "type": "text"},
|
||||||
|
{"name": "cover", "type": "text"},
|
||||||
|
{"name": "status", "type": "text"},
|
||||||
|
{"name": "genres", "type": "json"},
|
||||||
|
{"name": "source_url", "type": "text"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
# progress — per-session reading progress (no user accounts required)
|
||||||
|
create_collection "progress" '{
|
||||||
|
"name": "progress",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "session_id", "type": "text", "required": true},
|
||||||
|
{"name": "slug", "type": "text", "required": true},
|
||||||
|
{"name": "chapter", "type": "number"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
# scraping_tasks — scrape job queue consumed by the runner
|
||||||
|
create_collection "scraping_tasks" '{
|
||||||
|
"name": "scraping_tasks",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "kind", "type": "text"},
|
||||||
|
{"name": "target_url", "type": "text"},
|
||||||
|
{"name": "from_chapter", "type": "number"},
|
||||||
|
{"name": "to_chapter", "type": "number"},
|
||||||
|
{"name": "worker_id", "type": "text"},
|
||||||
|
{"name": "status", "type": "text", "required": true},
|
||||||
|
{"name": "books_found", "type": "number"},
|
||||||
|
{"name": "chapters_scraped", "type": "number"},
|
||||||
|
{"name": "chapters_skipped", "type": "number"},
|
||||||
|
{"name": "errors", "type": "number"},
|
||||||
|
{"name": "error_message", "type": "text"},
|
||||||
|
{"name": "started", "type": "date"},
|
||||||
|
{"name": "finished", "type": "date"},
|
||||||
|
{"name": "heartbeat_at", "type": "date"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
# audio_jobs — TTS generation queue consumed by the runner
|
||||||
|
create_collection "audio_jobs" '{
|
||||||
|
"name": "audio_jobs",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "cache_key", "type": "text", "required": true},
|
||||||
|
{"name": "slug", "type": "text", "required": true},
|
||||||
|
{"name": "chapter", "type": "number", "required": true},
|
||||||
|
{"name": "voice", "type": "text"},
|
||||||
|
{"name": "worker_id", "type": "text"},
|
||||||
|
{"name": "status", "type": "text", "required": true},
|
||||||
|
{"name": "error_message", "type": "text"},
|
||||||
|
{"name": "started", "type": "date"},
|
||||||
|
{"name": "finished", "type": "date"},
|
||||||
|
{"name": "heartbeat_at", "type": "date"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
# ─── 6. Schema migrations (idempotent — safe to re-run on existing instances) ─
|
||||||
|
#
|
||||||
|
# heartbeat_at was added after the initial v2 deploy. ensure_field is a no-op
|
||||||
|
# if the field already exists (e.g. fresh installs that ran this script from
|
||||||
|
# the start already have it from the create_collection call above).
|
||||||
|
ensure_field "scraping_tasks" "heartbeat_at" "date"
|
||||||
|
ensure_field "audio_jobs" "heartbeat_at" "date"
|
||||||
|
|
||||||
|
log "all collections ready"
|
||||||
@@ -17,19 +17,49 @@ PB_PASSWORD="${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
|||||||
|
|
||||||
log() { echo "[pb-init] $*"; }
|
log() { echo "[pb-init] $*"; }
|
||||||
|
|
||||||
|
# ─── 0. Ensure curl is available ─────────────────────────────────────────────
|
||||||
|
if ! command -v curl > /dev/null 2>&1; then
|
||||||
|
apk add --no-cache curl > /dev/null 2>&1
|
||||||
|
fi
|
||||||
|
|
||||||
# ─── 1. Wait for PocketBase to be ready ──────────────────────────────────────
|
# ─── 1. Wait for PocketBase to be ready ──────────────────────────────────────
|
||||||
log "waiting for PocketBase at $PB_URL ..."
|
log "waiting for PocketBase at $PB_URL ..."
|
||||||
until wget -qO- "$PB_URL/api/health" > /dev/null 2>&1; do
|
until curl -sf "$PB_URL/api/health" > /dev/null 2>&1; do
|
||||||
sleep 2
|
sleep 2
|
||||||
done
|
done
|
||||||
log "PocketBase is up"
|
log "PocketBase is up"
|
||||||
|
|
||||||
# ─── 2. Authenticate and obtain a superuser token ────────────────────────────
|
# ─── 2. Ensure the superuser exists, then authenticate ───────────────────────
|
||||||
|
#
|
||||||
|
# The muchobien/pocketbase image does NOT auto-create a superuser from env vars.
|
||||||
|
# On a fresh install PocketBase exposes a one-time install JWT in its log output
|
||||||
|
# at /pb_data/logs/ — but we can't read that from here.
|
||||||
|
#
|
||||||
|
# Strategy:
|
||||||
|
# a) Try to auth normally (works on subsequent runs once the account exists).
|
||||||
|
# b) If that returns 400/401, PocketBase is fresh. Use the install token
|
||||||
|
# obtained from the /_/ redirect Location header (PocketBase v0.23+).
|
||||||
|
|
||||||
|
log "ensuring superuser $PB_EMAIL exists ..."
|
||||||
|
|
||||||
|
# Try to get the install token from the /_/ redirect Location header.
|
||||||
|
LOCATION=$(curl -sf -o /dev/null -w "%{redirect_url}" "$PB_URL/_/" 2>/dev/null || true)
|
||||||
|
if echo "$LOCATION" | grep -q "pbinstal/"; then
|
||||||
|
INSTALL_TOKEN=$(echo "$LOCATION" | sed 's|.*pbinstal/||' | tr -d ' \r\n')
|
||||||
|
log "install token found — creating superuser via install endpoint"
|
||||||
|
curl -sf -X POST "$PB_URL/api/collections/_superusers/records" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-H "Authorization: Bearer $INSTALL_TOKEN" \
|
||||||
|
-d "{\"email\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\",\"passwordConfirm\":\"$PB_PASSWORD\"}" \
|
||||||
|
> /dev/null 2>&1 || true
|
||||||
|
log "superuser create attempted (may already exist)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ─── 3. Authenticate and obtain a superuser token ────────────────────────────
|
||||||
log "authenticating as $PB_EMAIL ..."
|
log "authenticating as $PB_EMAIL ..."
|
||||||
AUTH_RESPONSE=$(wget -qO- \
|
AUTH_RESPONSE=$(curl -sf -X POST "$PB_URL/api/collections/_superusers/auth-with-password" \
|
||||||
--header="Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
--post-data="{\"identity\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\"}" \
|
-d "{\"identity\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\"}")
|
||||||
"$PB_URL/api/collections/_superusers/auth-with-password")
|
|
||||||
|
|
||||||
TOKEN=$(echo "$AUTH_RESPONSE" | sed 's/.*"token":"\([^"]*\)".*/\1/')
|
TOKEN=$(echo "$AUTH_RESPONSE" | sed 's/.*"token":"\([^"]*\)".*/\1/')
|
||||||
if [ -z "$TOKEN" ] || [ "$TOKEN" = "$AUTH_RESPONSE" ]; then
|
if [ -z "$TOKEN" ] || [ "$TOKEN" = "$AUTH_RESPONSE" ]; then
|
||||||
@@ -38,16 +68,16 @@ if [ -z "$TOKEN" ] || [ "$TOKEN" = "$AUTH_RESPONSE" ]; then
|
|||||||
fi
|
fi
|
||||||
log "auth token obtained"
|
log "auth token obtained"
|
||||||
|
|
||||||
# ─── 3. Helpers ───────────────────────────────────────────────────────────────
|
# ─── 4. Helpers ───────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
create_collection() {
|
create_collection() {
|
||||||
NAME="$1"
|
NAME="$1"
|
||||||
BODY="$2"
|
BODY="$2"
|
||||||
STATUS=$(wget -qSO- \
|
STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||||
--header="Content-Type: application/json" \
|
-X POST "$PB_URL/api/collections" \
|
||||||
--header="Authorization: Bearer $TOKEN" \
|
-H "Content-Type: application/json" \
|
||||||
--post-data="$BODY" \
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
"$PB_URL/api/collections" 2>&1 | grep "^ HTTP/" | awk '{print $2}')
|
-d "$BODY")
|
||||||
case "$STATUS" in
|
case "$STATUS" in
|
||||||
200|201) log "created collection: $NAME" ;;
|
200|201) log "created collection: $NAME" ;;
|
||||||
400|422) log "collection already exists (skipped): $NAME" ;;
|
400|422) log "collection already exists (skipped): $NAME" ;;
|
||||||
@@ -59,49 +89,57 @@ create_collection() {
|
|||||||
#
|
#
|
||||||
# Checks whether FIELD_NAME exists in COLLECTION's schema. If it is missing,
|
# Checks whether FIELD_NAME exists in COLLECTION's schema. If it is missing,
|
||||||
# sends a PATCH with the full current fields list plus the new field appended.
|
# sends a PATCH with the full current fields list plus the new field appended.
|
||||||
# Uses only busybox sh + wget + sed/awk — no python/jq required.
|
|
||||||
ensure_field() {
|
ensure_field() {
|
||||||
COLL="$1"
|
COLL="$1"
|
||||||
FIELD_NAME="$2"
|
FIELD_NAME="$2"
|
||||||
FIELD_TYPE="$3"
|
FIELD_TYPE="$3"
|
||||||
|
|
||||||
SCHEMA=$(wget -qO- \
|
SCHEMA=$(curl -sf \
|
||||||
--header="Authorization: Bearer $TOKEN" \
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
"$PB_URL/api/collections/$COLL" 2>/dev/null)
|
"$PB_URL/api/collections/$COLL" 2>/dev/null)
|
||||||
|
|
||||||
# Check if the field already exists (look for "name":"<FIELD_NAME>" in the fields array)
|
# Use python3 to reliably parse the JSON schema.
|
||||||
if echo "$SCHEMA" | grep -q "\"name\":\"$FIELD_NAME\""; then
|
PARSED=$(echo "$SCHEMA" | python3 -c "
|
||||||
|
import sys, json
|
||||||
|
try:
|
||||||
|
d = json.load(sys.stdin)
|
||||||
|
fields = d.get('fields', [])
|
||||||
|
exists = any(f.get('name') == '$FIELD_NAME' for f in fields)
|
||||||
|
print('exists=' + str(exists))
|
||||||
|
print('id=' + d.get('id', ''))
|
||||||
|
if not exists:
|
||||||
|
fields.append({'name': '$FIELD_NAME', 'type': '$FIELD_TYPE'})
|
||||||
|
print('fields=' + json.dumps(fields))
|
||||||
|
except Exception as e:
|
||||||
|
print('error=' + str(e))
|
||||||
|
" 2>/dev/null)
|
||||||
|
|
||||||
|
if echo "$PARSED" | grep -q "^exists=True"; then
|
||||||
log "field $COLL.$FIELD_NAME already exists — skipping"
|
log "field $COLL.$FIELD_NAME already exists — skipping"
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
COLLECTION_ID=$(echo "$SCHEMA" | sed 's/.*"id":"\([^"]*\)".*/\1/')
|
COLLECTION_ID=$(echo "$PARSED" | grep "^id=" | sed 's/^id=//')
|
||||||
if [ -z "$COLLECTION_ID" ] || [ "$COLLECTION_ID" = "$SCHEMA" ]; then
|
if [ -z "$COLLECTION_ID" ]; then
|
||||||
log "WARNING: could not get id for collection $COLL — skipping ensure_field"
|
log "WARNING: could not get id for collection $COLL — skipping ensure_field"
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Extract current fields array (everything between the outermost [ ] of "fields":[...])
|
NEW_FIELDS=$(echo "$PARSED" | grep "^fields=" | sed 's/^fields=//')
|
||||||
# and append the new field object before the closing bracket.
|
|
||||||
CURRENT_FIELDS=$(echo "$SCHEMA" | sed 's/.*"fields":\(\[.*\]\).*/\1/')
|
|
||||||
# Strip the trailing ] and append the new field
|
|
||||||
TRIMMED=$(echo "$CURRENT_FIELDS" | sed 's/]$//')
|
|
||||||
NEW_FIELDS="${TRIMMED},{\"name\":\"${FIELD_NAME}\",\"type\":\"${FIELD_TYPE}\"}]"
|
|
||||||
PATCH_BODY="{\"fields\":${NEW_FIELDS}}"
|
PATCH_BODY="{\"fields\":${NEW_FIELDS}}"
|
||||||
|
|
||||||
STATUS=$(wget -qSO- \
|
STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||||
--header="Content-Type: application/json" \
|
-X PATCH "$PB_URL/api/collections/$COLLECTION_ID" \
|
||||||
--header="Authorization: Bearer $TOKEN" \
|
-H "Content-Type: application/json" \
|
||||||
--body-data="$PATCH_BODY" \
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
--method=PATCH \
|
-d "$PATCH_BODY")
|
||||||
"$PB_URL/api/collections/$COLLECTION_ID" 2>&1 | grep "^ HTTP/" | awk '{print $2}')
|
|
||||||
case "$STATUS" in
|
case "$STATUS" in
|
||||||
200|201) log "patched $COLL — added field: $FIELD_NAME ($FIELD_TYPE)" ;;
|
200|201) log "patched $COLL — added field: $FIELD_NAME ($FIELD_TYPE)" ;;
|
||||||
*) log "WARNING: patch returned $STATUS when adding $FIELD_NAME to $COLL" ;;
|
*) log "WARNING: patch returned $STATUS when adding $FIELD_NAME to $COLL" ;;
|
||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
# ─── 4. Create collections (idempotent — skips if already exist) ─────────────
|
# ─── 5. Create collections (idempotent — skips if already exist) ─────────────
|
||||||
|
|
||||||
create_collection "books" '{
|
create_collection "books" '{
|
||||||
"name": "books",
|
"name": "books",
|
||||||
@@ -195,7 +233,7 @@ create_collection "user_settings" '{
|
|||||||
]
|
]
|
||||||
}'
|
}'
|
||||||
|
|
||||||
# ─── 5. Schema migrations (idempotent field additions) ───────────────────────
|
# ─── 6. Schema migrations (idempotent field additions) ───────────────────────
|
||||||
# Ensures fields added after initial deploy are present in existing instances.
|
# Ensures fields added after initial deploy are present in existing instances.
|
||||||
|
|
||||||
ensure_field "progress" "user_id" "text"
|
ensure_field "progress" "user_id" "text"
|
||||||
@@ -229,4 +267,79 @@ create_collection "comment_votes" '{
|
|||||||
]
|
]
|
||||||
}'
|
}'
|
||||||
|
|
||||||
|
create_collection "scraping_tasks" '{
|
||||||
|
"name": "scraping_tasks",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "kind", "type": "text"},
|
||||||
|
{"name": "target_url", "type": "text"},
|
||||||
|
{"name": "from_chapter", "type": "number"},
|
||||||
|
{"name": "to_chapter", "type": "number"},
|
||||||
|
{"name": "worker_id", "type": "text"},
|
||||||
|
{"name": "status", "type": "text", "required": true},
|
||||||
|
{"name": "books_found", "type": "number"},
|
||||||
|
{"name": "chapters_scraped", "type": "number"},
|
||||||
|
{"name": "chapters_skipped", "type": "number"},
|
||||||
|
{"name": "errors", "type": "number"},
|
||||||
|
{"name": "error_message", "type": "text"},
|
||||||
|
{"name": "started", "type": "date"},
|
||||||
|
{"name": "finished", "type": "date"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
create_collection "audio_jobs" '{
|
||||||
|
"name": "audio_jobs",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "cache_key", "type": "text", "required": true},
|
||||||
|
{"name": "slug", "type": "text", "required": true},
|
||||||
|
{"name": "chapter", "type": "number", "required": true},
|
||||||
|
{"name": "voice", "type": "text"},
|
||||||
|
{"name": "worker_id", "type": "text"},
|
||||||
|
{"name": "status", "type": "text", "required": true},
|
||||||
|
{"name": "error_message", "type": "text"},
|
||||||
|
{"name": "started", "type": "date"},
|
||||||
|
{"name": "finished", "type": "date"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
create_collection "user_library" '{
|
||||||
|
"name": "user_library",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "session_id", "type": "text", "required": true},
|
||||||
|
{"name": "user_id", "type": "text"},
|
||||||
|
{"name": "slug", "type": "text", "required": true},
|
||||||
|
{"name": "saved_at", "type": "date"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
create_collection "user_sessions" '{
|
||||||
|
"name": "user_sessions",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "user_id", "type": "text", "required": true},
|
||||||
|
{"name": "session_id", "type": "text", "required": true},
|
||||||
|
{"name": "user_agent", "type": "text"},
|
||||||
|
{"name": "ip", "type": "text"},
|
||||||
|
{"name": "created_at", "type": "date"},
|
||||||
|
{"name": "last_seen", "type": "date"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
create_collection "user_subscriptions" '{
|
||||||
|
"name": "user_subscriptions",
|
||||||
|
"type": "base",
|
||||||
|
"fields": [
|
||||||
|
{"name": "follower_id", "type": "text", "required": true},
|
||||||
|
{"name": "followee_id", "type": "text", "required": true},
|
||||||
|
{"name": "created", "type": "date"}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
|
||||||
|
# ─── 7. Post-initial-deploy field additions ───────────────────────────────────
|
||||||
|
# heartbeat_at is used by the backend runner to detect stale tasks.
|
||||||
|
ensure_field "scraping_tasks" "heartbeat_at" "date"
|
||||||
|
ensure_field "audio_jobs" "heartbeat_at" "date"
|
||||||
|
|
||||||
log "all collections ready"
|
log "all collections ready"
|
||||||
|
|||||||
5
ui-v2/.dockerignore
Normal file
5
ui-v2/.dockerignore
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
node_modules
|
||||||
|
build
|
||||||
|
.svelte-kit
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
20
ui-v2/.env.example
Normal file
20
ui-v2/.env.example
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
# libnovel UI — environment variables
|
||||||
|
# Copy to .env and adjust; do NOT commit with real secrets.
|
||||||
|
|
||||||
|
# Public URL of the scraper API (used by SvelteKit server-side load functions)
|
||||||
|
# In docker-compose this is the internal service name
|
||||||
|
SCRAPER_API_URL=http://localhost:8080
|
||||||
|
|
||||||
|
# Public URL of PocketBase (used by SvelteKit server-side load functions)
|
||||||
|
POCKETBASE_URL=http://localhost:8090
|
||||||
|
|
||||||
|
# PocketBase admin credentials (server-side only, never exposed to browser)
|
||||||
|
POCKETBASE_ADMIN_EMAIL=admin@libnovel.local
|
||||||
|
POCKETBASE_ADMIN_PASSWORD=changeme123
|
||||||
|
|
||||||
|
# Public-facing MinIO URL (used to rewrite presigned URLs for the browser)
|
||||||
|
# In dev this is localhost; in prod set to your MinIO public domain
|
||||||
|
PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000
|
||||||
|
|
||||||
|
# Secret used to sign auth tokens stored in cookies (generate with: openssl rand -hex 32)
|
||||||
|
AUTH_SECRET=change_this_to_a_long_random_secret
|
||||||
23
ui-v2/.gitignore
vendored
Normal file
23
ui-v2/.gitignore
vendored
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
node_modules
|
||||||
|
|
||||||
|
# Output
|
||||||
|
.output
|
||||||
|
.vercel
|
||||||
|
.netlify
|
||||||
|
.wrangler
|
||||||
|
/.svelte-kit
|
||||||
|
/build
|
||||||
|
|
||||||
|
# OS
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# Env
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
!.env.example
|
||||||
|
!.env.test
|
||||||
|
|
||||||
|
# Vite
|
||||||
|
vite.config.js.timestamp-*
|
||||||
|
vite.config.ts.timestamp-*
|
||||||
1
ui-v2/.npmrc
Normal file
1
ui-v2/.npmrc
Normal file
@@ -0,0 +1 @@
|
|||||||
|
engine-strict=true
|
||||||
39
ui-v2/Dockerfile
Normal file
39
ui-v2/Dockerfile
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
# syntax=docker/dockerfile:1
|
||||||
|
FROM node:22-alpine AS builder
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Install dependencies in a separate layer so it is cached as long as
|
||||||
|
# package-lock.json does not change. The npm cache mount persists the
|
||||||
|
# ~/.npm cache across builds so packages are not re-downloaded.
|
||||||
|
COPY package.json package-lock.json ./
|
||||||
|
RUN --mount=type=cache,target=/root/.npm \
|
||||||
|
npm ci
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Build-time version info — injected by docker-compose or CI via --build-arg.
|
||||||
|
ARG BUILD_VERSION=dev
|
||||||
|
ARG BUILD_COMMIT=unknown
|
||||||
|
|
||||||
|
# Expose as PUBLIC_ env vars so SvelteKit's $env/dynamic/public can read them.
|
||||||
|
ENV PUBLIC_BUILD_VERSION=$BUILD_VERSION
|
||||||
|
ENV PUBLIC_BUILD_COMMIT=$BUILD_COMMIT
|
||||||
|
|
||||||
|
RUN npm run build
|
||||||
|
|
||||||
|
# ── Runtime image ──────────────────────────────────────────────────────────────
|
||||||
|
# adapter-node bundles all server-side dependencies into build/ — no npm install
|
||||||
|
# needed at runtime. We do need package.json (for "type": "module") so Node
|
||||||
|
# resolves the ESM output correctly when there is no parent package.json.
|
||||||
|
FROM node:22-alpine
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY --from=builder /app/build ./build
|
||||||
|
COPY --from=builder /app/package.json ./package.json
|
||||||
|
|
||||||
|
ENV NODE_ENV=production
|
||||||
|
ENV PORT=3000
|
||||||
|
ENV HOST=0.0.0.0
|
||||||
|
|
||||||
|
EXPOSE $PORT
|
||||||
|
CMD ["node", "build"]
|
||||||
42
ui-v2/README.md
Normal file
42
ui-v2/README.md
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
# sv
|
||||||
|
|
||||||
|
Everything you need to build a Svelte project, powered by [`sv`](https://github.com/sveltejs/cli).
|
||||||
|
|
||||||
|
## Creating a project
|
||||||
|
|
||||||
|
If you're seeing this, you've probably already done this step. Congrats!
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# create a new project
|
||||||
|
npx sv create my-app
|
||||||
|
```
|
||||||
|
|
||||||
|
To recreate this project with the same configuration:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# recreate this project
|
||||||
|
npx sv@0.12.4 create --template minimal --types ts --install npm ui
|
||||||
|
```
|
||||||
|
|
||||||
|
## Developing
|
||||||
|
|
||||||
|
Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
npm run dev
|
||||||
|
|
||||||
|
# or start the server and open the app in a new browser tab
|
||||||
|
npm run dev -- --open
|
||||||
|
```
|
||||||
|
|
||||||
|
## Building
|
||||||
|
|
||||||
|
To create a production version of your app:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
npm run build
|
||||||
|
```
|
||||||
|
|
||||||
|
You can preview the production build with `npm run preview`.
|
||||||
|
|
||||||
|
> To deploy your app, you may need to install an [adapter](https://svelte.dev/docs/kit/adapters) for your target environment.
|
||||||
4160
ui-v2/package-lock.json
generated
Normal file
4160
ui-v2/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
34
ui-v2/package.json
Normal file
34
ui-v2/package.json
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"name": "ui",
|
||||||
|
"private": true,
|
||||||
|
"version": "0.0.1",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "vite dev",
|
||||||
|
"build": "vite build",
|
||||||
|
"preview": "vite preview",
|
||||||
|
"prepare": "svelte-kit sync || echo ''",
|
||||||
|
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
|
||||||
|
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@sveltejs/adapter-auto": "^7.0.0",
|
||||||
|
"@sveltejs/adapter-node": "^5.5.4",
|
||||||
|
"@sveltejs/kit": "^2.50.2",
|
||||||
|
"@sveltejs/vite-plugin-svelte": "^6.2.4",
|
||||||
|
"@tailwindcss/vite": "^4.2.1",
|
||||||
|
"@types/node": "^25.3.3",
|
||||||
|
"svelte": "^5.51.0",
|
||||||
|
"svelte-check": "^4.4.2",
|
||||||
|
"tailwindcss": "^4.2.1",
|
||||||
|
"typescript": "^5.9.3",
|
||||||
|
"vite": "^7.3.1"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@aws-sdk/client-s3": "^3.1005.0",
|
||||||
|
"@aws-sdk/s3-request-presigner": "^3.1005.0",
|
||||||
|
"cropperjs": "^1.6.2",
|
||||||
|
"marked": "^17.0.3",
|
||||||
|
"pocketbase": "^0.26.8"
|
||||||
|
}
|
||||||
|
}
|
||||||
65
ui-v2/src/app.css
Normal file
65
ui-v2/src/app.css
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
@import "tailwindcss";
|
||||||
|
|
||||||
|
@theme {
|
||||||
|
--color-brand: #f59e0b; /* amber-400 */
|
||||||
|
--color-brand-dim: #d97706; /* amber-600 */
|
||||||
|
--color-surface: #18181b; /* zinc-900 */
|
||||||
|
--color-surface-2: #27272a; /* zinc-800 */
|
||||||
|
--color-surface-3: #3f3f46; /* zinc-700 */
|
||||||
|
--color-muted: #a1a1aa; /* zinc-400 */
|
||||||
|
--color-text: #f4f4f5; /* zinc-100 */
|
||||||
|
}
|
||||||
|
|
||||||
|
html {
|
||||||
|
background-color: var(--color-surface);
|
||||||
|
color: var(--color-text);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── Chapter prose ─────────────────────────────────────────────────── */
|
||||||
|
.prose-chapter {
|
||||||
|
max-width: 72ch;
|
||||||
|
line-height: 1.85;
|
||||||
|
font-size: 1.05rem;
|
||||||
|
color: #d4d4d8; /* zinc-300 */
|
||||||
|
}
|
||||||
|
|
||||||
|
.prose-chapter h1,
|
||||||
|
.prose-chapter h2,
|
||||||
|
.prose-chapter h3 {
|
||||||
|
color: #f4f4f5;
|
||||||
|
font-weight: 700;
|
||||||
|
margin-top: 1.5em;
|
||||||
|
margin-bottom: 0.5em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.prose-chapter h1 { font-size: 1.4rem; }
|
||||||
|
.prose-chapter h2 { font-size: 1.2rem; }
|
||||||
|
.prose-chapter h3 { font-size: 1.05rem; }
|
||||||
|
|
||||||
|
.prose-chapter p {
|
||||||
|
margin-bottom: 1.2em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.prose-chapter em {
|
||||||
|
color: #a1a1aa;
|
||||||
|
}
|
||||||
|
|
||||||
|
.prose-chapter strong {
|
||||||
|
color: #f4f4f5;
|
||||||
|
}
|
||||||
|
|
||||||
|
.prose-chapter hr {
|
||||||
|
border-color: #3f3f46;
|
||||||
|
margin: 2em 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── Navigation progress bar ───────────────────────────────────────── */
|
||||||
|
@keyframes progress-bar {
|
||||||
|
0% { width: 0%; opacity: 1; }
|
||||||
|
80% { width: 90%; opacity: 1; }
|
||||||
|
100% { width: 100%; opacity: 0; }
|
||||||
|
}
|
||||||
|
.animate-progress-bar {
|
||||||
|
animation: progress-bar 8s cubic-bezier(0.1, 0.05, 0.1, 1) forwards;
|
||||||
|
}
|
||||||
|
|
||||||
18
ui-v2/src/app.d.ts
vendored
Normal file
18
ui-v2/src/app.d.ts
vendored
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
// See https://svelte.dev/docs/kit/types#app.d.ts
|
||||||
|
// for information about these interfaces
|
||||||
|
declare global {
|
||||||
|
namespace App {
|
||||||
|
// interface Error {}
|
||||||
|
interface Locals {
|
||||||
|
sessionId: string;
|
||||||
|
user: { id: string; username: string; role: string; authSessionId: string } | null;
|
||||||
|
}
|
||||||
|
interface PageData {
|
||||||
|
user?: { id: string; username: string; role: string; authSessionId: string } | null;
|
||||||
|
}
|
||||||
|
// interface PageState {}
|
||||||
|
// interface Platform {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export {};
|
||||||
17
ui-v2/src/app.html
Normal file
17
ui-v2/src/app.html
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<link rel="icon" href="/favicon.ico" sizes="16x16 32x32" />
|
||||||
|
<link rel="icon" type="image/png" href="/favicon-32.png" sizes="32x32" />
|
||||||
|
<link rel="icon" type="image/png" href="/favicon-16.png" sizes="16x16" />
|
||||||
|
<link rel="apple-touch-icon" href="/apple-touch-icon.png" />
|
||||||
|
<link rel="icon" type="image/png" href="/icon-192.png" sizes="192x192" />
|
||||||
|
<link rel="icon" type="image/png" href="/icon-512.png" sizes="512x512" />
|
||||||
|
%sveltekit.head%
|
||||||
|
</head>
|
||||||
|
<body data-sveltekit-preload-data="hover">
|
||||||
|
<div style="display: contents">%sveltekit.body%</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
155
ui-v2/src/hooks.server.ts
Normal file
155
ui-v2/src/hooks.server.ts
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
import type { Handle } from '@sveltejs/kit';
|
||||||
|
import { randomBytes, createHmac } from 'node:crypto';
|
||||||
|
import { env } from '$env/dynamic/private';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
import { createUserSession, touchUserSession, isSessionRevoked } from '$lib/server/pocketbase';
|
||||||
|
import { drain as drainPresignCache } from '$lib/server/presignCache';
|
||||||
|
|
||||||
|
// ─── Graceful shutdown ────────────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// When Docker/Kubernetes sends SIGTERM (or the user sends SIGINT), we:
|
||||||
|
// 1. Set shuttingDown = true so new requests immediately receive 503.
|
||||||
|
// 2. Flush/drain in-process caches (presign URL cache).
|
||||||
|
// 3. Allow Node.js to exit naturally once in-flight requests finish.
|
||||||
|
//
|
||||||
|
// adapter-node does not provide a built-in hook for this, so we wire it here
|
||||||
|
// in hooks.server.ts which runs in the server Node.js process.
|
||||||
|
|
||||||
|
let shuttingDown = false;
|
||||||
|
|
||||||
|
function shutdown(signal: string) {
|
||||||
|
if (shuttingDown) return;
|
||||||
|
shuttingDown = true;
|
||||||
|
log.info('shutdown', `received ${signal}, draining in-flight requests`);
|
||||||
|
drainPresignCache();
|
||||||
|
// Don't call process.exit() — let Node exit naturally once the event loop
|
||||||
|
// is empty (adapter-node closes the HTTP server on its own).
|
||||||
|
}
|
||||||
|
|
||||||
|
process.once('SIGTERM', () => shutdown('SIGTERM'));
|
||||||
|
process.once('SIGINT', () => shutdown('SIGINT'));
|
||||||
|
|
||||||
|
const SESSION_COOKIE = 'libnovel_session';
|
||||||
|
const AUTH_COOKIE = 'libnovel_auth';
|
||||||
|
const ONE_YEAR = 60 * 60 * 24 * 365;
|
||||||
|
|
||||||
|
const AUTH_SECRET = env.AUTH_SECRET ?? 'dev_secret_change_in_production';
|
||||||
|
|
||||||
|
// ─── Token helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sign a payload string with HMAC-SHA256 using AUTH_SECRET.
|
||||||
|
* Returns "<payload>.<signature>".
|
||||||
|
*/
|
||||||
|
export function signToken(payload: string): string {
|
||||||
|
const sig = createHmac('sha256', AUTH_SECRET).update(payload).digest('hex');
|
||||||
|
return `${payload}.${sig}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verify a signed token. Returns the payload string on success, null on failure.
|
||||||
|
*/
|
||||||
|
export function verifyToken(token: string): string | null {
|
||||||
|
const lastDot = token.lastIndexOf('.');
|
||||||
|
if (lastDot < 0) return null;
|
||||||
|
const payload = token.slice(0, lastDot);
|
||||||
|
const expected = createHmac('sha256', AUTH_SECRET).update(payload).digest('hex');
|
||||||
|
const actual = token.slice(lastDot + 1);
|
||||||
|
// constant-time comparison
|
||||||
|
if (expected.length !== actual.length) return null;
|
||||||
|
let diff = 0;
|
||||||
|
for (let i = 0; i < expected.length; i++) {
|
||||||
|
diff |= expected.charCodeAt(i) ^ actual.charCodeAt(i);
|
||||||
|
}
|
||||||
|
return diff === 0 ? payload : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a signed auth token for a user.
|
||||||
|
* Payload format: "<userId>:<username>:<role>:<authSessionId>"
|
||||||
|
* authSessionId uniquely identifies this login session (for revocation).
|
||||||
|
*/
|
||||||
|
export function createAuthToken(userId: string, username: string, role: string, authSessionId: string): string {
|
||||||
|
return signToken(`${userId}:${username}:${role}:${authSessionId}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse a verified auth token into user data. Returns null if invalid.
|
||||||
|
* Supports both old format (3 segments) and new format (4 segments).
|
||||||
|
*/
|
||||||
|
export function parseAuthToken(token: string): { id: string; username: string; role: string; authSessionId: string } | null {
|
||||||
|
const payload = verifyToken(token);
|
||||||
|
if (!payload) return null;
|
||||||
|
const parts = payload.split(':');
|
||||||
|
// New format: userId:username:role:authSessionId (4 parts)
|
||||||
|
// Old format: userId:username:role (3 parts — legacy tokens before session tracking)
|
||||||
|
if (parts.length < 3) return null;
|
||||||
|
const id = parts[0];
|
||||||
|
const username = parts[1];
|
||||||
|
const role = parts[2];
|
||||||
|
const authSessionId = parts[3] ?? ''; // empty string for legacy tokens
|
||||||
|
if (!id || !username) return null;
|
||||||
|
return { id, username, role, authSessionId };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Hook ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export const handle: Handle = async ({ event, resolve }) => {
|
||||||
|
// During graceful shutdown, reject new requests immediately so the load
|
||||||
|
// balancer / Docker health-check can drain existing connections.
|
||||||
|
if (shuttingDown) {
|
||||||
|
return new Response('Service shutting down', { status: 503 });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Anonymous session cookie (for reading progress)
|
||||||
|
let sessionId = event.cookies.get(SESSION_COOKIE) ?? '';
|
||||||
|
if (!sessionId) {
|
||||||
|
sessionId = randomBytes(16).toString('hex');
|
||||||
|
event.cookies.set(SESSION_COOKIE, sessionId, {
|
||||||
|
path: '/',
|
||||||
|
httpOnly: true,
|
||||||
|
sameSite: 'lax',
|
||||||
|
maxAge: ONE_YEAR
|
||||||
|
});
|
||||||
|
}
|
||||||
|
event.locals.sessionId = sessionId;
|
||||||
|
|
||||||
|
// Auth cookie → resolve logged-in user
|
||||||
|
const authToken = event.cookies.get(AUTH_COOKIE);
|
||||||
|
if (authToken) {
|
||||||
|
const user = parseAuthToken(authToken);
|
||||||
|
if (!user) {
|
||||||
|
log.warn('auth', 'auth cookie present but failed to parse (malformed or tampered)');
|
||||||
|
event.locals.user = null;
|
||||||
|
} else {
|
||||||
|
// Validate session against DB (only for new-format tokens with authSessionId)
|
||||||
|
let sessionValid = true;
|
||||||
|
if (user.authSessionId) {
|
||||||
|
try {
|
||||||
|
const revoked = await isSessionRevoked(user.authSessionId);
|
||||||
|
if (revoked) {
|
||||||
|
log.info('auth', 'auth cookie references revoked session', {
|
||||||
|
userId: user.id,
|
||||||
|
authSessionId: user.authSessionId
|
||||||
|
});
|
||||||
|
sessionValid = false;
|
||||||
|
// Clear the invalid cookie
|
||||||
|
event.cookies.delete(AUTH_COOKIE, { path: '/' });
|
||||||
|
} else {
|
||||||
|
// Best-effort: update last_seen in the background
|
||||||
|
touchUserSession(user.authSessionId).catch(() => {});
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
// DB error — fail open to avoid locking everyone out
|
||||||
|
log.warn('auth', 'session check failed (fail open)', { err: String(err) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
event.locals.user = sessionValid ? user : null;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
event.locals.user = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return resolve(event);
|
||||||
|
};
|
||||||
|
|
||||||
1
ui-v2/src/lib/assets/favicon.svg
Normal file
1
ui-v2/src/lib/assets/favicon.svg
Normal file
@@ -0,0 +1 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="107" height="128" viewBox="0 0 107 128"><title>svelte-logo</title><path d="M94.157 22.819c-10.4-14.885-30.94-19.297-45.792-9.835L22.282 29.608A29.92 29.92 0 0 0 8.764 49.65a31.5 31.5 0 0 0 3.108 20.231 30 30 0 0 0-4.477 11.183 31.9 31.9 0 0 0 5.448 24.116c10.402 14.887 30.942 19.297 45.791 9.835l26.083-16.624A29.92 29.92 0 0 0 98.235 78.35a31.53 31.53 0 0 0-3.105-20.232 30 30 0 0 0 4.474-11.182 31.88 31.88 0 0 0-5.447-24.116" style="fill:#ff3e00"/><path d="M45.817 106.582a20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.503 18 18 0 0 1 .624-2.435l.49-1.498 1.337.981a33.6 33.6 0 0 0 10.203 5.098l.97.294-.09.968a5.85 5.85 0 0 0 1.052 3.878 6.24 6.24 0 0 0 6.695 2.485 5.8 5.8 0 0 0 1.603-.704L69.27 76.28a5.43 5.43 0 0 0 2.45-3.631 5.8 5.8 0 0 0-.987-4.371 6.24 6.24 0 0 0-6.698-2.487 5.7 5.7 0 0 0-1.6.704l-9.953 6.345a19 19 0 0 1-5.296 2.326 20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.502 17.99 17.99 0 0 1 8.13-12.052l26.081-16.623a19 19 0 0 1 5.3-2.329 20.72 20.72 0 0 1 22.237 8.243 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-.624 2.435l-.49 1.498-1.337-.98a33.6 33.6 0 0 0-10.203-5.1l-.97-.294.09-.968a5.86 5.86 0 0 0-1.052-3.878 6.24 6.24 0 0 0-6.696-2.485 5.8 5.8 0 0 0-1.602.704L37.73 51.72a5.42 5.42 0 0 0-2.449 3.63 5.79 5.79 0 0 0 .986 4.372 6.24 6.24 0 0 0 6.698 2.486 5.8 5.8 0 0 0 1.602-.704l9.952-6.342a19 19 0 0 1 5.295-2.328 20.72 20.72 0 0 1 22.237 8.242 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-8.13 12.053l-26.081 16.622a19 19 0 0 1-5.3 2.328" style="fill:#fff"/></svg>
|
||||||
|
After Width: | Height: | Size: 1.5 KiB |
146
ui-v2/src/lib/audio.svelte.ts
Normal file
146
ui-v2/src/lib/audio.svelte.ts
Normal file
@@ -0,0 +1,146 @@
|
|||||||
|
/**
|
||||||
|
* Global audio player state for libnovel.
|
||||||
|
*
|
||||||
|
* A single shared instance (module singleton) keeps audio playing across
|
||||||
|
* SvelteKit navigations. The layout mounts the <audio> element once and
|
||||||
|
* never unmounts it; the per-chapter AudioPlayer component is just a
|
||||||
|
* controller that reads/writes this state.
|
||||||
|
*
|
||||||
|
* Uses Svelte 5 runes ($state / $derived) — import only from .svelte files
|
||||||
|
* or other .svelte.ts files.
|
||||||
|
*
|
||||||
|
* ── State machine ────────────────────────────────────────────────────────────
|
||||||
|
*
|
||||||
|
* Current chapter (status):
|
||||||
|
* idle → loading → ready (fast path: audio exists in MinIO)
|
||||||
|
* idle → loading → generating → ready (slow path: Kokoro TTS)
|
||||||
|
* any → error
|
||||||
|
*
|
||||||
|
* Next chapter pre-fetch (nextStatus):
|
||||||
|
* 'none' – no next chapter, or auto-next is off
|
||||||
|
* 'prefetching' – POST /api/audio running for the next chapter
|
||||||
|
* 'prefetched' – next chapter audio is ready in MinIO
|
||||||
|
* 'failed' – pre-generation failed (will retry on navigate)
|
||||||
|
*
|
||||||
|
* Auto-next transition:
|
||||||
|
* onended fires → navigate to next chapter URL
|
||||||
|
* ↳ new chapter page mounts
|
||||||
|
* • if nextStatus === 'prefetched' → presign + play immediately
|
||||||
|
* • else → normal startPlayback() flow
|
||||||
|
*
|
||||||
|
* Pre-fetch is triggered when currentTime / duration >= 0.9 (90% mark).
|
||||||
|
* It only runs once per chapter (guarded by nextStatus !== 'none').
|
||||||
|
*/
|
||||||
|
|
||||||
|
export type AudioStatus = 'idle' | 'loading' | 'generating' | 'ready' | 'error';
|
||||||
|
export type NextStatus = 'none' | 'prefetching' | 'prefetched' | 'failed';
|
||||||
|
|
||||||
|
class AudioStore {
|
||||||
|
// ── What is loaded ──────────────────────────────────────────────────────
|
||||||
|
slug = $state('');
|
||||||
|
chapter = $state(0);
|
||||||
|
chapterTitle = $state('');
|
||||||
|
bookTitle = $state('');
|
||||||
|
voice = $state('af_bella');
|
||||||
|
speed = $state(1.0);
|
||||||
|
|
||||||
|
/** Cover image URL for the currently loaded book. */
|
||||||
|
cover = $state('');
|
||||||
|
|
||||||
|
/** Full chapter list for the currently loaded book (number + title). */
|
||||||
|
chapters = $state<{ number: number; title: string }[]>([]);
|
||||||
|
|
||||||
|
// ── Loading/generation state ────────────────────────────────────────────
|
||||||
|
status = $state<AudioStatus>('idle');
|
||||||
|
audioUrl = $state('');
|
||||||
|
errorMsg = $state('');
|
||||||
|
/** Pseudo-progress bar value 0–100 during generation */
|
||||||
|
progress = $state(0);
|
||||||
|
|
||||||
|
// ── Playback state (kept in sync with the <audio> element) ─────────────
|
||||||
|
currentTime = $state(0);
|
||||||
|
duration = $state(0);
|
||||||
|
isPlaying = $state(false);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Increment to signal the layout to toggle play/pause.
|
||||||
|
* The layout watches this with $effect and calls audioEl.play()/pause().
|
||||||
|
*/
|
||||||
|
toggleRequest = $state(0);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set to a number to seek the audio element to that time (seconds).
|
||||||
|
* The layout watches this with $effect and sets audioEl.currentTime.
|
||||||
|
* Reset to null after handling.
|
||||||
|
*/
|
||||||
|
seekRequest = $state<number | null>(null);
|
||||||
|
|
||||||
|
// ── Auto-next ────────────────────────────────────────────────────────────
|
||||||
|
/**
|
||||||
|
* When true, navigates to the next chapter when the current one ends
|
||||||
|
* and auto-starts its audio.
|
||||||
|
*/
|
||||||
|
autoNext = $state(false);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The next chapter number for the currently playing chapter, or null if
|
||||||
|
* there is no next chapter. Written by the chapter page's AudioPlayer.
|
||||||
|
* Stored here (not cleared on unmount) so onended can still read it after
|
||||||
|
* the component unmounts due to {#key} re-render on navigation.
|
||||||
|
*/
|
||||||
|
nextChapter = $state<number | null>(null);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set to the chapter number that should auto-start by the layout's onended
|
||||||
|
* handler (when autoNext fires a navigation). The AudioPlayer on the new
|
||||||
|
* page checks this on mount: if it matches the component's own chapter prop
|
||||||
|
* it starts playback and clears the value.
|
||||||
|
*
|
||||||
|
* Using the target chapter number (instead of a plain boolean) prevents the
|
||||||
|
* still-mounted outgoing AudioPlayer from reacting to the flag before the
|
||||||
|
* navigation completes — it only matches the incoming chapter's component.
|
||||||
|
*/
|
||||||
|
autoStartChapter = $state<number | null>(null);
|
||||||
|
|
||||||
|
// ── Next-chapter pre-fetch state ─────────────────────────────────────────
|
||||||
|
/**
|
||||||
|
* State of the background pre-generation for the next chapter.
|
||||||
|
* 'none' – nothing started (default / no next chapter)
|
||||||
|
* 'prefetching' – currently running POST /api/audio for next chapter
|
||||||
|
* 'prefetched' – next chapter audio confirmed ready in MinIO
|
||||||
|
* 'failed' – pre-generation failed (fallback: generate on navigate)
|
||||||
|
*/
|
||||||
|
nextStatus = $state<NextStatus>('none');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The presigned URL obtained during pre-fetch. When the user navigates
|
||||||
|
* to the next chapter, AudioPlayer picks this up and skips straight to play.
|
||||||
|
*/
|
||||||
|
nextAudioUrl = $state('');
|
||||||
|
|
||||||
|
/** Progress value (0–100) shown while pre-generating the next chapter. */
|
||||||
|
nextProgress = $state(0);
|
||||||
|
|
||||||
|
/** Which chapter number the pre-fetch state above belongs to. */
|
||||||
|
nextChapterPrefetched = $state<number | null>(null);
|
||||||
|
|
||||||
|
/** Whether the mini-bar at the bottom is visible */
|
||||||
|
get active(): boolean {
|
||||||
|
return this.status === 'ready' || this.status === 'generating' || this.status === 'loading';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True when the currently loaded track matches slug+chapter */
|
||||||
|
isCurrentChapter(slug: string, chapter: number): boolean {
|
||||||
|
return this.slug === slug && this.chapter === chapter;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Reset all next-chapter pre-fetch state. */
|
||||||
|
resetNextPrefetch() {
|
||||||
|
this.nextStatus = 'none';
|
||||||
|
this.nextAudioUrl = '';
|
||||||
|
this.nextProgress = 0;
|
||||||
|
this.nextChapterPrefetched = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const audioStore = new AudioStore();
|
||||||
862
ui-v2/src/lib/components/AudioPlayer.svelte
Normal file
862
ui-v2/src/lib/components/AudioPlayer.svelte
Normal file
@@ -0,0 +1,862 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
/**
|
||||||
|
* AudioPlayer — controller component.
|
||||||
|
*
|
||||||
|
* Does NOT own an <audio> element. Instead it reads/writes `audioStore`,
|
||||||
|
* which is shared with the layout's persistent <audio> element so audio
|
||||||
|
* survives SvelteKit navigations.
|
||||||
|
*
|
||||||
|
* ── Play flow ────────────────────────────────────────────────────────────
|
||||||
|
* On "Play narration" click / auto-start:
|
||||||
|
* 1. Populate store metadata (slug, chapter, titles, voice, speed).
|
||||||
|
* 2. If the pre-fetch already landed (nextStatus='prefetched' AND
|
||||||
|
* nextChapterPrefetched === chapter), use the cached URL immediately.
|
||||||
|
* 3. Otherwise try GET /api/presign/audio — if 200, set audioUrl → layout plays.
|
||||||
|
* 4. If 404, POST /api/audio/:slug/:n to generate. Drive pseudo progress bar.
|
||||||
|
* On success (200 or 202→done), presign and set audioUrl directly from MinIO.
|
||||||
|
*
|
||||||
|
* ── Voice selection ──────────────────────────────────────────────────────
|
||||||
|
* A "Change voice" panel lets users pick from the available Kokoro voices.
|
||||||
|
* Each voice shows a play button that streams a pre-generated sample from
|
||||||
|
* MinIO (GET /api/presign/voice-sample?voice=...). Samples are generated
|
||||||
|
* server-side via POST /api/audio/voice-samples.
|
||||||
|
*
|
||||||
|
* Changing voice updates audioStore.voice (saved to settings via layout).
|
||||||
|
* The currently loaded chapter audio is NOT re-generated automatically —
|
||||||
|
* the new voice takes effect on next "Play narration" click.
|
||||||
|
*
|
||||||
|
* ── Pre-fetch (immediate + 90% fallback) ────────────────────────────────
|
||||||
|
* When autoNext is on, prefetchNext() is called as soon as the current
|
||||||
|
* chapter starts playing (via maybeStartPrefetch() at the end of
|
||||||
|
* startPlayback()). This gives the maximum lead time for Kokoro to
|
||||||
|
* generate the next chapter so the transition is seamless.
|
||||||
|
*
|
||||||
|
* A $effect also watches currentTime/duration and fires prefetchNext() at
|
||||||
|
* the 90% mark as a fallback — covering the case where autoNext was toggled
|
||||||
|
* on mid-playback after startPlayback() had already returned.
|
||||||
|
* The nextStatus !== 'none' guard prevents double-runs in all cases.
|
||||||
|
*
|
||||||
|
* prefetchNext():
|
||||||
|
* • Calls POST /api/audio for next chapter (sets nextStatus='prefetching')
|
||||||
|
* • On success, presigns and stores URL in audioStore.nextAudioUrl
|
||||||
|
* (sets nextStatus='prefetched')
|
||||||
|
* • On failure, sets nextStatus='failed'
|
||||||
|
*
|
||||||
|
* ── Auto-next ────────────────────────────────────────────────────────────
|
||||||
|
* layout.svelte onended → sets autoStartPending=true → navigates.
|
||||||
|
* New chapter's AudioPlayer mounts → sees autoStartPending → startPlayback()
|
||||||
|
* which uses the prefetched URL if available.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { audioStore } from '$lib/audio.svelte';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
slug: string;
|
||||||
|
chapter: number;
|
||||||
|
chapterTitle?: string;
|
||||||
|
bookTitle?: string;
|
||||||
|
/** Cover image URL for the book (used in MediaSession for lock-screen art). */
|
||||||
|
cover?: string;
|
||||||
|
/** Next chapter number, or null/undefined if this is the last chapter. */
|
||||||
|
nextChapter?: number | null;
|
||||||
|
/** Full chapter list for the book (number + title). Written into the store. */
|
||||||
|
chapters?: { number: number; title: string }[];
|
||||||
|
/** List of available voices from the Kokoro API. */
|
||||||
|
voices?: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
let {
|
||||||
|
slug,
|
||||||
|
chapter,
|
||||||
|
chapterTitle = '',
|
||||||
|
bookTitle = '',
|
||||||
|
cover = '',
|
||||||
|
nextChapter = null,
|
||||||
|
chapters = [],
|
||||||
|
voices = []
|
||||||
|
}: Props = $props();
|
||||||
|
|
||||||
|
// ── Voice selector state ────────────────────────────────────────────────
|
||||||
|
let showVoicePanel = $state(false);
|
||||||
|
/** Voice whose sample is currently being fetched or playing. */
|
||||||
|
let samplePlayingVoice = $state<string | null>(null);
|
||||||
|
/** Currently active sample <audio> element — one at a time. */
|
||||||
|
let sampleAudio = $state<HTMLAudioElement | null>(null);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Human-readable label for a voice ID.
|
||||||
|
* e.g. "af_bella" → "Bella (US F)" | "bm_george" → "George (UK M)"
|
||||||
|
*/
|
||||||
|
function voiceLabel(v: string): string {
|
||||||
|
const langMap: Record<string, string> = {
|
||||||
|
af: 'US', am: 'US',
|
||||||
|
bf: 'UK', bm: 'UK',
|
||||||
|
ef: 'ES', em: 'ES',
|
||||||
|
ff: 'FR',
|
||||||
|
hf: 'IN', hm: 'IN',
|
||||||
|
'if': 'IT', im: 'IT',
|
||||||
|
jf: 'JP', jm: 'JP',
|
||||||
|
pf: 'PT', pm: 'PT',
|
||||||
|
zf: 'ZH', zm: 'ZH',
|
||||||
|
};
|
||||||
|
const genderMap: Record<string, string> = {
|
||||||
|
af: 'F', am: 'M',
|
||||||
|
bf: 'F', bm: 'M',
|
||||||
|
ef: 'F', em: 'M',
|
||||||
|
ff: 'F',
|
||||||
|
hf: 'F', hm: 'M',
|
||||||
|
'if': 'F', im: 'M',
|
||||||
|
jf: 'F', jm: 'M',
|
||||||
|
pf: 'F', pm: 'M',
|
||||||
|
zf: 'F', zm: 'M',
|
||||||
|
};
|
||||||
|
const prefix = v.slice(0, 2);
|
||||||
|
const name = v.slice(3);
|
||||||
|
// Capitalise and strip legacy v0 prefix.
|
||||||
|
const displayName = name
|
||||||
|
.replace(/^v0/, '')
|
||||||
|
.replace(/^([a-z])/, (c: string) => c.toUpperCase());
|
||||||
|
const lang = langMap[prefix] ?? prefix.toUpperCase();
|
||||||
|
const gender = genderMap[prefix] ?? '?';
|
||||||
|
return `${displayName} (${lang} ${gender})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Stop any currently playing sample. */
|
||||||
|
function stopSample() {
|
||||||
|
if (sampleAudio) {
|
||||||
|
sampleAudio.pause();
|
||||||
|
sampleAudio.src = '';
|
||||||
|
sampleAudio = null;
|
||||||
|
}
|
||||||
|
samplePlayingVoice = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Play a voice sample from MinIO. */
|
||||||
|
async function playSample(voice: string) {
|
||||||
|
// If this voice is already playing, stop it.
|
||||||
|
if (samplePlayingVoice === voice) {
|
||||||
|
stopSample();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
stopSample();
|
||||||
|
|
||||||
|
samplePlayingVoice = voice;
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/presign/voice-sample?voice=${encodeURIComponent(voice)}`);
|
||||||
|
if (res.status === 404) {
|
||||||
|
// Sample not generated yet — silently ignore
|
||||||
|
samplePlayingVoice = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!res.ok) throw new Error(`presign failed: ${res.status}`);
|
||||||
|
const data = (await res.json()) as { url: string };
|
||||||
|
|
||||||
|
const audio = new Audio(data.url);
|
||||||
|
sampleAudio = audio;
|
||||||
|
audio.onended = () => {
|
||||||
|
if (samplePlayingVoice === voice) stopSample();
|
||||||
|
};
|
||||||
|
audio.onerror = () => {
|
||||||
|
if (samplePlayingVoice === voice) stopSample();
|
||||||
|
};
|
||||||
|
await audio.play();
|
||||||
|
} catch {
|
||||||
|
samplePlayingVoice = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Select a voice and close the panel. */
|
||||||
|
function selectVoice(voice: string) {
|
||||||
|
stopSample();
|
||||||
|
audioStore.voice = voice;
|
||||||
|
showVoicePanel = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep nextChapter in the store so the layout's onended can navigate.
|
||||||
|
// NOTE: we do NOT clear on unmount here — the store retains the value so
|
||||||
|
// onended (which may fire after {#key} unmounts this component) can still
|
||||||
|
// read it. The value is superseded when the new chapter mounts.
|
||||||
|
$effect(() => {
|
||||||
|
audioStore.nextChapter = nextChapter ?? null;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Auto-start: if the layout navigated here via auto-next, kick off playback.
|
||||||
|
// We match against the chapter prop so the outgoing chapter's AudioPlayer
|
||||||
|
// (still mounted during the brief navigation window) never reacts to this.
|
||||||
|
$effect(() => {
|
||||||
|
if (audioStore.autoStartChapter === chapter) {
|
||||||
|
audioStore.autoStartChapter = null;
|
||||||
|
startPlayback();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Reset next-chapter prefetch state when this chapter changes (new page).
|
||||||
|
// Only reset if the prefetch belongs to neither the current chapter
|
||||||
|
// (about to be consumed by startPlayback) nor the next chapter (still valid).
|
||||||
|
// Any other value means stale data from a previous page.
|
||||||
|
$effect(() => {
|
||||||
|
const prefetchedFor = audioStore.nextChapterPrefetched;
|
||||||
|
if (
|
||||||
|
prefetchedFor !== null &&
|
||||||
|
prefetchedFor !== chapter &&
|
||||||
|
prefetchedFor !== (nextChapter ?? null)
|
||||||
|
) {
|
||||||
|
audioStore.resetNextPrefetch();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Close voice panel when user clicks outside (escape key).
|
||||||
|
function handleKeyDown(e: KeyboardEvent) {
|
||||||
|
if (e.key === 'Escape') {
|
||||||
|
stopSample();
|
||||||
|
showVoicePanel = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 90% pre-fetch trigger ─────────────────────────────────────────────────
|
||||||
|
// Watch playback progress; when >= 90% of current chapter, pre-generate
|
||||||
|
// the next chapter's audio so it's ready when we navigate.
|
||||||
|
$effect(() => {
|
||||||
|
const ct = audioStore.currentTime;
|
||||||
|
const dur = audioStore.duration;
|
||||||
|
const isCurrentlyPlaying = audioStore.isCurrentChapter(slug, chapter);
|
||||||
|
|
||||||
|
if (
|
||||||
|
!isCurrentlyPlaying ||
|
||||||
|
!audioStore.autoNext ||
|
||||||
|
nextChapter === null ||
|
||||||
|
nextChapter === undefined ||
|
||||||
|
dur <= 0 ||
|
||||||
|
ct / dur < 0.9 ||
|
||||||
|
audioStore.nextStatus !== 'none'
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trigger exactly once (nextStatus transitions away from 'none')
|
||||||
|
prefetchNext();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Pseudo progress helpers ────────────────────────────────────────────────
|
||||||
|
let progressRafId = 0;
|
||||||
|
|
||||||
|
function startProgress() {
|
||||||
|
audioStore.progress = 0;
|
||||||
|
let last = performance.now();
|
||||||
|
|
||||||
|
function tick(now: number) {
|
||||||
|
const dt = (now - last) / 1000;
|
||||||
|
last = now;
|
||||||
|
let rate: number;
|
||||||
|
if (audioStore.progress < 30) rate = 4;
|
||||||
|
else if (audioStore.progress < 60) rate = 12;
|
||||||
|
else if (audioStore.progress < 80) rate = 4;
|
||||||
|
else rate = 0.3;
|
||||||
|
|
||||||
|
audioStore.progress = Math.min(audioStore.progress + rate * dt, 99);
|
||||||
|
if (audioStore.progress < 99) {
|
||||||
|
progressRafId = requestAnimationFrame(tick);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
progressRafId = requestAnimationFrame(tick);
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopProgress() {
|
||||||
|
if (progressRafId) {
|
||||||
|
cancelAnimationFrame(progressRafId);
|
||||||
|
progressRafId = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function finishProgress() {
|
||||||
|
stopProgress();
|
||||||
|
const step = () => {
|
||||||
|
audioStore.progress = Math.min(audioStore.progress + 8, 100);
|
||||||
|
if (audioStore.progress < 100) {
|
||||||
|
progressRafId = requestAnimationFrame(step);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
progressRafId = requestAnimationFrame(step);
|
||||||
|
await new Promise((r) => setTimeout(r, 200));
|
||||||
|
stopProgress();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Next-chapter pseudo-progress helpers ──────────────────────────────────
|
||||||
|
let nextProgressRafId = 0;
|
||||||
|
|
||||||
|
function startNextProgress() {
|
||||||
|
audioStore.nextProgress = 0;
|
||||||
|
let last = performance.now();
|
||||||
|
|
||||||
|
function tick(now: number) {
|
||||||
|
const dt = (now - last) / 1000;
|
||||||
|
last = now;
|
||||||
|
let rate: number;
|
||||||
|
if (audioStore.nextProgress < 30) rate = 4;
|
||||||
|
else if (audioStore.nextProgress < 60) rate = 12;
|
||||||
|
else if (audioStore.nextProgress < 80) rate = 4;
|
||||||
|
else rate = 0.3;
|
||||||
|
|
||||||
|
audioStore.nextProgress = Math.min(audioStore.nextProgress + rate * dt, 99);
|
||||||
|
if (audioStore.nextProgress < 99) {
|
||||||
|
nextProgressRafId = requestAnimationFrame(tick);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nextProgressRafId = requestAnimationFrame(tick);
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopNextProgress() {
|
||||||
|
if (nextProgressRafId) {
|
||||||
|
cancelAnimationFrame(nextProgressRafId);
|
||||||
|
nextProgressRafId = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── API helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async function tryPresign(
|
||||||
|
targetSlug: string,
|
||||||
|
targetChapter: number,
|
||||||
|
targetVoice: string
|
||||||
|
): Promise<string | null> {
|
||||||
|
const params = new URLSearchParams({
|
||||||
|
slug: targetSlug,
|
||||||
|
n: String(targetChapter),
|
||||||
|
voice: targetVoice
|
||||||
|
});
|
||||||
|
const res = await fetch(`/api/presign/audio?${params}`);
|
||||||
|
if (res.status === 404) return null;
|
||||||
|
if (!res.ok) throw new Error(`presign HTTP ${res.status}`);
|
||||||
|
const data = (await res.json()) as { url: string };
|
||||||
|
return data.url;
|
||||||
|
}
|
||||||
|
|
||||||
|
type AudioStatusResponse =
|
||||||
|
| { status: 'done' }
|
||||||
|
| { status: 'pending' | 'generating'; job_id: string }
|
||||||
|
| { status: 'idle' }
|
||||||
|
| { status: 'failed'; error?: string };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Poll GET /api/audio/status/[slug]/[n]?voice=... every `intervalMs` ms
|
||||||
|
* until status is "done" or "failed" (or the caller cancels via signal).
|
||||||
|
*
|
||||||
|
* Returns the final status response, or throws on network error / cancellation.
|
||||||
|
*/
|
||||||
|
async function pollAudioStatus(
|
||||||
|
targetSlug: string,
|
||||||
|
targetChapter: number,
|
||||||
|
targetVoice: string,
|
||||||
|
intervalMs = 2000,
|
||||||
|
signal?: AbortSignal
|
||||||
|
): Promise<AudioStatusResponse> {
|
||||||
|
const qs = new URLSearchParams();
|
||||||
|
if (targetVoice) qs.set('voice', targetVoice);
|
||||||
|
const url = `/api/audio/status/${targetSlug}/${targetChapter}?${qs.toString()}`;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
if (signal?.aborted) throw new DOMException('Aborted', 'AbortError');
|
||||||
|
|
||||||
|
const res = await fetch(url, { signal });
|
||||||
|
if (!res.ok) throw new Error(`Status poll HTTP ${res.status}`);
|
||||||
|
const data = (await res.json()) as AudioStatusResponse;
|
||||||
|
|
||||||
|
if (data.status === 'done' || data.status === 'failed') {
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Still pending/generating — wait then retry.
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const timer = setTimeout(resolve, intervalMs);
|
||||||
|
signal?.addEventListener('abort', () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
reject(new DOMException('Aborted', 'AbortError'));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Pre-fetch next chapter ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async function prefetchNext() {
|
||||||
|
if (nextChapter === null || nextChapter === undefined) return;
|
||||||
|
if (audioStore.nextStatus !== 'none') return; // already running or done
|
||||||
|
|
||||||
|
const voice = audioStore.voice;
|
||||||
|
|
||||||
|
audioStore.nextStatus = 'prefetching';
|
||||||
|
audioStore.nextChapterPrefetched = nextChapter;
|
||||||
|
startNextProgress();
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Fast path: already generated
|
||||||
|
const url = await tryPresign(slug, nextChapter, voice);
|
||||||
|
if (url) {
|
||||||
|
stopNextProgress();
|
||||||
|
audioStore.nextProgress = 100;
|
||||||
|
audioStore.nextAudioUrl = url;
|
||||||
|
audioStore.nextStatus = 'prefetched';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Slow path: trigger Kokoro generation (non-blocking POST), then poll.
|
||||||
|
const res = await fetch(`/api/audio/${slug}/${nextChapter}`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ voice })
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(`Prefetch generation failed: HTTP ${res.status}`);
|
||||||
|
|
||||||
|
// Whether the server returned 200 (already cached) or 202 (enqueued),
|
||||||
|
// always presign — the status endpoint no longer returns a proxy URL.
|
||||||
|
if (res.status === 200) {
|
||||||
|
// Body is { status: 'done' } — audio confirmed in MinIO. Presign it.
|
||||||
|
await res.body?.cancel();
|
||||||
|
}
|
||||||
|
// else 202: generation enqueued — fall through to poll.
|
||||||
|
|
||||||
|
if (res.status !== 200) {
|
||||||
|
// 202: poll until done.
|
||||||
|
const final = await pollAudioStatus(slug, nextChapter, voice);
|
||||||
|
stopNextProgress();
|
||||||
|
audioStore.nextProgress = 100;
|
||||||
|
|
||||||
|
if (final.status === 'failed') {
|
||||||
|
throw new Error(`Prefetch failed: ${(final as { error?: string }).error ?? 'unknown'}`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
stopNextProgress();
|
||||||
|
audioStore.nextProgress = 100;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Audio is ready in MinIO — get a direct presigned URL.
|
||||||
|
const doneUrl = await tryPresign(slug, nextChapter, voice);
|
||||||
|
if (!doneUrl) throw new Error('Prefetch: audio done but presign returned 404');
|
||||||
|
|
||||||
|
audioStore.nextAudioUrl = doneUrl;
|
||||||
|
audioStore.nextStatus = 'prefetched';
|
||||||
|
} catch {
|
||||||
|
stopNextProgress();
|
||||||
|
audioStore.nextStatus = 'failed';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Media Session ──────────────────────────────────────────────────────────
|
||||||
|
// Sets the OS-level media metadata so the book cover, title, and chapter
|
||||||
|
// appear on the phone lock screen / notification center.
|
||||||
|
|
||||||
|
function setMediaSession() {
|
||||||
|
if (typeof navigator === 'undefined' || !('mediaSession' in navigator)) return;
|
||||||
|
|
||||||
|
const artwork: MediaImage[] = cover
|
||||||
|
? [
|
||||||
|
{ src: cover, sizes: '512x512', type: 'image/jpeg' },
|
||||||
|
{ src: cover, sizes: '256x256', type: 'image/jpeg' }
|
||||||
|
]
|
||||||
|
: [];
|
||||||
|
|
||||||
|
navigator.mediaSession.metadata = new MediaMetadata({
|
||||||
|
title: chapterTitle || `Chapter ${chapter}`,
|
||||||
|
artist: bookTitle,
|
||||||
|
album: bookTitle,
|
||||||
|
artwork
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Core play flow ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async function startPlayback() {
|
||||||
|
const voice = audioStore.voice;
|
||||||
|
|
||||||
|
// Populate store metadata so layout + mini-bar have track info.
|
||||||
|
audioStore.slug = slug;
|
||||||
|
audioStore.chapter = chapter;
|
||||||
|
audioStore.chapterTitle = chapterTitle;
|
||||||
|
audioStore.bookTitle = bookTitle;
|
||||||
|
audioStore.cover = cover;
|
||||||
|
audioStore.chapters = chapters;
|
||||||
|
|
||||||
|
// Update OS media session (lock screen / notification center).
|
||||||
|
setMediaSession();
|
||||||
|
|
||||||
|
audioStore.status = 'loading';
|
||||||
|
audioStore.errorMsg = '';
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Fast path A: pre-fetch already landed for THIS chapter.
|
||||||
|
if (
|
||||||
|
audioStore.nextStatus === 'prefetched' &&
|
||||||
|
audioStore.nextChapterPrefetched === chapter &&
|
||||||
|
audioStore.nextAudioUrl
|
||||||
|
) {
|
||||||
|
const url = audioStore.nextAudioUrl;
|
||||||
|
// Consume the pre-fetch — reset so it doesn't carry over
|
||||||
|
audioStore.resetNextPrefetch();
|
||||||
|
audioStore.audioUrl = url;
|
||||||
|
audioStore.status = 'ready';
|
||||||
|
// Don't restore saved time for auto-next; position is 0
|
||||||
|
// Immediately start pre-generating the chapter after this one.
|
||||||
|
maybeStartPrefetch();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fast path B: audio already in MinIO (presign check).
|
||||||
|
const url = await tryPresign(slug, chapter, voice);
|
||||||
|
if (url) {
|
||||||
|
audioStore.audioUrl = url;
|
||||||
|
audioStore.status = 'ready';
|
||||||
|
// Restore last saved position after the audio element loads
|
||||||
|
restoreSavedAudioTime();
|
||||||
|
// Immediately start pre-generating the next chapter in background.
|
||||||
|
maybeStartPrefetch();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Slow path: trigger Kokoro generation (non-blocking POST), then poll.
|
||||||
|
audioStore.status = 'generating';
|
||||||
|
startProgress();
|
||||||
|
|
||||||
|
const res = await fetch(`/api/audio/${slug}/${chapter}`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ voice })
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(`Generation failed: HTTP ${res.status}`);
|
||||||
|
|
||||||
|
if (res.status !== 200) {
|
||||||
|
// 202: generation enqueued — poll until done.
|
||||||
|
const final = await pollAudioStatus(slug, chapter, voice);
|
||||||
|
|
||||||
|
if (final.status === 'failed') {
|
||||||
|
throw new Error(
|
||||||
|
`Generation failed: ${(final as { error?: string }).error ?? 'unknown error'}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// 200: already cached — body is { status: 'done' }, no url needed.
|
||||||
|
await res.body?.cancel();
|
||||||
|
}
|
||||||
|
|
||||||
|
await finishProgress();
|
||||||
|
|
||||||
|
// Audio is ready in MinIO — always use a presigned URL for direct playback.
|
||||||
|
const doneUrl = await tryPresign(slug, chapter, voice);
|
||||||
|
if (!doneUrl) throw new Error('Audio generated but presign returned 404');
|
||||||
|
audioStore.audioUrl = doneUrl;
|
||||||
|
audioStore.status = 'ready';
|
||||||
|
// Don't restore time for freshly generated audio — position is 0
|
||||||
|
// Immediately start pre-generating the next chapter in background.
|
||||||
|
maybeStartPrefetch();
|
||||||
|
} catch (e) {
|
||||||
|
stopProgress();
|
||||||
|
audioStore.progress = 0;
|
||||||
|
audioStore.status = 'error';
|
||||||
|
audioStore.errorMsg = String(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start pre-fetching the next chapter if autoNext is on, there is a next
|
||||||
|
* chapter, and no prefetch is already running or completed.
|
||||||
|
* Called as soon as current-chapter playback begins so that the next
|
||||||
|
* chapter's audio is ready before we need it (seamless transition).
|
||||||
|
* The 90%-mark $effect acts as a fallback for cases where autoNext is
|
||||||
|
* toggled on mid-playback.
|
||||||
|
*/
|
||||||
|
function maybeStartPrefetch() {
|
||||||
|
if (
|
||||||
|
audioStore.autoNext &&
|
||||||
|
nextChapter !== null &&
|
||||||
|
nextChapter !== undefined &&
|
||||||
|
audioStore.nextStatus === 'none'
|
||||||
|
) {
|
||||||
|
prefetchNext();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch the saved audio time for this chapter and seek to it after a short
|
||||||
|
* delay (to allow the audio element to load the source).
|
||||||
|
*/
|
||||||
|
async function restoreSavedAudioTime() {
|
||||||
|
try {
|
||||||
|
const params = new URLSearchParams({ slug, chapter: String(chapter) });
|
||||||
|
const res = await fetch(`/api/progress/audio-time?${params}`);
|
||||||
|
if (!res.ok) return;
|
||||||
|
const data = (await res.json()) as { audioTime: number | null };
|
||||||
|
if (data.audioTime && data.audioTime > 5) {
|
||||||
|
// Small delay to let the <audio> element fully load the src before seeking
|
||||||
|
setTimeout(() => {
|
||||||
|
audioStore.seekRequest = data.audioTime as number;
|
||||||
|
}, 300);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Non-critical — silently ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handlePlay() {
|
||||||
|
const isCurrent = audioStore.isCurrentChapter(slug, chapter);
|
||||||
|
|
||||||
|
// Already loaded this chapter: toggle play/pause.
|
||||||
|
if (isCurrent && audioStore.status === 'ready') {
|
||||||
|
audioStore.toggleRequest = (audioStore.toggleRequest ?? 0) + 1;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not yet loaded — start the full flow.
|
||||||
|
await startPlayback();
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatTime(s: number): string {
|
||||||
|
if (!isFinite(s) || s < 0) return '0:00';
|
||||||
|
const m = Math.floor(s / 60);
|
||||||
|
const sec = Math.floor(s % 60);
|
||||||
|
return `${m}:${sec.toString().padStart(2, '0')}`;
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<svelte:window onkeydown={handleKeyDown} />
|
||||||
|
|
||||||
|
<div class="mt-6 p-4 rounded-lg bg-zinc-800 border border-zinc-700">
|
||||||
|
<div class="flex items-center justify-between gap-2 mb-3">
|
||||||
|
<div class="flex items-center gap-2">
|
||||||
|
<svg class="w-4 h-4 text-amber-400" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M12 3v10.55A4 4 0 1014 17V7h4V3h-6z"/>
|
||||||
|
</svg>
|
||||||
|
<span class="text-sm text-zinc-300 font-medium">Audio Narration</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Voice selector button -->
|
||||||
|
{#if voices.length > 0}
|
||||||
|
<button
|
||||||
|
onclick={() => { stopSample(); showVoicePanel = !showVoicePanel; }}
|
||||||
|
class="flex items-center gap-1.5 px-2 py-1 rounded text-xs font-medium transition-colors {showVoicePanel
|
||||||
|
? 'text-amber-400 bg-amber-400/15 hover:bg-amber-400/25'
|
||||||
|
: 'text-zinc-400 bg-zinc-700 hover:bg-zinc-600 hover:text-zinc-200'}"
|
||||||
|
title="Change voice"
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M12 14c1.66 0 3-1.34 3-3V5c0-1.66-1.34-3-3-3S9 3.34 9 5v6c0 1.66 1.34 3 3 3zm-1-9c0-.55.45-1 1-1s1 .45 1 1v6c0 .55-.45 1-1 1s-1-.45-1-1V5zm6 6c0 2.76-2.24 5-5 5s-5-2.24-5-5H5c0 3.53 2.61 6.43 6 6.92V21h2v-3.08c3.39-.49 6-3.39 6-6.92h-2z"/>
|
||||||
|
</svg>
|
||||||
|
<span class="max-w-[80px] truncate">{voiceLabel(audioStore.voice)}</span>
|
||||||
|
<svg class="w-3 h-3 flex-shrink-0 transition-transform {showVoicePanel ? 'rotate-180' : ''}" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M7 10l5 5 5-5z"/>
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- ── Voice selector panel ──────────────────────────────────────────── -->
|
||||||
|
{#if showVoicePanel && voices.length > 0}
|
||||||
|
<div class="mb-3 rounded-lg border border-zinc-600 bg-zinc-900 overflow-hidden">
|
||||||
|
<div class="px-3 py-2 border-b border-zinc-700 flex items-center justify-between">
|
||||||
|
<span class="text-xs font-semibold text-zinc-400 uppercase tracking-wider">Choose Voice</span>
|
||||||
|
<button
|
||||||
|
onclick={() => { stopSample(); showVoicePanel = false; }}
|
||||||
|
class="text-zinc-500 hover:text-zinc-300 transition-colors"
|
||||||
|
aria-label="Close voice selector"
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"/>
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class="max-h-64 overflow-y-auto">
|
||||||
|
{#each voices as v (v)}
|
||||||
|
<div
|
||||||
|
class="flex items-center gap-2 px-3 py-2 hover:bg-zinc-800 transition-colors cursor-pointer {audioStore.voice === v ? 'bg-amber-400/10' : ''}"
|
||||||
|
role="button"
|
||||||
|
tabindex="0"
|
||||||
|
onclick={() => selectVoice(v)}
|
||||||
|
onkeydown={(e) => e.key === 'Enter' && selectVoice(v)}
|
||||||
|
>
|
||||||
|
<!-- Selected indicator -->
|
||||||
|
<div class="w-4 flex-shrink-0">
|
||||||
|
{#if audioStore.voice === v}
|
||||||
|
<svg class="w-3.5 h-3.5 text-amber-400" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41L9 16.17z"/>
|
||||||
|
</svg>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Voice name -->
|
||||||
|
<span class="flex-1 text-xs {audioStore.voice === v ? 'text-amber-400 font-medium' : 'text-zinc-300'}">
|
||||||
|
{voiceLabel(v)}
|
||||||
|
</span>
|
||||||
|
<span class="text-zinc-600 text-xs font-mono">{v}</span>
|
||||||
|
|
||||||
|
<!-- Sample play button (stop propagation so click doesn't select) -->
|
||||||
|
<button
|
||||||
|
onclick={(e) => { e.stopPropagation(); playSample(v); }}
|
||||||
|
class="p-1 rounded transition-colors flex-shrink-0 {samplePlayingVoice === v
|
||||||
|
? 'text-amber-400 bg-amber-400/15 hover:bg-amber-400/25'
|
||||||
|
: 'text-zinc-500 hover:text-zinc-200 hover:bg-zinc-700'}"
|
||||||
|
title={samplePlayingVoice === v ? 'Stop sample' : 'Play sample'}
|
||||||
|
aria-label={samplePlayingVoice === v ? `Stop ${v} sample` : `Play ${v} sample`}
|
||||||
|
>
|
||||||
|
{#if samplePlayingVoice === v}
|
||||||
|
<!-- Stop icon -->
|
||||||
|
<svg class="w-3.5 h-3.5" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M6 6h12v12H6z"/>
|
||||||
|
</svg>
|
||||||
|
{:else}
|
||||||
|
<!-- Play icon -->
|
||||||
|
<svg class="w-3.5 h-3.5" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M8 5v14l11-7z"/>
|
||||||
|
</svg>
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
<div class="px-3 py-2 border-t border-zinc-700 bg-zinc-800/50">
|
||||||
|
<p class="text-xs text-zinc-500">
|
||||||
|
New voice applies on next "Play narration".
|
||||||
|
{#if voices.length > 0}
|
||||||
|
<a
|
||||||
|
href="/api/audio/voice-samples"
|
||||||
|
class="text-zinc-400 hover:text-amber-400 transition-colors underline"
|
||||||
|
onclick={(e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
fetch('/api/audio/voice-samples', { method: 'POST' }).catch(() => {});
|
||||||
|
}}
|
||||||
|
>Generate missing samples</a>
|
||||||
|
{/if}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
{#if audioStore.isCurrentChapter(slug, chapter)}
|
||||||
|
<!-- ── This chapter is the active one ── -->
|
||||||
|
|
||||||
|
{#if audioStore.status === 'idle' || audioStore.status === 'error'}
|
||||||
|
<!-- Should not normally reach here while current, but handle gracefully -->
|
||||||
|
{#if audioStore.status === 'error'}
|
||||||
|
<p class="text-red-400 text-sm mb-2">{audioStore.errorMsg || 'Failed to load audio.'}</p>
|
||||||
|
{/if}
|
||||||
|
<button
|
||||||
|
onclick={handlePlay}
|
||||||
|
class="px-4 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors flex items-center gap-2"
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5 ml-0.5" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M8 5v14l11-7z"/>
|
||||||
|
</svg>
|
||||||
|
Play narration
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{:else if audioStore.status === 'loading'}
|
||||||
|
<button
|
||||||
|
disabled
|
||||||
|
class="px-4 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold opacity-50 cursor-not-allowed flex items-center gap-2"
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5 animate-spin" fill="none" viewBox="0 0 24 24">
|
||||||
|
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
|
||||||
|
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
|
||||||
|
</svg>
|
||||||
|
Loading…
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{:else if audioStore.status === 'generating'}
|
||||||
|
<div class="space-y-2">
|
||||||
|
<p class="text-xs text-zinc-400">Generating narration…</p>
|
||||||
|
<div class="w-full h-1.5 bg-zinc-700 rounded-full overflow-hidden">
|
||||||
|
<div
|
||||||
|
class="h-full bg-amber-400 rounded-full transition-none"
|
||||||
|
style="width: {audioStore.progress}%"
|
||||||
|
></div>
|
||||||
|
</div>
|
||||||
|
<p class="text-xs text-zinc-500 tabular-nums">{Math.round(audioStore.progress)}%</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{:else if audioStore.status === 'ready'}
|
||||||
|
<!-- Mini-bar is the canonical control surface — show a compact indicator here -->
|
||||||
|
<div class="flex items-center justify-between gap-3">
|
||||||
|
<div class="flex items-center gap-2 text-xs text-zinc-400">
|
||||||
|
{#if audioStore.isPlaying}
|
||||||
|
<svg class="w-3.5 h-3.5 text-amber-400 flex-shrink-0" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M6 4h4v16H6V4zm8 0h4v16h-4V4z"/>
|
||||||
|
</svg>
|
||||||
|
<span>Playing — controls below</span>
|
||||||
|
{:else}
|
||||||
|
<svg class="w-3.5 h-3.5 flex-shrink-0 ml-0.5" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M8 5v14l11-7z"/>
|
||||||
|
</svg>
|
||||||
|
<span>Paused — controls below</span>
|
||||||
|
{/if}
|
||||||
|
<span class="tabular-nums text-zinc-500">
|
||||||
|
{formatTime(audioStore.currentTime)} / {formatTime(audioStore.duration)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Auto-next toggle (keep here as useful context) -->
|
||||||
|
{#if nextChapter !== null && nextChapter !== undefined}
|
||||||
|
<button
|
||||||
|
onclick={() => (audioStore.autoNext = !audioStore.autoNext)}
|
||||||
|
class="flex items-center gap-1.5 px-2 py-1 rounded text-xs font-medium transition-colors flex-shrink-0 {audioStore.autoNext
|
||||||
|
? 'text-amber-400 bg-amber-400/15 hover:bg-amber-400/25'
|
||||||
|
: 'text-zinc-500 bg-zinc-700 hover:bg-zinc-600 hover:text-zinc-200'}"
|
||||||
|
title={audioStore.autoNext ? `Auto-next on — will play Ch.${nextChapter} automatically` : 'Auto-next off'}
|
||||||
|
aria-pressed={audioStore.autoNext}
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M6 18l8.5-6L6 6v12zm8.5-6L23 6v12l-8.5-6z"/>
|
||||||
|
</svg>
|
||||||
|
Auto
|
||||||
|
</button>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Next chapter pre-fetch status (only when auto-next is on) -->
|
||||||
|
{#if audioStore.autoNext && nextChapter !== null && nextChapter !== undefined}
|
||||||
|
<div class="mt-2">
|
||||||
|
{#if audioStore.nextStatus === 'prefetching'}
|
||||||
|
<div class="flex items-center gap-2 text-xs text-zinc-500">
|
||||||
|
<svg class="w-3 h-3 animate-spin flex-shrink-0" fill="none" viewBox="0 0 24 24">
|
||||||
|
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
|
||||||
|
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
|
||||||
|
</svg>
|
||||||
|
<span>Preparing Ch.{nextChapter}… {Math.round(audioStore.nextProgress)}%</span>
|
||||||
|
</div>
|
||||||
|
{:else if audioStore.nextStatus === 'prefetched'}
|
||||||
|
<p class="text-xs text-zinc-500 flex items-center gap-1">
|
||||||
|
<svg class="w-3 h-3 text-amber-400 flex-shrink-0" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41L9 16.17z"/>
|
||||||
|
</svg>
|
||||||
|
Ch.{nextChapter} ready
|
||||||
|
</p>
|
||||||
|
{:else if audioStore.nextStatus === 'failed'}
|
||||||
|
<p class="text-xs text-zinc-600">Ch.{nextChapter} will generate on navigate</p>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
{:else if audioStore.active}
|
||||||
|
<!-- ── A different chapter is currently playing ── -->
|
||||||
|
<div class="flex items-center justify-between gap-3">
|
||||||
|
<p class="text-xs text-zinc-400">
|
||||||
|
Now playing: {audioStore.chapterTitle || `Ch.${audioStore.chapter}`}
|
||||||
|
</p>
|
||||||
|
<button
|
||||||
|
onclick={startPlayback}
|
||||||
|
class="px-3 py-1 rounded bg-zinc-700 text-zinc-200 text-xs font-medium hover:bg-zinc-600 transition-colors flex-shrink-0"
|
||||||
|
>
|
||||||
|
Load this chapter
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{:else}
|
||||||
|
<!-- ── Idle — nothing playing ── -->
|
||||||
|
<button
|
||||||
|
onclick={handlePlay}
|
||||||
|
class="px-4 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors flex items-center gap-2"
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5 ml-0.5" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M8 5v14l11-7z"/>
|
||||||
|
</svg>
|
||||||
|
Play narration
|
||||||
|
</button>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
117
ui-v2/src/lib/components/AvatarCropModal.svelte
Normal file
117
ui-v2/src/lib/components/AvatarCropModal.svelte
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { onDestroy } from 'svelte';
|
||||||
|
import Cropper from 'cropperjs';
|
||||||
|
import type { default as CropperType } from 'cropperjs';
|
||||||
|
import 'cropperjs/dist/cropper.css';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
file: File;
|
||||||
|
onconfirm: (blob: Blob, mimeType: string) => void;
|
||||||
|
oncancel: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
let { file, onconfirm, oncancel }: Props = $props();
|
||||||
|
|
||||||
|
let imgEl: HTMLImageElement | undefined = $state();
|
||||||
|
let cropper: CropperType | null = null;
|
||||||
|
let objectUrl = '';
|
||||||
|
|
||||||
|
// Initialize cropper once the img element is bound and the file is known.
|
||||||
|
// Use a $effect so it runs after the DOM is ready (replaces onMount).
|
||||||
|
$effect(() => {
|
||||||
|
if (!imgEl || !file) return;
|
||||||
|
|
||||||
|
// Create the object URL and set src directly on the element (not via reactive
|
||||||
|
// state) so cropperjs sees the correct src before the image load event fires.
|
||||||
|
objectUrl = URL.createObjectURL(file);
|
||||||
|
imgEl.src = objectUrl;
|
||||||
|
|
||||||
|
// Cropperjs must be initialised inside the image's load event so it can
|
||||||
|
// measure the natural dimensions — if we call new Cropper() before the image
|
||||||
|
// has loaded, the crop canvas is blank/invisible.
|
||||||
|
const handleLoad = () => {
|
||||||
|
cropper = new Cropper(imgEl!, {
|
||||||
|
aspectRatio: 1,
|
||||||
|
viewMode: 1,
|
||||||
|
dragMode: 'move',
|
||||||
|
autoCropArea: 0.8,
|
||||||
|
restore: false,
|
||||||
|
guides: false,
|
||||||
|
center: true,
|
||||||
|
highlight: false,
|
||||||
|
cropBoxMovable: true,
|
||||||
|
cropBoxResizable: true,
|
||||||
|
toggleDragModeOnDblclick: false,
|
||||||
|
background: false
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
imgEl.addEventListener('load', handleLoad, { once: true });
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
imgEl?.removeEventListener('load', handleLoad);
|
||||||
|
cropper?.destroy();
|
||||||
|
cropper = null;
|
||||||
|
URL.revokeObjectURL(objectUrl);
|
||||||
|
objectUrl = '';
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
onDestroy(() => {
|
||||||
|
cropper?.destroy();
|
||||||
|
if (objectUrl) URL.revokeObjectURL(objectUrl);
|
||||||
|
});
|
||||||
|
|
||||||
|
function confirm() {
|
||||||
|
if (!cropper) return;
|
||||||
|
const canvas = cropper.getCroppedCanvas({ width: 400, height: 400 });
|
||||||
|
const mimeType = file.type === 'image/webp' ? 'image/webp' : 'image/jpeg';
|
||||||
|
canvas.toBlob(
|
||||||
|
(blob: Blob | null) => {
|
||||||
|
if (blob) onconfirm(blob, mimeType);
|
||||||
|
},
|
||||||
|
mimeType,
|
||||||
|
0.9
|
||||||
|
);
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<!-- Backdrop -->
|
||||||
|
<div
|
||||||
|
class="fixed inset-0 z-50 flex items-center justify-center bg-black/70 backdrop-blur-sm p-4"
|
||||||
|
role="dialog"
|
||||||
|
aria-modal="true"
|
||||||
|
aria-label="Crop profile picture"
|
||||||
|
>
|
||||||
|
<div class="bg-zinc-900 rounded-2xl border border-zinc-700 shadow-2xl w-full max-w-sm flex flex-col gap-4 p-5">
|
||||||
|
<h2 class="text-base font-semibold text-zinc-100">Crop profile picture</h2>
|
||||||
|
|
||||||
|
<!-- Cropper image container — overflow must be visible so cropperjs can
|
||||||
|
render the crop canvas outside the natural image bounds. The fixed
|
||||||
|
height gives cropperjs a stable container to size itself against. -->
|
||||||
|
<div class="rounded-xl bg-zinc-800" style="height: 300px; position: relative;">
|
||||||
|
<img
|
||||||
|
bind:this={imgEl}
|
||||||
|
alt="Crop preview"
|
||||||
|
style="display:block; max-width:100%; max-height:100%;"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p class="text-xs text-zinc-500 text-center">Drag to reposition · pinch or scroll to zoom · drag corners to resize</p>
|
||||||
|
|
||||||
|
<div class="flex gap-3">
|
||||||
|
<button
|
||||||
|
onclick={oncancel}
|
||||||
|
class="flex-1 py-2 rounded-lg border border-zinc-600 text-zinc-300 text-sm font-medium hover:bg-zinc-700 transition-colors"
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onclick={confirm}
|
||||||
|
class="flex-1 py-2 rounded-lg bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors"
|
||||||
|
>
|
||||||
|
Use photo
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
560
ui-v2/src/lib/components/CommentsSection.svelte
Normal file
560
ui-v2/src/lib/components/CommentsSection.svelte
Normal file
@@ -0,0 +1,560 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
interface BookComment {
|
||||||
|
id: string;
|
||||||
|
slug: string;
|
||||||
|
user_id: string;
|
||||||
|
username: string;
|
||||||
|
body: string;
|
||||||
|
upvotes: number;
|
||||||
|
downvotes: number;
|
||||||
|
created: string;
|
||||||
|
parent_id?: string;
|
||||||
|
replies?: BookComment[];
|
||||||
|
}
|
||||||
|
|
||||||
|
let {
|
||||||
|
slug,
|
||||||
|
isLoggedIn = false,
|
||||||
|
currentUserId = ''
|
||||||
|
}: {
|
||||||
|
slug: string;
|
||||||
|
isLoggedIn?: boolean;
|
||||||
|
currentUserId?: string;
|
||||||
|
} = $props();
|
||||||
|
|
||||||
|
// ── State ─────────────────────────────────────────────────────────────────
|
||||||
|
let comments = $state<BookComment[]>([]);
|
||||||
|
let myVotes = $state<Record<string, 'up' | 'down'>>({});
|
||||||
|
let avatarUrls = $state<Record<string, string>>({});
|
||||||
|
let loading = $state(true);
|
||||||
|
let loadError = $state('');
|
||||||
|
|
||||||
|
// Top-level new comment
|
||||||
|
let newBody = $state('');
|
||||||
|
let posting = $state(false);
|
||||||
|
let postError = $state('');
|
||||||
|
|
||||||
|
// Sort
|
||||||
|
let sort = $state<'new' | 'top'>('top');
|
||||||
|
|
||||||
|
// Reply state: which comment is being replied to
|
||||||
|
let replyingTo = $state<string | null>(null); // comment id
|
||||||
|
let replyBody = $state('');
|
||||||
|
let replyPosting = $state(false);
|
||||||
|
let replyError = $state('');
|
||||||
|
|
||||||
|
// Delete in-flight set
|
||||||
|
let deletingIds = $state(new Set<string>());
|
||||||
|
|
||||||
|
// Per-comment vote inflight set (prevents double-clicks)
|
||||||
|
let votingIds = $state(new Set<string>());
|
||||||
|
|
||||||
|
// ── Load comments ─────────────────────────────────────────────────────────
|
||||||
|
async function loadComments() {
|
||||||
|
loading = true;
|
||||||
|
loadError = '';
|
||||||
|
try {
|
||||||
|
const res = await fetch(
|
||||||
|
`/api/comments/${encodeURIComponent(slug)}?sort=${sort}`
|
||||||
|
);
|
||||||
|
if (!res.ok) throw new Error(`${res.status}`);
|
||||||
|
const data = await res.json();
|
||||||
|
comments = data.comments ?? [];
|
||||||
|
myVotes = data.myVotes ?? {};
|
||||||
|
avatarUrls = data.avatarUrls ?? {};
|
||||||
|
} catch (e) {
|
||||||
|
loadError = 'Failed to load comments.';
|
||||||
|
} finally {
|
||||||
|
loading = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$effect(() => {
|
||||||
|
loadComments();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Re-load when sort changes (after initial mount)
|
||||||
|
let firstLoad = true;
|
||||||
|
$effect(() => {
|
||||||
|
// Read sort to create a dependency
|
||||||
|
const _ = sort;
|
||||||
|
if (firstLoad) { firstLoad = false; return; }
|
||||||
|
loadComments();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Post top-level comment ────────────────────────────────────────────────
|
||||||
|
async function postComment() {
|
||||||
|
const text = newBody.trim();
|
||||||
|
if (!text || posting) return;
|
||||||
|
if (text.length > 2000) { postError = 'Comment is too long (max 2000 characters).'; return; }
|
||||||
|
posting = true;
|
||||||
|
postError = '';
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/comments/${encodeURIComponent(slug)}`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ body: text })
|
||||||
|
});
|
||||||
|
if (res.status === 401) { postError = 'You must be logged in to comment.'; return; }
|
||||||
|
if (!res.ok) {
|
||||||
|
const err = await res.json().catch(() => ({}));
|
||||||
|
postError = err.message ?? 'Failed to post comment.';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const created: BookComment = await res.json();
|
||||||
|
created.replies = [];
|
||||||
|
// Prepend for 'new', or re-sort for 'top'
|
||||||
|
if (sort === 'new') {
|
||||||
|
comments = [created, ...comments];
|
||||||
|
} else {
|
||||||
|
comments = [created, ...comments]; // new comment has 0 score, goes to end after sort would happen
|
||||||
|
}
|
||||||
|
newBody = '';
|
||||||
|
} catch {
|
||||||
|
postError = 'Failed to post comment.';
|
||||||
|
} finally {
|
||||||
|
posting = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Post reply ────────────────────────────────────────────────────────────
|
||||||
|
async function postReply(parentId: string) {
|
||||||
|
const text = replyBody.trim();
|
||||||
|
if (!text || replyPosting) return;
|
||||||
|
if (text.length > 2000) { replyError = 'Reply is too long (max 2000 characters).'; return; }
|
||||||
|
replyPosting = true;
|
||||||
|
replyError = '';
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/comments/${encodeURIComponent(slug)}`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ body: text, parent_id: parentId })
|
||||||
|
});
|
||||||
|
if (res.status === 401) { replyError = 'You must be logged in to reply.'; return; }
|
||||||
|
if (!res.ok) {
|
||||||
|
const err = await res.json().catch(() => ({}));
|
||||||
|
replyError = err.message ?? 'Failed to post reply.';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const created: BookComment = await res.json();
|
||||||
|
// Append to the parent's replies list
|
||||||
|
comments = comments.map((c) => {
|
||||||
|
if (c.id !== parentId) return c;
|
||||||
|
return { ...c, replies: [...(c.replies ?? []), created] };
|
||||||
|
});
|
||||||
|
replyBody = '';
|
||||||
|
replyingTo = null;
|
||||||
|
} catch {
|
||||||
|
replyError = 'Failed to post reply.';
|
||||||
|
} finally {
|
||||||
|
replyPosting = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Delete ────────────────────────────────────────────────────────────────
|
||||||
|
async function deleteComment(commentId: string, parentId?: string) {
|
||||||
|
if (deletingIds.has(commentId)) return;
|
||||||
|
deletingIds = new Set([...deletingIds, commentId]);
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/comment/${commentId}`, { method: 'DELETE' });
|
||||||
|
if (!res.ok) return;
|
||||||
|
if (parentId) {
|
||||||
|
// Remove reply from parent
|
||||||
|
comments = comments.map((c) => {
|
||||||
|
if (c.id !== parentId) return c;
|
||||||
|
return { ...c, replies: (c.replies ?? []).filter((r) => r.id !== commentId) };
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// Remove top-level comment
|
||||||
|
comments = comments.filter((c) => c.id !== commentId);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
const next = new Set(deletingIds);
|
||||||
|
next.delete(commentId);
|
||||||
|
deletingIds = next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Vote ──────────────────────────────────────────────────────────────────
|
||||||
|
async function vote(commentId: string, v: 'up' | 'down', parentId?: string) {
|
||||||
|
if (votingIds.has(commentId)) return;
|
||||||
|
votingIds = new Set([...votingIds, commentId]);
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/comment/${commentId}/vote`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ vote: v })
|
||||||
|
});
|
||||||
|
if (!res.ok) return;
|
||||||
|
const updated: BookComment = await res.json();
|
||||||
|
// Update comment in list (handle both top-level and replies)
|
||||||
|
if (parentId) {
|
||||||
|
comments = comments.map((c) => {
|
||||||
|
if (c.id !== parentId) return c;
|
||||||
|
return {
|
||||||
|
...c,
|
||||||
|
replies: (c.replies ?? []).map((r) => (r.id === commentId ? updated : r))
|
||||||
|
};
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
comments = comments.map((c) => (c.id === commentId ? { ...updated, replies: c.replies } : c));
|
||||||
|
}
|
||||||
|
// Update myVotes: toggle off if same, else set new vote
|
||||||
|
const prev = myVotes[commentId];
|
||||||
|
if (prev === v) {
|
||||||
|
const next = { ...myVotes };
|
||||||
|
delete next[commentId];
|
||||||
|
myVotes = next;
|
||||||
|
} else {
|
||||||
|
myVotes = { ...myVotes, [commentId]: v };
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
const next = new Set(votingIds);
|
||||||
|
next.delete(commentId);
|
||||||
|
votingIds = next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Helpers ───────────────────────────────────────────────────────────────
|
||||||
|
function initials(username: string): string {
|
||||||
|
const name = username.trim() || '?';
|
||||||
|
return name.slice(0, 2).toUpperCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatDate(iso: string): string {
|
||||||
|
try {
|
||||||
|
const date = new Date(iso);
|
||||||
|
const now = Date.now();
|
||||||
|
const diffMs = now - date.getTime();
|
||||||
|
const diffMins = Math.floor(diffMs / 60_000);
|
||||||
|
if (diffMins < 1) return 'just now';
|
||||||
|
if (diffMins < 60) return `${diffMins}m ago`;
|
||||||
|
const diffHours = Math.floor(diffMins / 60);
|
||||||
|
if (diffHours < 24) return `${diffHours}h ago`;
|
||||||
|
const diffDays = Math.floor(diffHours / 24);
|
||||||
|
if (diffDays < 30) return `${diffDays}d ago`;
|
||||||
|
return date.toLocaleDateString(undefined, { year: 'numeric', month: 'short', day: 'numeric' });
|
||||||
|
} catch {
|
||||||
|
return iso;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const charCount = $derived(newBody.length);
|
||||||
|
const charOver = $derived(charCount > 2000);
|
||||||
|
const replyCharCount = $derived(replyBody.length);
|
||||||
|
const replyCharOver = $derived(replyCharCount > 2000);
|
||||||
|
|
||||||
|
const totalCount = $derived(
|
||||||
|
comments.reduce((n, c) => n + 1 + (c.replies?.length ?? 0), 0)
|
||||||
|
);
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div class="mt-10">
|
||||||
|
<!-- Header + sort controls -->
|
||||||
|
<div class="flex items-center justify-between gap-3 mb-4 flex-wrap">
|
||||||
|
<h2 class="text-base font-semibold text-zinc-200">
|
||||||
|
Comments
|
||||||
|
{#if !loading && totalCount > 0}
|
||||||
|
<span class="text-zinc-500 font-normal text-sm ml-1">({totalCount})</span>
|
||||||
|
{/if}
|
||||||
|
</h2>
|
||||||
|
|
||||||
|
<!-- Sort tabs -->
|
||||||
|
{#if !loading && comments.length > 0}
|
||||||
|
<div class="flex items-center gap-1 text-xs rounded-lg bg-zinc-800/60 p-1">
|
||||||
|
<button
|
||||||
|
onclick={() => (sort = 'top')}
|
||||||
|
class="px-2.5 py-1 rounded-md transition-colors {sort === 'top'
|
||||||
|
? 'bg-zinc-700 text-zinc-100'
|
||||||
|
: 'text-zinc-500 hover:text-zinc-300'}"
|
||||||
|
>
|
||||||
|
Top
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onclick={() => (sort = 'new')}
|
||||||
|
class="px-2.5 py-1 rounded-md transition-colors {sort === 'new'
|
||||||
|
? 'bg-zinc-700 text-zinc-100'
|
||||||
|
: 'text-zinc-500 hover:text-zinc-300'}"
|
||||||
|
>
|
||||||
|
New
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Post form -->
|
||||||
|
<div class="mb-6">
|
||||||
|
{#if isLoggedIn}
|
||||||
|
<div class="flex flex-col gap-2">
|
||||||
|
<textarea
|
||||||
|
bind:value={newBody}
|
||||||
|
placeholder="Write a comment…"
|
||||||
|
rows="3"
|
||||||
|
class="w-full px-3 py-2 rounded-lg bg-zinc-800 border border-zinc-700 text-zinc-200 text-sm placeholder-zinc-500 resize-none focus:outline-none focus:border-amber-400 transition-colors"
|
||||||
|
></textarea>
|
||||||
|
<div class="flex items-center justify-between gap-3">
|
||||||
|
<span class="text-xs {charOver ? 'text-red-400' : 'text-zinc-600'} tabular-nums">
|
||||||
|
{charCount}/2000
|
||||||
|
</span>
|
||||||
|
<div class="flex items-center gap-3">
|
||||||
|
{#if postError}
|
||||||
|
<span class="text-xs text-red-400">{postError}</span>
|
||||||
|
{/if}
|
||||||
|
<button
|
||||||
|
onclick={postComment}
|
||||||
|
disabled={posting || !newBody.trim() || charOver}
|
||||||
|
class="px-4 py-1.5 rounded-lg text-sm font-medium transition-colors
|
||||||
|
{posting || !newBody.trim() || charOver
|
||||||
|
? 'bg-zinc-700 text-zinc-500 cursor-not-allowed'
|
||||||
|
: 'bg-amber-400 text-zinc-900 hover:bg-amber-300'}"
|
||||||
|
>
|
||||||
|
{posting ? 'Posting…' : 'Post'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{:else}
|
||||||
|
<p class="text-sm text-zinc-500">
|
||||||
|
<a href="/auth/login" class="text-amber-400 hover:text-amber-300 transition-colors">Log in</a>
|
||||||
|
to leave a comment.
|
||||||
|
</p>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Comment list -->
|
||||||
|
{#if loading}
|
||||||
|
<div class="flex flex-col gap-3">
|
||||||
|
{#each Array(3) as _}
|
||||||
|
<div class="rounded-lg bg-zinc-800/50 p-4 animate-pulse">
|
||||||
|
<div class="h-3 w-24 bg-zinc-700 rounded mb-3"></div>
|
||||||
|
<div class="h-3 w-full bg-zinc-700/60 rounded mb-2"></div>
|
||||||
|
<div class="h-3 w-3/4 bg-zinc-700/60 rounded"></div>
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
{:else if loadError}
|
||||||
|
<p class="text-sm text-red-400">{loadError}</p>
|
||||||
|
{:else if comments.length === 0}
|
||||||
|
<p class="text-sm text-zinc-500">No comments yet. Be the first!</p>
|
||||||
|
{:else}
|
||||||
|
<div class="flex flex-col gap-3">
|
||||||
|
{#each comments as comment (comment.id)}
|
||||||
|
{@const myVote = myVotes[comment.id]}
|
||||||
|
{@const voting = votingIds.has(comment.id)}
|
||||||
|
{@const deleting = deletingIds.has(comment.id)}
|
||||||
|
{@const isOwner = isLoggedIn && currentUserId === comment.user_id}
|
||||||
|
|
||||||
|
<div class="rounded-lg bg-zinc-800/50 border border-zinc-700/50 px-4 py-3 flex flex-col gap-2 {deleting ? 'opacity-50' : ''}">
|
||||||
|
<!-- Header -->
|
||||||
|
<div class="flex items-center gap-2 flex-wrap">
|
||||||
|
{#if avatarUrls[comment.user_id]}
|
||||||
|
<img src={avatarUrls[comment.user_id]} alt={comment.username} class="w-6 h-6 rounded-full object-cover flex-shrink-0" />
|
||||||
|
{:else}
|
||||||
|
<div class="w-6 h-6 rounded-full bg-zinc-700 flex items-center justify-center flex-shrink-0">
|
||||||
|
<span class="text-[9px] font-semibold text-zinc-300 leading-none">{initials(comment.username)}</span>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
{#if comment.username}
|
||||||
|
<a href="/users/{comment.username}" class="text-sm font-medium text-zinc-200 hover:text-amber-400 transition-colors">{comment.username}</a>
|
||||||
|
{:else}
|
||||||
|
<span class="text-sm font-medium text-zinc-400">Anonymous</span>
|
||||||
|
{/if}
|
||||||
|
<span class="text-zinc-600 text-xs">·</span>
|
||||||
|
<span class="text-xs text-zinc-500">{formatDate(comment.created)}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Body -->
|
||||||
|
<p class="text-sm text-zinc-300 leading-relaxed whitespace-pre-wrap break-words">{comment.body}</p>
|
||||||
|
|
||||||
|
<!-- Actions row: votes + reply + delete -->
|
||||||
|
<div class="flex items-center gap-3 pt-1 flex-wrap">
|
||||||
|
<!-- Upvote -->
|
||||||
|
<button
|
||||||
|
onclick={() => vote(comment.id, 'up')}
|
||||||
|
disabled={voting}
|
||||||
|
title="Upvote"
|
||||||
|
class="flex items-center gap-1 text-xs transition-colors disabled:opacity-50
|
||||||
|
{myVote === 'up' ? 'text-amber-400' : 'text-zinc-500 hover:text-zinc-300'}"
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" d="M14 10h4.764a2 2 0 011.789 2.894l-3.5 7A2 2 0 0115.263 21h-4.017c-.163 0-.326-.02-.485-.06L7 20m7-10V5a2 2 0 00-2-2h-.095c-.5 0-.905.405-.905.905 0 .714-.211 1.412-.608 2.006L7 11v9m7-10h-2M7 20H5a2 2 0 01-2-2v-6a2 2 0 012-2h2.5"/>
|
||||||
|
</svg>
|
||||||
|
<span class="tabular-nums">{comment.upvotes ?? 0}</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<!-- Downvote -->
|
||||||
|
<button
|
||||||
|
onclick={() => vote(comment.id, 'down')}
|
||||||
|
disabled={voting}
|
||||||
|
title="Downvote"
|
||||||
|
class="flex items-center gap-1 text-xs transition-colors disabled:opacity-50
|
||||||
|
{myVote === 'down' ? 'text-red-400' : 'text-zinc-500 hover:text-zinc-300'}"
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" d="M10 14H5.236a2 2 0 01-1.789-2.894l3.5-7A2 2 0 018.736 3h4.018a2 2 0 01.485.06l3.76.94m-7 10v5a2 2 0 002 2h.096c.5 0 .905-.405.905-.904 0-.715.211-1.413.608-2.008L17 13V4m-7 10h2m5-10h2a2 2 0 012 2v6a2 2 0 01-2 2h-2.5"/>
|
||||||
|
</svg>
|
||||||
|
<span class="tabular-nums">{comment.downvotes ?? 0}</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<!-- Reply button -->
|
||||||
|
{#if isLoggedIn}
|
||||||
|
<button
|
||||||
|
onclick={() => {
|
||||||
|
if (replyingTo === comment.id) {
|
||||||
|
replyingTo = null;
|
||||||
|
replyBody = '';
|
||||||
|
replyError = '';
|
||||||
|
} else {
|
||||||
|
replyingTo = comment.id;
|
||||||
|
replyBody = '';
|
||||||
|
replyError = '';
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
class="flex items-center gap-1 text-xs transition-colors
|
||||||
|
{replyingTo === comment.id
|
||||||
|
? 'text-amber-400'
|
||||||
|
: 'text-zinc-500 hover:text-zinc-300'}"
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" d="M3 10h10a8 8 0 018 8v2M3 10l6 6m-6-6l6-6"/>
|
||||||
|
</svg>
|
||||||
|
Reply
|
||||||
|
</button>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<!-- Delete (owner only) -->
|
||||||
|
{#if isOwner}
|
||||||
|
<button
|
||||||
|
onclick={() => deleteComment(comment.id)}
|
||||||
|
disabled={deleting}
|
||||||
|
class="flex items-center gap-1 text-xs text-zinc-600 hover:text-red-400 transition-colors disabled:opacity-50 ml-auto"
|
||||||
|
title="Delete comment"
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"/>
|
||||||
|
</svg>
|
||||||
|
Delete
|
||||||
|
</button>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Inline reply form -->
|
||||||
|
{#if replyingTo === comment.id}
|
||||||
|
<div class="mt-1 flex flex-col gap-2 pl-2 border-l-2 border-zinc-700">
|
||||||
|
<textarea
|
||||||
|
bind:value={replyBody}
|
||||||
|
placeholder="Write a reply…"
|
||||||
|
rows="2"
|
||||||
|
class="w-full px-3 py-2 rounded-lg bg-zinc-900 border border-zinc-700 text-zinc-200 text-sm placeholder-zinc-500 resize-none focus:outline-none focus:border-amber-400 transition-colors"
|
||||||
|
></textarea>
|
||||||
|
<div class="flex items-center justify-between gap-2">
|
||||||
|
<span class="text-xs {replyCharOver ? 'text-red-400' : 'text-zinc-600'} tabular-nums">
|
||||||
|
{replyCharCount}/2000
|
||||||
|
</span>
|
||||||
|
<div class="flex items-center gap-2">
|
||||||
|
{#if replyError}
|
||||||
|
<span class="text-xs text-red-400">{replyError}</span>
|
||||||
|
{/if}
|
||||||
|
<button
|
||||||
|
onclick={() => { replyingTo = null; replyBody = ''; replyError = ''; }}
|
||||||
|
class="px-3 py-1 rounded-lg text-xs text-zinc-400 hover:text-zinc-200 transition-colors"
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onclick={() => postReply(comment.id)}
|
||||||
|
disabled={replyPosting || !replyBody.trim() || replyCharOver}
|
||||||
|
class="px-3 py-1.5 rounded-lg text-xs font-medium transition-colors
|
||||||
|
{replyPosting || !replyBody.trim() || replyCharOver
|
||||||
|
? 'bg-zinc-700 text-zinc-500 cursor-not-allowed'
|
||||||
|
: 'bg-amber-400 text-zinc-900 hover:bg-amber-300'}"
|
||||||
|
>
|
||||||
|
{replyPosting ? 'Posting…' : 'Reply'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<!-- Replies -->
|
||||||
|
{#if comment.replies && comment.replies.length > 0}
|
||||||
|
<div class="mt-1 flex flex-col gap-2 pl-3 border-l-2 border-zinc-700/60">
|
||||||
|
{#each comment.replies as reply (reply.id)}
|
||||||
|
{@const replyVote = myVotes[reply.id]}
|
||||||
|
{@const replyVoting = votingIds.has(reply.id)}
|
||||||
|
{@const replyDeleting = deletingIds.has(reply.id)}
|
||||||
|
{@const replyIsOwner = isLoggedIn && currentUserId === reply.user_id}
|
||||||
|
|
||||||
|
<div class="rounded-md bg-zinc-800/30 px-3 py-2.5 flex flex-col gap-1.5 {replyDeleting ? 'opacity-50' : ''}">
|
||||||
|
<!-- Reply header -->
|
||||||
|
<div class="flex items-center gap-2 flex-wrap">
|
||||||
|
{#if avatarUrls[reply.user_id]}
|
||||||
|
<img src={avatarUrls[reply.user_id]} alt={reply.username} class="w-5 h-5 rounded-full object-cover flex-shrink-0" />
|
||||||
|
{:else}
|
||||||
|
<div class="w-5 h-5 rounded-full bg-zinc-700 flex items-center justify-center flex-shrink-0">
|
||||||
|
<span class="text-[8px] font-semibold text-zinc-300 leading-none">{initials(reply.username)}</span>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
{#if reply.username}
|
||||||
|
<a href="/users/{reply.username}" class="text-xs font-medium text-zinc-300 hover:text-amber-400 transition-colors">{reply.username}</a>
|
||||||
|
{:else}
|
||||||
|
<span class="text-xs font-medium text-zinc-400">Anonymous</span>
|
||||||
|
{/if}
|
||||||
|
<span class="text-zinc-600 text-xs">·</span>
|
||||||
|
<span class="text-xs text-zinc-500">{formatDate(reply.created)}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Reply body -->
|
||||||
|
<p class="text-sm text-zinc-300 leading-relaxed whitespace-pre-wrap break-words">{reply.body}</p>
|
||||||
|
|
||||||
|
<!-- Reply actions -->
|
||||||
|
<div class="flex items-center gap-3 pt-0.5">
|
||||||
|
<button
|
||||||
|
onclick={() => vote(reply.id, 'up', comment.id)}
|
||||||
|
disabled={replyVoting}
|
||||||
|
title="Upvote"
|
||||||
|
class="flex items-center gap-1 text-xs transition-colors disabled:opacity-50
|
||||||
|
{replyVote === 'up' ? 'text-amber-400' : 'text-zinc-500 hover:text-zinc-300'}"
|
||||||
|
>
|
||||||
|
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" d="M14 10h4.764a2 2 0 011.789 2.894l-3.5 7A2 2 0 0115.263 21h-4.017c-.163 0-.326-.02-.485-.06L7 20m7-10V5a2 2 0 00-2-2h-.095c-.5 0-.905.405-.905.905 0 .714-.211 1.412-.608 2.006L7 11v9m7-10h-2M7 20H5a2 2 0 01-2-2v-6a2 2 0 012-2h2.5"/>
|
||||||
|
</svg>
|
||||||
|
<span class="tabular-nums">{reply.upvotes ?? 0}</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<button
|
||||||
|
onclick={() => vote(reply.id, 'down', comment.id)}
|
||||||
|
disabled={replyVoting}
|
||||||
|
title="Downvote"
|
||||||
|
class="flex items-center gap-1 text-xs transition-colors disabled:opacity-50
|
||||||
|
{replyVote === 'down' ? 'text-red-400' : 'text-zinc-500 hover:text-zinc-300'}"
|
||||||
|
>
|
||||||
|
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" d="M10 14H5.236a2 2 0 01-1.789-2.894l3.5-7A2 2 0 018.736 3h4.018a2 2 0 01.485.06l3.76.94m-7 10v5a2 2 0 002 2h.096c.5 0 .905-.405.905-.904 0-.715.211-1.413.608-2.008L17 13V4m-7 10h2m5-10h2a2 2 0 012 2v6a2 2 0 01-2 2h-2.5"/>
|
||||||
|
</svg>
|
||||||
|
<span class="tabular-nums">{reply.downvotes ?? 0}</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{#if replyIsOwner}
|
||||||
|
<button
|
||||||
|
onclick={() => deleteComment(reply.id, comment.id)}
|
||||||
|
disabled={replyDeleting}
|
||||||
|
class="flex items-center gap-1 text-xs text-zinc-600 hover:text-red-400 transition-colors disabled:opacity-50 ml-auto"
|
||||||
|
title="Delete reply"
|
||||||
|
>
|
||||||
|
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"/>
|
||||||
|
</svg>
|
||||||
|
Delete
|
||||||
|
</button>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
1
ui-v2/src/lib/index.ts
Normal file
1
ui-v2/src/lib/index.ts
Normal file
@@ -0,0 +1 @@
|
|||||||
|
// place files you want to import through the `$lib` alias in this folder.
|
||||||
37
ui-v2/src/lib/server/logger.ts
Normal file
37
ui-v2/src/lib/server/logger.ts
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
/**
|
||||||
|
* Structured server-side logger.
|
||||||
|
*
|
||||||
|
* Emits JSON lines to stderr so they appear in container/process logs without
|
||||||
|
* polluting stdout (which Node's HTTP layer uses for responses).
|
||||||
|
*
|
||||||
|
* Format mirrors Go's log/slog default JSON output:
|
||||||
|
* {"time":"…","level":"ERROR","msg":"…","context":"pocketbase",...extra}
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* import { log } from '$lib/server/logger';
|
||||||
|
* log.error('pocketbase', 'auth failed', { status: 401, url });
|
||||||
|
* log.warn('minio', 'presign slow', { slug, n, ms: elapsed });
|
||||||
|
* log.info('auth', 'user registered', { username });
|
||||||
|
*/
|
||||||
|
|
||||||
|
type Level = 'DEBUG' | 'INFO' | 'WARN' | 'ERROR';
|
||||||
|
type Extra = Record<string, unknown>;
|
||||||
|
|
||||||
|
function emit(level: Level, context: string, msg: string, extra?: Extra): void {
|
||||||
|
const entry: Record<string, unknown> = {
|
||||||
|
time: new Date().toISOString(),
|
||||||
|
level,
|
||||||
|
context,
|
||||||
|
msg,
|
||||||
|
...extra
|
||||||
|
};
|
||||||
|
// Write to stderr — never stdout
|
||||||
|
process.stderr.write(JSON.stringify(entry) + '\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
export const log = {
|
||||||
|
debug: (context: string, msg: string, extra?: Extra) => emit('DEBUG', context, msg, extra),
|
||||||
|
info: (context: string, msg: string, extra?: Extra) => emit('INFO', context, msg, extra),
|
||||||
|
warn: (context: string, msg: string, extra?: Extra) => emit('WARN', context, msg, extra),
|
||||||
|
error: (context: string, msg: string, extra?: Extra) => emit('ERROR', context, msg, extra),
|
||||||
|
};
|
||||||
185
ui-v2/src/lib/server/minio.ts
Normal file
185
ui-v2/src/lib/server/minio.ts
Normal file
@@ -0,0 +1,185 @@
|
|||||||
|
/**
|
||||||
|
* Server-side MinIO presign helper.
|
||||||
|
* Calls the scraper API to get presigned URLs, then optionally rewrites
|
||||||
|
* the MinIO host to the public-facing URL for browser use.
|
||||||
|
*
|
||||||
|
* Never import this from client-side code.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { env } from '$env/dynamic/private';
|
||||||
|
import { env as pubEnv } from '$env/dynamic/public';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||||
|
// Public MinIO URL — used to rewrite presigned URLs so the browser can reach MinIO directly.
|
||||||
|
// In docker-compose this would differ from the internal endpoint.
|
||||||
|
const MINIO_PUBLIC_URL = pubEnv.PUBLIC_MINIO_PUBLIC_URL ?? 'http://localhost:9000';
|
||||||
|
|
||||||
|
// ─── Avatar helpers ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
function extFromMime(mime: string): string {
|
||||||
|
if (mime.includes('png')) return 'png';
|
||||||
|
if (mime.includes('webp')) return 'webp';
|
||||||
|
if (mime.includes('gif')) return 'gif';
|
||||||
|
return 'jpg';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a short-lived presigned PUT URL for uploading an avatar directly to MinIO,
|
||||||
|
* along with the object key to record in PocketBase after upload completes.
|
||||||
|
* Routed through the Go scraper which holds MinIO credentials.
|
||||||
|
*/
|
||||||
|
export async function presignAvatarUploadUrl(userId: string, mimeType: string): Promise<{ uploadUrl: string; key: string }> {
|
||||||
|
const ext = extFromMime(mimeType);
|
||||||
|
const res = await fetch(`${SCRAPER_URL}/api/presign/avatar-upload/${encodeURIComponent(userId)}?ext=${ext}`);
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.text().catch(() => '');
|
||||||
|
throw new Error(`presign avatar upload failed: ${res.status} ${body}`);
|
||||||
|
}
|
||||||
|
const data = (await res.json()) as { upload_url: string; key: string };
|
||||||
|
return { uploadUrl: data.upload_url, key: data.key };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a presigned GET URL for a user's avatar, rewritten to the public URL.
|
||||||
|
* Returns null if no avatar exists.
|
||||||
|
*/
|
||||||
|
export async function presignAvatarUrl(userId: string): Promise<string | null> {
|
||||||
|
const res = await fetch(`${SCRAPER_URL}/api/presign/avatar/${encodeURIComponent(userId)}`);
|
||||||
|
if (res.status === 404) return null;
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.text().catch(() => '');
|
||||||
|
throw new Error(`presign avatar failed: ${res.status} ${body}`);
|
||||||
|
}
|
||||||
|
const data = (await res.json()) as { url: string };
|
||||||
|
return data.url ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Rewrites the MinIO host in a presigned URL to the public-facing URL.
|
||||||
|
*
|
||||||
|
* The Go backend presigns URLs against its internal endpoint (e.g. minio:9000)
|
||||||
|
* when PUBLIC_MINIO_PUBLIC_URL is not set or equals the internal endpoint.
|
||||||
|
* In that case the browser must reach MinIO via the public URL (e.g.
|
||||||
|
* localhost:9000 in dev), so we swap the origin.
|
||||||
|
*
|
||||||
|
* NOTE: AWS Signature V4 DOES include the Host header in the canonical request
|
||||||
|
* (via X-Amz-SignedHeaders=host). Rewriting the host here would break the
|
||||||
|
* signature. This function is therefore only a no-op safety net — in
|
||||||
|
* production the Go backend is configured with MINIO_PUBLIC_ENDPOINT equal to
|
||||||
|
* the externally-reachable hostname, so presigned URLs already carry the right
|
||||||
|
* host and no rewrite is needed.
|
||||||
|
*
|
||||||
|
* For local dev: MINIO_PUBLIC_ENDPOINT=http://localhost:9000 and the backend
|
||||||
|
* presigns with localhost:9000 (the public client), so this rewrite is again
|
||||||
|
* a no-op (origins already match).
|
||||||
|
*/
|
||||||
|
function rewriteHost(presignedUrl: string): string {
|
||||||
|
try {
|
||||||
|
const u = new URL(presignedUrl);
|
||||||
|
const pub = new URL(MINIO_PUBLIC_URL);
|
||||||
|
// No-op if already pointing at the right origin.
|
||||||
|
if (u.protocol === pub.protocol && u.hostname === pub.hostname && u.port === pub.port) {
|
||||||
|
return presignedUrl;
|
||||||
|
}
|
||||||
|
u.protocol = pub.protocol;
|
||||||
|
u.hostname = pub.hostname;
|
||||||
|
u.port = pub.port;
|
||||||
|
return u.toString();
|
||||||
|
} catch {
|
||||||
|
return presignedUrl;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a presigned URL for a chapter markdown file.
|
||||||
|
* URL is valid for ~15 minutes (set by the scraper).
|
||||||
|
*
|
||||||
|
* @param rewrite - if true, rewrites the MinIO host to PUBLIC_MINIO_PUBLIC_URL
|
||||||
|
* (for browser use). Defaults to false — the server-side load function fetches
|
||||||
|
* the URL directly from the internal MinIO endpoint.
|
||||||
|
*/
|
||||||
|
export async function presignChapter(slug: string, n: number, rewrite = false): Promise<string> {
|
||||||
|
log.debug('minio', 'presigning chapter', { slug, n });
|
||||||
|
let res: Response;
|
||||||
|
try {
|
||||||
|
res = await fetch(`${SCRAPER_URL}/api/presign/chapter/${slug}/${n}`);
|
||||||
|
} catch (e) {
|
||||||
|
log.error('minio', 'presign chapter network error', { slug, n, err: String(e) });
|
||||||
|
throw new Error(`presign chapter ${slug}/${n}: network error`);
|
||||||
|
}
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.text().catch(() => '');
|
||||||
|
log.error('minio', 'presign chapter failed', { slug, n, status: res.status, body });
|
||||||
|
throw new Error(`presign chapter ${slug}/${n}: ${res.status}`);
|
||||||
|
}
|
||||||
|
const data = (await res.json()) as { url: string };
|
||||||
|
log.debug('minio', 'presign chapter ok', { slug, n });
|
||||||
|
return rewrite ? rewriteHost(data.url) : data.url;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a presigned URL for a voice sample audio file.
|
||||||
|
* URL is valid for ~1 hour. The URL is returned to the browser for direct streaming.
|
||||||
|
* Throws with { status: 404 } when the sample has not been generated yet.
|
||||||
|
*/
|
||||||
|
export async function presignVoiceSample(voice: string): Promise<string> {
|
||||||
|
log.debug('minio', 'presigning voice sample', { voice });
|
||||||
|
let res: Response;
|
||||||
|
try {
|
||||||
|
res = await fetch(`${SCRAPER_URL}/api/presign/voice-sample/${encodeURIComponent(voice)}`);
|
||||||
|
} catch (e) {
|
||||||
|
log.error('minio', 'presign voice sample network error', { voice, err: String(e) });
|
||||||
|
throw new Error(`presign voice sample ${voice}: network error`);
|
||||||
|
}
|
||||||
|
if (res.status === 404) {
|
||||||
|
const err = new Error(`presign voice sample ${voice}: not found`) as Error & { status: number };
|
||||||
|
err.status = 404;
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.text().catch(() => '');
|
||||||
|
log.error('minio', 'presign voice sample failed', { voice, status: res.status, body });
|
||||||
|
throw new Error(`presign voice sample ${voice}: ${res.status}`);
|
||||||
|
}
|
||||||
|
const data = (await res.json()) as { url: string };
|
||||||
|
log.debug('minio', 'presign voice sample ok', { voice });
|
||||||
|
return rewriteHost(data.url);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a presigned URL for an audio file.
|
||||||
|
* URL is valid for ~1 hour. The URL is returned to the browser for direct streaming.
|
||||||
|
* Throws with { status: 404 } when the audio object has not been generated yet.
|
||||||
|
*/
|
||||||
|
export async function presignAudio(
|
||||||
|
slug: string,
|
||||||
|
n: number,
|
||||||
|
voice?: string
|
||||||
|
): Promise<string> {
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
if (voice) params.set('voice', voice);
|
||||||
|
const qs = params.toString() ? `?${params.toString()}` : '';
|
||||||
|
log.debug('minio', 'presigning audio', { slug, n, voice });
|
||||||
|
let res: Response;
|
||||||
|
try {
|
||||||
|
res = await fetch(`${SCRAPER_URL}/api/presign/audio/${slug}/${n}${qs}`);
|
||||||
|
} catch (e) {
|
||||||
|
log.error('minio', 'presign audio network error', { slug, n, err: String(e) });
|
||||||
|
throw new Error(`presign audio ${slug}/${n}: network error`);
|
||||||
|
}
|
||||||
|
if (res.status === 404) {
|
||||||
|
// Audio hasn't been generated / uploaded yet — caller should surface this as 404.
|
||||||
|
const err = new Error(`presign audio ${slug}/${n}: not found`) as Error & { status: number };
|
||||||
|
err.status = 404;
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.text().catch(() => '');
|
||||||
|
log.error('minio', 'presign audio failed', { slug, n, status: res.status, body });
|
||||||
|
throw new Error(`presign audio ${slug}/${n}: ${res.status}`);
|
||||||
|
}
|
||||||
|
const data = (await res.json()) as { url: string };
|
||||||
|
log.debug('minio', 'presign audio ok', { slug, n });
|
||||||
|
return rewriteHost(data.url);
|
||||||
|
}
|
||||||
1349
ui-v2/src/lib/server/pocketbase.ts
Normal file
1349
ui-v2/src/lib/server/pocketbase.ts
Normal file
File diff suppressed because it is too large
Load Diff
96
ui-v2/src/lib/server/presignCache.ts
Normal file
96
ui-v2/src/lib/server/presignCache.ts
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
/**
|
||||||
|
* In-process presign URL cache.
|
||||||
|
*
|
||||||
|
* MinIO presigned audio URLs are valid for 1 hour (set by the backend).
|
||||||
|
* We cache them for 50 minutes so the browser always gets a URL with at
|
||||||
|
* least 10 minutes of remaining validity, while avoiding a round-trip to
|
||||||
|
* the backend + MinIO presign API on every "Play" click.
|
||||||
|
*
|
||||||
|
* The cache is a plain Map in the Node.js module scope — it lives for the
|
||||||
|
* lifetime of the SvelteKit server process and is shared across all requests.
|
||||||
|
* No persistence, no distributed cache needed: each SvelteKit instance
|
||||||
|
* maintains its own cache and entries expire naturally.
|
||||||
|
*
|
||||||
|
* Voice-sample URLs use the same cache with key "sample:<voice>".
|
||||||
|
*/
|
||||||
|
|
||||||
|
const AUDIO_TTL_MS = 50 * 60 * 1000; // 50 minutes
|
||||||
|
|
||||||
|
interface CacheEntry {
|
||||||
|
url: string;
|
||||||
|
expiresAt: number; // Date.now() ms
|
||||||
|
}
|
||||||
|
|
||||||
|
const cache = new Map<string, CacheEntry>();
|
||||||
|
|
||||||
|
// ── Periodic sweep ────────────────────────────────────────────────────────────
|
||||||
|
// Remove stale entries every 10 minutes so the Map doesn't grow unboundedly
|
||||||
|
// in long-running processes. Uses unref() so it never prevents Node from
|
||||||
|
// exiting cleanly.
|
||||||
|
let sweepTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
|
|
||||||
|
function startSweep() {
|
||||||
|
if (sweepTimer) return;
|
||||||
|
sweepTimer = setInterval(() => {
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [key, entry] of cache) {
|
||||||
|
if (entry.expiresAt <= now) cache.delete(key);
|
||||||
|
}
|
||||||
|
}, 10 * 60 * 1000);
|
||||||
|
// Don't block Node.js exit
|
||||||
|
sweepTimer.unref?.();
|
||||||
|
}
|
||||||
|
|
||||||
|
startSweep();
|
||||||
|
|
||||||
|
// ── Public API ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Cache key for a chapter audio presigned URL. */
|
||||||
|
export function audioKey(slug: string, n: number, voice: string): string {
|
||||||
|
return `audio:${slug}:${n}:${voice}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Cache key for a voice-sample presigned URL. */
|
||||||
|
export function sampleKey(voice: string): string {
|
||||||
|
return `sample:${voice}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Return the cached URL for key, or null if absent / expired. */
|
||||||
|
export function get(key: string): string | null {
|
||||||
|
const entry = cache.get(key);
|
||||||
|
if (!entry) return null;
|
||||||
|
if (entry.expiresAt <= Date.now()) {
|
||||||
|
cache.delete(key);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return entry.url;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Store a presigned URL under key for TTL_MS milliseconds. */
|
||||||
|
export function set(key: string, url: string, ttlMs = AUDIO_TTL_MS): void {
|
||||||
|
cache.set(key, { url, expiresAt: Date.now() + ttlMs });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Invalidate a specific key (e.g. after audio generation to force refresh). */
|
||||||
|
export function invalidate(key: string): void {
|
||||||
|
cache.delete(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drain all entries — called on graceful shutdown to release memory. */
|
||||||
|
export function drain(): void {
|
||||||
|
cache.clear();
|
||||||
|
if (sweepTimer) {
|
||||||
|
clearInterval(sweepTimer);
|
||||||
|
sweepTimer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Current number of live (non-expired) cached entries. For health/debug. */
|
||||||
|
export function size(): number {
|
||||||
|
const now = Date.now();
|
||||||
|
let n = 0;
|
||||||
|
for (const entry of cache.values()) {
|
||||||
|
if (entry.expiresAt > now) n++;
|
||||||
|
}
|
||||||
|
return n;
|
||||||
|
}
|
||||||
32
ui-v2/src/routes/+layout.server.ts
Normal file
32
ui-v2/src/routes/+layout.server.ts
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
import { redirect } from '@sveltejs/kit';
|
||||||
|
import type { LayoutServerLoad } from './$types';
|
||||||
|
import { getSettings } from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
// Routes that are accessible without being logged in
|
||||||
|
const PUBLIC_ROUTES = new Set(['/login']);
|
||||||
|
|
||||||
|
export const load: LayoutServerLoad = async ({ locals, url }) => {
|
||||||
|
if (!PUBLIC_ROUTES.has(url.pathname) && !locals.user) {
|
||||||
|
redirect(302, `/login`);
|
||||||
|
}
|
||||||
|
|
||||||
|
let settings = { autoNext: false, voice: 'af_bella', speed: 1.0 };
|
||||||
|
try {
|
||||||
|
const row = await getSettings(locals.sessionId, locals.user?.id);
|
||||||
|
if (row) {
|
||||||
|
settings = {
|
||||||
|
autoNext: row.auto_next ?? false,
|
||||||
|
voice: row.voice ?? 'af_bella',
|
||||||
|
speed: row.speed ?? 1.0
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
log.warn('layout', 'failed to load settings', { err: String(e) });
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
user: locals.user,
|
||||||
|
settings
|
||||||
|
};
|
||||||
|
};
|
||||||
628
ui-v2/src/routes/+layout.svelte
Normal file
628
ui-v2/src/routes/+layout.svelte
Normal file
@@ -0,0 +1,628 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import '../app.css';
|
||||||
|
import { page, navigating } from '$app/state';
|
||||||
|
import { goto } from '$app/navigation';
|
||||||
|
import type { Snippet } from 'svelte';
|
||||||
|
import type { LayoutData } from './$types';
|
||||||
|
import { audioStore } from '$lib/audio.svelte';
|
||||||
|
import { env } from '$env/dynamic/public';
|
||||||
|
|
||||||
|
let { children, data }: { children: Snippet; data: LayoutData } = $props();
|
||||||
|
|
||||||
|
// Mobile nav drawer state
|
||||||
|
let menuOpen = $state(false);
|
||||||
|
|
||||||
|
// Chapter list drawer state for the mini-player
|
||||||
|
let chapterDrawerOpen = $state(false);
|
||||||
|
|
||||||
|
// The single <audio> element that persists across navigations.
|
||||||
|
// AudioPlayer components in chapter pages control it via audioStore.
|
||||||
|
let audioEl = $state<HTMLAudioElement | null>(null);
|
||||||
|
|
||||||
|
// Apply persisted settings once on mount (server-loaded data).
|
||||||
|
let settingsApplied = false;
|
||||||
|
$effect(() => {
|
||||||
|
if (!settingsApplied && data.settings) {
|
||||||
|
settingsApplied = true;
|
||||||
|
audioStore.autoNext = data.settings.autoNext;
|
||||||
|
audioStore.voice = data.settings.voice;
|
||||||
|
audioStore.speed = data.settings.speed;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Persist settings changes (debounced 800ms) ──────────────────────────
|
||||||
|
let settingsSaveTimer = 0;
|
||||||
|
$effect(() => {
|
||||||
|
// Subscribe to the three settings fields
|
||||||
|
const autoNext = audioStore.autoNext;
|
||||||
|
const voice = audioStore.voice;
|
||||||
|
const speed = audioStore.speed;
|
||||||
|
|
||||||
|
// Skip saving until settings have been applied from the server
|
||||||
|
if (!settingsApplied) return;
|
||||||
|
|
||||||
|
clearTimeout(settingsSaveTimer);
|
||||||
|
settingsSaveTimer = setTimeout(() => {
|
||||||
|
fetch('/api/settings', {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ autoNext, voice, speed })
|
||||||
|
}).catch(() => {});
|
||||||
|
}, 800) as unknown as number;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Keep the audio element's playback rate in sync with the store speed.
|
||||||
|
$effect(() => {
|
||||||
|
if (audioEl) audioEl.playbackRate = audioStore.speed;
|
||||||
|
});
|
||||||
|
|
||||||
|
// When audioUrl changes, load the new source.
|
||||||
|
// Use a local variable to track which URL is currently loaded so we never
|
||||||
|
// compare against audioEl.src (browsers normalise it, causing false mismatches).
|
||||||
|
let loadedUrl = '';
|
||||||
|
$effect(() => {
|
||||||
|
if (!audioEl) return;
|
||||||
|
const url = audioStore.audioUrl;
|
||||||
|
if (url && url !== loadedUrl) {
|
||||||
|
loadedUrl = url;
|
||||||
|
audioEl.src = url;
|
||||||
|
audioEl.load();
|
||||||
|
audioEl.playbackRate = audioStore.speed;
|
||||||
|
audioEl.play().catch(() => {});
|
||||||
|
} else if (!url) {
|
||||||
|
loadedUrl = '';
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Handle toggle requests from AudioPlayer controller.
|
||||||
|
$effect(() => {
|
||||||
|
// Read toggleRequest to subscribe; ignore value 0 (initial).
|
||||||
|
const _req = audioStore.toggleRequest;
|
||||||
|
if (!audioEl || _req === 0) return;
|
||||||
|
if (audioStore.isPlaying) {
|
||||||
|
audioEl.pause();
|
||||||
|
} else {
|
||||||
|
audioEl.play().catch(() => {});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Handle seek requests from AudioPlayer controller.
|
||||||
|
$effect(() => {
|
||||||
|
const t = audioStore.seekRequest;
|
||||||
|
if (!audioEl || t === null) return;
|
||||||
|
audioEl.currentTime = t;
|
||||||
|
audioStore.seekRequest = null;
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Save audio time on pause/end (debounced 2s) ─────────────────────────
|
||||||
|
let audioTimeSaveTimer = 0;
|
||||||
|
function saveAudioTime() {
|
||||||
|
if (!audioStore.slug || !audioStore.chapter) return;
|
||||||
|
const slug = audioStore.slug;
|
||||||
|
const chapter = audioStore.chapter;
|
||||||
|
const currentTime = audioStore.currentTime;
|
||||||
|
clearTimeout(audioTimeSaveTimer);
|
||||||
|
audioTimeSaveTimer = setTimeout(() => {
|
||||||
|
fetch('/api/progress/audio-time', {
|
||||||
|
method: 'PATCH',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ slug, chapter, audioTime: currentTime })
|
||||||
|
}).catch(() => {});
|
||||||
|
}, 2000) as unknown as number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatTime(s: number): string {
|
||||||
|
if (!isFinite(s) || s < 0) return '0:00';
|
||||||
|
const m = Math.floor(s / 60);
|
||||||
|
const sec = Math.floor(s % 60);
|
||||||
|
return `${m}:${sec.toString().padStart(2, '0')}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function togglePlay() {
|
||||||
|
if (!audioEl) return;
|
||||||
|
if (audioStore.isPlaying) {
|
||||||
|
audioEl.pause();
|
||||||
|
} else {
|
||||||
|
audioEl.play().catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function seek(e: Event) {
|
||||||
|
if (!audioEl) return;
|
||||||
|
audioEl.currentTime = parseFloat((e.target as HTMLInputElement).value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function skipBack() {
|
||||||
|
if (!audioEl) return;
|
||||||
|
audioEl.currentTime = Math.max(0, audioEl.currentTime - 15);
|
||||||
|
}
|
||||||
|
|
||||||
|
function skipForward() {
|
||||||
|
if (!audioEl) return;
|
||||||
|
audioEl.currentTime = Math.min(audioEl.duration || 0, audioEl.currentTime + 30);
|
||||||
|
}
|
||||||
|
|
||||||
|
const speedSteps = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0];
|
||||||
|
|
||||||
|
function cycleSpeed() {
|
||||||
|
const idx = speedSteps.indexOf(audioStore.speed);
|
||||||
|
audioStore.speed = speedSteps[(idx + 1) % speedSteps.length];
|
||||||
|
}
|
||||||
|
|
||||||
|
function dismiss() {
|
||||||
|
if (audioEl) {
|
||||||
|
audioEl.pause();
|
||||||
|
audioEl.src = '';
|
||||||
|
}
|
||||||
|
audioStore.status = 'idle';
|
||||||
|
audioStore.audioUrl = '';
|
||||||
|
audioStore.slug = '';
|
||||||
|
audioStore.chapter = 0;
|
||||||
|
audioStore.isPlaying = false;
|
||||||
|
audioStore.currentTime = 0;
|
||||||
|
audioStore.duration = 0;
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<svelte:head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
|
<title>libnovel</title>
|
||||||
|
</svelte:head>
|
||||||
|
|
||||||
|
<!-- Persistent audio element — always in the DOM, never conditionally unmounted.
|
||||||
|
Conditional rendering ({#if}) would destroy/recreate it when reactive state
|
||||||
|
changes (e.g. currentTime ticking), triggering onpause and restarting audio. -->
|
||||||
|
<audio
|
||||||
|
bind:this={audioEl}
|
||||||
|
bind:currentTime={audioStore.currentTime}
|
||||||
|
bind:duration={audioStore.duration}
|
||||||
|
onplay={() => (audioStore.isPlaying = true)}
|
||||||
|
onpause={() => {
|
||||||
|
audioStore.isPlaying = false;
|
||||||
|
saveAudioTime();
|
||||||
|
}}
|
||||||
|
onended={() => {
|
||||||
|
audioStore.isPlaying = false;
|
||||||
|
saveAudioTime();
|
||||||
|
if (audioStore.autoNext && audioStore.nextChapter !== null && audioStore.slug) {
|
||||||
|
// Capture values synchronously before any async work — the AudioPlayer
|
||||||
|
// component will unmount during navigation, but we've already read what
|
||||||
|
// we need.
|
||||||
|
const targetSlug = audioStore.slug;
|
||||||
|
const targetChapter = audioStore.nextChapter;
|
||||||
|
// Store the target chapter number so only the newly-mounted AudioPlayer
|
||||||
|
// for that chapter reacts — not the outgoing chapter's component.
|
||||||
|
audioStore.autoStartChapter = targetChapter;
|
||||||
|
goto(`/books/${targetSlug}/chapters/${targetChapter}`).catch(() => {
|
||||||
|
audioStore.autoStartChapter = null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
preload="metadata"
|
||||||
|
style="display:none"
|
||||||
|
></audio>
|
||||||
|
|
||||||
|
<div class="min-h-screen flex flex-col" class:pb-24={audioStore.active}>
|
||||||
|
<!-- Navigation progress bar — shown while SSR is running for any page transition -->
|
||||||
|
{#if navigating}
|
||||||
|
<div class="fixed top-0 left-0 right-0 z-[100] h-1 bg-zinc-800">
|
||||||
|
<div class="h-full bg-amber-400 animate-progress-bar"></div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
<header class="border-b border-zinc-700 bg-zinc-900 sticky top-0 z-50">
|
||||||
|
<nav class="max-w-6xl mx-auto px-4 h-14 flex items-center gap-6">
|
||||||
|
<a href="/" class="text-amber-400 font-bold text-lg tracking-tight hover:text-amber-300 shrink-0">
|
||||||
|
libnovel
|
||||||
|
</a>
|
||||||
|
|
||||||
|
{#if page.data.book?.title && /\/books\/[^/]+\/chapters\//.test(page.url.pathname)}
|
||||||
|
<span class="text-zinc-400 text-sm truncate min-w-0 flex-1 sm:flex-none sm:max-w-xs">
|
||||||
|
{page.data.book.title}
|
||||||
|
</span>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
{#if data.user}
|
||||||
|
<!-- Desktop nav links (hidden on mobile) -->
|
||||||
|
<a
|
||||||
|
href="/books"
|
||||||
|
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/books') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Library
|
||||||
|
</a>
|
||||||
|
<a
|
||||||
|
href="/browse"
|
||||||
|
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/browse') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Discover
|
||||||
|
</a>
|
||||||
|
|
||||||
|
<div class="ml-auto flex items-center gap-4">
|
||||||
|
<!-- Desktop: admin + profile + sign out (hidden on mobile) -->
|
||||||
|
{#if data.user?.role === 'admin'}
|
||||||
|
<a
|
||||||
|
href="/admin/scrape"
|
||||||
|
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/admin/scrape') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Scrape
|
||||||
|
</a>
|
||||||
|
<a
|
||||||
|
href="/admin/audio"
|
||||||
|
class="hidden sm:block text-sm transition-colors {page.url.pathname === '/admin/audio' ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Audio cache
|
||||||
|
</a>
|
||||||
|
<a
|
||||||
|
href="/admin/audio-jobs"
|
||||||
|
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/admin/audio-jobs') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Audio jobs
|
||||||
|
</a>
|
||||||
|
{/if}
|
||||||
|
<a
|
||||||
|
href="/profile"
|
||||||
|
class="hidden sm:block text-sm transition-colors {page.url.pathname === '/profile' ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
{data.user.username}
|
||||||
|
</a>
|
||||||
|
<form method="POST" action="/logout" class="hidden sm:block">
|
||||||
|
<button type="submit" class="text-zinc-400 hover:text-zinc-100 text-sm transition-colors">
|
||||||
|
Sign out
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<!-- Mobile: hamburger button -->
|
||||||
|
<button
|
||||||
|
onclick={() => (menuOpen = !menuOpen)}
|
||||||
|
aria-label="Toggle menu"
|
||||||
|
aria-expanded={menuOpen}
|
||||||
|
class="sm:hidden p-2 -mr-1 rounded text-zinc-400 hover:text-zinc-100 transition-colors"
|
||||||
|
>
|
||||||
|
{#if menuOpen}
|
||||||
|
<!-- X icon -->
|
||||||
|
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
|
||||||
|
</svg>
|
||||||
|
{:else}
|
||||||
|
<!-- Hamburger icon -->
|
||||||
|
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 6h16M4 12h16M4 18h16" />
|
||||||
|
</svg>
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{:else}
|
||||||
|
<div class="ml-auto">
|
||||||
|
<a
|
||||||
|
href="/login"
|
||||||
|
class="text-sm px-3 py-1.5 rounded bg-amber-400 text-zinc-900 font-semibold hover:bg-amber-300 transition-colors"
|
||||||
|
>
|
||||||
|
Sign in
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<!-- Mobile drawer (full-width, below the bar) -->
|
||||||
|
{#if data.user && menuOpen}
|
||||||
|
<div class="sm:hidden border-t border-zinc-700 bg-zinc-900 px-4 py-3 flex flex-col gap-1">
|
||||||
|
<a
|
||||||
|
href="/books"
|
||||||
|
onclick={() => (menuOpen = false)}
|
||||||
|
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/books') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Library
|
||||||
|
</a>
|
||||||
|
<a
|
||||||
|
href="/browse"
|
||||||
|
onclick={() => (menuOpen = false)}
|
||||||
|
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/browse') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Discover
|
||||||
|
</a>
|
||||||
|
<a
|
||||||
|
href="/profile"
|
||||||
|
onclick={() => (menuOpen = false)}
|
||||||
|
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname === '/profile' ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Profile <span class="text-zinc-500 font-normal">({data.user.username})</span>
|
||||||
|
</a>
|
||||||
|
{#if data.user?.role === 'admin'}
|
||||||
|
<div class="my-1 border-t border-zinc-700/60"></div>
|
||||||
|
<p class="px-3 pt-1 pb-0.5 text-xs text-zinc-600 uppercase tracking-widest">Admin</p>
|
||||||
|
<a
|
||||||
|
href="/admin/scrape"
|
||||||
|
onclick={() => (menuOpen = false)}
|
||||||
|
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/admin/scrape') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Scrape tasks
|
||||||
|
</a>
|
||||||
|
<a
|
||||||
|
href="/admin/audio"
|
||||||
|
onclick={() => (menuOpen = false)}
|
||||||
|
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname === '/admin/audio' ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Audio cache
|
||||||
|
</a>
|
||||||
|
<a
|
||||||
|
href="/admin/audio-jobs"
|
||||||
|
onclick={() => (menuOpen = false)}
|
||||||
|
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/admin/audio-jobs') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
|
||||||
|
>
|
||||||
|
Audio jobs
|
||||||
|
</a>
|
||||||
|
{/if}
|
||||||
|
<div class="my-1 border-t border-zinc-700/60"></div>
|
||||||
|
<form method="POST" action="/logout">
|
||||||
|
<button
|
||||||
|
type="submit"
|
||||||
|
class="w-full text-left px-3 py-2.5 rounded-lg text-sm font-medium text-red-400 hover:bg-zinc-800 transition-colors"
|
||||||
|
>
|
||||||
|
Sign out
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<main class="flex-1 max-w-6xl mx-auto w-full px-4 py-8">
|
||||||
|
{#key page.url.pathname + page.url.search}
|
||||||
|
{@render children()}
|
||||||
|
{/key}
|
||||||
|
</main>
|
||||||
|
|
||||||
|
<footer class="border-t border-zinc-800 mt-auto">
|
||||||
|
<div class="max-w-6xl mx-auto px-4 py-6 flex flex-col items-center gap-4 text-xs text-zinc-600">
|
||||||
|
<!-- Top row: site links -->
|
||||||
|
<nav class="flex flex-wrap items-center justify-center gap-x-5 gap-y-2">
|
||||||
|
<a href="/books" class="hover:text-zinc-400 transition-colors">Library</a>
|
||||||
|
<a href="/browse" class="hover:text-zinc-400 transition-colors">Discover</a>
|
||||||
|
<a
|
||||||
|
href="https://novelfire.net"
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
class="hover:text-zinc-400 transition-colors flex items-center gap-1"
|
||||||
|
>
|
||||||
|
novelfire.net
|
||||||
|
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||||
|
d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14" />
|
||||||
|
</svg>
|
||||||
|
</a>
|
||||||
|
</nav>
|
||||||
|
<!-- Bottom row: legal links + copyright -->
|
||||||
|
<div class="flex flex-wrap items-center justify-center gap-x-5 gap-y-2 text-zinc-700">
|
||||||
|
<a href="/disclaimer" class="hover:text-zinc-500 transition-colors">Disclaimer</a>
|
||||||
|
<a href="/privacy" class="hover:text-zinc-500 transition-colors">Privacy</a>
|
||||||
|
<a href="/dmca" class="hover:text-zinc-500 transition-colors">DMCA</a>
|
||||||
|
<span>© {new Date().getFullYear()} libnovel</span>
|
||||||
|
{#if env.PUBLIC_BUILD_VERSION && env.PUBLIC_BUILD_VERSION !== 'dev'}
|
||||||
|
<span class="text-zinc-800">{env.PUBLIC_BUILD_VERSION}+{env.PUBLIC_BUILD_COMMIT?.slice(0, 7)}</span>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</footer>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- ── Persistent mini-player bar ─────────────────────────────────────────── -->
|
||||||
|
{#if audioStore.active}
|
||||||
|
<div class="fixed bottom-0 left-0 right-0 z-50 bg-zinc-900 border-t border-zinc-700 shadow-2xl">
|
||||||
|
|
||||||
|
<!-- Chapter list drawer (slides up above the mini-bar) -->
|
||||||
|
{#if chapterDrawerOpen && audioStore.chapters.length > 0}
|
||||||
|
<div class="border-b border-zinc-700 bg-zinc-900 max-h-[32rem] overflow-y-auto">
|
||||||
|
<div class="max-w-6xl mx-auto px-4">
|
||||||
|
<div class="flex items-center justify-between py-2 border-b border-zinc-800 sticky top-0 bg-zinc-900">
|
||||||
|
<span class="text-xs font-semibold text-zinc-400 uppercase tracking-wider">Chapters</span>
|
||||||
|
<button
|
||||||
|
onclick={() => (chapterDrawerOpen = false)}
|
||||||
|
class="text-zinc-600 hover:text-zinc-300 transition-colors p-1"
|
||||||
|
aria-label="Close chapter list"
|
||||||
|
>
|
||||||
|
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"/>
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{#each audioStore.chapters as ch (ch.number)}
|
||||||
|
<a
|
||||||
|
href="/books/{audioStore.slug}/chapters/{ch.number}"
|
||||||
|
onclick={() => (chapterDrawerOpen = false)}
|
||||||
|
class="flex items-center gap-2 py-2 text-xs transition-colors hover:text-zinc-100 {ch.number === audioStore.chapter
|
||||||
|
? 'text-amber-400 font-semibold'
|
||||||
|
: 'text-zinc-400'}"
|
||||||
|
>
|
||||||
|
<span class="tabular-nums text-zinc-600 w-8 shrink-0 text-right">
|
||||||
|
{ch.number}
|
||||||
|
</span>
|
||||||
|
<span class="truncate">{ch.title || `Chapter ${ch.number}`}</span>
|
||||||
|
{#if ch.number === audioStore.chapter}
|
||||||
|
<svg class="w-3 h-3 shrink-0 text-amber-400" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M8 5v14l11-7z"/>
|
||||||
|
</svg>
|
||||||
|
{/if}
|
||||||
|
</a>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<!-- Generation progress bar (sits at very top of the bar) -->
|
||||||
|
{#if audioStore.status === 'generating' || audioStore.status === 'loading'}
|
||||||
|
<div class="h-0.5 bg-zinc-800">
|
||||||
|
<div
|
||||||
|
class="h-full bg-amber-400 transition-none"
|
||||||
|
style="width: {audioStore.progress}%"
|
||||||
|
></div>
|
||||||
|
</div>
|
||||||
|
{:else if audioStore.status === 'ready'}
|
||||||
|
<!-- Seek bar flush at top — tappable on mobile -->
|
||||||
|
<div class="px-0">
|
||||||
|
<input
|
||||||
|
type="range"
|
||||||
|
min="0"
|
||||||
|
max={audioStore.duration || 0}
|
||||||
|
value={audioStore.currentTime}
|
||||||
|
oninput={seek}
|
||||||
|
class="w-full h-1 accent-amber-400 cursor-pointer block"
|
||||||
|
style="margin: 0; border-radius: 0;"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<div class="max-w-6xl mx-auto px-4 py-2 flex items-center gap-3">
|
||||||
|
|
||||||
|
<!-- Track info (click to open chapter list drawer) -->
|
||||||
|
<button
|
||||||
|
class="flex-1 min-w-0 text-left rounded px-1 -ml-1 hover:bg-zinc-800 transition-colors"
|
||||||
|
onclick={() => { if (audioStore.chapters.length > 0) chapterDrawerOpen = !chapterDrawerOpen; }}
|
||||||
|
aria-label={audioStore.chapters.length > 0 ? 'Toggle chapter list' : undefined}
|
||||||
|
title={audioStore.chapters.length > 0 ? 'Chapter list' : undefined}
|
||||||
|
>
|
||||||
|
{#if audioStore.chapterTitle}
|
||||||
|
<p class="text-xs text-zinc-100 truncate leading-tight">{audioStore.chapterTitle}</p>
|
||||||
|
{/if}
|
||||||
|
{#if audioStore.bookTitle}
|
||||||
|
<p class="text-xs text-zinc-500 truncate leading-tight">{audioStore.bookTitle}</p>
|
||||||
|
{/if}
|
||||||
|
{#if audioStore.status === 'generating'}
|
||||||
|
<p class="text-xs text-amber-400 leading-tight">
|
||||||
|
Generating… {Math.round(audioStore.progress)}%
|
||||||
|
</p>
|
||||||
|
{:else if audioStore.status === 'ready'}
|
||||||
|
<p class="text-xs text-zinc-500 tabular-nums leading-tight">
|
||||||
|
{formatTime(audioStore.currentTime)} / {formatTime(audioStore.duration)}
|
||||||
|
</p>
|
||||||
|
{:else if audioStore.status === 'loading'}
|
||||||
|
<p class="text-xs text-zinc-500 leading-tight">Loading…</p>
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{#if audioStore.status === 'ready'}
|
||||||
|
<!-- Skip back 15s -->
|
||||||
|
<button
|
||||||
|
onclick={skipBack}
|
||||||
|
class="text-zinc-400 hover:text-zinc-100 transition-colors p-1.5 rounded"
|
||||||
|
title="Back 15s"
|
||||||
|
aria-label="Rewind 15 seconds"
|
||||||
|
>
|
||||||
|
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M11.99 5V1l-5 5 5 5V7c3.31 0 6 2.69 6 6s-2.69 6-6 6-6-2.69-6-6h-2c0 4.42 3.58 8 8 8s8-3.58 8-8-3.58-8-8-8z"/>
|
||||||
|
<text x="8.5" y="14.5" font-size="5" font-family="sans-serif" font-weight="bold" fill="currentColor">15</text>
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<!-- Play / Pause -->
|
||||||
|
<button
|
||||||
|
onclick={togglePlay}
|
||||||
|
class="w-10 h-10 rounded-full bg-amber-400 text-zinc-900 flex items-center justify-center hover:bg-amber-300 transition-colors flex-shrink-0"
|
||||||
|
aria-label={audioStore.isPlaying ? 'Pause' : 'Play'}
|
||||||
|
>
|
||||||
|
{#if audioStore.isPlaying}
|
||||||
|
<svg class="w-4 h-4" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M6 4h4v16H6V4zm8 0h4v16h-4V4z"/>
|
||||||
|
</svg>
|
||||||
|
{:else}
|
||||||
|
<svg class="w-4 h-4 ml-0.5" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M8 5v14l11-7z"/>
|
||||||
|
</svg>
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<!-- Skip forward 30s -->
|
||||||
|
<button
|
||||||
|
onclick={skipForward}
|
||||||
|
class="text-zinc-400 hover:text-zinc-100 transition-colors p-1.5 rounded"
|
||||||
|
title="Forward 30s"
|
||||||
|
aria-label="Skip 30 seconds"
|
||||||
|
>
|
||||||
|
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M12 5V1l5 5-5 5V7c-3.31 0-6 2.69-6 6s2.69 6 6 6 6-2.69 6-6h2c0 4.42-3.58 8-8 8s-8-3.58-8-8 3.58-8 8-8z"/>
|
||||||
|
<text x="8.5" y="14.5" font-size="5" font-family="sans-serif" font-weight="bold" fill="currentColor">30</text>
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<!-- Speed control -->
|
||||||
|
<button
|
||||||
|
onclick={cycleSpeed}
|
||||||
|
class="text-xs font-semibold text-zinc-300 hover:text-amber-400 transition-colors px-2 py-1 rounded bg-zinc-800 hover:bg-zinc-700 flex-shrink-0 tabular-nums w-12 text-center"
|
||||||
|
title="Change playback speed"
|
||||||
|
aria-label="Playback speed {audioStore.speed}x"
|
||||||
|
>
|
||||||
|
{audioStore.speed}×
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<!-- Auto-next toggle (with prefetch indicator) -->
|
||||||
|
<button
|
||||||
|
onclick={() => (audioStore.autoNext = !audioStore.autoNext)}
|
||||||
|
class="relative p-1.5 rounded flex-shrink-0 transition-colors {audioStore.autoNext
|
||||||
|
? 'text-amber-400 bg-amber-400/15 hover:bg-amber-400/25'
|
||||||
|
: 'text-zinc-600 hover:text-zinc-300 hover:bg-zinc-800'}"
|
||||||
|
title={audioStore.autoNext
|
||||||
|
? audioStore.nextStatus === 'prefetched'
|
||||||
|
? `Auto-next on — Ch.${audioStore.nextChapter} ready`
|
||||||
|
: audioStore.nextStatus === 'prefetching'
|
||||||
|
? `Auto-next on — preparing Ch.${audioStore.nextChapter}…`
|
||||||
|
: 'Auto-next on'
|
||||||
|
: 'Auto-next off'}
|
||||||
|
aria-label="Auto-next {audioStore.autoNext ? 'on' : 'off'}"
|
||||||
|
aria-pressed={audioStore.autoNext}
|
||||||
|
>
|
||||||
|
<!-- "skip to end" / auto-advance icon -->
|
||||||
|
<svg class="w-4 h-4" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M6 18l8.5-6L6 6v12zm8.5-6L23 6v12l-8.5-6z"/>
|
||||||
|
</svg>
|
||||||
|
<!-- Prefetch status dot -->
|
||||||
|
{#if audioStore.autoNext && audioStore.nextStatus === 'prefetching'}
|
||||||
|
<span class="absolute top-0.5 right-0.5 w-1.5 h-1.5 rounded-full bg-amber-400 animate-pulse"></span>
|
||||||
|
{:else if audioStore.autoNext && audioStore.nextStatus === 'prefetched'}
|
||||||
|
<span class="absolute top-0.5 right-0.5 w-1.5 h-1.5 rounded-full bg-green-400"></span>
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
|
{:else if audioStore.status === 'generating'}
|
||||||
|
<!-- Spinner during generation -->
|
||||||
|
<svg class="w-6 h-6 text-amber-400 animate-spin flex-shrink-0" fill="none" viewBox="0 0 24 24">
|
||||||
|
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
|
||||||
|
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
|
||||||
|
</svg>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<!-- Cover thumbnail / go-to-chapter link -->
|
||||||
|
{#if audioStore.slug && audioStore.chapter > 0}
|
||||||
|
<a
|
||||||
|
href="/books/{audioStore.slug}/chapters/{audioStore.chapter}"
|
||||||
|
class="shrink-0 rounded overflow-hidden hover:opacity-80 transition-opacity"
|
||||||
|
title="Go to chapter"
|
||||||
|
aria-label="Go to chapter"
|
||||||
|
>
|
||||||
|
{#if audioStore.cover}
|
||||||
|
<img
|
||||||
|
src={audioStore.cover}
|
||||||
|
alt=""
|
||||||
|
class="w-8 h-11 object-cover rounded"
|
||||||
|
/>
|
||||||
|
{:else}
|
||||||
|
<!-- Fallback book icon -->
|
||||||
|
<div class="w-8 h-11 flex items-center justify-center bg-zinc-800 rounded border border-zinc-700">
|
||||||
|
<svg class="w-4 h-4 text-zinc-500" fill="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path d="M18 2H6c-1.1 0-2 .9-2 2v16c0 1.1.9 2 2 2h12c1.1 0 2-.9 2-2V4c0-1.1-.9-2-2-2zm-2 14H8v-2h8v2zm0-4H8v-2h8v2zm0-4H8V6h8v2z"/>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</a>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<!-- Dismiss -->
|
||||||
|
<button
|
||||||
|
onclick={dismiss}
|
||||||
|
class="text-zinc-600 hover:text-zinc-400 transition-colors p-1.5 rounded flex-shrink-0"
|
||||||
|
title="Close player"
|
||||||
|
aria-label="Close player"
|
||||||
|
>
|
||||||
|
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"/>
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
59
ui-v2/src/routes/+page.server.ts
Normal file
59
ui-v2/src/routes/+page.server.ts
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
import type { PageServerLoad } from './$types';
|
||||||
|
import {
|
||||||
|
listBooks,
|
||||||
|
recentlyAddedBooks,
|
||||||
|
allProgress,
|
||||||
|
getHomeStats,
|
||||||
|
getSubscriptionFeed
|
||||||
|
} from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
import type { Book, Progress } from '$lib/server/pocketbase';
|
||||||
|
|
||||||
|
export const load: PageServerLoad = async ({ locals }) => {
|
||||||
|
let allBooks: Book[] = [];
|
||||||
|
let recentBooks: Book[] = [];
|
||||||
|
let progressList: Progress[] = [];
|
||||||
|
let stats = { totalBooks: 0, totalChapters: 0 };
|
||||||
|
|
||||||
|
try {
|
||||||
|
[allBooks, recentBooks, progressList, stats] = await Promise.all([
|
||||||
|
listBooks(),
|
||||||
|
recentlyAddedBooks(8),
|
||||||
|
allProgress(locals.sessionId, locals.user?.id),
|
||||||
|
getHomeStats()
|
||||||
|
]);
|
||||||
|
} catch (e) {
|
||||||
|
log.error('home', 'failed to load home data', { err: String(e) });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build slug → book lookup
|
||||||
|
const bookMap = new Map<string, Book>(allBooks.map((b) => [b.slug, b]));
|
||||||
|
|
||||||
|
// Continue reading: progress entries joined with book data, most recent first
|
||||||
|
const continueReading = progressList
|
||||||
|
.filter((p) => bookMap.has(p.slug))
|
||||||
|
.slice(0, 6)
|
||||||
|
.map((p) => ({ book: bookMap.get(p.slug)!, chapter: p.chapter }));
|
||||||
|
|
||||||
|
// Recently updated: deduplicate against continueReading slugs
|
||||||
|
const inProgressSlugs = new Set(continueReading.map((c) => c.book.slug));
|
||||||
|
const recentlyUpdated = recentBooks.filter((b) => !inProgressSlugs.has(b.slug)).slice(0, 6);
|
||||||
|
|
||||||
|
// Subscription feed — only when logged in
|
||||||
|
const subscriptionFeed = locals.user
|
||||||
|
? await getSubscriptionFeed(locals.user.id, 12).catch((e) => {
|
||||||
|
log.error('home', 'failed to load subscription feed', { err: String(e) });
|
||||||
|
return [] as Awaited<ReturnType<typeof getSubscriptionFeed>>;
|
||||||
|
})
|
||||||
|
: [];
|
||||||
|
|
||||||
|
return {
|
||||||
|
continueReading,
|
||||||
|
recentlyUpdated,
|
||||||
|
subscriptionFeed,
|
||||||
|
stats: {
|
||||||
|
...stats,
|
||||||
|
booksInProgress: continueReading.length
|
||||||
|
}
|
||||||
|
};
|
||||||
|
};
|
||||||
202
ui-v2/src/routes/+page.svelte
Normal file
202
ui-v2/src/routes/+page.svelte
Normal file
@@ -0,0 +1,202 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import type { PageData } from './$types';
|
||||||
|
|
||||||
|
let { data }: { data: PageData } = $props();
|
||||||
|
|
||||||
|
function parseGenres(genres: string[] | string | null | undefined): string[] {
|
||||||
|
if (!genres) return [];
|
||||||
|
if (Array.isArray(genres)) return genres;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(genres);
|
||||||
|
return Array.isArray(parsed) ? parsed : [];
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<svelte:head>
|
||||||
|
<title>libnovel</title>
|
||||||
|
</svelte:head>
|
||||||
|
|
||||||
|
<!-- Stats bar -->
|
||||||
|
<div class="flex gap-6 mb-8 text-center">
|
||||||
|
<div class="flex-1 rounded-lg bg-zinc-800 border border-zinc-700 py-4 px-6">
|
||||||
|
<p class="text-2xl font-bold text-amber-400">{data.stats.totalBooks}</p>
|
||||||
|
<p class="text-xs text-zinc-400 mt-0.5">Books</p>
|
||||||
|
</div>
|
||||||
|
<div class="flex-1 rounded-lg bg-zinc-800 border border-zinc-700 py-4 px-6">
|
||||||
|
<p class="text-2xl font-bold text-amber-400">{data.stats.totalChapters.toLocaleString()}</p>
|
||||||
|
<p class="text-xs text-zinc-400 mt-0.5">Chapters</p>
|
||||||
|
</div>
|
||||||
|
<div class="flex-1 rounded-lg bg-zinc-800 border border-zinc-700 py-4 px-6">
|
||||||
|
<p class="text-2xl font-bold text-amber-400">{data.stats.booksInProgress}</p>
|
||||||
|
<p class="text-xs text-zinc-400 mt-0.5">In progress</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Continue Reading -->
|
||||||
|
{#if data.continueReading.length > 0}
|
||||||
|
<section class="mb-10">
|
||||||
|
<div class="flex items-baseline justify-between mb-3">
|
||||||
|
<h2 class="text-lg font-bold text-zinc-100">Continue Reading</h2>
|
||||||
|
<a href="/books" class="text-xs text-amber-400 hover:text-amber-300">View all</a>
|
||||||
|
</div>
|
||||||
|
<div class="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-6 gap-4">
|
||||||
|
{#each data.continueReading as { book, chapter }}
|
||||||
|
<a
|
||||||
|
href="/books/{book.slug}/chapters/{chapter}"
|
||||||
|
class="group flex flex-col rounded-lg overflow-hidden bg-zinc-800 hover:bg-zinc-700 transition-colors border border-zinc-700 hover:border-zinc-500"
|
||||||
|
>
|
||||||
|
<div class="aspect-[2/3] bg-zinc-900 overflow-hidden relative">
|
||||||
|
{#if book.cover}
|
||||||
|
<img
|
||||||
|
src={book.cover}
|
||||||
|
alt={book.title}
|
||||||
|
class="w-full h-full object-cover group-hover:scale-105 transition-transform duration-300"
|
||||||
|
loading="lazy"
|
||||||
|
/>
|
||||||
|
{:else}
|
||||||
|
<div class="w-full h-full flex items-center justify-center text-zinc-600">
|
||||||
|
<svg class="w-10 h-10" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"
|
||||||
|
d="M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.332.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253" />
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
<!-- Chapter badge overlay -->
|
||||||
|
<span class="absolute bottom-1.5 right-1.5 text-xs bg-amber-400 text-zinc-900 font-bold px-1.5 py-0.5 rounded">
|
||||||
|
ch.{chapter}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div class="p-2">
|
||||||
|
<h3 class="text-xs font-semibold text-zinc-100 line-clamp-2 leading-snug">{book.title ?? ''}</h3>
|
||||||
|
{#if book.author}
|
||||||
|
<p class="text-xs text-zinc-500 truncate mt-0.5">{book.author}</p>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<!-- Recently Updated -->
|
||||||
|
{#if data.recentlyUpdated.length > 0}
|
||||||
|
<section class="mb-10">
|
||||||
|
<div class="flex items-baseline justify-between mb-3">
|
||||||
|
<h2 class="text-lg font-bold text-zinc-100">Recently Updated</h2>
|
||||||
|
<a href="/books" class="text-xs text-amber-400 hover:text-amber-300">View all</a>
|
||||||
|
</div>
|
||||||
|
<div class="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-6 gap-4">
|
||||||
|
{#each data.recentlyUpdated as book}
|
||||||
|
{@const genres = parseGenres(book.genres)}
|
||||||
|
<a
|
||||||
|
href="/books/{book.slug}"
|
||||||
|
class="group flex flex-col rounded-lg overflow-hidden bg-zinc-800 hover:bg-zinc-700 transition-colors border border-zinc-700 hover:border-zinc-500"
|
||||||
|
>
|
||||||
|
<div class="aspect-[2/3] bg-zinc-900 overflow-hidden">
|
||||||
|
{#if book.cover}
|
||||||
|
<img
|
||||||
|
src={book.cover}
|
||||||
|
alt={book.title}
|
||||||
|
class="w-full h-full object-cover group-hover:scale-105 transition-transform duration-300"
|
||||||
|
loading="lazy"
|
||||||
|
/>
|
||||||
|
{:else}
|
||||||
|
<div class="w-full h-full flex items-center justify-center text-zinc-600">
|
||||||
|
<svg class="w-10 h-10" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"
|
||||||
|
d="M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.332.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253" />
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
<div class="p-2 flex flex-col gap-1">
|
||||||
|
<h3 class="text-xs font-semibold text-zinc-100 line-clamp-2 leading-snug">{book.title ?? ''}</h3>
|
||||||
|
{#if book.author}
|
||||||
|
<p class="text-xs text-zinc-400 truncate">{book.author}</p>
|
||||||
|
{/if}
|
||||||
|
{#if book.status}
|
||||||
|
<span class="text-xs px-1.5 py-0.5 rounded bg-zinc-700 text-zinc-300 self-start">{book.status}</span>
|
||||||
|
{/if}
|
||||||
|
{#if genres.length > 0}
|
||||||
|
<div class="flex flex-wrap gap-1 mt-auto pt-1">
|
||||||
|
{#each genres.slice(0, 2) as genre}
|
||||||
|
<span class="text-xs px-1 py-0.5 rounded bg-zinc-900 text-zinc-500">{genre}</span>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<!-- Empty state -->
|
||||||
|
{#if data.continueReading.length === 0 && data.recentlyUpdated.length === 0}
|
||||||
|
<div class="text-center py-20 text-zinc-500">
|
||||||
|
<p class="text-lg font-semibold text-zinc-300 mb-2">Your library is empty</p>
|
||||||
|
<p class="text-sm mb-6">Discover novels and scrape them into your library.</p>
|
||||||
|
<a
|
||||||
|
href="/browse"
|
||||||
|
class="inline-block px-6 py-3 bg-amber-400 text-zinc-900 font-semibold rounded hover:bg-amber-300 transition-colors"
|
||||||
|
>
|
||||||
|
Discover Novels
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<!-- From Subscriptions -->
|
||||||
|
{#if data.subscriptionFeed.length > 0}
|
||||||
|
<section class="mb-10">
|
||||||
|
<div class="flex items-baseline justify-between mb-3">
|
||||||
|
<h2 class="text-lg font-bold text-zinc-100">From People You Follow</h2>
|
||||||
|
</div>
|
||||||
|
<div class="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-6 gap-4">
|
||||||
|
{#each data.subscriptionFeed as { book, readerUsername }}
|
||||||
|
{@const genres = parseGenres(book.genres)}
|
||||||
|
<a
|
||||||
|
href="/books/{book.slug}"
|
||||||
|
class="group flex flex-col rounded-lg overflow-hidden bg-zinc-800 hover:bg-zinc-700 transition-colors border border-zinc-700 hover:border-zinc-500"
|
||||||
|
>
|
||||||
|
<div class="aspect-[2/3] bg-zinc-900 overflow-hidden">
|
||||||
|
{#if book.cover}
|
||||||
|
<img
|
||||||
|
src={book.cover}
|
||||||
|
alt={book.title}
|
||||||
|
class="w-full h-full object-cover group-hover:scale-105 transition-transform duration-300"
|
||||||
|
loading="lazy"
|
||||||
|
/>
|
||||||
|
{:else}
|
||||||
|
<div class="w-full h-full flex items-center justify-center text-zinc-600">
|
||||||
|
<svg class="w-10 h-10" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||||
|
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"
|
||||||
|
d="M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.332.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253" />
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
<div class="p-2 flex flex-col gap-1">
|
||||||
|
<h3 class="text-xs font-semibold text-zinc-100 line-clamp-2 leading-snug">{book.title ?? ''}</h3>
|
||||||
|
{#if book.author}
|
||||||
|
<p class="text-xs text-zinc-400 truncate">{book.author}</p>
|
||||||
|
{/if}
|
||||||
|
<!-- Reader attribution -->
|
||||||
|
<p class="text-xs text-zinc-600 truncate mt-0.5">
|
||||||
|
via <span class="text-amber-500/70">{readerUsername}</span>
|
||||||
|
</p>
|
||||||
|
{#if genres.length > 0}
|
||||||
|
<div class="flex flex-wrap gap-1 mt-auto pt-1">
|
||||||
|
{#each genres.slice(0, 1) as genre}
|
||||||
|
<span class="text-xs px-1 py-0.5 rounded bg-zinc-900 text-zinc-500">{genre}</span>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
{/if}
|
||||||
17
ui-v2/src/routes/admin/audio-jobs/+page.server.ts
Normal file
17
ui-v2/src/routes/admin/audio-jobs/+page.server.ts
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
import { redirect } from '@sveltejs/kit';
|
||||||
|
import type { PageServerLoad } from './$types';
|
||||||
|
import { listAudioJobs } from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
export const load: PageServerLoad = async ({ locals }) => {
|
||||||
|
if (locals.user?.role !== 'admin') {
|
||||||
|
redirect(302, '/');
|
||||||
|
}
|
||||||
|
|
||||||
|
const jobs = await listAudioJobs().catch((e) => {
|
||||||
|
log.warn('admin/audio-jobs', 'failed to load audio jobs', { err: String(e) });
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
|
||||||
|
return { jobs };
|
||||||
|
};
|
||||||
153
ui-v2/src/routes/admin/audio-jobs/+page.svelte
Normal file
153
ui-v2/src/routes/admin/audio-jobs/+page.svelte
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { untrack } from 'svelte';
|
||||||
|
import type { PageData } from './$types';
|
||||||
|
|
||||||
|
let { data }: { data: PageData } = $props();
|
||||||
|
|
||||||
|
let jobs = $state(untrack(() => data.jobs));
|
||||||
|
|
||||||
|
// ── Live-poll: refresh while any job is in-flight ────────────────────────────
|
||||||
|
let hasInFlight = $derived(jobs.some((j) => j.status === 'pending' || j.status === 'generating'));
|
||||||
|
|
||||||
|
$effect(() => {
|
||||||
|
if (!hasInFlight) return;
|
||||||
|
const id = setInterval(async () => {
|
||||||
|
const res = await fetch('/admin/audio-jobs?__data=1').catch(() => null);
|
||||||
|
if (res?.ok) {
|
||||||
|
// SvelteKit invalidateAll is cleaner — just trigger a soft navigation reload.
|
||||||
|
import('$app/navigation').then(({ invalidateAll }) => invalidateAll());
|
||||||
|
}
|
||||||
|
}, 3000);
|
||||||
|
return () => clearInterval(id);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Keep local state in sync when server re-loads
|
||||||
|
$effect(() => {
|
||||||
|
jobs = data.jobs;
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||||
|
function statusColor(status: string) {
|
||||||
|
if (status === 'done') return 'text-green-400';
|
||||||
|
if (status === 'generating') return 'text-amber-400 animate-pulse';
|
||||||
|
if (status === 'pending') return 'text-sky-400 animate-pulse';
|
||||||
|
if (status === 'failed') return 'text-red-400';
|
||||||
|
return 'text-zinc-300';
|
||||||
|
}
|
||||||
|
|
||||||
|
function fmtDate(s: string) {
|
||||||
|
if (!s) return '—';
|
||||||
|
return new Date(s).toLocaleString(undefined, {
|
||||||
|
month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function duration(started: string, finished: string) {
|
||||||
|
if (!started || !finished) return '—';
|
||||||
|
const ms = new Date(finished).getTime() - new Date(started).getTime();
|
||||||
|
if (ms < 0) return '—';
|
||||||
|
const s = Math.floor(ms / 1000);
|
||||||
|
if (s < 60) return `${s}s`;
|
||||||
|
const m = Math.floor(s / 60);
|
||||||
|
return `${m}m ${s % 60}s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Search ───────────────────────────────────────────────────────────────────
|
||||||
|
let q = $state('');
|
||||||
|
let filtered = $derived(
|
||||||
|
q.trim()
|
||||||
|
? jobs.filter(
|
||||||
|
(j) =>
|
||||||
|
j.slug.toLowerCase().includes(q.toLowerCase().trim()) ||
|
||||||
|
j.voice.toLowerCase().includes(q.toLowerCase().trim()) ||
|
||||||
|
j.status.toLowerCase().includes(q.toLowerCase().trim())
|
||||||
|
)
|
||||||
|
: jobs
|
||||||
|
);
|
||||||
|
|
||||||
|
// ── Stats ────────────────────────────────────────────────────────────────────
|
||||||
|
let stats = $derived({
|
||||||
|
total: jobs.length,
|
||||||
|
done: jobs.filter((j) => j.status === 'done').length,
|
||||||
|
failed: jobs.filter((j) => j.status === 'failed').length,
|
||||||
|
inFlight: jobs.filter((j) => j.status === 'pending' || j.status === 'generating').length
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<svelte:head>
|
||||||
|
<title>Audio jobs — libnovel admin</title>
|
||||||
|
</svelte:head>
|
||||||
|
|
||||||
|
<div class="space-y-6">
|
||||||
|
<div class="flex items-start justify-between flex-wrap gap-3">
|
||||||
|
<div>
|
||||||
|
<h1 class="text-2xl font-bold text-zinc-100">Audio jobs</h1>
|
||||||
|
<p class="text-zinc-400 text-sm mt-1">
|
||||||
|
{stats.total} total ·
|
||||||
|
<span class="text-green-400">{stats.done} done</span> ·
|
||||||
|
{#if stats.failed > 0}
|
||||||
|
<span class="text-red-400">{stats.failed} failed</span> ·
|
||||||
|
{/if}
|
||||||
|
{#if stats.inFlight > 0}
|
||||||
|
<span class="text-amber-400 animate-pulse">{stats.inFlight} in-flight</span>
|
||||||
|
{:else}
|
||||||
|
<span class="text-zinc-500">0 in-flight</span>
|
||||||
|
{/if}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Search -->
|
||||||
|
<input
|
||||||
|
type="search"
|
||||||
|
bind:value={q}
|
||||||
|
placeholder="Filter by slug, voice or status…"
|
||||||
|
class="w-full max-w-sm bg-zinc-800 border border-zinc-700 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
|
||||||
|
/>
|
||||||
|
|
||||||
|
{#if filtered.length === 0}
|
||||||
|
<p class="text-zinc-500 text-sm py-8 text-center">
|
||||||
|
{q.trim() ? 'No results.' : 'No audio jobs yet.'}
|
||||||
|
</p>
|
||||||
|
{:else}
|
||||||
|
<div class="overflow-x-auto rounded-xl border border-zinc-700">
|
||||||
|
<table class="w-full text-sm">
|
||||||
|
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide">
|
||||||
|
<tr>
|
||||||
|
<th class="px-4 py-3 text-left">Book</th>
|
||||||
|
<th class="px-4 py-3 text-right">Ch.</th>
|
||||||
|
<th class="px-4 py-3 text-left">Voice</th>
|
||||||
|
<th class="px-4 py-3 text-left">Status</th>
|
||||||
|
<th class="px-4 py-3 text-left">Started</th>
|
||||||
|
<th class="px-4 py-3 text-left">Duration</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody class="divide-y divide-zinc-700/50">
|
||||||
|
{#each filtered as job}
|
||||||
|
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
|
||||||
|
<td class="px-4 py-3 text-zinc-200 font-medium">
|
||||||
|
<a href="/books/{job.slug}" class="hover:text-amber-400 transition-colors">
|
||||||
|
{job.slug}
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td class="px-4 py-3 text-right text-zinc-400">{job.chapter}</td>
|
||||||
|
<td class="px-4 py-3 text-zinc-400 font-mono text-xs">{job.voice}</td>
|
||||||
|
<td class="px-4 py-3">
|
||||||
|
<span class="font-medium {statusColor(job.status)}">{job.status}</span>
|
||||||
|
</td>
|
||||||
|
<td class="px-4 py-3 text-zinc-400">{fmtDate(job.started)}</td>
|
||||||
|
<td class="px-4 py-3 text-zinc-400">{duration(job.started, job.finished)}</td>
|
||||||
|
</tr>
|
||||||
|
{#if job.error_message}
|
||||||
|
<tr class="bg-red-950/20">
|
||||||
|
<td colspan="6" class="px-4 py-2 text-xs text-red-400 font-mono"
|
||||||
|
>{job.error_message}</td
|
||||||
|
>
|
||||||
|
</tr>
|
||||||
|
{/if}
|
||||||
|
{/each}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
17
ui-v2/src/routes/admin/audio/+page.server.ts
Normal file
17
ui-v2/src/routes/admin/audio/+page.server.ts
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
import { redirect } from '@sveltejs/kit';
|
||||||
|
import type { PageServerLoad } from './$types';
|
||||||
|
import { listAudioCache } from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
export const load: PageServerLoad = async ({ locals }) => {
|
||||||
|
if (locals.user?.role !== 'admin') {
|
||||||
|
redirect(302, '/');
|
||||||
|
}
|
||||||
|
|
||||||
|
const entries = await listAudioCache().catch((e) => {
|
||||||
|
log.warn('admin/audio', 'failed to load audio cache', { err: String(e) });
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
|
||||||
|
return { entries };
|
||||||
|
};
|
||||||
93
ui-v2/src/routes/admin/audio/+page.svelte
Normal file
93
ui-v2/src/routes/admin/audio/+page.svelte
Normal file
@@ -0,0 +1,93 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { untrack } from 'svelte';
|
||||||
|
import type { PageData } from './$types';
|
||||||
|
|
||||||
|
let { data }: { data: PageData } = $props();
|
||||||
|
|
||||||
|
let entries = $state(untrack(() => data.entries));
|
||||||
|
|
||||||
|
// ── Parse cache_key ─────────────────────────────────────────────────────────
|
||||||
|
// cache_key format: "slug/chapter/voice"
|
||||||
|
function parseKey(key: string) {
|
||||||
|
const parts = key.split('/');
|
||||||
|
if (parts.length >= 3) {
|
||||||
|
return { slug: parts[0], chapter: parts[1], voice: parts.slice(2).join('/') };
|
||||||
|
}
|
||||||
|
return { slug: key, chapter: '—', voice: '—' };
|
||||||
|
}
|
||||||
|
|
||||||
|
function fmtDate(s: string) {
|
||||||
|
if (!s) return '—';
|
||||||
|
return new Date(s).toLocaleString(undefined, {
|
||||||
|
month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Search ──────────────────────────────────────────────────────────────────
|
||||||
|
let q = $state('');
|
||||||
|
let filtered = $derived(
|
||||||
|
q.trim()
|
||||||
|
? entries.filter((e) => e.cache_key.toLowerCase().includes(q.toLowerCase().trim()))
|
||||||
|
: entries
|
||||||
|
);
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<svelte:head>
|
||||||
|
<title>Audio cache — libnovel admin</title>
|
||||||
|
</svelte:head>
|
||||||
|
|
||||||
|
<div class="space-y-6">
|
||||||
|
<div>
|
||||||
|
<h1 class="text-2xl font-bold text-zinc-100">Audio cache</h1>
|
||||||
|
<p class="text-zinc-400 text-sm mt-1">{entries.length} cached audio file{entries.length !== 1 ? 's' : ''}</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Search -->
|
||||||
|
<input
|
||||||
|
type="search"
|
||||||
|
bind:value={q}
|
||||||
|
placeholder="Filter by slug, chapter or voice…"
|
||||||
|
class="w-full max-w-sm bg-zinc-800 border border-zinc-700 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
|
||||||
|
/>
|
||||||
|
|
||||||
|
{#if filtered.length === 0}
|
||||||
|
<p class="text-zinc-500 text-sm py-8 text-center">
|
||||||
|
{q.trim() ? 'No results.' : 'Audio cache is empty.'}
|
||||||
|
</p>
|
||||||
|
{:else}
|
||||||
|
<div class="overflow-x-auto rounded-xl border border-zinc-700">
|
||||||
|
<table class="w-full text-sm">
|
||||||
|
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide">
|
||||||
|
<tr>
|
||||||
|
<th class="px-4 py-3 text-left">Book</th>
|
||||||
|
<th class="px-4 py-3 text-left">Chapter</th>
|
||||||
|
<th class="px-4 py-3 text-left">Voice</th>
|
||||||
|
<th class="px-4 py-3 text-left">Filename</th>
|
||||||
|
<th class="px-4 py-3 text-left">Updated</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody class="divide-y divide-zinc-700/50">
|
||||||
|
{#each filtered as entry}
|
||||||
|
{@const parts = parseKey(entry.cache_key)}
|
||||||
|
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
|
||||||
|
<td class="px-4 py-3 text-zinc-200 font-medium">
|
||||||
|
<a
|
||||||
|
href="/books/{parts.slug}"
|
||||||
|
class="hover:text-amber-400 transition-colors"
|
||||||
|
>
|
||||||
|
{parts.slug}
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td class="px-4 py-3 text-zinc-400">{parts.chapter}</td>
|
||||||
|
<td class="px-4 py-3 text-zinc-400 font-mono text-xs">{parts.voice}</td>
|
||||||
|
<td class="px-4 py-3 text-zinc-500 font-mono text-xs truncate max-w-[14rem]" title={entry.filename}>
|
||||||
|
{entry.filename}
|
||||||
|
</td>
|
||||||
|
<td class="px-4 py-3 text-zinc-400">{fmtDate(entry.updated)}</td>
|
||||||
|
</tr>
|
||||||
|
{/each}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
29
ui-v2/src/routes/admin/scrape/+page.server.ts
Normal file
29
ui-v2/src/routes/admin/scrape/+page.server.ts
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
import { redirect } from '@sveltejs/kit';
|
||||||
|
import type { PageServerLoad } from './$types';
|
||||||
|
import { listScrapingTasks } from '$lib/server/pocketbase';
|
||||||
|
import { env } from '$env/dynamic/private';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||||
|
|
||||||
|
export const load: PageServerLoad = async ({ locals }) => {
|
||||||
|
if (locals.user?.role !== 'admin') {
|
||||||
|
redirect(302, '/');
|
||||||
|
}
|
||||||
|
|
||||||
|
const [tasks, statusRes] = await Promise.all([
|
||||||
|
listScrapingTasks().catch((e) => {
|
||||||
|
log.warn('admin/scrape', 'failed to load tasks', { err: String(e) });
|
||||||
|
return [];
|
||||||
|
}),
|
||||||
|
fetch(`${SCRAPER_URL}/api/scrape/status`).catch(() => null)
|
||||||
|
]);
|
||||||
|
|
||||||
|
let running = false;
|
||||||
|
if (statusRes?.ok) {
|
||||||
|
const body = await statusRes.json().catch(() => null);
|
||||||
|
running = body?.running ?? false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { tasks, running };
|
||||||
|
};
|
||||||
238
ui-v2/src/routes/admin/scrape/+page.svelte
Normal file
238
ui-v2/src/routes/admin/scrape/+page.svelte
Normal file
@@ -0,0 +1,238 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { untrack } from 'svelte';
|
||||||
|
import { invalidateAll } from '$app/navigation';
|
||||||
|
import type { PageData } from './$types';
|
||||||
|
import type { ScrapingTask } from '$lib/server/pocketbase';
|
||||||
|
|
||||||
|
let { data }: { data: PageData } = $props();
|
||||||
|
|
||||||
|
// ── Live-poll status ────────────────────────────────────────────────────────
|
||||||
|
let running = $state(untrack(() => data.running));
|
||||||
|
let tasks = $state(untrack(() => data.tasks));
|
||||||
|
let polling = $state(false);
|
||||||
|
|
||||||
|
// Poll every 5 s while a job is running
|
||||||
|
$effect(() => {
|
||||||
|
if (!running) return;
|
||||||
|
const id = setInterval(async () => {
|
||||||
|
const res = await fetch('/api/admin/scrape').catch(() => null);
|
||||||
|
if (res?.ok) {
|
||||||
|
const body = await res.json().catch(() => null);
|
||||||
|
running = body?.running ?? false;
|
||||||
|
if (!running) {
|
||||||
|
// Refresh tasks list once job finishes
|
||||||
|
await invalidateAll();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, 5000);
|
||||||
|
return () => clearInterval(id);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Keep local state in sync when server re-loads
|
||||||
|
$effect(() => {
|
||||||
|
running = data.running;
|
||||||
|
tasks = data.tasks;
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Trigger scrape ──────────────────────────────────────────────────────────
|
||||||
|
let scrapeUrl = $state('');
|
||||||
|
let scrapeError = $state('');
|
||||||
|
let scraping = $state(false);
|
||||||
|
|
||||||
|
async function triggerScrape(url?: string) {
|
||||||
|
if (running || scraping) return;
|
||||||
|
scraping = true;
|
||||||
|
scrapeError = '';
|
||||||
|
try {
|
||||||
|
const body = url ? { url } : {};
|
||||||
|
const res = await fetch('/api/scrape', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify(body)
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
const data = await res.json().catch(() => ({}));
|
||||||
|
scrapeError = data.error ?? data.message ?? `Error ${res.status}`;
|
||||||
|
} else {
|
||||||
|
running = true;
|
||||||
|
if (url) scrapeUrl = '';
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
scrapeError = 'Network error.';
|
||||||
|
} finally {
|
||||||
|
scraping = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Cancel task ─────────────────────────────────────────────────────────────
|
||||||
|
// Tracks which task IDs are currently being cancelled (to disable the button).
|
||||||
|
let cancellingIds = $state(new Set<string>());
|
||||||
|
let cancelErrors: Record<string, string> = $state({});
|
||||||
|
|
||||||
|
async function cancelTask(id: string) {
|
||||||
|
if (cancellingIds.has(id)) return;
|
||||||
|
cancellingIds = new Set([...cancellingIds, id]);
|
||||||
|
delete cancelErrors[id];
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/scrape/cancel/${encodeURIComponent(id)}`, { method: 'POST' });
|
||||||
|
if (!res.ok) {
|
||||||
|
const body = await res.json().catch(() => ({}));
|
||||||
|
cancelErrors = { ...cancelErrors, [id]: body.error ?? body.message ?? `Error ${res.status}` };
|
||||||
|
} else {
|
||||||
|
// Optimistically flip status in the local list so the button disappears immediately.
|
||||||
|
tasks = tasks.map((t: ScrapingTask) => (t.id === id ? { ...t, status: 'cancelled' } : t));
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
cancelErrors = { ...cancelErrors, [id]: 'Network error.' };
|
||||||
|
} finally {
|
||||||
|
cancellingIds = new Set([...cancellingIds].filter((x) => x !== id));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||||
|
function statusColor(status: string) {
|
||||||
|
if (status === 'done') return 'text-green-400';
|
||||||
|
if (status === 'running') return 'text-amber-400 animate-pulse';
|
||||||
|
if (status === 'failed') return 'text-red-400';
|
||||||
|
if (status === 'cancelled') return 'text-zinc-400';
|
||||||
|
return 'text-zinc-300';
|
||||||
|
}
|
||||||
|
|
||||||
|
function fmtDate(s: string) {
|
||||||
|
if (!s) return '—';
|
||||||
|
return new Date(s).toLocaleString(undefined, {
|
||||||
|
month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function duration(started: string, finished: string) {
|
||||||
|
if (!started || !finished) return '—';
|
||||||
|
const ms = new Date(finished).getTime() - new Date(started).getTime();
|
||||||
|
if (ms < 0) return '—';
|
||||||
|
const s = Math.floor(ms / 1000);
|
||||||
|
if (s < 60) return `${s}s`;
|
||||||
|
const m = Math.floor(s / 60);
|
||||||
|
return `${m}m ${s % 60}s`;
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<svelte:head>
|
||||||
|
<title>Scrape tasks — libnovel admin</title>
|
||||||
|
</svelte:head>
|
||||||
|
|
||||||
|
<div class="space-y-8">
|
||||||
|
<div class="flex items-center justify-between flex-wrap gap-3">
|
||||||
|
<div>
|
||||||
|
<h1 class="text-2xl font-bold text-zinc-100">Scrape tasks</h1>
|
||||||
|
<p class="text-zinc-400 text-sm mt-1">
|
||||||
|
Job status:
|
||||||
|
{#if running}
|
||||||
|
<span class="text-amber-400 font-medium animate-pulse">Running</span>
|
||||||
|
{:else}
|
||||||
|
<span class="text-green-400 font-medium">Idle</span>
|
||||||
|
{/if}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Trigger controls -->
|
||||||
|
<div class="flex flex-wrap gap-3 items-start">
|
||||||
|
<button
|
||||||
|
onclick={() => triggerScrape()}
|
||||||
|
disabled={running || scraping}
|
||||||
|
class="px-4 py-2 rounded-lg bg-amber-400 text-zinc-900 font-semibold text-sm hover:bg-amber-300 transition-colors disabled:opacity-50"
|
||||||
|
>
|
||||||
|
Full catalogue scrape
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Single book scrape -->
|
||||||
|
<div class="bg-zinc-800 rounded-xl border border-zinc-700 p-5 space-y-3">
|
||||||
|
<h2 class="text-sm font-semibold text-zinc-300">Scrape a single book</h2>
|
||||||
|
<div class="flex gap-2">
|
||||||
|
<input
|
||||||
|
type="url"
|
||||||
|
bind:value={scrapeUrl}
|
||||||
|
placeholder="https://novelfire.net/book/..."
|
||||||
|
class="flex-1 bg-zinc-700 border border-zinc-600 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
|
||||||
|
/>
|
||||||
|
<button
|
||||||
|
onclick={() => triggerScrape(scrapeUrl.trim() || undefined)}
|
||||||
|
disabled={!scrapeUrl.trim() || running || scraping}
|
||||||
|
class="px-4 py-2 rounded-lg bg-zinc-600 text-zinc-100 font-semibold text-sm hover:bg-zinc-500 transition-colors disabled:opacity-50"
|
||||||
|
>
|
||||||
|
Scrape
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{#if scrapeError}
|
||||||
|
<p class="text-sm text-red-400">{scrapeError}</p>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Tasks table -->
|
||||||
|
{#if tasks.length === 0}
|
||||||
|
<p class="text-zinc-500 text-sm py-8 text-center">No scrape tasks yet.</p>
|
||||||
|
{:else}
|
||||||
|
<div class="overflow-x-auto rounded-xl border border-zinc-700">
|
||||||
|
<table class="w-full text-sm">
|
||||||
|
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide">
|
||||||
|
<tr>
|
||||||
|
<th class="px-4 py-3 text-left">Kind</th>
|
||||||
|
<th class="px-4 py-3 text-left">Status</th>
|
||||||
|
<th class="px-4 py-3 text-right">Books</th>
|
||||||
|
<th class="px-4 py-3 text-right">Chapters</th>
|
||||||
|
<th class="px-4 py-3 text-right">Skipped</th>
|
||||||
|
<th class="px-4 py-3 text-right">Errors</th>
|
||||||
|
<th class="px-4 py-3 text-left">Started</th>
|
||||||
|
<th class="px-4 py-3 text-left">Duration</th>
|
||||||
|
<th class="px-4 py-3 text-left">Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody class="divide-y divide-zinc-700/50">
|
||||||
|
{#each tasks as task}
|
||||||
|
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
|
||||||
|
<td class="px-4 py-3 font-mono text-xs text-zinc-300">
|
||||||
|
{task.kind}
|
||||||
|
{#if task.target_url}
|
||||||
|
<br />
|
||||||
|
<span class="text-zinc-500 truncate max-w-[16rem] block" title={task.target_url}>
|
||||||
|
{task.target_url.replace('https://novelfire.net/book/', '')}
|
||||||
|
</span>
|
||||||
|
{/if}
|
||||||
|
</td>
|
||||||
|
<td class="px-4 py-3">
|
||||||
|
<span class="font-medium {statusColor(task.status)}">{task.status}</span>
|
||||||
|
</td>
|
||||||
|
<td class="px-4 py-3 text-right text-zinc-300">{task.books_found ?? 0}</td>
|
||||||
|
<td class="px-4 py-3 text-right text-zinc-300">{task.chapters_scraped ?? 0}</td>
|
||||||
|
<td class="px-4 py-3 text-right text-zinc-400">{task.chapters_skipped ?? 0}</td>
|
||||||
|
<td class="px-4 py-3 text-right {task.errors > 0 ? 'text-red-400' : 'text-zinc-400'}">{task.errors ?? 0}</td>
|
||||||
|
<td class="px-4 py-3 text-zinc-400">{fmtDate(task.started)}</td>
|
||||||
|
<td class="px-4 py-3 text-zinc-400">{duration(task.started, task.finished)}</td>
|
||||||
|
<td class="px-4 py-3">
|
||||||
|
{#if task.status === 'pending'}
|
||||||
|
<button
|
||||||
|
onclick={() => cancelTask(task.id)}
|
||||||
|
disabled={cancellingIds.has(task.id)}
|
||||||
|
class="px-2 py-1 rounded text-xs font-medium bg-zinc-700 text-zinc-300 hover:bg-red-900 hover:text-red-300 transition-colors disabled:opacity-50"
|
||||||
|
title="Cancel this task"
|
||||||
|
>
|
||||||
|
{cancellingIds.has(task.id) ? 'Cancelling…' : 'Cancel'}
|
||||||
|
</button>
|
||||||
|
{#if cancelErrors[task.id]}
|
||||||
|
<p class="text-xs text-red-400 mt-1">{cancelErrors[task.id]}</p>
|
||||||
|
{/if}
|
||||||
|
{/if}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{#if task.error_message}
|
||||||
|
<tr class="bg-red-950/20">
|
||||||
|
<td colspan="9" class="px-4 py-2 text-xs text-red-400 font-mono">{task.error_message}</td>
|
||||||
|
</tr>
|
||||||
|
{/if}
|
||||||
|
{/each}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
23
ui-v2/src/routes/api/admin/scrape/+server.ts
Normal file
23
ui-v2/src/routes/api/admin/scrape/+server.ts
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { env } from '$env/dynamic/private';
|
||||||
|
|
||||||
|
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/admin/scrape/status
|
||||||
|
* Admin-only proxy to the Go scraper's /api/scrape/status endpoint.
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ locals }) => {
|
||||||
|
if (!locals.user || locals.user.role !== 'admin') {
|
||||||
|
throw error(403, 'Forbidden');
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${SCRAPER_URL}/api/scrape/status`);
|
||||||
|
if (!res.ok) return json({ running: false });
|
||||||
|
const data = await res.json();
|
||||||
|
return json({ running: data.running ?? false });
|
||||||
|
} catch {
|
||||||
|
return json({ running: false });
|
||||||
|
}
|
||||||
|
};
|
||||||
100
ui-v2/src/routes/api/audio/[slug]/[n]/+server.ts
Normal file
100
ui-v2/src/routes/api/audio/[slug]/[n]/+server.ts
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
import { error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { env } from '$env/dynamic/private';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/audio/[slug]/[n]
|
||||||
|
* Proxies the audio generation request to the scraper's /api/audio endpoint.
|
||||||
|
* Keeps the scraper URL server-side — the browser never needs to know it.
|
||||||
|
*
|
||||||
|
* Body: { voice?: string }
|
||||||
|
*
|
||||||
|
* Responses:
|
||||||
|
* 200 { status: "done" } — audio already cached; client should call
|
||||||
|
* GET /api/presign/audio to obtain a direct MinIO presigned URL.
|
||||||
|
* 202 { task_id: string, status: "pending"|"generating" } — generation
|
||||||
|
* enqueued; poll GET /api/audio/status/[slug]/[n]?voice=... until done.
|
||||||
|
*/
|
||||||
|
export const POST: RequestHandler = async ({ params, request }) => {
|
||||||
|
const { slug, n } = params;
|
||||||
|
const chapter = parseInt(n, 10);
|
||||||
|
if (!slug || !chapter || chapter < 1) {
|
||||||
|
error(400, 'Invalid slug or chapter number');
|
||||||
|
}
|
||||||
|
|
||||||
|
let body: { voice?: string } = {};
|
||||||
|
try {
|
||||||
|
body = await request.json();
|
||||||
|
} catch {
|
||||||
|
// empty body is fine — scraper will use defaults
|
||||||
|
}
|
||||||
|
|
||||||
|
const scraperRes = await fetch(`${SCRAPER_URL}/api/audio/${slug}/${chapter}`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify(body)
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!scraperRes.ok) {
|
||||||
|
const text = await scraperRes.text().catch(() => '');
|
||||||
|
log.error('audio', 'scraper audio generation failed', { slug, chapter, status: scraperRes.status, body: text });
|
||||||
|
error(scraperRes.status as Parameters<typeof error>[0], text || 'Audio generation failed');
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = (await scraperRes.json()) as
|
||||||
|
| { url: string; status: 'done' }
|
||||||
|
| { task_id: string; status: string };
|
||||||
|
|
||||||
|
// 202 Accepted: generation enqueued — return task_id + status for polling.
|
||||||
|
if (scraperRes.status === 202 || 'task_id' in data) {
|
||||||
|
return new Response(JSON.stringify(data), {
|
||||||
|
status: 202,
|
||||||
|
headers: { 'Content-Type': 'application/json' }
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// 200: audio was already cached.
|
||||||
|
// Return status only — no url — so the client calls GET /api/presign/audio
|
||||||
|
// and streams directly from MinIO instead of through the Node.js server.
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({ status: 'done' }),
|
||||||
|
{ headers: { 'Content-Type': 'application/json' } }
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/audio/[slug]/[n]?voice=...
|
||||||
|
* Proxies the audio stream from the scraper's /api/audio-proxy endpoint.
|
||||||
|
* Kept as a fallback but no longer used as the primary playback path —
|
||||||
|
* AudioPlayer fetches a presigned MinIO URL directly via /api/presign/audio.
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ params, url }) => {
|
||||||
|
const { slug, n } = params;
|
||||||
|
const chapter = parseInt(n, 10);
|
||||||
|
if (!slug || !chapter || chapter < 1) {
|
||||||
|
error(400, 'Invalid slug or chapter number');
|
||||||
|
}
|
||||||
|
|
||||||
|
const voice = url.searchParams.get('voice') ?? '';
|
||||||
|
const qs = new URLSearchParams();
|
||||||
|
if (voice) qs.set('voice', voice);
|
||||||
|
|
||||||
|
const scraperRes = await fetch(`${SCRAPER_URL}/api/audio-proxy/${slug}/${chapter}?${qs.toString()}`);
|
||||||
|
|
||||||
|
if (!scraperRes.ok) {
|
||||||
|
log.error('audio', 'scraper audio proxy failed', { slug, chapter, status: scraperRes.status });
|
||||||
|
error(scraperRes.status as Parameters<typeof error>[0], 'Audio not found');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stream the audio body through — preserve Content-Type and Content-Length.
|
||||||
|
const headers = new Headers();
|
||||||
|
headers.set('Content-Type', scraperRes.headers.get('Content-Type') ?? 'audio/mpeg');
|
||||||
|
headers.set('Cache-Control', 'public, max-age=3600');
|
||||||
|
const cl = scraperRes.headers.get('Content-Length');
|
||||||
|
if (cl) headers.set('Content-Length', cl);
|
||||||
|
|
||||||
|
return new Response(scraperRes.body, { headers });
|
||||||
|
};
|
||||||
66
ui-v2/src/routes/api/audio/status/[slug]/[n]/+server.ts
Normal file
66
ui-v2/src/routes/api/audio/status/[slug]/[n]/+server.ts
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
import { error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { env } from '$env/dynamic/private';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/audio/status/[slug]/[n]?voice=...
|
||||||
|
* Proxies the audio generation status check to the scraper's
|
||||||
|
* GET /api/audio/status/{slug}/{n} endpoint.
|
||||||
|
*
|
||||||
|
* Possible responses passed through to the client:
|
||||||
|
* {"status":"done"} — audio ready; no url
|
||||||
|
* {"status":"pending"|"generating","task_id":"..."} — in progress
|
||||||
|
* {"status":"idle"} — no job yet
|
||||||
|
* {"status":"failed","error":"..."} — last job failed
|
||||||
|
*
|
||||||
|
* When status is "done" the scraper's internal proxy URL is stripped — the
|
||||||
|
* client must call GET /api/presign/audio to obtain a direct MinIO presigned
|
||||||
|
* URL. This avoids streaming audio through the Node.js server.
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ params, url }) => {
|
||||||
|
const { slug, n } = params;
|
||||||
|
const chapter = parseInt(n, 10);
|
||||||
|
if (!slug || !chapter || chapter < 1) {
|
||||||
|
error(400, 'Invalid slug or chapter number');
|
||||||
|
}
|
||||||
|
|
||||||
|
const voice = url.searchParams.get('voice') ?? '';
|
||||||
|
const qs = new URLSearchParams();
|
||||||
|
if (voice) qs.set('voice', voice);
|
||||||
|
|
||||||
|
const scraperRes = await fetch(
|
||||||
|
`${SCRAPER_URL}/api/audio/status/${slug}/${chapter}?${qs.toString()}`
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!scraperRes.ok) {
|
||||||
|
const text = await scraperRes.text().catch(() => '');
|
||||||
|
log.error('audio', 'scraper audio status check failed', {
|
||||||
|
slug,
|
||||||
|
chapter,
|
||||||
|
status: scraperRes.status,
|
||||||
|
body: text
|
||||||
|
});
|
||||||
|
error(scraperRes.status as Parameters<typeof error>[0], text || 'Status check failed');
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = (await scraperRes.json()) as {
|
||||||
|
status: string;
|
||||||
|
task_id?: string;
|
||||||
|
url?: string;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Strip the scraper's internal proxy URL from "done" responses.
|
||||||
|
// The client will call GET /api/presign/audio to get a direct MinIO URL,
|
||||||
|
// avoiding streaming audio through the Node.js server.
|
||||||
|
if (data.status === 'done') {
|
||||||
|
delete data.url;
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Response(JSON.stringify(data), {
|
||||||
|
headers: { 'Content-Type': 'application/json' }
|
||||||
|
});
|
||||||
|
};
|
||||||
11
ui-v2/src/routes/api/audio/voice-samples/+server.ts
Normal file
11
ui-v2/src/routes/api/audio/voice-samples/+server.ts
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
import { json } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/audio/voice-samples
|
||||||
|
* The new backend does not expose a voice-samples generation endpoint.
|
||||||
|
* Return 501 so callers get a clear signal rather than a 502 proxy error.
|
||||||
|
*/
|
||||||
|
export const POST: RequestHandler = async () => {
|
||||||
|
return json({ error: 'Voice sample pre-generation is not supported by this backend.' }, { status: 501 });
|
||||||
|
};
|
||||||
47
ui-v2/src/routes/api/auth/change-password/+server.ts
Normal file
47
ui-v2/src/routes/api/auth/change-password/+server.ts
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { changePassword } from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/auth/change-password
|
||||||
|
* Body: { currentPassword: string, newPassword: string }
|
||||||
|
* Requires authentication.
|
||||||
|
*/
|
||||||
|
export const POST: RequestHandler = async ({ request, locals }) => {
|
||||||
|
if (!locals.user) {
|
||||||
|
error(401, 'Not authenticated');
|
||||||
|
}
|
||||||
|
|
||||||
|
let body: { currentPassword?: string; newPassword?: string };
|
||||||
|
try {
|
||||||
|
body = await request.json();
|
||||||
|
} catch {
|
||||||
|
error(400, 'Invalid JSON body');
|
||||||
|
}
|
||||||
|
|
||||||
|
const currentPassword = body.currentPassword ?? '';
|
||||||
|
const newPassword = body.newPassword ?? '';
|
||||||
|
|
||||||
|
if (!currentPassword || !newPassword) {
|
||||||
|
error(400, 'currentPassword and newPassword are required');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (newPassword.length < 4) {
|
||||||
|
error(400, 'New password must be at least 4 characters');
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const ok = await changePassword(locals.user.id, currentPassword, newPassword);
|
||||||
|
if (!ok) {
|
||||||
|
error(401, 'Current password is incorrect');
|
||||||
|
}
|
||||||
|
} catch (e: unknown) {
|
||||||
|
// Re-throw SvelteKit errors as-is
|
||||||
|
if (e && typeof e === 'object' && 'status' in e) throw e;
|
||||||
|
log.error('api/auth/change-password', 'unexpected error', { err: String(e) });
|
||||||
|
error(500, 'An error occurred. Please try again.');
|
||||||
|
}
|
||||||
|
|
||||||
|
return json({ ok: true });
|
||||||
|
};
|
||||||
75
ui-v2/src/routes/api/auth/login/+server.ts
Normal file
75
ui-v2/src/routes/api/auth/login/+server.ts
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { loginUser, mergeSessionProgress, createUserSession } from '$lib/server/pocketbase';
|
||||||
|
import { createAuthToken } from '../../../../hooks.server';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
import { randomBytes } from 'node:crypto';
|
||||||
|
|
||||||
|
const AUTH_COOKIE = 'libnovel_auth';
|
||||||
|
const ONE_YEAR = 60 * 60 * 24 * 365;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/auth/login
|
||||||
|
* Body: { username: string, password: string }
|
||||||
|
* Returns: { token: string, user: { id, username, role } }
|
||||||
|
*
|
||||||
|
* Sets the libnovel_auth cookie and returns the raw token value so the
|
||||||
|
* iOS app can persist it for subsequent requests.
|
||||||
|
*/
|
||||||
|
export const POST: RequestHandler = async ({ request, cookies, locals }) => {
|
||||||
|
let body: { username?: string; password?: string };
|
||||||
|
try {
|
||||||
|
body = await request.json();
|
||||||
|
} catch {
|
||||||
|
error(400, 'Invalid JSON body');
|
||||||
|
}
|
||||||
|
|
||||||
|
const username = (body.username ?? '').trim();
|
||||||
|
const password = body.password ?? '';
|
||||||
|
|
||||||
|
if (!username || !password) {
|
||||||
|
error(400, 'Username and password are required');
|
||||||
|
}
|
||||||
|
|
||||||
|
let user;
|
||||||
|
try {
|
||||||
|
user = await loginUser(username, password);
|
||||||
|
} catch (e) {
|
||||||
|
log.error('api/auth/login', 'unexpected error', { username, err: String(e) });
|
||||||
|
error(500, 'An error occurred. Please try again.');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!user) {
|
||||||
|
error(401, 'Invalid username or password');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge anonymous session progress (non-fatal)
|
||||||
|
mergeSessionProgress(locals.sessionId, user.id).catch((e) =>
|
||||||
|
log.warn('api/auth/login', 'mergeSessionProgress failed (non-fatal)', { err: String(e) })
|
||||||
|
);
|
||||||
|
|
||||||
|
const authSessionId = randomBytes(16).toString('hex');
|
||||||
|
|
||||||
|
const userAgent = request.headers.get('user-agent') ?? '';
|
||||||
|
const ip =
|
||||||
|
request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ??
|
||||||
|
request.headers.get('x-real-ip') ??
|
||||||
|
'';
|
||||||
|
createUserSession(user.id, authSessionId, userAgent, ip).catch((e) =>
|
||||||
|
log.warn('api/auth/login', 'createUserSession failed (non-fatal)', { err: String(e) })
|
||||||
|
);
|
||||||
|
|
||||||
|
const token = createAuthToken(user.id, user.username, user.role ?? 'user', authSessionId);
|
||||||
|
|
||||||
|
cookies.set(AUTH_COOKIE, token, {
|
||||||
|
path: '/',
|
||||||
|
httpOnly: true,
|
||||||
|
sameSite: 'lax',
|
||||||
|
maxAge: ONE_YEAR
|
||||||
|
});
|
||||||
|
|
||||||
|
return json({
|
||||||
|
token,
|
||||||
|
user: { id: user.id, username: user.username, role: user.role ?? 'user' }
|
||||||
|
});
|
||||||
|
};
|
||||||
15
ui-v2/src/routes/api/auth/logout/+server.ts
Normal file
15
ui-v2/src/routes/api/auth/logout/+server.ts
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
import { json } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
|
||||||
|
const AUTH_COOKIE = 'libnovel_auth';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/auth/logout
|
||||||
|
* Clears the auth cookie and returns { ok: true }.
|
||||||
|
* Does not revoke the session record from PocketBase —
|
||||||
|
* for full revocation use DELETE /api/sessions/[id] first.
|
||||||
|
*/
|
||||||
|
export const POST: RequestHandler = async ({ cookies }) => {
|
||||||
|
cookies.delete(AUTH_COOKIE, { path: '/' });
|
||||||
|
return json({ ok: true });
|
||||||
|
};
|
||||||
22
ui-v2/src/routes/api/auth/me/+server.ts
Normal file
22
ui-v2/src/routes/api/auth/me/+server.ts
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { getUserByUsername } from '$lib/server/pocketbase';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/auth/me
|
||||||
|
* Returns the currently authenticated user from the request's auth cookie.
|
||||||
|
* Returns 401 if not authenticated.
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ locals }) => {
|
||||||
|
if (!locals.user) {
|
||||||
|
error(401, 'Not authenticated');
|
||||||
|
}
|
||||||
|
// Fetch full record from PocketBase to get avatar_url
|
||||||
|
const record = await getUserByUsername(locals.user.username).catch(() => null);
|
||||||
|
return json({
|
||||||
|
id: locals.user.id,
|
||||||
|
username: locals.user.username,
|
||||||
|
role: locals.user.role,
|
||||||
|
avatar_url: record?.avatar_url ?? null
|
||||||
|
});
|
||||||
|
};
|
||||||
84
ui-v2/src/routes/api/auth/register/+server.ts
Normal file
84
ui-v2/src/routes/api/auth/register/+server.ts
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { createUser, mergeSessionProgress, createUserSession } from '$lib/server/pocketbase';
|
||||||
|
import { createAuthToken } from '../../../../hooks.server';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
import { randomBytes } from 'node:crypto';
|
||||||
|
|
||||||
|
const AUTH_COOKIE = 'libnovel_auth';
|
||||||
|
const ONE_YEAR = 60 * 60 * 24 * 365;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/auth/register
|
||||||
|
* Body: { username: string, password: string }
|
||||||
|
* Returns: { token: string, user: { id, username, role } }
|
||||||
|
*
|
||||||
|
* Sets the libnovel_auth cookie and returns the raw token value so the
|
||||||
|
* iOS app can persist it for subsequent requests.
|
||||||
|
*/
|
||||||
|
export const POST: RequestHandler = async ({ request, cookies, locals }) => {
|
||||||
|
let body: { username?: string; password?: string };
|
||||||
|
try {
|
||||||
|
body = await request.json();
|
||||||
|
} catch {
|
||||||
|
error(400, 'Invalid JSON body');
|
||||||
|
}
|
||||||
|
|
||||||
|
const username = (body.username ?? '').trim();
|
||||||
|
const password = body.password ?? '';
|
||||||
|
|
||||||
|
if (!username || !password) {
|
||||||
|
error(400, 'Username and password are required');
|
||||||
|
}
|
||||||
|
if (username.length < 3 || username.length > 32) {
|
||||||
|
error(400, 'Username must be between 3 and 32 characters');
|
||||||
|
}
|
||||||
|
if (!/^[a-zA-Z0-9_-]+$/.test(username)) {
|
||||||
|
error(400, 'Username may only contain letters, numbers, underscores and hyphens');
|
||||||
|
}
|
||||||
|
if (password.length < 8) {
|
||||||
|
error(400, 'Password must be at least 8 characters');
|
||||||
|
}
|
||||||
|
|
||||||
|
let user;
|
||||||
|
try {
|
||||||
|
user = await createUser(username, password);
|
||||||
|
} catch (e: unknown) {
|
||||||
|
const msg = e instanceof Error ? e.message : 'Registration failed.';
|
||||||
|
if (msg.includes('Username already taken')) {
|
||||||
|
error(409, 'That username is already taken');
|
||||||
|
}
|
||||||
|
log.error('api/auth/register', 'unexpected error', { username, err: String(e) });
|
||||||
|
error(500, 'An error occurred. Please try again.');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge anonymous session progress (non-fatal)
|
||||||
|
mergeSessionProgress(locals.sessionId, user.id).catch((e) =>
|
||||||
|
log.warn('api/auth/register', 'mergeSessionProgress failed (non-fatal)', { err: String(e) })
|
||||||
|
);
|
||||||
|
|
||||||
|
const authSessionId = randomBytes(16).toString('hex');
|
||||||
|
|
||||||
|
const userAgent = request.headers.get('user-agent') ?? '';
|
||||||
|
const ip =
|
||||||
|
request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ??
|
||||||
|
request.headers.get('x-real-ip') ??
|
||||||
|
'';
|
||||||
|
createUserSession(user.id, authSessionId, userAgent, ip).catch((e) =>
|
||||||
|
log.warn('api/auth/register', 'createUserSession failed (non-fatal)', { err: String(e) })
|
||||||
|
);
|
||||||
|
|
||||||
|
const token = createAuthToken(user.id, user.username, user.role ?? 'user', authSessionId);
|
||||||
|
|
||||||
|
cookies.set(AUTH_COOKIE, token, {
|
||||||
|
path: '/',
|
||||||
|
httpOnly: true,
|
||||||
|
sameSite: 'lax',
|
||||||
|
maxAge: ONE_YEAR
|
||||||
|
});
|
||||||
|
|
||||||
|
return json({
|
||||||
|
token,
|
||||||
|
user: { id: user.id, username: user.username, role: user.role ?? 'user' }
|
||||||
|
});
|
||||||
|
};
|
||||||
111
ui-v2/src/routes/api/book/[slug]/+server.ts
Normal file
111
ui-v2/src/routes/api/book/[slug]/+server.ts
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { getBook, listChapterIdx, getProgress, isBookSaved } from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
import { env } from '$env/dynamic/private';
|
||||||
|
|
||||||
|
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/book/[slug]
|
||||||
|
* Returns book metadata, chapter list, progress, and library status.
|
||||||
|
*
|
||||||
|
* If the book is not yet in PocketBase, asks the backend to enqueue a scrape
|
||||||
|
* task and returns 202 with { scraping: true, task_id }.
|
||||||
|
* The client should poll and retry once the task completes.
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ params, locals }) => {
|
||||||
|
const { slug } = params;
|
||||||
|
|
||||||
|
// Try PocketBase first
|
||||||
|
let book = await getBook(slug).catch((e) => {
|
||||||
|
log.error('api/book', 'getBook failed', { slug, err: String(e) });
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (book) {
|
||||||
|
let chapters, progress, saved;
|
||||||
|
try {
|
||||||
|
[chapters, progress, saved] = await Promise.all([
|
||||||
|
listChapterIdx(slug),
|
||||||
|
getProgress(locals.sessionId, slug, locals.user?.id),
|
||||||
|
isBookSaved(locals.sessionId, slug, locals.user?.id)
|
||||||
|
]);
|
||||||
|
} catch (e) {
|
||||||
|
log.error('api/book', 'failed to load book detail data', { slug, err: String(e) });
|
||||||
|
error(500, 'Failed to load book');
|
||||||
|
}
|
||||||
|
|
||||||
|
return json({
|
||||||
|
book,
|
||||||
|
chapters,
|
||||||
|
in_lib: true,
|
||||||
|
saved,
|
||||||
|
last_chapter: progress?.chapter ?? null,
|
||||||
|
scraping: false,
|
||||||
|
task_id: null
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to backend: enqueue scrape task if not in library.
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${SCRAPER_URL}/api/book-preview/${encodeURIComponent(slug)}`);
|
||||||
|
|
||||||
|
if (res.status === 202) {
|
||||||
|
const body: { task_id: string; message: string } = await res.json();
|
||||||
|
log.info('api/book', 'scrape task enqueued', { slug, task_id: body.task_id });
|
||||||
|
return json({ scraping: true, task_id: body.task_id, in_lib: false }, { status: 202 });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
log.warn('api/book', 'book-preview returned error', { slug, status: res.status });
|
||||||
|
error(404, `Book "${slug}" not found`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 200 — book was already in library
|
||||||
|
const preview: {
|
||||||
|
in_lib: boolean;
|
||||||
|
meta: {
|
||||||
|
slug: string;
|
||||||
|
title: string;
|
||||||
|
author: string;
|
||||||
|
cover: string;
|
||||||
|
status: string;
|
||||||
|
genres: string[];
|
||||||
|
summary: string;
|
||||||
|
total_chapters: number;
|
||||||
|
source_url: string;
|
||||||
|
};
|
||||||
|
chapters: { number: number; title: string; date?: string }[];
|
||||||
|
} = await res.json();
|
||||||
|
|
||||||
|
const previewBook = {
|
||||||
|
id: '',
|
||||||
|
slug: preview.meta.slug || slug,
|
||||||
|
title: preview.meta.title,
|
||||||
|
author: preview.meta.author,
|
||||||
|
cover: preview.meta.cover,
|
||||||
|
status: preview.meta.status,
|
||||||
|
genres: preview.meta.genres ?? [],
|
||||||
|
summary: preview.meta.summary,
|
||||||
|
total_chapters: preview.meta.total_chapters,
|
||||||
|
source_url: preview.meta.source_url,
|
||||||
|
ranking: 0,
|
||||||
|
meta_updated: ''
|
||||||
|
};
|
||||||
|
|
||||||
|
return json({
|
||||||
|
book: previewBook,
|
||||||
|
chapters: preview.chapters,
|
||||||
|
in_lib: true,
|
||||||
|
saved: false,
|
||||||
|
last_chapter: null,
|
||||||
|
scraping: false,
|
||||||
|
task_id: null
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof Error && 'status' in e) throw e;
|
||||||
|
log.error('api/book', 'book-preview fetch failed', { slug, err: String(e) });
|
||||||
|
error(404, `Book "${slug}" not found`);
|
||||||
|
}
|
||||||
|
};
|
||||||
37
ui-v2/src/routes/api/browse-page/+server.ts
Normal file
37
ui-v2/src/routes/api/browse-page/+server.ts
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { env } from '$env/dynamic/private';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/browse-page?page=2&genre=all&sort=popular&status=all
|
||||||
|
*
|
||||||
|
* Thin proxy to the Go scraper's /api/browse endpoint.
|
||||||
|
* Used by the infinite-scroll browse page to append subsequent pages
|
||||||
|
* without a full SSR navigation.
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ url }) => {
|
||||||
|
const page = url.searchParams.get('page') ?? '1';
|
||||||
|
const genre = url.searchParams.get('genre') ?? 'all';
|
||||||
|
const sort = url.searchParams.get('sort') ?? 'popular';
|
||||||
|
const status = url.searchParams.get('status') ?? 'all';
|
||||||
|
|
||||||
|
const params = new URLSearchParams({ page, genre, sort, status });
|
||||||
|
const apiURL = `${SCRAPER_URL}/api/browse?${params.toString()}`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await fetch(apiURL);
|
||||||
|
if (!res.ok) {
|
||||||
|
log.error('browse-page', 'scraper returned error', { status: res.status });
|
||||||
|
throw error(502, `Browse fetch failed: ${res.status}`);
|
||||||
|
}
|
||||||
|
const data = await res.json();
|
||||||
|
return json(data);
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof Error && 'status' in e) throw e;
|
||||||
|
log.error('browse-page', 'network error', { err: String(e) });
|
||||||
|
throw error(502, 'Could not reach browse service');
|
||||||
|
}
|
||||||
|
};
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
import { error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { env } from '$env/dynamic/private';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/chapter-text-preview/[slug]/[n]
|
||||||
|
* Proxies to the scraper's /api/chapter-text-preview endpoint.
|
||||||
|
* Used client-side when the normal chapter path returns no content
|
||||||
|
* (chapter indexed but not yet scraped to MinIO).
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ params, url }) => {
|
||||||
|
const { slug, n } = params;
|
||||||
|
const chapter = parseInt(n, 10);
|
||||||
|
if (!slug || !chapter || chapter < 1) {
|
||||||
|
error(400, 'Invalid slug or chapter number');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Forward optional query params (chapter_url, title) if present
|
||||||
|
const qs = new URLSearchParams();
|
||||||
|
const chapterUrl = url.searchParams.get('chapter_url');
|
||||||
|
const title = url.searchParams.get('title');
|
||||||
|
if (chapterUrl) qs.set('chapter_url', chapterUrl);
|
||||||
|
if (title) qs.set('title', title);
|
||||||
|
|
||||||
|
const scraperRes = await fetch(
|
||||||
|
`${SCRAPER_URL}/api/chapter-text-preview/${encodeURIComponent(slug)}/${chapter}?${qs.toString()}`
|
||||||
|
).catch((e) => {
|
||||||
|
log.error('chapter-preview', 'scraper fetch failed', { slug, chapter, err: String(e) });
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!scraperRes || !scraperRes.ok) {
|
||||||
|
const status = scraperRes?.status ?? 502;
|
||||||
|
log.error('chapter-preview', 'scraper returned error', { slug, chapter, status });
|
||||||
|
error(status as Parameters<typeof error>[0], 'Chapter preview not available');
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = await scraperRes.json();
|
||||||
|
|
||||||
|
return new Response(JSON.stringify(data), {
|
||||||
|
headers: { 'Content-Type': 'application/json' }
|
||||||
|
});
|
||||||
|
};
|
||||||
128
ui-v2/src/routes/api/chapter/[slug]/[n]/+server.ts
Normal file
128
ui-v2/src/routes/api/chapter/[slug]/[n]/+server.ts
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import { marked } from 'marked';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { getBook, listChapterIdx } from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
import { env } from '$env/dynamic/private';
|
||||||
|
|
||||||
|
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/chapter/[slug]/[n]
|
||||||
|
* Returns rendered chapter HTML, navigation info, and voice list.
|
||||||
|
* Supports ?preview=1&chapter_url=...&title=... for un-scraped books.
|
||||||
|
*
|
||||||
|
* Response shape mirrors ChapterResponse in the iOS APIClient.
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ params, url, locals }) => {
|
||||||
|
const { slug } = params;
|
||||||
|
const n = parseInt(params.n, 10);
|
||||||
|
|
||||||
|
if (!n || n < 1) error(400, 'Invalid chapter number');
|
||||||
|
|
||||||
|
const isPreview = url.searchParams.get('preview') === '1';
|
||||||
|
const chapterUrl = url.searchParams.get('chapter_url') ?? '';
|
||||||
|
const chapterTitle = url.searchParams.get('title') ?? '';
|
||||||
|
|
||||||
|
if (isPreview) {
|
||||||
|
// Preview path: scrape live, nothing from PocketBase/MinIO
|
||||||
|
const previewParams = new URLSearchParams();
|
||||||
|
if (chapterUrl) previewParams.set('chapter_url', chapterUrl);
|
||||||
|
if (chapterTitle) previewParams.set('title', chapterTitle);
|
||||||
|
|
||||||
|
let chapterData: { slug: string; number: number; title: string; text: string; url: string };
|
||||||
|
try {
|
||||||
|
const res = await fetch(
|
||||||
|
`${SCRAPER_URL}/api/chapter-text-preview/${encodeURIComponent(slug)}/${n}?${previewParams.toString()}`
|
||||||
|
);
|
||||||
|
if (!res.ok) {
|
||||||
|
log.error('api/chapter', 'chapter-text-preview returned error', { slug, n, status: res.status });
|
||||||
|
error(404, `Chapter ${n} not found`);
|
||||||
|
}
|
||||||
|
chapterData = await res.json();
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof Error && 'status' in e) throw e;
|
||||||
|
log.error('api/chapter', 'chapter-text-preview fetch failed', { slug, n, err: String(e) });
|
||||||
|
error(502, 'Could not fetch chapter preview');
|
||||||
|
}
|
||||||
|
|
||||||
|
const html = chapterData.text
|
||||||
|
? '<p>' + chapterData.text.replace(/\n{2,}/g, '</p><p>').replace(/\n/g, '<br>') + '</p>'
|
||||||
|
: '';
|
||||||
|
|
||||||
|
let voices: string[] = [];
|
||||||
|
try {
|
||||||
|
const vRes = await fetch(`${SCRAPER_URL}/api/voices`);
|
||||||
|
if (vRes.ok) {
|
||||||
|
const d = (await vRes.json()) as { voices: string[] };
|
||||||
|
voices = d.voices ?? [];
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Non-critical
|
||||||
|
}
|
||||||
|
|
||||||
|
const pb = await getBook(slug).catch(() => null);
|
||||||
|
|
||||||
|
return json({
|
||||||
|
book: { slug, title: pb?.title ?? slug, cover: pb?.cover ?? '' },
|
||||||
|
chapter: { id: '', slug, number: n, title: chapterData.title || `Chapter ${n}`, date_label: '' },
|
||||||
|
html,
|
||||||
|
voices,
|
||||||
|
prev: null,
|
||||||
|
next: null,
|
||||||
|
chapters: [],
|
||||||
|
is_preview: true
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normal path: PocketBase + MinIO
|
||||||
|
const [book, chapters, voicesRes] = await Promise.all([
|
||||||
|
getBook(slug),
|
||||||
|
listChapterIdx(slug),
|
||||||
|
fetch(`${SCRAPER_URL}/api/voices`).catch(() => null)
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!book) error(404, `Book "${slug}" not found`);
|
||||||
|
|
||||||
|
const chapterIdx = chapters.find((c) => c.number === n);
|
||||||
|
if (!chapterIdx) error(404, `Chapter ${n} not found`);
|
||||||
|
|
||||||
|
let voices: string[] = [];
|
||||||
|
try {
|
||||||
|
if (voicesRes?.ok) {
|
||||||
|
const data = (await voicesRes.json()) as { voices: string[] };
|
||||||
|
voices = data.voices ?? [];
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Non-critical
|
||||||
|
}
|
||||||
|
|
||||||
|
let html = '';
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${SCRAPER_URL}/api/chapter-markdown/${encodeURIComponent(slug)}/${n}`);
|
||||||
|
if (!res.ok) {
|
||||||
|
log.error('api/chapter', 'chapter-markdown returned error', { slug, n, status: res.status });
|
||||||
|
error(res.status === 404 ? 404 : 502, res.status === 404 ? `Chapter ${n} not found` : 'Could not fetch chapter content');
|
||||||
|
}
|
||||||
|
const markdown = await res.text();
|
||||||
|
html = marked(markdown) as string;
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof Error && 'status' in e) throw e;
|
||||||
|
log.error('api/chapter', 'failed to fetch chapter content', { slug, n, err: String(e) });
|
||||||
|
error(502, 'Could not fetch chapter content');
|
||||||
|
}
|
||||||
|
|
||||||
|
const prevChapter = chapters.find((c) => c.number === n - 1) ?? null;
|
||||||
|
const nextChapter = chapters.find((c) => c.number === n + 1) ?? null;
|
||||||
|
|
||||||
|
return json({
|
||||||
|
book: { slug: book.slug, title: book.title, cover: book.cover ?? '' },
|
||||||
|
chapter: chapterIdx,
|
||||||
|
html,
|
||||||
|
voices,
|
||||||
|
prev: prevChapter ? prevChapter.number : null,
|
||||||
|
next: nextChapter ? nextChapter.number : null,
|
||||||
|
chapters: chapters.map((c) => ({ number: c.number, title: c.title })),
|
||||||
|
is_preview: false
|
||||||
|
});
|
||||||
|
};
|
||||||
26
ui-v2/src/routes/api/comment/[id]/+server.ts
Normal file
26
ui-v2/src/routes/api/comment/[id]/+server.ts
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
import { error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { deleteComment } from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DELETE /api/comment/[id]
|
||||||
|
* Deletes a comment and its replies. Only the comment owner may delete.
|
||||||
|
* Requires authentication.
|
||||||
|
*/
|
||||||
|
export const DELETE: RequestHandler = async ({ params, locals }) => {
|
||||||
|
if (!locals.user) error(401, 'Login required');
|
||||||
|
|
||||||
|
const { id } = params;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await deleteComment(id, locals.user.id);
|
||||||
|
return new Response(null, { status: 204 });
|
||||||
|
} catch (e) {
|
||||||
|
const msg = String(e);
|
||||||
|
if (msg.includes('Not authorized')) error(403, 'Not authorized to delete this comment');
|
||||||
|
if (msg.includes('not found')) error(404, 'Comment not found');
|
||||||
|
log.error('api/comment/[id]', 'deleteComment failed', { id, err: msg });
|
||||||
|
error(500, 'Failed to delete comment');
|
||||||
|
}
|
||||||
|
};
|
||||||
33
ui-v2/src/routes/api/comment/[id]/vote/+server.ts
Normal file
33
ui-v2/src/routes/api/comment/[id]/vote/+server.ts
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { voteComment } from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/comment/[id]/vote
|
||||||
|
* Body: { vote: 'up' | 'down' }
|
||||||
|
* Casts, changes, or toggles off a vote on a comment.
|
||||||
|
* Works for both authenticated and anonymous users (session-scoped).
|
||||||
|
* Returns the updated comment.
|
||||||
|
*/
|
||||||
|
export const POST: RequestHandler = async ({ params, request, locals }) => {
|
||||||
|
const { id } = params;
|
||||||
|
let body: { vote?: string };
|
||||||
|
try {
|
||||||
|
body = await request.json();
|
||||||
|
} catch {
|
||||||
|
error(400, 'Invalid JSON body');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (body.vote !== 'up' && body.vote !== 'down') {
|
||||||
|
error(400, 'vote must be "up" or "down"');
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const updated = await voteComment(id, body.vote, locals.sessionId, locals.user?.id);
|
||||||
|
return json(updated);
|
||||||
|
} catch (e) {
|
||||||
|
log.error('api/comment/[id]/vote', 'voteComment failed', { id, err: String(e) });
|
||||||
|
error(500, 'Failed to record vote');
|
||||||
|
}
|
||||||
|
};
|
||||||
102
ui-v2/src/routes/api/comments/[slug]/+server.ts
Normal file
102
ui-v2/src/routes/api/comments/[slug]/+server.ts
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import {
|
||||||
|
listComments,
|
||||||
|
listReplies,
|
||||||
|
createComment,
|
||||||
|
getMyVotes,
|
||||||
|
type CommentSort
|
||||||
|
} from '$lib/server/pocketbase';
|
||||||
|
import { presignAvatarUrl } from '$lib/server/minio';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/comments/[slug]?sort=new|top
|
||||||
|
* Returns top-level comments + their replies + current visitor's votes + avatar URLs.
|
||||||
|
* Response: { comments: BookComment[], myVotes: Record<string, 'up'|'down'>, avatarUrls: Record<string, string> }
|
||||||
|
* Each top-level comment has a `replies` array attached.
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ params, url, locals }) => {
|
||||||
|
const { slug } = params;
|
||||||
|
const sortParam = url.searchParams.get('sort') ?? 'new';
|
||||||
|
const sort: CommentSort = sortParam === 'top' ? 'top' : 'new';
|
||||||
|
|
||||||
|
try {
|
||||||
|
const topLevel = await listComments(slug, sort);
|
||||||
|
|
||||||
|
// Fetch replies for all top-level comments in parallel
|
||||||
|
const repliesPerComment = await Promise.all(topLevel.map((c) => listReplies(c.id)));
|
||||||
|
const allReplies = repliesPerComment.flat();
|
||||||
|
|
||||||
|
// Build comment+reply list for vote lookup
|
||||||
|
const allIds = [...topLevel.map((c) => c.id), ...allReplies.map((r) => r.id)];
|
||||||
|
const myVotes = await getMyVotes(allIds, locals.sessionId, locals.user?.id);
|
||||||
|
|
||||||
|
// Attach replies to each top-level comment
|
||||||
|
const comments = topLevel.map((c, i) => ({
|
||||||
|
...c,
|
||||||
|
replies: repliesPerComment[i]
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Batch-resolve avatar presign URLs for all unique user_ids
|
||||||
|
const allComments = [...topLevel, ...allReplies];
|
||||||
|
const uniqueUserIds = [...new Set(allComments.map((c) => c.user_id).filter(Boolean))];
|
||||||
|
const avatarEntries = await Promise.all(
|
||||||
|
uniqueUserIds.map(async (userId) => {
|
||||||
|
try {
|
||||||
|
const url = await presignAvatarUrl(userId);
|
||||||
|
return [userId, url] as [string, string | null];
|
||||||
|
} catch {
|
||||||
|
return [userId, null] as [string, null];
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
const avatarUrls: Record<string, string> = {};
|
||||||
|
for (const [userId, url] of avatarEntries) {
|
||||||
|
if (url) avatarUrls[userId] = url;
|
||||||
|
}
|
||||||
|
|
||||||
|
return json({ comments, myVotes, avatarUrls });
|
||||||
|
} catch (e) {
|
||||||
|
log.error('api/comments/[slug]', 'listComments failed', { slug, err: String(e) });
|
||||||
|
error(500, 'Failed to load comments');
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /api/comments/[slug]
|
||||||
|
* Body: { body: string, parent_id?: string }
|
||||||
|
* Creates a new comment or reply. Requires authentication.
|
||||||
|
*/
|
||||||
|
export const POST: RequestHandler = async ({ params, request, locals }) => {
|
||||||
|
if (!locals.user) error(401, 'Login required to comment');
|
||||||
|
|
||||||
|
const { slug } = params;
|
||||||
|
let body: { body?: string; parent_id?: string };
|
||||||
|
try {
|
||||||
|
body = await request.json();
|
||||||
|
} catch {
|
||||||
|
error(400, 'Invalid JSON body');
|
||||||
|
}
|
||||||
|
|
||||||
|
const text = (body.body ?? '').trim();
|
||||||
|
if (!text) error(400, 'Comment body is required');
|
||||||
|
if (text.length > 2000) error(400, 'Comment is too long (max 2000 characters)');
|
||||||
|
|
||||||
|
// Enforce 1-level depth: parent_id must be a top-level comment
|
||||||
|
const parentId = body.parent_id?.trim() || undefined;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const comment = await createComment(
|
||||||
|
slug,
|
||||||
|
text,
|
||||||
|
locals.user.id,
|
||||||
|
locals.user.username,
|
||||||
|
parentId
|
||||||
|
);
|
||||||
|
return json(comment, { status: 201 });
|
||||||
|
} catch (e) {
|
||||||
|
log.error('api/comments/[slug]', 'createComment failed', { slug, err: String(e) });
|
||||||
|
error(500, 'Failed to post comment');
|
||||||
|
}
|
||||||
|
};
|
||||||
65
ui-v2/src/routes/api/home/+server.ts
Normal file
65
ui-v2/src/routes/api/home/+server.ts
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
import { json } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import {
|
||||||
|
listBooks,
|
||||||
|
recentlyAddedBooks,
|
||||||
|
allProgress,
|
||||||
|
getHomeStats,
|
||||||
|
getSubscriptionFeed
|
||||||
|
} from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
import type { Book, Progress } from '$lib/server/pocketbase';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/home
|
||||||
|
* Returns home screen data: continue-reading list, recently updated books, stats,
|
||||||
|
* and subscription feed (books recently read by followed users).
|
||||||
|
* Requires authentication (enforced by layout guard).
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ locals }) => {
|
||||||
|
let allBooks: Book[] = [];
|
||||||
|
let recentBooks: Book[] = [];
|
||||||
|
let progressList: Progress[] = [];
|
||||||
|
let stats = { totalBooks: 0, totalChapters: 0 };
|
||||||
|
|
||||||
|
try {
|
||||||
|
[allBooks, recentBooks, progressList, stats] = await Promise.all([
|
||||||
|
listBooks(),
|
||||||
|
recentlyAddedBooks(8),
|
||||||
|
allProgress(locals.sessionId, locals.user?.id),
|
||||||
|
getHomeStats()
|
||||||
|
]);
|
||||||
|
} catch (e) {
|
||||||
|
log.error('api/home', 'failed to load home data', { err: String(e) });
|
||||||
|
}
|
||||||
|
|
||||||
|
const bookMap = new Map<string, Book>(allBooks.map((b) => [b.slug, b]));
|
||||||
|
|
||||||
|
const continueReading = progressList
|
||||||
|
.filter((p) => bookMap.has(p.slug))
|
||||||
|
.slice(0, 6)
|
||||||
|
.map((p) => ({ book: bookMap.get(p.slug)!, chapter: p.chapter }));
|
||||||
|
|
||||||
|
const inProgressSlugs = new Set(continueReading.map((c) => c.book.slug));
|
||||||
|
const recentlyUpdated = recentBooks.filter((b) => !inProgressSlugs.has(b.slug)).slice(0, 6);
|
||||||
|
|
||||||
|
// Subscription feed — only available for logged-in users with following
|
||||||
|
let subscriptionFeed: Array<{ book: Book; readerUsername: string }> = [];
|
||||||
|
if (locals.user?.id) {
|
||||||
|
subscriptionFeed = await getSubscriptionFeed(locals.user.id).catch(() => []);
|
||||||
|
}
|
||||||
|
|
||||||
|
return json({
|
||||||
|
continue_reading: continueReading,
|
||||||
|
recently_updated: recentlyUpdated,
|
||||||
|
stats: {
|
||||||
|
totalBooks: stats.totalBooks,
|
||||||
|
totalChapters: stats.totalChapters,
|
||||||
|
booksInProgress: continueReading.length
|
||||||
|
},
|
||||||
|
subscription_feed: subscriptionFeed.map((item) => ({
|
||||||
|
book: item.book,
|
||||||
|
readerUsername: item.readerUsername
|
||||||
|
}))
|
||||||
|
});
|
||||||
|
};
|
||||||
61
ui-v2/src/routes/api/library/+server.ts
Normal file
61
ui-v2/src/routes/api/library/+server.ts
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
import { json } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { listBooks, allProgress, getSavedSlugs } from '$lib/server/pocketbase';
|
||||||
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/library
|
||||||
|
* Returns the user's library: books they have started reading or explicitly saved.
|
||||||
|
* Each item includes the book record, the last chapter read, and saved_at timestamp.
|
||||||
|
*
|
||||||
|
* Response shape mirrors LibraryItem in the iOS APIClient.
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ locals }) => {
|
||||||
|
let allBooks: Awaited<ReturnType<typeof listBooks>>;
|
||||||
|
let progressList: Awaited<ReturnType<typeof allProgress>>;
|
||||||
|
let savedSlugs: Set<string>;
|
||||||
|
|
||||||
|
try {
|
||||||
|
[allBooks, progressList, savedSlugs] = await Promise.all([
|
||||||
|
listBooks(),
|
||||||
|
allProgress(locals.sessionId, locals.user?.id),
|
||||||
|
getSavedSlugs(locals.sessionId, locals.user?.id)
|
||||||
|
]);
|
||||||
|
} catch (e) {
|
||||||
|
log.error('api/library', 'failed to load library data', { err: String(e) });
|
||||||
|
allBooks = [];
|
||||||
|
progressList = [];
|
||||||
|
savedSlugs = new Set();
|
||||||
|
}
|
||||||
|
|
||||||
|
const progressMap: Record<string, number> = {};
|
||||||
|
const progressUpdatedMap: Record<string, string> = {};
|
||||||
|
for (const p of progressList) {
|
||||||
|
progressMap[p.slug] = p.chapter;
|
||||||
|
progressUpdatedMap[p.slug] = p.updated;
|
||||||
|
}
|
||||||
|
|
||||||
|
const progressSlugs = new Set(progressList.map((p) => p.slug));
|
||||||
|
const books = allBooks.filter((b) => progressSlugs.has(b.slug) || savedSlugs.has(b.slug));
|
||||||
|
|
||||||
|
const withProgress = books.filter((b) => progressSlugs.has(b.slug));
|
||||||
|
const savedOnly = books
|
||||||
|
.filter((b) => !progressSlugs.has(b.slug))
|
||||||
|
.sort((a, b) => (a.title ?? '').localeCompare(b.title ?? ''));
|
||||||
|
|
||||||
|
withProgress.sort((a, b) => {
|
||||||
|
const ta = progressUpdatedMap[a.slug] ?? '';
|
||||||
|
const tb = progressUpdatedMap[b.slug] ?? '';
|
||||||
|
return tb.localeCompare(ta);
|
||||||
|
});
|
||||||
|
|
||||||
|
const ordered = [...withProgress, ...savedOnly];
|
||||||
|
|
||||||
|
const items = ordered.map((book) => ({
|
||||||
|
book,
|
||||||
|
last_chapter: progressMap[book.slug] ?? null,
|
||||||
|
saved_at: progressUpdatedMap[book.slug] ?? new Date().toISOString()
|
||||||
|
}));
|
||||||
|
|
||||||
|
return json(items);
|
||||||
|
};
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user