Compare commits

..

1 Commits
v1.1.0 ... v2

Author SHA1 Message Date
Admin
1a49cb5e75 fix(scraper): add Brotli decompression to HTTP client
Some checks failed
CI / Scraper / Lint (push) Failing after 29s
CI / Scraper / Lint (pull_request) Failing after 29s
CI / Scraper / Test (push) Failing after 38s
CI / Scraper / Docker Push (push) Has been skipped
CI / UI / Build (pull_request) Successful in 47s
CI / UI / Docker Push (pull_request) Has been skipped
CI / Scraper / Test (pull_request) Successful in 54s
CI / Scraper / Docker Push (pull_request) Has been skipped
iOS CI / Build (pull_request) Successful in 3m35s
iOS CI / Test (pull_request) Successful in 5m47s
novelfire.net responds with Content-Encoding: br when the scraper
advertises 'gzip, deflate, br'. The client only handled gzip, so
Brotli-compressed bytes were fed raw into the HTML parser producing
garbage — empty titles, zero chapters, and selector failures.

Added github.com/andybalholm/brotli and wired it into GetContent
alongside the existing gzip path.
2026-03-20 11:20:50 +05:00
145 changed files with 35 additions and 22771 deletions

View File

@@ -1,23 +1,6 @@
# libnovel scraper — environment overrides
# Copy to .env and adjust values; do NOT commit this file with real secrets.
# ── Docker BuildKit ───────────────────────────────────────────────────────────
# Required for the backend/Dockerfile cache mounts (--mount=type=cache).
# BuildKit is the default in Docker Engine 23+, but Colima users may need this.
#
# If you see: "the --mount option requires BuildKit", enable it one of two ways:
#
# Option A — per-project (recommended, zero restart needed):
# Uncomment the line below and copy this file to .env.
# Docker Compose reads .env automatically, so BuildKit will be active for
# every `docker compose build` / `docker compose up --build` in this project.
#
# Option B — system-wide for Colima (persists across restarts):
# echo '{"features":{"buildkit":true}}' > ~/.colima/default/daemon.json
# colima stop && colima start
#
# DOCKER_BUILDKIT=1
# ── Service ports (host-side) ─────────────────────────────────────────────────
# Port the scraper HTTP API listens on (default 8080)
SCRAPER_PORT=8080

View File

@@ -1,163 +0,0 @@
name: Release / v2
on:
push:
tags:
- "v*"
concurrency:
group: ${{ gitea.workflow }}-${{ gitea.ref }}
cancel-in-progress: true
jobs:
# ── backend: lint & test ─────────────────────────────────────────────────────
test-backend:
name: Test backend
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: backend/go.mod
cache-dependency-path: backend/go.sum
- name: go vet
working-directory: backend
run: go vet ./...
- name: Run tests
working-directory: backend
run: go test -short -race -count=1 -timeout=60s ./...
# ── ui-v2: type-check & build ────────────────────────────────────────────────
build-ui:
name: Build ui-v2
runs-on: ubuntu-latest
defaults:
run:
working-directory: ui-v2
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
cache-dependency-path: ui-v2/package-lock.json
- name: Install dependencies
run: npm ci
- name: Type check
run: npm run check
- name: Build
run: npm run build
# ── docker: backend ──────────────────────────────────────────────────────────
docker-backend:
name: Docker / backend
runs-on: ubuntu-latest
needs: [test-backend]
steps:
- uses: actions/checkout@v4
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ secrets.DOCKER_USER }}/libnovel-backend
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=raw,value=latest
- name: Build and push
uses: docker/build-push-action@v6
with:
context: backend
target: backend
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
COMMIT=${{ gitea.sha }}
# ── docker: runner ───────────────────────────────────────────────────────────
docker-runner:
name: Docker / runner
runs-on: ubuntu-latest
needs: [test-backend]
steps:
- uses: actions/checkout@v4
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ secrets.DOCKER_USER }}/libnovel-runner
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=raw,value=latest
- name: Build and push
uses: docker/build-push-action@v6
with:
context: backend
target: runner
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
COMMIT=${{ gitea.sha }}
# ── docker: ui-v2 ────────────────────────────────────────────────────────────
docker-ui:
name: Docker / ui-v2
runs-on: ubuntu-latest
needs: [build-ui]
steps:
- uses: actions/checkout@v4
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ secrets.DOCKER_USER }}/libnovel-ui-v2
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=raw,value=latest
- name: Build and push
uses: docker/build-push-action@v6
with:
context: ui-v2
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_VERSION=${{ steps.meta.outputs.version }}
BUILD_COMMIT=${{ gitea.sha }}

View File

@@ -1,13 +0,0 @@
# Exclude compiled binaries
bin/
# Exclude test binaries produced by `go test -c`
*.test
# Git history is not needed inside the image
.git/
# Editor/OS noise
.DS_Store
*.swp
*.swo

View File

@@ -1,42 +0,0 @@
# syntax=docker/dockerfile:1
FROM golang:1.26.1-alpine AS builder
WORKDIR /app
# Download modules into the BuildKit cache so they survive across builds.
# This layer is only invalidated when go.mod or go.sum changes.
COPY go.mod go.sum ./
RUN --mount=type=cache,target=/root/go/pkg/mod \
go mod download
COPY . .
ARG VERSION=dev
ARG COMMIT=unknown
# Build all three binaries in a single layer so the Go compiler can reuse
# intermediate object files. Both cache mounts are preserved between builds:
# /root/go/pkg/mod — downloaded module source
# /root/.cache/go-build — compiled package objects (incremental recompile)
RUN --mount=type=cache,target=/root/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
CGO_ENABLED=0 GOOS=linux go build \
-ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT}" \
-o /out/backend ./cmd/backend && \
CGO_ENABLED=0 GOOS=linux go build \
-ldflags="-s -w -X main.version=${VERSION} -X main.commit=${COMMIT}" \
-o /out/runner ./cmd/runner && \
CGO_ENABLED=0 GOOS=linux go build \
-ldflags="-s -w" \
-o /out/healthcheck ./cmd/healthcheck
# ── backend service ──────────────────────────────────────────────────────────
FROM gcr.io/distroless/static:nonroot AS backend
COPY --from=builder /out/healthcheck /healthcheck
COPY --from=builder /out/backend /backend
ENTRYPOINT ["/backend"]
# ── runner service ───────────────────────────────────────────────────────────
FROM gcr.io/distroless/static:nonroot AS runner
COPY --from=builder /out/healthcheck /healthcheck
COPY --from=builder /out/runner /runner
ENTRYPOINT ["/runner"]

View File

@@ -1,125 +0,0 @@
// Command backend is the LibNovel HTTP API server.
//
// It exposes all endpoints consumed by the SvelteKit UI: book/chapter reads,
// scrape-task creation, presigned MinIO URLs, audio-task creation, reading
// progress, live novelfire.net browse/search, and Kokoro voice list.
//
// All heavy lifting (scraping, TTS generation) is delegated to the runner
// binary via PocketBase task records. The backend never scrapes directly.
//
// Usage:
//
// backend # start HTTP server (blocks until SIGINT/SIGTERM)
package main
import (
"context"
"fmt"
"log/slog"
"os"
"os/signal"
"syscall"
"github.com/libnovel/backend/internal/backend"
"github.com/libnovel/backend/internal/config"
"github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/storage"
)
// version and commit are set at build time via -ldflags.
var (
version = "dev"
commit = "unknown"
)
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "backend: fatal: %v\n", err)
os.Exit(1)
}
}
func run() error {
cfg := config.Load()
// ── Logger ───────────────────────────────────────────────────────────────
log := buildLogger(cfg.LogLevel)
log.Info("backend starting",
"version", version,
"commit", commit,
"addr", cfg.HTTP.Addr,
)
// ── Context: cancel on SIGINT / SIGTERM ──────────────────────────────────
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
// ── Storage ──────────────────────────────────────────────────────────────
store, err := storage.NewStore(ctx, cfg, log)
if err != nil {
return fmt.Errorf("init storage: %w", err)
}
// ── Kokoro (voice list only; audio generation is done by the runner) ─────
var kokoroClient kokoro.Client
if cfg.Kokoro.URL != "" {
kokoroClient = kokoro.New(cfg.Kokoro.URL)
log.Info("kokoro voices enabled", "url", cfg.Kokoro.URL)
} else {
log.Info("KOKORO_URL not set — voice list will use built-in fallback")
kokoroClient = &noopKokoro{}
}
// ── Backend server ───────────────────────────────────────────────────────
srv := backend.New(
backend.Config{
Addr: cfg.HTTP.Addr,
DefaultVoice: cfg.Kokoro.DefaultVoice,
Version: version,
Commit: commit,
},
backend.Dependencies{
BookReader: store,
RankingStore: store,
AudioStore: store,
PresignStore: store,
ProgressStore: store,
Producer: store,
TaskReader: store,
Kokoro: kokoroClient,
Log: log,
},
)
return srv.ListenAndServe(ctx)
}
// ── Helpers ───────────────────────────────────────────────────────────────────
func buildLogger(level string) *slog.Logger {
var lvl slog.Level
switch level {
case "debug":
lvl = slog.LevelDebug
case "warn":
lvl = slog.LevelWarn
case "error":
lvl = slog.LevelError
default:
lvl = slog.LevelInfo
}
return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl}))
}
// noopKokoro is a no-op implementation used when KOKORO_URL is not set.
// The backend only uses Kokoro for the voice list; audio generation is the
// runner's responsibility. With no URL the built-in fallback list is served.
type noopKokoro struct{}
func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)")
}
func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) {
return nil, nil
}

View File

@@ -1,57 +0,0 @@
package main
import (
"os"
"testing"
)
// TestBuildLogger verifies that buildLogger returns a non-nil logger for each
// supported log level string and for unknown values.
func TestBuildLogger(t *testing.T) {
for _, level := range []string{"debug", "info", "warn", "error", "unknown", ""} {
l := buildLogger(level)
if l == nil {
t.Errorf("buildLogger(%q) returned nil", level)
}
}
}
// TestNoopKokoro verifies that the no-op Kokoro stub returns the expected
// sentinel error from GenerateAudio and nil, nil from ListVoices.
func TestNoopKokoro(t *testing.T) {
noop := &noopKokoro{}
_, err := noop.GenerateAudio(t.Context(), "text", "af_bella")
if err == nil {
t.Fatal("noopKokoro.GenerateAudio: expected error, got nil")
}
voices, err := noop.ListVoices(t.Context())
if err != nil {
t.Fatalf("noopKokoro.ListVoices: unexpected error: %v", err)
}
if voices != nil {
t.Fatalf("noopKokoro.ListVoices: expected nil slice, got %v", voices)
}
}
// TestRunStorageUnreachable verifies that run() fails fast and returns a
// descriptive error when PocketBase is unreachable.
func TestRunStorageUnreachable(t *testing.T) {
// Point at an address nothing is listening on.
t.Setenv("POCKETBASE_URL", "http://127.0.0.1:19999")
// Use a fast listen address so we don't accidentally start a real server.
t.Setenv("BACKEND_HTTP_ADDR", "127.0.0.1:0")
err := run()
if err == nil {
t.Fatal("run() should have returned an error when storage is unreachable")
}
t.Logf("got expected error: %v", err)
}
// TestMain runs the test suite. No special setup required.
func TestMain(m *testing.M) {
os.Exit(m.Run())
}

View File

@@ -1,89 +0,0 @@
// healthcheck is a static binary used by Docker HEALTHCHECK CMD in distroless
// images (which have no shell, wget, or curl).
//
// Two modes:
//
// 1. HTTP mode (default):
// /healthcheck <url>
// Performs GET <url>; exits 0 if HTTP 2xx/3xx, 1 otherwise.
// Example: /healthcheck http://localhost:8080/health
//
// 2. File-liveness mode:
// /healthcheck file <path> <max_age_seconds>
// Reads <path>, parses its content as RFC3339 timestamp, and exits 1 if the
// timestamp is older than <max_age_seconds>. Used by the runner service which
// writes /tmp/runner.alive on every successful poll.
// Example: /healthcheck file /tmp/runner.alive 120
package main
import (
"fmt"
"net/http"
"os"
"strconv"
"time"
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "file" {
checkFile()
return
}
checkHTTP()
}
// checkHTTP performs a GET request and exits 0 on success, 1 on failure.
func checkHTTP() {
url := "http://localhost:8080/health"
if len(os.Args) > 1 {
url = os.Args[1]
}
resp, err := http.Get(url) //nolint:gosec,noctx
if err != nil {
fmt.Fprintf(os.Stderr, "healthcheck: %v\n", err)
os.Exit(1)
}
resp.Body.Close()
if resp.StatusCode >= 400 {
fmt.Fprintf(os.Stderr, "healthcheck: status %d\n", resp.StatusCode)
os.Exit(1)
}
}
// checkFile reads a timestamp from a file and exits 1 if it is older than the
// given max age. Usage: /healthcheck file <path> <max_age_seconds>
func checkFile() {
if len(os.Args) < 4 {
fmt.Fprintln(os.Stderr, "healthcheck file: usage: /healthcheck file <path> <max_age_seconds>")
os.Exit(1)
}
path := os.Args[2]
maxAgeSec, err := strconv.ParseInt(os.Args[3], 10, 64)
if err != nil {
fmt.Fprintf(os.Stderr, "healthcheck file: invalid max_age_seconds %q: %v\n", os.Args[3], err)
os.Exit(1)
}
data, err := os.ReadFile(path)
if err != nil {
fmt.Fprintf(os.Stderr, "healthcheck file: cannot read %s: %v\n", path, err)
os.Exit(1)
}
ts, err := time.Parse(time.RFC3339, string(data))
if err != nil {
// Fallback: use file mtime if content is not a valid timestamp.
info, statErr := os.Stat(path)
if statErr != nil {
fmt.Fprintf(os.Stderr, "healthcheck file: cannot stat %s: %v\n", path, statErr)
os.Exit(1)
}
ts = info.ModTime()
}
age := time.Since(ts)
if age > time.Duration(maxAgeSec)*time.Second {
fmt.Fprintf(os.Stderr, "healthcheck file: %s is %.0fs old (max %ds)\n", path, age.Seconds(), maxAgeSec)
os.Exit(1)
}
}

View File

@@ -1,139 +0,0 @@
// Command runner is the homelab worker binary.
//
// It polls PocketBase for pending scrape and audio tasks, executes them, and
// writes results back. It connects directly to PocketBase and MinIO using
// admin credentials loaded from environment variables.
//
// Usage:
//
// runner # start polling loop (blocks until SIGINT/SIGTERM)
package main
import (
"context"
"fmt"
"log/slog"
"os"
"os/signal"
"runtime"
"syscall"
"time"
"github.com/libnovel/backend/internal/browser"
"github.com/libnovel/backend/internal/config"
"github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/novelfire"
"github.com/libnovel/backend/internal/runner"
"github.com/libnovel/backend/internal/storage"
)
// version and commit are set at build time via -ldflags.
var (
version = "dev"
commit = "unknown"
)
func main() {
if err := run(); err != nil {
fmt.Fprintf(os.Stderr, "runner: fatal: %v\n", err)
os.Exit(1)
}
}
func run() error {
cfg := config.Load()
// ── Logger ──────────────────────────────────────────────────────────────
log := buildLogger(cfg.LogLevel)
log.Info("runner starting",
"version", version,
"commit", commit,
"worker_id", cfg.Runner.WorkerID,
)
// ── Context: cancel on SIGINT / SIGTERM ─────────────────────────────────
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
// ── Storage ─────────────────────────────────────────────────────────────
store, err := storage.NewStore(ctx, cfg, log)
if err != nil {
return fmt.Errorf("init storage: %w", err)
}
// ── Browser / Scraper ───────────────────────────────────────────────────
workers := cfg.Runner.Workers
if workers <= 0 {
workers = runtime.NumCPU()
}
timeout := cfg.Runner.Timeout
if timeout <= 0 {
timeout = 90 * time.Second
}
browserClient := browser.NewDirectClient(browser.Config{
MaxConcurrent: workers,
Timeout: timeout,
ProxyURL: cfg.Runner.ProxyURL,
})
novel := novelfire.New(browserClient, log)
// ── Kokoro ──────────────────────────────────────────────────────────────
var kokoroClient kokoro.Client
if cfg.Kokoro.URL != "" {
kokoroClient = kokoro.New(cfg.Kokoro.URL)
log.Info("kokoro TTS enabled", "url", cfg.Kokoro.URL)
} else {
log.Warn("KOKORO_URL not set — audio tasks will fail")
kokoroClient = &noopKokoro{}
}
// ── Runner ──────────────────────────────────────────────────────────────
rCfg := runner.Config{
WorkerID: cfg.Runner.WorkerID,
PollInterval: cfg.Runner.PollInterval,
MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape,
MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio,
OrchestratorWorkers: workers,
}
deps := runner.Dependencies{
Consumer: store,
BookWriter: store,
BookReader: store,
AudioStore: store,
Novel: novel,
Kokoro: kokoroClient,
Log: log,
}
r := runner.New(rCfg, deps)
return r.Run(ctx)
}
// ── Helpers ───────────────────────────────────────────────────────────────────
func buildLogger(level string) *slog.Logger {
var lvl slog.Level
switch level {
case "debug":
lvl = slog.LevelDebug
case "warn":
lvl = slog.LevelWarn
case "error":
lvl = slog.LevelError
default:
lvl = slog.LevelInfo
}
return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: lvl}))
}
// noopKokoro is a no-op implementation used when KOKORO_URL is not set.
type noopKokoro struct{}
func (n *noopKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
return nil, fmt.Errorf("kokoro not configured (KOKORO_URL is empty)")
}
func (n *noopKokoro) ListVoices(_ context.Context) ([]string, error) {
return nil, nil
}

View File

@@ -1,29 +0,0 @@
module github.com/libnovel/backend
go 1.26.1
require (
github.com/minio/minio-go/v7 v7.0.98
golang.org/x/net v0.51.0
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View File

@@ -1,45 +0,0 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

Binary file not shown.

View File

@@ -1,937 +0,0 @@
package backend
// handlers.go — all HTTP request handlers for the backend server.
//
// Handler naming mirrors the route table in server.go:
// handleScrapeCatalogue, handleScrapeBook, handleScrapeBookRange
// handleScrapeStatus, handleScrapeTasks
// handleBrowse, handleSearch
// handleGetRanking, handleGetCover
// handleBookPreview, handleChapterText, handleReindex
// handleChapterText, handleReindex
// handleAudioGenerate, handleAudioStatus, handleAudioProxy
// handleVoices
// handlePresignChapter, handlePresignAudio, handlePresignVoiceSample
// handlePresignAvatarUpload, handlePresignAvatar
// handleGetProgress, handleSetProgress, handleDeleteProgress
//
// Key design choices vs. old scraper:
// - POST /scrape* creates a PocketBase task record and returns 202 with the
// task_id — it does NOT run the orchestrator inline.
// - POST /api/audio creates a PocketBase audio task and returns 202 — the
// runner binary executes TTS generation asynchronously.
// - GET /api/audio/status polls PocketBase for the task record status.
// - GET /api/audio-proxy reads the completed audio object from MinIO via a
// presigned URL redirect (the runner has already uploaded the bytes).
// - GET /api/browse and /api/search fetch novelfire.net live (no MinIO cache).
// - GET /api/cover redirects to the source cover URL live.
// - GET /api/ranking reads from the PocketBase ranking collection (populated
// by the runner after each catalogue scrape).
// - GET /api/book-preview returns stored data when in library, or enqueues a
// scrape task and returns 202 when not. The backend never scrapes directly.
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/kokoro"
)
const (
novelFireBase = "https://novelfire.net"
novelFireDomain = "novelfire.net"
)
// ── Scrape task creation ───────────────────────────────────────────────────────
// handleScrapeCatalogue handles POST /scrape.
// Creates a "catalogue" scrape task in PocketBase and returns 202 with the task ID.
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "catalogue", "", 0, 0)
if err != nil {
s.deps.Log.Error("handleScrapeCatalogue: CreateScrapeTask failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to create task")
return
}
writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"})
}
// handleScrapeBook handles POST /scrape/book.
// Body: {"url": "https://novelfire.net/book/..."}
func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
var body struct {
URL string `json:"url"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
jsonError(w, http.StatusBadRequest, `request body must be JSON with "url" field`)
return
}
taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "book", body.URL, 0, 0)
if err != nil {
s.deps.Log.Error("handleScrapeBook: CreateScrapeTask failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to create task")
return
}
writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"})
}
// handleScrapeBookRange handles POST /scrape/book/range.
// Body: {"url": "...", "from": N, "to": M}
func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) {
var body struct {
URL string `json:"url"`
From int `json:"from"`
To int `json:"to"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
jsonError(w, http.StatusBadRequest, `request body must be JSON with "url" field`)
return
}
taskID, err := s.deps.Producer.CreateScrapeTask(r.Context(), "book_range", body.URL, body.From, body.To)
if err != nil {
s.deps.Log.Error("handleScrapeBookRange: CreateScrapeTask failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to create task")
return
}
writeJSON(w, http.StatusAccepted, map[string]string{"task_id": taskID, "status": "accepted"})
}
// handleCancelTask handles POST /api/cancel-task/{id}.
// Transitions a pending task (scrape or audio) to status=cancelled.
// Returns 404 if the task does not exist, 409 if it cannot be cancelled
// (e.g. already running/done).
func (s *Server) handleCancelTask(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
if id == "" {
jsonError(w, http.StatusBadRequest, "missing task id")
return
}
if err := s.deps.Producer.CancelTask(r.Context(), id); err != nil {
s.deps.Log.Warn("handleCancelTask: CancelTask failed", "id", id, "err", err)
jsonError(w, http.StatusConflict, "could not cancel task: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]string{"status": "cancelled", "id": id})
}
// ── Scrape task status / history ───────────────────────────────────────────────
// handleScrapeStatus handles GET /api/scrape/status.
// Returns the most recent scrape task status (or {"running":false} if none).
func (s *Server) handleScrapeStatus(w http.ResponseWriter, r *http.Request) {
tasks, err := s.deps.TaskReader.ListScrapeTasks(r.Context())
if err != nil {
s.deps.Log.Error("handleScrapeStatus: ListScrapeTasks failed", "err", err)
writeJSON(w, 0, map[string]bool{"running": false})
return
}
running := false
for _, t := range tasks {
if t.Status == domain.TaskStatusRunning || t.Status == domain.TaskStatusPending {
running = true
break
}
}
writeJSON(w, 0, map[string]bool{"running": running})
}
// handleScrapeTasks handles GET /api/scrape/tasks.
// Returns all scrape task records from PocketBase, newest first.
func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) {
tasks, err := s.deps.TaskReader.ListScrapeTasks(r.Context())
if err != nil {
s.deps.Log.Error("handleScrapeTasks: ListScrapeTasks failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to list tasks")
return
}
if tasks == nil {
tasks = []domain.ScrapeTask{}
}
writeJSON(w, 0, tasks)
}
// ── Browse & search ────────────────────────────────────────────────────────────
// NovelListing represents a single novel entry from the novelfire browse/search page.
type NovelListing struct {
Slug string `json:"slug"`
Title string `json:"title"`
Cover string `json:"cover"`
Rank string `json:"rank"`
Rating string `json:"rating"`
Chapters string `json:"chapters"`
URL string `json:"url"`
}
// handleBrowse handles GET /api/browse.
// Fetches novelfire.net live (no MinIO cache in the new backend).
// Query params: page (default 1), genre (default "all"), sort (default "popular"),
// status (default "all"), type (default "all-novel")
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
page := q.Get("page")
if page == "" {
page = "1"
}
genre := q.Get("genre")
if genre == "" {
genre = "all"
}
sortBy := q.Get("sort")
if sortBy == "" {
sortBy = "popular"
}
status := q.Get("status")
if status == "" {
status = "all"
}
novelType := q.Get("type")
if novelType == "" {
novelType = "all-novel"
}
pageNum, _ := strconv.Atoi(page)
if pageNum <= 0 {
pageNum = 1
}
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
novelFireBase, genre, sortBy, status, novelType, pageNum)
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
defer cancel()
novels, hasNext, err := s.fetchBrowsePage(ctx, targetURL)
if err != nil {
s.deps.Log.Error("handleBrowse: fetch failed", "url", targetURL, "err", err)
jsonError(w, http.StatusBadGateway, err.Error())
return
}
w.Header().Set("Cache-Control", "public, max-age=300")
writeJSON(w, 0, map[string]any{
"novels": novels,
"page": pageNum,
"hasNext": hasNext,
})
}
// handleSearch handles GET /api/search.
// Query params: q (min 2 chars), source ("local"|"remote"|"all", default "all")
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query().Get("q")
if len([]rune(q)) < 2 {
jsonError(w, http.StatusBadRequest, "query must be at least 2 characters")
return
}
source := r.URL.Query().Get("source")
if source == "" {
source = "all"
}
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
defer cancel()
var localResults, remoteResults []NovelListing
// Local search (PocketBase books)
if source == "local" || source == "all" {
books, err := s.deps.BookReader.ListBooks(ctx)
if err != nil {
s.deps.Log.Warn("search: ListBooks failed", "err", err)
} else {
qLower := strings.ToLower(q)
for _, b := range books {
if strings.Contains(strings.ToLower(b.Title), qLower) ||
strings.Contains(strings.ToLower(b.Author), qLower) {
localResults = append(localResults, NovelListing{
Slug: b.Slug,
Title: b.Title,
Cover: b.Cover,
URL: b.SourceURL,
})
}
}
}
}
// Remote search (novelfire.net)
if source == "remote" || source == "all" {
searchURL := novelFireBase + "/search?keyword=" + url.QueryEscape(q)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
if err == nil {
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
if resp, fetchErr := http.DefaultClient.Do(req); fetchErr == nil {
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
parsed, _ := parseBrowsePage(resp.Body)
remoteResults = parsed
}
}
}
}
// Merge: local first, de-duplicate remote
localSlugs := make(map[string]bool, len(localResults))
for _, item := range localResults {
localSlugs[item.Slug] = true
}
combined := make([]NovelListing, 0, len(localResults)+len(remoteResults))
combined = append(combined, localResults...)
for _, item := range remoteResults {
if !localSlugs[item.Slug] {
combined = append(combined, item)
}
}
writeJSON(w, 0, map[string]any{
"results": combined,
"local_count": len(localResults),
"remote_count": len(remoteResults),
})
}
// ── Ranking ────────────────────────────────────────────────────────────────────
// handleGetRanking handles GET /api/ranking.
// Returns all ranking items sorted by rank ascending.
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
items, err := s.deps.RankingStore.ReadRankingItems(r.Context())
if err != nil {
s.deps.Log.Error("handleGetRanking: ReadRankingItems failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to read ranking")
return
}
if items == nil {
items = []domain.RankingItem{}
}
writeJSON(w, 0, items)
}
// handleGetCover handles GET /api/cover/{domain}/{slug}.
// The new backend does not cache covers in MinIO. Instead it redirects the
// client to the novelfire.net source URL. The domain path segment is kept for
// API compatibility with the old scraper.
func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
if slug == "" {
http.Error(w, "missing slug", http.StatusBadRequest)
return
}
// Redirect to the standard novelfire cover CDN URL. If the caller has the
// actual cover URL stored in metadata they should use it directly; this
// endpoint is a best-effort fallback.
coverURL := fmt.Sprintf("https://cdn.novelfire.net/covers/%s.jpg", slug)
http.Redirect(w, r, coverURL, http.StatusFound)
}
// ── Preview (live scrape, no store writes) ─────────────────────────────────────
// handleBookPreview handles GET /api/book-preview/{slug}.
//
// If the book is already in the library (PocketBase), returns its metadata and
// chapter index immediately (200).
//
// If the book is not yet in the library, enqueues a "book" scrape task and
// returns 202 Accepted with the task_id. The runner will scrape the book
// asynchronously; the client should poll GET /api/scrape/status or
// GET /api/scrape/tasks to detect completion, then re-request this endpoint.
//
// The backend never scrapes directly — all scraping is the runner's job.
func (s *Server) handleBookPreview(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
if slug == "" {
jsonError(w, http.StatusBadRequest, "missing slug")
return
}
ctx := r.Context()
meta, inLib, err := s.deps.BookReader.ReadMetadata(ctx, slug)
if err != nil {
s.deps.Log.Warn("book-preview: ReadMetadata failed", "slug", slug, "err", err)
inLib = false
}
if inLib {
// Fast path: book is already scraped — return stored data.
chapters, cerr := s.deps.BookReader.ListChapters(ctx, slug)
if cerr != nil {
s.deps.Log.Warn("book-preview: ListChapters failed", "slug", slug, "err", cerr)
}
writeJSON(w, 0, map[string]any{
"in_lib": true,
"meta": meta,
"chapters": chapters,
})
return
}
// Book not in library — enqueue a range scrape task for the first 20 chapters
// so the user can start reading quickly. Remaining chapters can be scraped
// later via the book detail page or the admin scrape panel.
bookURL := r.URL.Query().Get("source_url")
if bookURL == "" {
bookURL = fmt.Sprintf("%s/book/%s", novelFireBase, slug)
}
const previewFrom, previewTo = 1, 20
taskID, err := s.deps.Producer.CreateScrapeTask(ctx, "book_range", bookURL, previewFrom, previewTo)
if err != nil {
s.deps.Log.Error("book-preview: CreateScrapeTask failed", "slug", slug, "err", err)
jsonError(w, http.StatusInternalServerError, "failed to enqueue scrape task")
return
}
s.deps.Log.Info("book-preview: enqueued range scrape task", "slug", slug, "task_id", taskID,
"from", previewFrom, "to", previewTo)
writeJSON(w, http.StatusAccepted, map[string]any{
"in_lib": false,
"task_id": taskID,
"message": fmt.Sprintf("scraping first %d chapters; poll /api/scrape/tasks for completion", previewTo),
})
}
// ── Chapter text ───────────────────────────────────────────────────────────────
// handleChapterText handles GET /api/chapter-text/{slug}/{n}.
// Returns plain text (markdown stripped) of a stored chapter.
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 {
http.NotFound(w, r)
return
}
raw, err := s.deps.BookReader.ReadChapter(r.Context(), slug, n)
if err != nil {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
fmt.Fprint(w, stripMarkdown(raw))
}
// handleChapterMarkdown handles GET /api/chapter-markdown/{slug}/{n}.
//
// Returns the raw markdown content of a stored chapter directly from MinIO.
// This is used by the SvelteKit UI as a simpler alternative to presign+fetch:
// it avoids the need for the SvelteKit server to reach MinIO directly, and
// gives a clean 404 when the chapter has not been scraped yet.
func (s *Server) handleChapterMarkdown(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
return
}
raw, err := s.deps.BookReader.ReadChapter(r.Context(), slug, n)
if err != nil {
s.deps.Log.Warn("chapter-markdown: not found in MinIO", "slug", slug, "n", n, "err", err)
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "text/markdown; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
fmt.Fprint(w, raw)
}
// handleReindex handles POST /api/reindex/{slug}.
// Rebuilds the chapters_idx PocketBase collection for a book from MinIO objects.
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
if slug == "" {
jsonError(w, http.StatusBadRequest, "missing slug")
return
}
count, err := s.deps.BookReader.ReindexChapters(r.Context(), slug)
if err != nil {
s.deps.Log.Error("reindex failed", "slug", slug, "indexed", count, "err", err)
writeJSON(w, http.StatusInternalServerError, map[string]any{
"error": err.Error(),
"indexed": count,
})
return
}
s.deps.Log.Info("reindex complete", "slug", slug, "indexed", count)
writeJSON(w, 0, map[string]any{"slug": slug, "indexed": count})
}
// ── Audio ──────────────────────────────────────────────────────────────────────
// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
// Creates an audio_jobs task in PocketBase (runner executes asynchronously).
// Returns 200 immediately if audio already exists in MinIO.
// Returns 202 with the task_id if a new task was created.
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 {
jsonError(w, http.StatusBadRequest, "invalid chapter")
return
}
voice := s.cfg.DefaultVoice
var body struct {
Voice string `json:"voice"`
}
if r.Body != nil {
_ = json.NewDecoder(r.Body).Decode(&body)
}
if body.Voice != "" {
voice = body.Voice
}
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
// Fast path: audio already in MinIO
audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice)
if s.deps.AudioStore.AudioExists(r.Context(), audioKey) {
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
writeJSON(w, 0, map[string]string{"url": proxyURL, "status": "done"})
return
}
// Check if a task is already pending/running
task, found, _ := s.deps.TaskReader.GetAudioTask(r.Context(), cacheKey)
if found && (task.Status == domain.TaskStatusPending || task.Status == domain.TaskStatusRunning) {
writeJSON(w, http.StatusAccepted, map[string]string{
"task_id": task.ID,
"status": string(task.Status),
})
return
}
// Create a new audio task
taskID, err := s.deps.Producer.CreateAudioTask(r.Context(), slug, n, voice)
if err != nil {
s.deps.Log.Error("handleAudioGenerate: CreateAudioTask failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to create audio task")
return
}
writeJSON(w, http.StatusAccepted, map[string]string{
"task_id": taskID,
"status": "pending",
})
}
// handleAudioStatus handles GET /api/audio/status/{slug}/{n}.
// Polls PocketBase for the audio task status.
// Query params: voice (optional, defaults to DefaultVoice)
func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
jsonError(w, http.StatusBadRequest, "invalid params")
return
}
voice := r.URL.Query().Get("voice")
if voice == "" {
voice = s.cfg.DefaultVoice
}
// Fast path: audio exists in MinIO
audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice)
if s.deps.AudioStore.AudioExists(r.Context(), audioKey) {
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s", slug, n, voice)
writeJSON(w, 0, map[string]string{
"status": "done",
"url": proxyURL,
})
return
}
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, voice)
task, found, _ := s.deps.TaskReader.GetAudioTask(r.Context(), cacheKey)
if !found {
writeJSON(w, 0, map[string]string{"status": "idle"})
return
}
resp := map[string]string{
"status": string(task.Status),
"task_id": task.ID,
}
if task.Status == domain.TaskStatusFailed && task.ErrorMessage != "" {
resp["error"] = task.ErrorMessage
}
writeJSON(w, 0, resp)
}
// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}.
// Redirects to a presigned MinIO URL for the generated audio object.
// Query params: voice (optional, defaults to DefaultVoice)
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 {
http.NotFound(w, r)
return
}
voice := r.URL.Query().Get("voice")
if voice == "" {
voice = s.cfg.DefaultVoice
}
audioKey := s.deps.AudioStore.AudioObjectKey(slug, n, voice)
if !s.deps.AudioStore.AudioExists(r.Context(), audioKey) {
http.Error(w, "audio not generated yet", http.StatusNotFound)
return
}
presignURL, err := s.deps.PresignStore.PresignAudio(r.Context(), audioKey, 1*time.Hour)
if err != nil {
s.deps.Log.Error("handleAudioProxy: PresignAudio failed", "slug", slug, "n", n, "err", err)
http.Error(w, "presign failed", http.StatusInternalServerError)
return
}
http.Redirect(w, r, presignURL, http.StatusFound)
}
// ── Voices ─────────────────────────────────────────────────────────────────────
// handleVoices handles GET /api/voices.
// Returns {"voices": [...]} — fetched from Kokoro with built-in fallback.
func (s *Server) handleVoices(w http.ResponseWriter, r *http.Request) {
writeJSON(w, 0, map[string]any{"voices": s.voices(r.Context())})
}
// ── Presigned URLs ─────────────────────────────────────────────────────────────
// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}.
func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
jsonError(w, http.StatusBadRequest, "invalid params")
return
}
u, err := s.deps.PresignStore.PresignChapter(r.Context(), slug, n, 15*time.Minute)
if err != nil {
s.deps.Log.Error("presign chapter failed", "slug", slug, "n", n, "err", err)
jsonError(w, http.StatusInternalServerError, "presign failed")
return
}
writeJSON(w, 0, map[string]string{"url": u})
}
// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}.
// Query params: voice (optional)
func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
jsonError(w, http.StatusBadRequest, "invalid params")
return
}
voice := r.URL.Query().Get("voice")
if voice == "" {
voice = s.cfg.DefaultVoice
}
key := s.deps.AudioStore.AudioObjectKey(slug, n, voice)
if !s.deps.AudioStore.AudioExists(r.Context(), key) {
http.NotFound(w, r)
return
}
u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour)
if err != nil {
s.deps.Log.Error("presign audio failed", "slug", slug, "n", n, "err", err)
jsonError(w, http.StatusInternalServerError, "presign failed")
return
}
writeJSON(w, 0, map[string]string{"url": u})
}
// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}.
func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) {
voice := r.PathValue("voice")
if voice == "" {
jsonError(w, http.StatusBadRequest, "missing voice")
return
}
key := kokoro.VoiceSampleKey(voice)
if !s.deps.AudioStore.AudioExists(r.Context(), key) {
http.NotFound(w, r)
return
}
u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour)
if err != nil {
s.deps.Log.Error("presign voice sample failed", "voice", voice, "err", err)
jsonError(w, http.StatusInternalServerError, "presign failed")
return
}
writeJSON(w, 0, map[string]string{"url": u})
}
// handlePresignAvatarUpload handles GET /api/presign/avatar-upload/{userId}.
// Query params: ext (jpg|png|webp, defaults to jpg)
func (s *Server) handlePresignAvatarUpload(w http.ResponseWriter, r *http.Request) {
userID := r.PathValue("userId")
if userID == "" {
jsonError(w, http.StatusBadRequest, "missing userId")
return
}
ext := r.URL.Query().Get("ext")
switch ext {
case "jpg", "jpeg":
ext = "jpg"
case "png":
ext = "png"
case "webp":
ext = "webp"
default:
ext = "jpg"
}
uploadURL, key, err := s.deps.PresignStore.PresignAvatarUpload(r.Context(), userID, ext)
if err != nil {
s.deps.Log.Error("presign avatar upload failed", "userId", userID, "err", err)
jsonError(w, http.StatusInternalServerError, "presign failed")
return
}
writeJSON(w, 0, map[string]string{"upload_url": uploadURL, "key": key})
}
// handlePresignAvatar handles GET /api/presign/avatar/{userId}.
func (s *Server) handlePresignAvatar(w http.ResponseWriter, r *http.Request) {
userID := r.PathValue("userId")
if userID == "" {
jsonError(w, http.StatusBadRequest, "missing userId")
return
}
u, found, err := s.deps.PresignStore.PresignAvatarURL(r.Context(), userID)
if err != nil {
s.deps.Log.Error("presign avatar failed", "userId", userID, "err", err)
jsonError(w, http.StatusInternalServerError, "presign failed")
return
}
if !found {
http.NotFound(w, r)
return
}
writeJSON(w, 0, map[string]string{"url": u})
}
// ── Progress ───────────────────────────────────────────────────────────────────
// handleGetProgress handles GET /api/progress.
// Returns {"slug": chapterNum, "slug_ts": timestampMs, ...}
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
sid := ensureSession(w, r)
entries, err := s.deps.ProgressStore.AllProgress(r.Context(), sid)
if err != nil {
s.deps.Log.Error("AllProgress failed", "err", err)
entries = nil
}
progress := make(map[string]any, len(entries)*2)
for _, p := range entries {
progress[p.Slug] = p.Chapter
progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
}
writeJSON(w, 0, progress)
}
// handleSetProgress handles POST /api/progress/{slug}.
// Body: {"chapter": N}
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
sid := ensureSession(w, r)
slug := r.PathValue("slug")
if slug == "" {
jsonError(w, http.StatusBadRequest, "missing slug")
return
}
var body struct {
Chapter int `json:"chapter"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
jsonError(w, http.StatusBadRequest, "invalid body")
return
}
p := domain.ReadingProgress{
Slug: slug,
Chapter: body.Chapter,
UpdatedAt: time.Now(),
}
if err := s.deps.ProgressStore.SetProgress(r.Context(), sid, p); err != nil {
s.deps.Log.Error("SetProgress failed", "slug", slug, "err", err)
jsonError(w, http.StatusInternalServerError, "store error")
return
}
writeJSON(w, 0, map[string]string{})
}
// handleDeleteProgress handles DELETE /api/progress/{slug}.
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
sid := ensureSession(w, r)
slug := r.PathValue("slug")
if slug == "" {
jsonError(w, http.StatusBadRequest, "missing slug")
return
}
if err := s.deps.ProgressStore.DeleteProgress(r.Context(), sid, slug); err != nil {
s.deps.Log.Error("DeleteProgress failed", "slug", slug, "err", err)
// non-fatal
}
writeJSON(w, 0, map[string]string{})
}
// ── Browse page parsing helpers ────────────────────────────────────────────────
// fetchBrowsePage fetches pageURL and parses NovelListings from the HTML.
func (s *Server) fetchBrowsePage(ctx context.Context, pageURL string) ([]NovelListing, bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
if err != nil {
return nil, false, fmt.Errorf("build request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, resp.Body)
return nil, false, fmt.Errorf("upstream returned %d", resp.StatusCode)
}
novels, hasNext := parseBrowsePage(resp.Body)
return novels, hasNext, nil
}
// parseBrowsePage parses a novelfire HTML body and returns novel listings.
// It uses a simple string-scanning approach to avoid importing golang.org/x/net/html
// in this package (that dependency is only in internal/novelfire).
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
data, err := io.ReadAll(r)
if err != nil {
return nil, false
}
body := string(data)
var novels []NovelListing
hasNext := false
// Detect "next page" link
if strings.Contains(body, `rel="next"`) ||
strings.Contains(body, `aria-label="Next"`) ||
strings.Contains(body, `class="next"`) {
hasNext = true
}
// Extract novel slugs and titles using simple regex patterns.
// novelfire.net novel items: <li class="novel-item">...</li>
// Each contains an anchor like <a href="/book/{slug}">
slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
slugMatches := slugRe.FindAllStringSubmatch(body, -1)
titleMatches := titleRe.FindAllStringSubmatch(body, -1)
coverMatches := coverRe.FindAllStringSubmatch(body, -1)
seen := make(map[string]bool)
for i, sm := range slugMatches {
slug := sm[1]
if seen[slug] {
continue
}
seen[slug] = true
novel := NovelListing{
Slug: slug,
URL: novelFireBase + "/book/" + slug,
}
if i < len(titleMatches) {
novel.Title = strings.TrimSpace(titleMatches[i][1])
}
if i < len(coverMatches) {
novel.Cover = coverMatches[i][1]
}
if novel.Title != "" {
novels = append(novels, novel)
}
}
return novels, hasNext
}
// ── Markdown stripping ─────────────────────────────────────────────────────────
// stripMarkdown removes common markdown syntax from src, returning plain text.
func stripMarkdown(src string) string {
src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "")
src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "")
src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "")
src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "")
src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1")
src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "")
src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "")
src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "")
src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n")
return strings.TrimSpace(src)
}
// ── Hardcoded Kokoro voice fallback ───────────────────────────────────────────
// kokoroVoices is the built-in fallback list used when the Kokoro service is
// unavailable. Matches the list in the old scraper helpers.go.
var kokoroVoices = []string{
// American English
"af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia",
"af_jessica", "af_kore", "af_nicole", "af_nova", "af_river",
"af_sarah", "af_sky",
"am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam",
"am_michael", "am_onyx", "am_puck",
// British English
"bf_alice", "bf_emma", "bf_lily",
"bm_daniel", "bm_fable", "bm_george", "bm_lewis",
// Spanish
"ef_dora", "em_alex",
// French
"ff_siwis",
// Hindi
"hf_alpha", "hf_beta", "hm_omega", "hm_psi",
// Italian
"if_sara", "im_nicola",
// Japanese
"jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo",
// Portuguese
"pf_dora", "pm_alex",
// Chinese
"zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi",
"zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang",
}

View File

@@ -1,285 +0,0 @@
// Package backend implements the HTTP API server for the LibNovel backend.
//
// The server exposes all endpoints consumed by the SvelteKit UI:
// - Book/chapter reads from PocketBase/MinIO via bookstore interfaces
// - Task creation (scrape + audio) via taskqueue.Producer — the runner binary
// picks up and executes those tasks asynchronously
// - Presigned MinIO URLs for media playback/upload
// - Session-scoped reading progress
// - Live novelfire.net browse/search (no scraper interface needed; direct HTTP)
// - Kokoro voice list
//
// The backend never scrapes directly. All scraping (metadata, chapter list,
// chapter text, audio TTS) is delegated to the runner binary via PocketBase
// task records. GET /api/book-preview enqueues a task when the book is absent.
//
// All external dependencies are injected as interfaces; concrete types live in
// internal/storage and are wired by cmd/backend/main.go.
package backend
import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"sync"
"time"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/taskqueue"
)
// Dependencies holds all external services the backend server depends on.
// Every field is an interface so test doubles can be injected freely.
type Dependencies struct {
// BookReader reads book metadata and chapter text from PocketBase/MinIO.
BookReader bookstore.BookReader
// RankingStore reads ranking data from PocketBase.
RankingStore bookstore.RankingStore
// AudioStore checks audio object existence and computes MinIO keys.
AudioStore bookstore.AudioStore
// PresignStore generates short-lived MinIO URLs.
PresignStore bookstore.PresignStore
// ProgressStore reads/writes per-session reading progress.
ProgressStore bookstore.ProgressStore
// Producer creates scrape/audio tasks in PocketBase.
Producer taskqueue.Producer
// TaskReader reads scrape/audio task records from PocketBase.
TaskReader taskqueue.Reader
// Kokoro is the TTS client (used for voice list only in the backend;
// audio generation is done by the runner).
Kokoro kokoro.Client
// Log is the structured logger.
Log *slog.Logger
}
// Config holds HTTP server tuning parameters.
type Config struct {
// Addr is the listen address, e.g. ":8080".
Addr string
// DefaultVoice is used when no voice is specified in audio requests.
DefaultVoice string
// Version and Commit are embedded in /health and /api/version responses.
Version string
Commit string
}
// Server is the HTTP API server.
type Server struct {
cfg Config
deps Dependencies
// voiceMu guards cachedVoices. Populated lazily on first GET /api/voices.
voiceMu sync.RWMutex
cachedVoices []string
}
// New creates a Server from cfg and deps.
func New(cfg Config, deps Dependencies) *Server {
if cfg.DefaultVoice == "" {
cfg.DefaultVoice = "af_bella"
}
if deps.Log == nil {
deps.Log = slog.Default()
}
return &Server{cfg: cfg, deps: deps}
}
// ListenAndServe registers all routes and starts the HTTP server.
// It blocks until ctx is cancelled, then performs a graceful shutdown.
func (s *Server) ListenAndServe(ctx context.Context) error {
mux := http.NewServeMux()
// Health / version
mux.HandleFunc("GET /health", s.handleHealth)
mux.HandleFunc("GET /api/version", s.handleVersion)
// Scrape task creation (202 Accepted — runner executes asynchronously)
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
mux.HandleFunc("POST /scrape/book/range", s.handleScrapeBookRange)
// Scrape task status / history
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
// Cancel a pending task (scrape or audio)
mux.HandleFunc("POST /api/cancel-task/{id}", s.handleCancelTask)
// Browse & search (live novelfire.net)
mux.HandleFunc("GET /api/browse", s.handleBrowse)
mux.HandleFunc("GET /api/search", s.handleSearch)
// Ranking (from PocketBase)
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
// Cover proxy (live URL redirect)
mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
// Book preview (enqueues scrape task if not in library; returns stored data if already scraped)
mux.HandleFunc("GET /api/book-preview/{slug}", s.handleBookPreview)
// Chapter text (served from MinIO via PocketBase index)
mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText)
// Raw markdown chapter content — served directly from MinIO by the backend.
// Use this instead of presign+fetch to avoid SvelteKit→MinIO network path.
mux.HandleFunc("GET /api/chapter-markdown/{slug}/{n}", s.handleChapterMarkdown)
// Reindex chapters_idx from MinIO
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
// Audio task creation (backend creates task; runner executes)
mux.HandleFunc("POST /api/audio/{slug}/{n}", s.handleAudioGenerate)
mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus)
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
// Voices list
mux.HandleFunc("GET /api/voices", s.handleVoices)
// Presigned URLs
mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter)
mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio)
mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample)
mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload)
mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar)
// Reading progress
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
srv := &http.Server{
Addr: s.cfg.Addr,
Handler: mux,
ReadTimeout: 15 * time.Second,
WriteTimeout: 60 * time.Second,
IdleTimeout: 60 * time.Second,
}
errCh := make(chan error, 1)
go func() { errCh <- srv.ListenAndServe() }()
s.deps.Log.Info("backend: HTTP server listening", "addr", s.cfg.Addr)
select {
case <-ctx.Done():
s.deps.Log.Info("backend: context cancelled, starting graceful shutdown")
shutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := srv.Shutdown(shutCtx); err != nil {
s.deps.Log.Error("backend: graceful shutdown failed", "err", err)
return err
}
s.deps.Log.Info("backend: shutdown complete")
return nil
case err := <-errCh:
return err
}
}
// ── Session cookie helpers ─────────────────────────────────────────────────────
const sessionCookieName = "libnovel_session"
func sessionID(r *http.Request) string {
c, err := r.Cookie(sessionCookieName)
if err != nil {
return ""
}
return c.Value
}
func newSessionID() (string, error) {
b := make([]byte, 16)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
func ensureSession(w http.ResponseWriter, r *http.Request) string {
if id := sessionID(r); id != "" {
return id
}
id, err := newSessionID()
if err != nil {
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
}
http.SetCookie(w, &http.Cookie{
Name: sessionCookieName,
Value: id,
Path: "/",
HttpOnly: true,
SameSite: http.SameSiteLaxMode,
MaxAge: 365 * 24 * 60 * 60,
})
return id
}
// ── Utility helpers ────────────────────────────────────────────────────────────
// writeJSON writes v as a JSON response with status code. Status 0 → 200.
func writeJSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
if status != 0 {
w.WriteHeader(status)
}
_ = json.NewEncoder(w).Encode(v)
}
// jsonError writes a JSON error body and the given status code.
func jsonError(w http.ResponseWriter, status int, msg string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
}
// voices returns the list of available Kokoro voices. On the first call it
// fetches from the Kokoro service and caches the result. Falls back to the
// hardcoded list on error.
func (s *Server) voices(ctx context.Context) []string {
s.voiceMu.RLock()
cached := s.cachedVoices
s.voiceMu.RUnlock()
if len(cached) > 0 {
return cached
}
if s.deps.Kokoro == nil {
return kokoroVoices
}
fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
list, err := s.deps.Kokoro.ListVoices(fetchCtx)
if err != nil || len(list) == 0 {
s.deps.Log.Warn("backend: could not fetch kokoro voices, using built-in list", "err", err)
return kokoroVoices
}
s.voiceMu.Lock()
s.cachedVoices = list
s.voiceMu.Unlock()
s.deps.Log.Info("backend: fetched kokoro voices", "count", len(list))
return list
}
// handleHealth handles GET /health.
func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, 0, map[string]string{
"status": "ok",
"version": s.cfg.Version,
"commit": s.cfg.Commit,
})
}
// handleVersion handles GET /api/version.
func (s *Server) handleVersion(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, 0, map[string]string{
"version": s.cfg.Version,
"commit": s.cfg.Commit,
})
}

View File

@@ -1,125 +0,0 @@
// Package bookstore defines the segregated read/write interfaces for book,
// chapter, ranking, progress, audio, and presign data.
//
// Interface segregation:
// - BookWriter — used by the runner to persist scraped data.
// - BookReader — used by the backend to serve book/chapter data.
// - RankingStore — used by both runner (write) and backend (read).
// - PresignStore — used only by the backend for URL signing.
// - AudioStore — used by the runner to store audio; backend for presign.
// - ProgressStore— used only by the backend for reading progress.
//
// Concrete implementations live in internal/storage.
package bookstore
import (
"context"
"time"
"github.com/libnovel/backend/internal/domain"
)
// BookWriter is the write side used by the runner after scraping a book.
type BookWriter interface {
// WriteMetadata upserts all bibliographic fields for a book.
WriteMetadata(ctx context.Context, meta domain.BookMeta) error
// WriteChapter stores a fully-scraped chapter's text in MinIO and
// updates the chapters_idx record in PocketBase.
WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error
// WriteChapterRefs persists chapter metadata (number + title) into
// chapters_idx without fetching or storing chapter text.
WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error
// ChapterExists returns true if the markdown object for ref already exists.
ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool
}
// BookReader is the read side used by the backend to serve content.
type BookReader interface {
// ReadMetadata returns the metadata for slug.
// Returns (zero, false, nil) when not found.
ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error)
// ListBooks returns all books sorted alphabetically by title.
ListBooks(ctx context.Context) ([]domain.BookMeta, error)
// LocalSlugs returns the set of slugs that have metadata stored.
LocalSlugs(ctx context.Context) (map[string]bool, error)
// MetadataMtime returns the Unix-second mtime of the metadata record, or 0.
MetadataMtime(ctx context.Context, slug string) int64
// ReadChapter returns the raw markdown for chapter number n.
ReadChapter(ctx context.Context, slug string, n int) (string, error)
// ListChapters returns all stored chapters for slug, sorted by number.
ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error)
// CountChapters returns the count of stored chapters.
CountChapters(ctx context.Context, slug string) int
// ReindexChapters rebuilds chapters_idx from MinIO objects for slug.
ReindexChapters(ctx context.Context, slug string) (int, error)
}
// RankingStore covers ranking reads and writes.
type RankingStore interface {
// WriteRankingItem upserts a single ranking entry (keyed on Slug).
WriteRankingItem(ctx context.Context, item domain.RankingItem) error
// ReadRankingItems returns all ranking items sorted by rank ascending.
ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error)
// RankingFreshEnough returns true when ranking rows exist and the most
// recent Updated timestamp is within maxAge.
RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error)
}
// AudioStore covers audio object storage (runner writes; backend reads).
type AudioStore interface {
// AudioObjectKey returns the MinIO object key for a cached audio file.
AudioObjectKey(slug string, n int, voice string) string
// AudioExists returns true when the audio object is present in MinIO.
AudioExists(ctx context.Context, key string) bool
// PutAudio stores raw audio bytes under the given MinIO object key.
PutAudio(ctx context.Context, key string, data []byte) error
}
// PresignStore generates short-lived URLs — used exclusively by the backend.
type PresignStore interface {
// PresignChapter returns a presigned GET URL for a chapter markdown object.
PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error)
// PresignAudio returns a presigned GET URL for an audio object.
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
// PresignAvatarUpload returns a short-lived presigned PUT URL for uploading
// an avatar image. ext should be "jpg", "png", or "webp".
PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error)
// PresignAvatarURL returns a presigned GET URL for a user's avatar.
// Returns ("", false, nil) when no avatar exists.
PresignAvatarURL(ctx context.Context, userID string) (string, bool, error)
// DeleteAvatar removes all avatar objects for a user.
DeleteAvatar(ctx context.Context, userID string) error
}
// ProgressStore covers per-session reading progress — backend only.
type ProgressStore interface {
// GetProgress returns the reading progress for the given session + slug.
GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool)
// SetProgress saves or updates reading progress.
SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error
// AllProgress returns all progress entries for a session.
AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error)
// DeleteProgress removes progress for a specific slug.
DeleteProgress(ctx context.Context, sessionID, slug string) error
}

View File

@@ -1,138 +0,0 @@
package bookstore_test
import (
"context"
"testing"
"time"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/domain"
)
// ── Mock that satisfies all bookstore interfaces ──────────────────────────────
type mockStore struct{}
// BookWriter
func (m *mockStore) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil }
func (m *mockStore) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error { return nil }
func (m *mockStore) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error {
return nil
}
func (m *mockStore) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool {
return false
}
// BookReader
func (m *mockStore) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) {
return domain.BookMeta{}, false, nil
}
func (m *mockStore) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil }
func (m *mockStore) LocalSlugs(_ context.Context) (map[string]bool, error) {
return map[string]bool{}, nil
}
func (m *mockStore) MetadataMtime(_ context.Context, _ string) int64 { return 0 }
func (m *mockStore) ReadChapter(_ context.Context, _ string, _ int) (string, error) {
return "", nil
}
func (m *mockStore) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) {
return nil, nil
}
func (m *mockStore) CountChapters(_ context.Context, _ string) int { return 0 }
func (m *mockStore) ReindexChapters(_ context.Context, _ string) (int, error) { return 0, nil }
// RankingStore
func (m *mockStore) WriteRankingItem(_ context.Context, _ domain.RankingItem) error { return nil }
func (m *mockStore) ReadRankingItems(_ context.Context) ([]domain.RankingItem, error) {
return nil, nil
}
func (m *mockStore) RankingFreshEnough(_ context.Context, _ time.Duration) (bool, error) {
return false, nil
}
// AudioStore
func (m *mockStore) AudioObjectKey(_ string, _ int, _ string) string { return "" }
func (m *mockStore) AudioExists(_ context.Context, _ string) bool { return false }
func (m *mockStore) PutAudio(_ context.Context, _ string, _ []byte) error { return nil }
// PresignStore
func (m *mockStore) PresignChapter(_ context.Context, _ string, _ int, _ time.Duration) (string, error) {
return "", nil
}
func (m *mockStore) PresignAudio(_ context.Context, _ string, _ time.Duration) (string, error) {
return "", nil
}
func (m *mockStore) PresignAvatarUpload(_ context.Context, _, _ string) (string, string, error) {
return "", "", nil
}
func (m *mockStore) PresignAvatarURL(_ context.Context, _ string) (string, bool, error) {
return "", false, nil
}
func (m *mockStore) DeleteAvatar(_ context.Context, _ string) error { return nil }
// ProgressStore
func (m *mockStore) GetProgress(_ context.Context, _, _ string) (domain.ReadingProgress, bool) {
return domain.ReadingProgress{}, false
}
func (m *mockStore) SetProgress(_ context.Context, _ string, _ domain.ReadingProgress) error {
return nil
}
func (m *mockStore) AllProgress(_ context.Context, _ string) ([]domain.ReadingProgress, error) {
return nil, nil
}
func (m *mockStore) DeleteProgress(_ context.Context, _, _ string) error { return nil }
// ── Compile-time interface satisfaction ───────────────────────────────────────
var _ bookstore.BookWriter = (*mockStore)(nil)
var _ bookstore.BookReader = (*mockStore)(nil)
var _ bookstore.RankingStore = (*mockStore)(nil)
var _ bookstore.AudioStore = (*mockStore)(nil)
var _ bookstore.PresignStore = (*mockStore)(nil)
var _ bookstore.ProgressStore = (*mockStore)(nil)
// ── Behavioural tests ─────────────────────────────────────────────────────────
func TestBookWriter_WriteMetadata_ReturnsNilError(t *testing.T) {
var w bookstore.BookWriter = &mockStore{}
if err := w.WriteMetadata(context.Background(), domain.BookMeta{Slug: "test"}); err != nil {
t.Errorf("unexpected error: %v", err)
}
}
func TestBookReader_ReadMetadata_NotFound(t *testing.T) {
var r bookstore.BookReader = &mockStore{}
_, found, err := r.ReadMetadata(context.Background(), "unknown")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if found {
t.Error("expected not found")
}
}
func TestRankingStore_RankingFreshEnough_ReturnsFalse(t *testing.T) {
var s bookstore.RankingStore = &mockStore{}
fresh, err := s.RankingFreshEnough(context.Background(), time.Hour)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if fresh {
t.Error("expected false")
}
}
func TestAudioStore_AudioExists_ReturnsFalse(t *testing.T) {
var s bookstore.AudioStore = &mockStore{}
if s.AudioExists(context.Background(), "audio/slug/1/af_bella.mp3") {
t.Error("expected false")
}
}
func TestProgressStore_GetProgress_NotFound(t *testing.T) {
var s bookstore.ProgressStore = &mockStore{}
_, found := s.GetProgress(context.Background(), "session-1", "slug")
if found {
t.Error("expected not found")
}
}

View File

@@ -1,206 +0,0 @@
// Package browser provides a rate-limited HTTP client for web scraping.
// The Client interface is the only thing the rest of the codebase depends on;
// the concrete DirectClient can be swapped for any other implementation
// (e.g. a Browserless-backed client) without touching callers.
package browser
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"sync"
"time"
)
// ErrRateLimit is returned by GetContent when the server responds with 429.
// It carries the suggested retry delay (from Retry-After header, or a default).
var ErrRateLimit = errors.New("rate limited (429)")
// RateLimitError wraps ErrRateLimit and carries the suggested wait duration.
type RateLimitError struct {
// RetryAfter is how long the caller should wait before retrying.
// Derived from the Retry-After response header when present; otherwise a default.
RetryAfter time.Duration
}
func (e *RateLimitError) Error() string {
return fmt.Sprintf("rate limited (429): retry after %s", e.RetryAfter)
}
func (e *RateLimitError) Is(target error) bool { return target == ErrRateLimit }
// defaultRateLimitDelay is used when the server returns 429 with no Retry-After header.
const defaultRateLimitDelay = 60 * time.Second
// Client is the interface used by scrapers to fetch raw page HTML.
// Implementations must be safe for concurrent use.
type Client interface {
// GetContent fetches the URL and returns the full response body as a string.
// It should respect the provided context for cancellation and timeouts.
GetContent(ctx context.Context, pageURL string) (string, error)
}
// Config holds tunable parameters for the direct HTTP client.
type Config struct {
// MaxConcurrent limits the number of simultaneous in-flight requests.
// Defaults to 5 when 0.
MaxConcurrent int
// Timeout is the per-request deadline. Defaults to 90s when 0.
Timeout time.Duration
// ProxyURL is an optional outbound proxy, e.g. "http://user:pass@host:3128".
// Falls back to HTTP_PROXY / HTTPS_PROXY environment variables when empty.
ProxyURL string
}
// DirectClient is a plain net/http-based Client with a concurrency semaphore.
type DirectClient struct {
http *http.Client
semaphore chan struct{}
}
// NewDirectClient returns a DirectClient configured by cfg.
func NewDirectClient(cfg Config) *DirectClient {
if cfg.MaxConcurrent <= 0 {
cfg.MaxConcurrent = 5
}
if cfg.Timeout <= 0 {
cfg.Timeout = 90 * time.Second
}
transport := &http.Transport{
MaxIdleConnsPerHost: cfg.MaxConcurrent * 2,
DisableCompression: false,
}
if cfg.ProxyURL != "" {
proxyParsed, err := url.Parse(cfg.ProxyURL)
if err == nil {
transport.Proxy = http.ProxyURL(proxyParsed)
}
} else {
transport.Proxy = http.ProxyFromEnvironment
}
return &DirectClient{
http: &http.Client{
Transport: transport,
Timeout: cfg.Timeout,
},
semaphore: make(chan struct{}, cfg.MaxConcurrent),
}
}
// GetContent fetches pageURL respecting the concurrency limit.
func (c *DirectClient) GetContent(ctx context.Context, pageURL string) (string, error) {
// Acquire semaphore slot.
select {
case c.semaphore <- struct{}{}:
case <-ctx.Done():
return "", ctx.Err()
}
defer func() { <-c.semaphore }()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
if err != nil {
return "", fmt.Errorf("browser: build request %s: %w", pageURL, err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
resp, err := c.http.Do(req)
if err != nil {
return "", fmt.Errorf("browser: GET %s: %w", pageURL, err)
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusTooManyRequests {
delay := defaultRateLimitDelay
if ra := resp.Header.Get("Retry-After"); ra != "" {
if secs, err := strconv.Atoi(ra); err == nil && secs > 0 {
delay = time.Duration(secs) * time.Second
}
}
return "", &RateLimitError{RetryAfter: delay}
}
if resp.StatusCode >= 400 {
return "", fmt.Errorf("browser: GET %s returned %d", pageURL, resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("browser: read body %s: %w", pageURL, err)
}
return string(body), nil
}
// Do implements httputil.Client so DirectClient can be passed to RetryGet.
func (c *DirectClient) Do(req *http.Request) (*http.Response, error) {
select {
case c.semaphore <- struct{}{}:
case <-req.Context().Done():
return nil, req.Context().Err()
}
defer func() { <-c.semaphore }()
return c.http.Do(req)
}
// ── Stub for testing ──────────────────────────────────────────────────────────
// StubClient is a test double for Client. It returns pre-configured responses
// keyed on URL. Calls to unknown URLs return an error.
type StubClient struct {
mu sync.Mutex
pages map[string]string
errors map[string]error
callLog []string
}
// NewStub creates a StubClient with no pages pre-loaded.
func NewStub() *StubClient {
return &StubClient{
pages: make(map[string]string),
errors: make(map[string]error),
}
}
// SetPage registers a URL → HTML body mapping.
func (s *StubClient) SetPage(u, html string) {
s.mu.Lock()
s.pages[u] = html
s.mu.Unlock()
}
// SetError registers a URL → error mapping (returned instead of a body).
func (s *StubClient) SetError(u string, err error) {
s.mu.Lock()
s.errors[u] = err
s.mu.Unlock()
}
// CallLog returns the ordered list of URLs that were requested.
func (s *StubClient) CallLog() []string {
s.mu.Lock()
defer s.mu.Unlock()
out := make([]string, len(s.callLog))
copy(out, s.callLog)
return out
}
// GetContent returns the registered page or an error for the URL.
func (s *StubClient) GetContent(_ context.Context, pageURL string) (string, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.callLog = append(s.callLog, pageURL)
if err, ok := s.errors[pageURL]; ok {
return "", err
}
if html, ok := s.pages[pageURL]; ok {
return html, nil
}
return "", fmt.Errorf("stub: no page registered for %q", pageURL)
}

View File

@@ -1,141 +0,0 @@
package browser_test
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/libnovel/backend/internal/browser"
)
func TestDirectClient_GetContent_Success(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("<html>hello</html>"))
}))
defer srv.Close()
c := browser.NewDirectClient(browser.Config{MaxConcurrent: 2, Timeout: 5 * time.Second})
body, err := c.GetContent(context.Background(), srv.URL)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if body != "<html>hello</html>" {
t.Errorf("want <html>hello</html>, got %q", body)
}
}
func TestDirectClient_GetContent_4xxReturnsError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer srv.Close()
c := browser.NewDirectClient(browser.Config{})
_, err := c.GetContent(context.Background(), srv.URL)
if err == nil {
t.Fatal("expected error for 404")
}
}
func TestDirectClient_SemaphoreBlocksConcurrency(t *testing.T) {
const maxConcurrent = 2
var inflight atomic.Int32
var peak atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := inflight.Add(1)
if int(n) > int(peak.Load()) {
peak.Store(n)
}
time.Sleep(20 * time.Millisecond)
inflight.Add(-1)
w.Write([]byte("ok"))
}))
defer srv.Close()
c := browser.NewDirectClient(browser.Config{MaxConcurrent: maxConcurrent, Timeout: 5 * time.Second})
var wg sync.WaitGroup
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
c.GetContent(context.Background(), srv.URL)
}()
}
wg.Wait()
if int(peak.Load()) > maxConcurrent {
t.Errorf("concurrent requests exceeded limit: peak=%d, limit=%d", peak.Load(), maxConcurrent)
}
}
func TestDirectClient_ContextCancel(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(200 * time.Millisecond)
w.Write([]byte("ok"))
}))
defer srv.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel before making the request
c := browser.NewDirectClient(browser.Config{})
_, err := c.GetContent(ctx, srv.URL)
if err == nil {
t.Fatal("expected context cancellation error")
}
}
// ── StubClient ────────────────────────────────────────────────────────────────
func TestStubClient_ReturnsRegisteredPage(t *testing.T) {
stub := browser.NewStub()
stub.SetPage("http://example.com/page1", "<html>page1</html>")
body, err := stub.GetContent(context.Background(), "http://example.com/page1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if body != "<html>page1</html>" {
t.Errorf("want page1 html, got %q", body)
}
}
func TestStubClient_ReturnsRegisteredError(t *testing.T) {
stub := browser.NewStub()
want := errors.New("network failure")
stub.SetError("http://example.com/bad", want)
_, err := stub.GetContent(context.Background(), "http://example.com/bad")
if err == nil {
t.Fatal("expected error")
}
}
func TestStubClient_UnknownURLReturnsError(t *testing.T) {
stub := browser.NewStub()
_, err := stub.GetContent(context.Background(), "http://unknown.example.com/")
if err == nil {
t.Fatal("expected error for unknown URL")
}
}
func TestStubClient_CallLog(t *testing.T) {
stub := browser.NewStub()
stub.SetPage("http://example.com/a", "a")
stub.SetPage("http://example.com/b", "b")
stub.GetContent(context.Background(), "http://example.com/a")
stub.GetContent(context.Background(), "http://example.com/b")
log := stub.CallLog()
if len(log) != 2 || log[0] != "http://example.com/a" || log[1] != "http://example.com/b" {
t.Errorf("unexpected call log: %v", log)
}
}

View File

@@ -1,183 +0,0 @@
// Package config loads all service configuration from environment variables.
// Both the runner and backend binaries call config.Load() at startup; each
// uses only the sub-struct relevant to it.
//
// Every field has a documented default so the service starts sensibly without
// any environment configuration (useful for local development).
package config
import (
"os"
"strconv"
"strings"
"time"
)
// PocketBase holds connection settings for the remote PocketBase instance.
type PocketBase struct {
// URL is the base URL of the PocketBase instance, e.g. https://pb.libnovel.cc
URL string
// AdminEmail is the admin account email used for API authentication.
AdminEmail string
// AdminPassword is the admin account password.
AdminPassword string
}
// MinIO holds connection settings for the remote MinIO / S3-compatible store.
type MinIO struct {
// Endpoint is the host:port of the MinIO S3 API, e.g. storage.libnovel.cc:443
Endpoint string
// PublicEndpoint is the browser-visible endpoint used for presigned URLs.
// Falls back to Endpoint when empty.
PublicEndpoint string
// AccessKey is the MinIO access key.
AccessKey string
// SecretKey is the MinIO secret key.
SecretKey string
// UseSSL enables TLS for the internal MinIO connection.
UseSSL bool
// PublicUseSSL enables TLS for presigned URL generation.
PublicUseSSL bool
// BucketChapters is the bucket that holds chapter markdown objects.
BucketChapters string
// BucketAudio is the bucket that holds generated audio MP3 objects.
BucketAudio string
// BucketAvatars is the bucket that holds user avatar images.
BucketAvatars string
}
// Kokoro holds connection settings for the Kokoro-FastAPI TTS service.
type Kokoro struct {
// URL is the base URL of the Kokoro service, e.g. https://kokoro.libnovel.cc
// An empty string disables TTS generation.
URL string
// DefaultVoice is the voice used when none is specified.
DefaultVoice string
}
// HTTP holds settings for the HTTP server (backend only).
type HTTP struct {
// Addr is the listen address, e.g. ":8080"
Addr string
}
// Runner holds settings specific to the runner/worker binary.
type Runner struct {
// PollInterval is how often the runner checks PocketBase for pending tasks.
PollInterval time.Duration
// MaxConcurrentScrape limits simultaneous book-scrape goroutines.
MaxConcurrentScrape int
// MaxConcurrentAudio limits simultaneous audio-generation goroutines.
MaxConcurrentAudio int
// WorkerID is a unique identifier for this runner instance.
// Defaults to the system hostname.
WorkerID string
// Workers is the number of chapter-scraping goroutines per book.
Workers int
// Timeout is the per-request HTTP timeout for scraping.
Timeout time.Duration
// ProxyURL is an optional outbound proxy for scraper HTTP requests.
ProxyURL string
}
// Config is the top-level configuration struct consumed by both binaries.
type Config struct {
PocketBase PocketBase
MinIO MinIO
Kokoro Kokoro
HTTP HTTP
Runner Runner
// LogLevel is one of "debug", "info", "warn", "error".
LogLevel string
}
// Load reads all configuration from environment variables and returns a
// populated Config. Missing variables fall back to documented defaults.
func Load() Config {
workerID, _ := os.Hostname()
if workerID == "" {
workerID = "runner-default"
}
return Config{
LogLevel: envOr("LOG_LEVEL", "info"),
PocketBase: PocketBase{
URL: envOr("POCKETBASE_URL", "http://localhost:8090"),
AdminEmail: envOr("POCKETBASE_ADMIN_EMAIL", "admin@libnovel.local"),
AdminPassword: envOr("POCKETBASE_ADMIN_PASSWORD", "changeme123"),
},
MinIO: MinIO{
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
PublicEndpoint: envOr("MINIO_PUBLIC_ENDPOINT", ""),
AccessKey: envOr("MINIO_ACCESS_KEY", "admin"),
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
UseSSL: envBool("MINIO_USE_SSL", false),
PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true),
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"),
},
Kokoro: Kokoro{
URL: envOr("KOKORO_URL", ""),
DefaultVoice: envOr("KOKORO_VOICE", "af_bella"),
},
HTTP: HTTP{
Addr: envOr("BACKEND_HTTP_ADDR", ":8080"),
},
Runner: Runner{
PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second),
MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1),
MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1),
WorkerID: envOr("RUNNER_WORKER_ID", workerID),
Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU()
Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second),
ProxyURL: envOr("SCRAPER_PROXY", ""),
},
}
}
// ── helpers ───────────────────────────────────────────────────────────────────
func envOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
func envBool(key string, fallback bool) bool {
v := os.Getenv(key)
if v == "" {
return fallback
}
return strings.ToLower(v) == "true"
}
func envInt(key string, fallback int) int {
v := os.Getenv(key)
if v == "" {
return fallback
}
n, err := strconv.Atoi(v)
if err != nil || n < 0 {
return fallback
}
return n
}
func envDuration(key string, fallback time.Duration) time.Duration {
v := os.Getenv(key)
if v == "" {
return fallback
}
d, err := time.ParseDuration(v)
if err != nil {
return fallback
}
return d
}

View File

@@ -1,127 +0,0 @@
package config_test
import (
"os"
"testing"
"time"
"github.com/libnovel/backend/internal/config"
)
func TestLoad_Defaults(t *testing.T) {
// Unset all relevant vars so we test pure defaults.
unset := []string{
"LOG_LEVEL",
"POCKETBASE_URL", "POCKETBASE_ADMIN_EMAIL", "POCKETBASE_ADMIN_PASSWORD",
"MINIO_ENDPOINT", "MINIO_PUBLIC_ENDPOINT", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY",
"MINIO_USE_SSL", "MINIO_PUBLIC_USE_SSL",
"MINIO_BUCKET_CHAPTERS", "MINIO_BUCKET_AUDIO", "MINIO_BUCKET_AVATARS",
"KOKORO_URL", "KOKORO_VOICE",
"BACKEND_HTTP_ADDR",
"RUNNER_POLL_INTERVAL", "RUNNER_MAX_CONCURRENT_SCRAPE", "RUNNER_MAX_CONCURRENT_AUDIO",
"RUNNER_WORKER_ID", "RUNNER_WORKERS", "RUNNER_TIMEOUT", "SCRAPER_PROXY",
}
for _, k := range unset {
t.Setenv(k, "")
}
cfg := config.Load()
if cfg.LogLevel != "info" {
t.Errorf("LogLevel: want info, got %q", cfg.LogLevel)
}
if cfg.PocketBase.URL != "http://localhost:8090" {
t.Errorf("PocketBase.URL: want http://localhost:8090, got %q", cfg.PocketBase.URL)
}
if cfg.MinIO.BucketChapters != "libnovel-chapters" {
t.Errorf("MinIO.BucketChapters: want libnovel-chapters, got %q", cfg.MinIO.BucketChapters)
}
if cfg.MinIO.UseSSL != false {
t.Errorf("MinIO.UseSSL: want false, got %v", cfg.MinIO.UseSSL)
}
if cfg.MinIO.PublicUseSSL != true {
t.Errorf("MinIO.PublicUseSSL: want true, got %v", cfg.MinIO.PublicUseSSL)
}
if cfg.Kokoro.DefaultVoice != "af_bella" {
t.Errorf("Kokoro.DefaultVoice: want af_bella, got %q", cfg.Kokoro.DefaultVoice)
}
if cfg.HTTP.Addr != ":8080" {
t.Errorf("HTTP.Addr: want :8080, got %q", cfg.HTTP.Addr)
}
if cfg.Runner.PollInterval != 30*time.Second {
t.Errorf("Runner.PollInterval: want 30s, got %v", cfg.Runner.PollInterval)
}
if cfg.Runner.MaxConcurrentScrape != 1 {
t.Errorf("Runner.MaxConcurrentScrape: want 1, got %d", cfg.Runner.MaxConcurrentScrape)
}
if cfg.Runner.MaxConcurrentAudio != 1 {
t.Errorf("Runner.MaxConcurrentAudio: want 1, got %d", cfg.Runner.MaxConcurrentAudio)
}
}
func TestLoad_EnvOverride(t *testing.T) {
t.Setenv("LOG_LEVEL", "debug")
t.Setenv("POCKETBASE_URL", "https://pb.libnovel.cc")
t.Setenv("MINIO_USE_SSL", "true")
t.Setenv("MINIO_PUBLIC_USE_SSL", "false")
t.Setenv("RUNNER_POLL_INTERVAL", "1m")
t.Setenv("RUNNER_MAX_CONCURRENT_SCRAPE", "5")
t.Setenv("RUNNER_WORKER_ID", "homelab-01")
t.Setenv("BACKEND_HTTP_ADDR", ":9090")
t.Setenv("KOKORO_URL", "https://kokoro.libnovel.cc")
cfg := config.Load()
if cfg.LogLevel != "debug" {
t.Errorf("LogLevel: want debug, got %q", cfg.LogLevel)
}
if cfg.PocketBase.URL != "https://pb.libnovel.cc" {
t.Errorf("PocketBase.URL: want https://pb.libnovel.cc, got %q", cfg.PocketBase.URL)
}
if !cfg.MinIO.UseSSL {
t.Error("MinIO.UseSSL: want true")
}
if cfg.MinIO.PublicUseSSL {
t.Error("MinIO.PublicUseSSL: want false")
}
if cfg.Runner.PollInterval != time.Minute {
t.Errorf("Runner.PollInterval: want 1m, got %v", cfg.Runner.PollInterval)
}
if cfg.Runner.MaxConcurrentScrape != 5 {
t.Errorf("Runner.MaxConcurrentScrape: want 5, got %d", cfg.Runner.MaxConcurrentScrape)
}
if cfg.Runner.WorkerID != "homelab-01" {
t.Errorf("Runner.WorkerID: want homelab-01, got %q", cfg.Runner.WorkerID)
}
if cfg.HTTP.Addr != ":9090" {
t.Errorf("HTTP.Addr: want :9090, got %q", cfg.HTTP.Addr)
}
if cfg.Kokoro.URL != "https://kokoro.libnovel.cc" {
t.Errorf("Kokoro.URL: want https://kokoro.libnovel.cc, got %q", cfg.Kokoro.URL)
}
}
func TestLoad_InvalidInt_FallsToDefault(t *testing.T) {
t.Setenv("RUNNER_MAX_CONCURRENT_SCRAPE", "notanumber")
cfg := config.Load()
if cfg.Runner.MaxConcurrentScrape != 1 {
t.Errorf("want default 1, got %d", cfg.Runner.MaxConcurrentScrape)
}
}
func TestLoad_InvalidDuration_FallsToDefault(t *testing.T) {
t.Setenv("RUNNER_POLL_INTERVAL", "notaduration")
cfg := config.Load()
if cfg.Runner.PollInterval != 30*time.Second {
t.Errorf("want default 30s, got %v", cfg.Runner.PollInterval)
}
}
func TestLoad_WorkerID_FallsToHostname(t *testing.T) {
t.Setenv("RUNNER_WORKER_ID", "")
cfg := config.Load()
host, _ := os.Hostname()
if host != "" && cfg.Runner.WorkerID != host {
t.Errorf("want hostname %q, got %q", host, cfg.Runner.WorkerID)
}
}

View File

@@ -1,131 +0,0 @@
// Package domain contains the core value types shared across all packages
// in this module. It has zero internal imports — only the standard library.
// Every other package imports domain; domain imports nothing from this module.
package domain
import "time"
// ── Book types ────────────────────────────────────────────────────────────────
// BookMeta carries all bibliographic information about a novel.
type BookMeta struct {
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author"`
Cover string `json:"cover,omitempty"`
Status string `json:"status,omitempty"`
Genres []string `json:"genres,omitempty"`
Summary string `json:"summary,omitempty"`
TotalChapters int `json:"total_chapters,omitempty"`
SourceURL string `json:"source_url"`
Ranking int `json:"ranking,omitempty"`
}
// CatalogueEntry is a lightweight book reference returned by catalogue pages.
type CatalogueEntry struct {
Title string `json:"title"`
URL string `json:"url"`
}
// ChapterRef is a reference to a single chapter returned by chapter-list pages.
type ChapterRef struct {
Number int `json:"number"`
Title string `json:"title"`
URL string `json:"url"`
Volume int `json:"volume,omitempty"`
}
// Chapter contains the fully-extracted text of a single chapter.
type Chapter struct {
Ref ChapterRef `json:"ref"`
Text string `json:"text"`
}
// RankingItem represents a single entry in the novel ranking list.
type RankingItem struct {
Rank int `json:"rank"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author,omitempty"`
Cover string `json:"cover,omitempty"`
Status string `json:"status,omitempty"`
Genres []string `json:"genres,omitempty"`
SourceURL string `json:"source_url,omitempty"`
Updated time.Time `json:"updated,omitempty"`
}
// ── Storage record types ──────────────────────────────────────────────────────
// ChapterInfo is a lightweight chapter descriptor stored in the index.
type ChapterInfo struct {
Number int `json:"number"`
Title string `json:"title"`
Date string `json:"date,omitempty"`
}
// ReadingProgress holds a single user's reading position for one book.
type ReadingProgress struct {
Slug string `json:"slug"`
Chapter int `json:"chapter"`
UpdatedAt time.Time `json:"updated_at"`
}
// ── Task record types ─────────────────────────────────────────────────────────
// TaskStatus enumerates the lifecycle states of any task.
type TaskStatus string
const (
TaskStatusPending TaskStatus = "pending"
TaskStatusRunning TaskStatus = "running"
TaskStatusDone TaskStatus = "done"
TaskStatusFailed TaskStatus = "failed"
TaskStatusCancelled TaskStatus = "cancelled"
)
// ScrapeTask represents a book-scraping job stored in PocketBase.
type ScrapeTask struct {
ID string `json:"id"`
Kind string `json:"kind"` // "catalogue" | "book" | "book_range"
TargetURL string `json:"target_url"` // non-empty for single-book tasks
FromChapter int `json:"from_chapter,omitempty"`
ToChapter int `json:"to_chapter,omitempty"`
WorkerID string `json:"worker_id,omitempty"`
Status TaskStatus `json:"status"`
BooksFound int `json:"books_found"`
ChaptersScraped int `json:"chapters_scraped"`
ChaptersSkipped int `json:"chapters_skipped"`
Errors int `json:"errors"`
Started time.Time `json:"started"`
Finished time.Time `json:"finished,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
}
// ScrapeResult is the outcome reported by the runner after finishing a ScrapeTask.
type ScrapeResult struct {
BooksFound int `json:"books_found"`
ChaptersScraped int `json:"chapters_scraped"`
ChaptersSkipped int `json:"chapters_skipped"`
Errors int `json:"errors"`
ErrorMessage string `json:"error_message,omitempty"`
}
// AudioTask represents an audio-generation job stored in PocketBase.
type AudioTask struct {
ID string `json:"id"`
CacheKey string `json:"cache_key"` // "slug/chapter/voice"
Slug string `json:"slug"`
Chapter int `json:"chapter"`
Voice string `json:"voice"`
WorkerID string `json:"worker_id,omitempty"`
Status TaskStatus `json:"status"`
ErrorMessage string `json:"error_message,omitempty"`
Started time.Time `json:"started"`
Finished time.Time `json:"finished,omitempty"`
}
// AudioResult is the outcome reported by the runner after finishing an AudioTask.
type AudioResult struct {
ObjectKey string `json:"object_key,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
}

View File

@@ -1,104 +0,0 @@
package domain_test
import (
"encoding/json"
"testing"
"time"
"github.com/libnovel/backend/internal/domain"
)
func TestBookMeta_JSONRoundtrip(t *testing.T) {
orig := domain.BookMeta{
Slug: "a-great-novel",
Title: "A Great Novel",
Author: "Jane Doe",
Cover: "https://example.com/cover.jpg",
Status: "Ongoing",
Genres: []string{"Fantasy", "Action"},
Summary: "A thrilling tale.",
TotalChapters: 120,
SourceURL: "https://novelfire.net/book/a-great-novel",
Ranking: 3,
}
b, err := json.Marshal(orig)
if err != nil {
t.Fatalf("marshal: %v", err)
}
var got domain.BookMeta
if err := json.Unmarshal(b, &got); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if got.Slug != orig.Slug {
t.Errorf("Slug: want %q, got %q", orig.Slug, got.Slug)
}
if got.TotalChapters != orig.TotalChapters {
t.Errorf("TotalChapters: want %d, got %d", orig.TotalChapters, got.TotalChapters)
}
if len(got.Genres) != len(orig.Genres) {
t.Errorf("Genres len: want %d, got %d", len(orig.Genres), len(got.Genres))
}
}
func TestChapterRef_JSONRoundtrip(t *testing.T) {
orig := domain.ChapterRef{Number: 42, Title: "The Battle", URL: "https://example.com/ch-42", Volume: 2}
b, _ := json.Marshal(orig)
var got domain.ChapterRef
json.Unmarshal(b, &got)
if got != orig {
t.Errorf("want %+v, got %+v", orig, got)
}
}
func TestRankingItem_JSONRoundtrip(t *testing.T) {
now := time.Now().Truncate(time.Second)
orig := domain.RankingItem{
Rank: 1,
Slug: "top-novel",
Title: "Top Novel",
SourceURL: "https://novelfire.net/book/top-novel",
Updated: now,
}
b, _ := json.Marshal(orig)
var got domain.RankingItem
json.Unmarshal(b, &got)
if got.Rank != orig.Rank || got.Slug != orig.Slug {
t.Errorf("want %+v, got %+v", orig, got)
}
}
func TestScrapeResult_JSONRoundtrip(t *testing.T) {
orig := domain.ScrapeResult{BooksFound: 10, ChaptersScraped: 200, ChaptersSkipped: 5, Errors: 1, ErrorMessage: "one error"}
b, _ := json.Marshal(orig)
var got domain.ScrapeResult
json.Unmarshal(b, &got)
if got != orig {
t.Errorf("want %+v, got %+v", orig, got)
}
}
func TestAudioResult_JSONRoundtrip(t *testing.T) {
orig := domain.AudioResult{ObjectKey: "audio/slug/1/af_bella.mp3"}
b, _ := json.Marshal(orig)
var got domain.AudioResult
json.Unmarshal(b, &got)
if got != orig {
t.Errorf("want %+v, got %+v", orig, got)
}
}
func TestTaskStatus_Values(t *testing.T) {
cases := []domain.TaskStatus{
domain.TaskStatusPending,
domain.TaskStatusRunning,
domain.TaskStatusDone,
domain.TaskStatusFailed,
domain.TaskStatusCancelled,
}
for _, s := range cases {
if s == "" {
t.Errorf("TaskStatus constant must not be empty")
}
}
}

View File

@@ -1,124 +0,0 @@
// Package httputil provides shared HTTP helpers used by both the runner and
// backend binaries. It has no imports from this module — only the standard
// library — so it is safe to import from anywhere in the dependency graph.
package httputil
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"time"
)
// Client is the minimal interface for making HTTP GET requests.
// *http.Client satisfies this interface.
type Client interface {
Do(req *http.Request) (*http.Response, error)
}
// ErrMaxRetries is returned when RetryGet exhausts all attempts.
var ErrMaxRetries = errors.New("httputil: max retries exceeded")
// errClientError is returned by doGet for 4xx responses; it signals that the
// request should NOT be retried (the client is at fault).
var errClientError = errors.New("httputil: client error")
// RetryGet fetches url using client, retrying on network errors or 5xx
// responses with exponential backoff. It returns the full response body as a
// string on success.
//
// - maxAttempts: total number of attempts (must be >= 1)
// - baseDelay: initial wait before the second attempt; doubles each retry
func RetryGet(ctx context.Context, client Client, url string, maxAttempts int, baseDelay time.Duration) (string, error) {
if maxAttempts < 1 {
maxAttempts = 1
}
delay := baseDelay
var lastErr error
for attempt := 0; attempt < maxAttempts; attempt++ {
if attempt > 0 {
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(delay):
}
delay *= 2
}
body, err := doGet(ctx, client, url)
if err == nil {
return body, nil
}
lastErr = err
// Do not retry on context cancellation.
if ctx.Err() != nil {
return "", ctx.Err()
}
// Do not retry on 4xx — the client is at fault.
if errors.Is(err, errClientError) {
return "", err
}
}
return "", fmt.Errorf("%w after %d attempts: %w", ErrMaxRetries, maxAttempts, lastErr)
}
func doGet(ctx context.Context, client Client, url string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return "", fmt.Errorf("build request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("GET %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode >= 500 {
return "", fmt.Errorf("GET %s: server error %d", url, resp.StatusCode)
}
if resp.StatusCode >= 400 {
return "", fmt.Errorf("%w: GET %s: client error %d", errClientError, url, resp.StatusCode)
}
raw, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("read body %s: %w", url, err)
}
return string(raw), nil
}
// WriteJSON writes v as JSON to w with the given HTTP status code and sets the
// Content-Type header to application/json.
func WriteJSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(v)
}
// WriteError writes a JSON error object {"error": msg} with the given status.
func WriteError(w http.ResponseWriter, status int, msg string) {
WriteJSON(w, status, map[string]string{"error": msg})
}
// maxBodyBytes is the limit applied by DecodeJSON to prevent unbounded reads.
const maxBodyBytes = 1 << 20 // 1 MiB
// DecodeJSON decodes a JSON request body into v. It enforces a 1 MiB size
// limit and returns a descriptive error on any failure.
func DecodeJSON(r *http.Request, v any) error {
r.Body = http.MaxBytesReader(nil, r.Body, maxBodyBytes)
dec := json.NewDecoder(r.Body)
dec.DisallowUnknownFields()
if err := dec.Decode(v); err != nil {
return fmt.Errorf("decode JSON body: %w", err)
}
return nil
}

View File

@@ -1,181 +0,0 @@
package httputil_test
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/libnovel/backend/internal/httputil"
)
// ── RetryGet ──────────────────────────────────────────────────────────────────
func TestRetryGet_ImmediateSuccess(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("hello"))
}))
defer srv.Close()
body, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 3, time.Millisecond)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if body != "hello" {
t.Errorf("want hello, got %q", body)
}
}
func TestRetryGet_RetriesOn5xx(t *testing.T) {
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
if calls < 3 {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
w.Write([]byte("ok"))
}))
defer srv.Close()
body, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 5, time.Millisecond)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if body != "ok" {
t.Errorf("want ok, got %q", body)
}
if calls != 3 {
t.Errorf("want 3 calls, got %d", calls)
}
}
func TestRetryGet_MaxAttemptsExceeded(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer srv.Close()
_, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 3, time.Millisecond)
if err == nil {
t.Fatal("expected error, got nil")
}
}
func TestRetryGet_ContextCancelDuringBackoff(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusServiceUnavailable)
}))
defer srv.Close()
ctx, cancel := context.WithCancel(context.Background())
// Cancel after first failed attempt hits the backoff wait.
go func() { time.Sleep(5 * time.Millisecond); cancel() }()
_, err := httputil.RetryGet(ctx, srv.Client(), srv.URL, 10, 500*time.Millisecond)
if err == nil {
t.Fatal("expected context cancellation error")
}
}
func TestRetryGet_NoRetryOn4xx(t *testing.T) {
calls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
w.WriteHeader(http.StatusNotFound)
}))
defer srv.Close()
_, err := httputil.RetryGet(context.Background(), srv.Client(), srv.URL, 5, time.Millisecond)
if err == nil {
t.Fatal("expected error for 404")
}
// 4xx is NOT retried — should be exactly 1 call.
if calls != 1 {
t.Errorf("want 1 call for 4xx, got %d", calls)
}
}
// ── WriteJSON ─────────────────────────────────────────────────────────────────
func TestWriteJSON_SetsHeadersAndStatus(t *testing.T) {
rr := httptest.NewRecorder()
httputil.WriteJSON(rr, http.StatusCreated, map[string]string{"key": "val"})
if rr.Code != http.StatusCreated {
t.Errorf("status: want 201, got %d", rr.Code)
}
if ct := rr.Header().Get("Content-Type"); ct != "application/json" {
t.Errorf("Content-Type: want application/json, got %q", ct)
}
var got map[string]string
if err := json.NewDecoder(rr.Body).Decode(&got); err != nil {
t.Fatalf("decode body: %v", err)
}
if got["key"] != "val" {
t.Errorf("body key: want val, got %q", got["key"])
}
}
// ── WriteError ────────────────────────────────────────────────────────────────
func TestWriteError_Format(t *testing.T) {
rr := httptest.NewRecorder()
httputil.WriteError(rr, http.StatusBadRequest, "bad input")
if rr.Code != http.StatusBadRequest {
t.Errorf("status: want 400, got %d", rr.Code)
}
var got map[string]string
json.NewDecoder(rr.Body).Decode(&got)
if got["error"] != "bad input" {
t.Errorf("error field: want bad input, got %q", got["error"])
}
}
// ── DecodeJSON ────────────────────────────────────────────────────────────────
func TestDecodeJSON_HappyPath(t *testing.T) {
body := `{"name":"test","value":42}`
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
req.Header.Set("Content-Type", "application/json")
var payload struct {
Name string `json:"name"`
Value int `json:"value"`
}
if err := httputil.DecodeJSON(req, &payload); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if payload.Name != "test" || payload.Value != 42 {
t.Errorf("unexpected payload: %+v", payload)
}
}
func TestDecodeJSON_UnknownFieldReturnsError(t *testing.T) {
body := `{"name":"test","unknown_field":"boom"}`
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
var payload struct {
Name string `json:"name"`
}
if err := httputil.DecodeJSON(req, &payload); err == nil {
t.Fatal("expected error for unknown field, got nil")
}
}
func TestDecodeJSON_BodyTooLarge(t *testing.T) {
// Build a body > 1 MiB.
big := bytes.Repeat([]byte("a"), 2<<20)
req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(big))
var payload map[string]any
if err := httputil.DecodeJSON(req, &payload); err == nil {
t.Fatal("expected error for oversized body, got nil")
}
}

View File

@@ -1,160 +0,0 @@
// Package kokoro provides a client for the Kokoro-FastAPI TTS service.
//
// The Kokoro API is an OpenAI-compatible audio speech API that returns a
// download link (X-Download-Path header) instead of streaming audio directly.
// GenerateAudio handles the two-step flow: POST /v1/audio/speech → GET /v1/download/{file}.
package kokoro
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
)
// Client is the interface for interacting with the Kokoro TTS service.
type Client interface {
// GenerateAudio synthesises text using voice and returns raw MP3 bytes.
GenerateAudio(ctx context.Context, text, voice string) ([]byte, error)
// ListVoices returns the available voice IDs. Falls back to an empty slice
// on error — callers should treat an empty list as "service unavailable".
ListVoices(ctx context.Context) ([]string, error)
}
// httpClient is the concrete Kokoro HTTP client.
type httpClient struct {
baseURL string
http *http.Client
}
// New returns a Kokoro Client targeting baseURL (e.g. "https://kokoro.example.com").
func New(baseURL string) Client {
return &httpClient{
baseURL: strings.TrimRight(baseURL, "/"),
http: &http.Client{Timeout: 10 * time.Minute},
}
}
// GenerateAudio calls POST /v1/audio/speech (return_download_link=true) and then
// downloads the resulting MP3 from GET /v1/download/{filename}.
func (c *httpClient) GenerateAudio(ctx context.Context, text, voice string) ([]byte, error) {
if text == "" {
return nil, fmt.Errorf("kokoro: empty text")
}
if voice == "" {
voice = "af_bella"
}
// ── Step 1: request generation ────────────────────────────────────────────
reqBody, err := json.Marshal(map[string]any{
"model": "kokoro",
"input": text,
"voice": voice,
"response_format": "mp3",
"speed": 1.0,
"stream": false,
"return_download_link": true,
})
if err != nil {
return nil, fmt.Errorf("kokoro: marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
c.baseURL+"/v1/audio/speech", bytes.NewReader(reqBody))
if err != nil {
return nil, fmt.Errorf("kokoro: build speech request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := c.http.Do(req)
if err != nil {
return nil, fmt.Errorf("kokoro: speech request: %w", err)
}
defer resp.Body.Close()
_, _ = io.Copy(io.Discard, resp.Body)
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("kokoro: speech returned %d", resp.StatusCode)
}
dlPath := resp.Header.Get("X-Download-Path")
if dlPath == "" {
return nil, fmt.Errorf("kokoro: no X-Download-Path header in response")
}
filename := dlPath
if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
filename = dlPath[idx+1:]
}
if filename == "" {
return nil, fmt.Errorf("kokoro: empty filename in X-Download-Path: %q", dlPath)
}
// ── Step 2: download the generated file ───────────────────────────────────
dlURL := c.baseURL + "/v1/download/" + filename
dlReq, err := http.NewRequestWithContext(ctx, http.MethodGet, dlURL, nil)
if err != nil {
return nil, fmt.Errorf("kokoro: build download request: %w", err)
}
dlResp, err := c.http.Do(dlReq)
if err != nil {
return nil, fmt.Errorf("kokoro: download request: %w", err)
}
defer dlResp.Body.Close()
if dlResp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("kokoro: download returned %d", dlResp.StatusCode)
}
data, err := io.ReadAll(dlResp.Body)
if err != nil {
return nil, fmt.Errorf("kokoro: read download body: %w", err)
}
return data, nil
}
// ListVoices calls GET /v1/audio/voices and returns the list of voice IDs.
func (c *httpClient) ListVoices(ctx context.Context) ([]string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
c.baseURL+"/v1/audio/voices", nil)
if err != nil {
return nil, fmt.Errorf("kokoro: build voices request: %w", err)
}
resp, err := c.http.Do(req)
if err != nil {
return nil, fmt.Errorf("kokoro: voices request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, resp.Body)
return nil, fmt.Errorf("kokoro: voices returned %d", resp.StatusCode)
}
var result struct {
Voices []string `json:"voices"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, fmt.Errorf("kokoro: decode voices response: %w", err)
}
return result.Voices, nil
}
// VoiceSampleKey returns the MinIO object key for a voice sample MP3.
// Key: _voice-samples/{voice}.mp3 (sanitised).
func VoiceSampleKey(voice string) string {
safe := strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
return r
}
return '_'
}, voice)
return fmt.Sprintf("_voice-samples/%s.mp3", safe)
}

View File

@@ -1,291 +0,0 @@
package kokoro_test
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/libnovel/backend/internal/kokoro"
)
// ── VoiceSampleKey ────────────────────────────────────────────────────────────
func TestVoiceSampleKey(t *testing.T) {
tests := []struct {
voice string
want string
}{
{"af_bella", "_voice-samples/af_bella.mp3"},
{"am_echo", "_voice-samples/am_echo.mp3"},
{"voice with spaces", "_voice-samples/voice_with_spaces.mp3"},
{"special!@#chars", "_voice-samples/special___chars.mp3"},
{"", "_voice-samples/.mp3"},
}
for _, tt := range tests {
t.Run(tt.voice, func(t *testing.T) {
got := kokoro.VoiceSampleKey(tt.voice)
if got != tt.want {
t.Errorf("VoiceSampleKey(%q) = %q, want %q", tt.voice, got, tt.want)
}
})
}
}
// ── GenerateAudio ─────────────────────────────────────────────────────────────
func TestGenerateAudio_EmptyText(t *testing.T) {
srv := httptest.NewServer(http.NotFoundHandler())
defer srv.Close()
c := kokoro.New(srv.URL)
_, err := c.GenerateAudio(context.Background(), "", "af_bella")
if err == nil {
t.Fatal("expected error for empty text, got nil")
}
if !strings.Contains(err.Error(), "empty text") {
t.Errorf("expected 'empty text' in error, got: %v", err)
}
}
func TestGenerateAudio_DefaultVoice(t *testing.T) {
// Tracks that the voice defaults to af_bella when empty.
var capturedBody string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/audio/speech" {
buf := make([]byte, 512)
n, _ := r.Body.Read(buf)
capturedBody = string(buf[:n])
w.Header().Set("X-Download-Path", "/download/test_file.mp3")
w.WriteHeader(http.StatusOK)
return
}
if strings.HasPrefix(r.URL.Path, "/v1/download/") {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("fake-mp3-data"))
return
}
http.NotFound(w, r)
}))
defer srv.Close()
c := kokoro.New(srv.URL)
data, err := c.GenerateAudio(context.Background(), "hello world", "")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(data) != "fake-mp3-data" {
t.Errorf("unexpected data: %q", string(data))
}
if !strings.Contains(capturedBody, `"af_bella"`) {
t.Errorf("expected default voice af_bella in request body, got: %s", capturedBody)
}
}
func TestGenerateAudio_SpeechNon200(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/audio/speech" {
w.WriteHeader(http.StatusInternalServerError)
return
}
http.NotFound(w, r)
}))
defer srv.Close()
c := kokoro.New(srv.URL)
_, err := c.GenerateAudio(context.Background(), "text", "af_bella")
if err == nil {
t.Fatal("expected error for non-200 speech response")
}
if !strings.Contains(err.Error(), "500") {
t.Errorf("expected 500 in error, got: %v", err)
}
}
func TestGenerateAudio_NoDownloadPathHeader(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/audio/speech" {
// No X-Download-Path header
w.WriteHeader(http.StatusOK)
return
}
http.NotFound(w, r)
}))
defer srv.Close()
c := kokoro.New(srv.URL)
_, err := c.GenerateAudio(context.Background(), "text", "af_bella")
if err == nil {
t.Fatal("expected error for missing X-Download-Path")
}
if !strings.Contains(err.Error(), "X-Download-Path") {
t.Errorf("expected X-Download-Path in error, got: %v", err)
}
}
func TestGenerateAudio_DownloadFails(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/audio/speech" {
w.Header().Set("X-Download-Path", "/v1/download/speech.mp3")
w.WriteHeader(http.StatusOK)
return
}
if strings.HasPrefix(r.URL.Path, "/v1/download/") {
w.WriteHeader(http.StatusNotFound)
return
}
http.NotFound(w, r)
}))
defer srv.Close()
c := kokoro.New(srv.URL)
_, err := c.GenerateAudio(context.Background(), "text", "af_bella")
if err == nil {
t.Fatal("expected error for failed download")
}
if !strings.Contains(err.Error(), "404") {
t.Errorf("expected 404 in error, got: %v", err)
}
}
func TestGenerateAudio_FullPath(t *testing.T) {
// X-Download-Path with a full path: extract just filename.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/audio/speech" {
w.Header().Set("X-Download-Path", "/some/nested/path/audio_abc123.mp3")
w.WriteHeader(http.StatusOK)
return
}
if r.URL.Path == "/v1/download/audio_abc123.mp3" {
_, _ = w.Write([]byte("audio-bytes"))
return
}
http.NotFound(w, r)
}))
defer srv.Close()
c := kokoro.New(srv.URL)
data, err := c.GenerateAudio(context.Background(), "text", "af_bella")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(data) != "audio-bytes" {
t.Errorf("unexpected data: %q", string(data))
}
}
func TestGenerateAudio_ContextCancelled(t *testing.T) {
// Server that hangs — context should cancel before we get a response.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Never respond.
select {}
}))
defer srv.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel immediately
c := kokoro.New(srv.URL)
_, err := c.GenerateAudio(ctx, "text", "af_bella")
if err == nil {
t.Fatal("expected error for cancelled context")
}
}
// ── ListVoices ────────────────────────────────────────────────────────────────
func TestListVoices_Success(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/audio/voices" {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"voices":["af_bella","am_adam","bf_emma"]}`))
return
}
http.NotFound(w, r)
}))
defer srv.Close()
c := kokoro.New(srv.URL)
voices, err := c.ListVoices(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(voices) != 3 {
t.Errorf("expected 3 voices, got %d: %v", len(voices), voices)
}
if voices[0] != "af_bella" {
t.Errorf("expected first voice to be af_bella, got %q", voices[0])
}
}
func TestListVoices_Non200(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusServiceUnavailable)
}))
defer srv.Close()
c := kokoro.New(srv.URL)
_, err := c.ListVoices(context.Background())
if err == nil {
t.Fatal("expected error for non-200 response")
}
if !strings.Contains(err.Error(), "503") {
t.Errorf("expected 503 in error, got: %v", err)
}
}
func TestListVoices_MalformedJSON(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`not-json`))
}))
defer srv.Close()
c := kokoro.New(srv.URL)
_, err := c.ListVoices(context.Background())
if err == nil {
t.Fatal("expected error for malformed JSON")
}
}
func TestListVoices_EmptyVoices(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"voices":[]}`))
}))
defer srv.Close()
c := kokoro.New(srv.URL)
voices, err := c.ListVoices(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(voices) != 0 {
t.Errorf("expected 0 voices, got %d", len(voices))
}
}
// ── New ───────────────────────────────────────────────────────────────────────
func TestNew_TrailingSlashStripped(t *testing.T) {
// Verify that a trailing slash on baseURL doesn't produce double-slash paths.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/audio/voices" {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"voices":["af_bella"]}`))
return
}
http.NotFound(w, r)
}))
defer srv.Close()
c := kokoro.New(srv.URL + "/") // trailing slash
voices, err := c.ListVoices(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(voices) == 0 {
t.Error("expected at least one voice")
}
}

View File

@@ -1,228 +0,0 @@
// Package htmlutil provides helper functions for parsing HTML with
// golang.org/x/net/html and extracting values by Selector descriptors.
package htmlutil
import (
"net/url"
"regexp"
"strings"
"github.com/libnovel/backend/internal/scraper"
"golang.org/x/net/html"
)
// ResolveURL returns an absolute URL. If href is already absolute it is
// returned unchanged. Otherwise it is resolved against base.
func ResolveURL(base, href string) string {
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
return href
}
b, err := url.Parse(base)
if err != nil {
return base + href
}
ref, err := url.Parse(href)
if err != nil {
return base + href
}
return b.ResolveReference(ref).String()
}
// ParseHTML parses raw HTML and returns the root node.
func ParseHTML(raw string) (*html.Node, error) {
return html.Parse(strings.NewReader(raw))
}
// selectorMatches reports whether node n matches sel.
func selectorMatches(n *html.Node, sel scraper.Selector) bool {
if n.Type != html.ElementNode {
return false
}
if sel.Tag != "" && n.Data != sel.Tag {
return false
}
if sel.ID != "" {
for _, a := range n.Attr {
if a.Key == "id" && a.Val == sel.ID {
goto checkClass
}
}
return false
}
checkClass:
if sel.Class != "" {
for _, a := range n.Attr {
if a.Key == "class" {
for _, cls := range strings.Fields(a.Val) {
if cls == sel.Class {
goto matched
}
}
}
}
return false
}
matched:
return true
}
// AttrVal returns the value of attribute key from node n.
func AttrVal(n *html.Node, key string) string {
for _, a := range n.Attr {
if a.Key == key {
return a.Val
}
}
return ""
}
// TextContent returns the concatenated text content of all descendant text nodes.
func TextContent(n *html.Node) string {
var sb strings.Builder
var walk func(*html.Node)
walk = func(cur *html.Node) {
if cur.Type == html.TextNode {
sb.WriteString(cur.Data)
}
for c := cur.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(n)
return strings.TrimSpace(sb.String())
}
// FindFirst returns the first node matching sel within root.
func FindFirst(root *html.Node, sel scraper.Selector) *html.Node {
var found *html.Node
var walk func(*html.Node) bool
walk = func(n *html.Node) bool {
if selectorMatches(n, sel) {
found = n
return true
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
if walk(c) {
return true
}
}
return false
}
walk(root)
return found
}
// FindAll returns all nodes matching sel within root.
func FindAll(root *html.Node, sel scraper.Selector) []*html.Node {
var results []*html.Node
var walk func(*html.Node)
walk = func(n *html.Node) {
if selectorMatches(n, sel) {
results = append(results, n)
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(root)
return results
}
// ExtractText extracts a string value from node n using sel.
// If sel.Attr is set the attribute value is returned; otherwise the inner text.
func ExtractText(n *html.Node, sel scraper.Selector) string {
if sel.Attr != "" {
return AttrVal(n, sel.Attr)
}
return TextContent(n)
}
// ExtractFirst locates the first match in root and returns its text/attr value.
func ExtractFirst(root *html.Node, sel scraper.Selector) string {
n := FindFirst(root, sel)
if n == nil {
return ""
}
return ExtractText(n, sel)
}
// ExtractAll locates all matches in root and returns their text/attr values.
func ExtractAll(root *html.Node, sel scraper.Selector) []string {
nodes := FindAll(root, sel)
out := make([]string, 0, len(nodes))
for _, n := range nodes {
if v := ExtractText(n, sel); v != "" {
out = append(out, v)
}
}
return out
}
// NodeToMarkdown converts the children of an HTML node to a plain-text/Markdown
// representation suitable for chapter storage.
func NodeToMarkdown(n *html.Node) string {
var sb strings.Builder
nodeToMD(n, &sb)
out := multiBlankLine.ReplaceAllString(sb.String(), "\n\n")
return strings.TrimSpace(out)
}
var multiBlankLine = regexp.MustCompile(`\n(\s*\n){2,}`)
var blockElements = map[string]bool{
"p": true, "div": true, "br": true, "h1": true, "h2": true,
"h3": true, "h4": true, "h5": true, "h6": true, "li": true,
"blockquote": true, "pre": true, "hr": true,
}
func nodeToMD(n *html.Node, sb *strings.Builder) {
switch n.Type {
case html.TextNode:
sb.WriteString(n.Data)
case html.ElementNode:
tag := n.Data
switch tag {
case "br":
sb.WriteString("\n")
case "hr":
sb.WriteString("\n---\n")
case "h1", "h2", "h3", "h4", "h5", "h6":
level := int(tag[1] - '0')
sb.WriteString("\n" + strings.Repeat("#", level) + " ")
for c := n.FirstChild; c != nil; c = c.NextSibling {
nodeToMD(c, sb)
}
sb.WriteString("\n\n")
return
case "p", "div", "blockquote":
sb.WriteString("\n")
for c := n.FirstChild; c != nil; c = c.NextSibling {
nodeToMD(c, sb)
}
sb.WriteString("\n")
return
case "em", "i":
sb.WriteString("*")
for c := n.FirstChild; c != nil; c = c.NextSibling {
nodeToMD(c, sb)
}
sb.WriteString("*")
return
case "strong", "b":
sb.WriteString("**")
for c := n.FirstChild; c != nil; c = c.NextSibling {
nodeToMD(c, sb)
}
sb.WriteString("**")
return
case "script", "style", "noscript":
return // drop
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
nodeToMD(c, sb)
}
if blockElements[tag] {
sb.WriteString("\n")
}
}
}

View File

@@ -1,498 +0,0 @@
// Package novelfire provides a NovelScraper implementation for novelfire.net.
//
// Site structure (as of 2025):
//
// Catalogue : https://novelfire.net/genre-all/sort-new/status-all/all-novel?page=N
// Book page : https://novelfire.net/book/{slug}
// Chapters : https://novelfire.net/book/{slug}/chapters?page=N
// Chapter : https://novelfire.net/book/{slug}/{chapter-slug}
package novelfire
import (
"context"
"errors"
"fmt"
"log/slog"
"net/url"
"path"
"strconv"
"strings"
"time"
"github.com/libnovel/backend/internal/browser"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/novelfire/htmlutil"
"github.com/libnovel/backend/internal/scraper"
"golang.org/x/net/html"
)
const (
baseURL = "https://novelfire.net"
cataloguePath = "/genre-all/sort-new/status-all/all-novel"
rankingPath = "/genre-all/sort-popular/status-all/all-novel"
)
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
type Scraper struct {
client browser.Client
log *slog.Logger
}
// Compile-time interface check.
var _ scraper.NovelScraper = (*Scraper)(nil)
// New returns a new novelfire Scraper backed by client.
func New(client browser.Client, log *slog.Logger) *Scraper {
if log == nil {
log = slog.Default()
}
return &Scraper{client: client, log: log}
}
// SourceName implements NovelScraper.
func (s *Scraper) SourceName() string { return "novelfire.net" }
// ── CatalogueProvider ─────────────────────────────────────────────────────────
// ScrapeCatalogue streams all CatalogueEntry values across all catalogue pages.
func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
entries := make(chan domain.CatalogueEntry, 64)
errs := make(chan error, 16)
go func() {
defer close(entries)
defer close(errs)
pageURL := baseURL + cataloguePath
page := 1
for pageURL != "" {
select {
case <-ctx.Done():
return
default:
}
s.log.Info("scraping catalogue page", "page", page, "url", pageURL)
raw, err := s.client.GetContent(ctx, pageURL)
if err != nil {
errs <- fmt.Errorf("catalogue page %d: %w", page, err)
return
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
errs <- fmt.Errorf("catalogue page %d parse: %w", page, err)
return
}
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
if len(cards) == 0 {
s.log.Warn("no novel cards found, stopping pagination", "page", page)
return
}
for _, card := range cards {
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"})
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})
var title, href string
if linkNode != nil {
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
}
if titleNode != nil {
title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{}))
}
if href == "" || title == "" {
continue
}
bookURL := resolveURL(baseURL, href)
select {
case <-ctx.Done():
return
case entries <- domain.CatalogueEntry{Title: title, URL: bookURL}:
}
}
if !hasNextPageLink(root) {
break
}
nextHref := ""
for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) {
if htmlutil.AttrVal(a, "rel") == "next" {
nextHref = htmlutil.AttrVal(a, "href")
break
}
}
if nextHref == "" {
break
}
pageURL = resolveURL(baseURL, nextHref)
page++
}
}()
return entries, errs
}
// ── MetadataProvider ──────────────────────────────────────────────────────────
// ScrapeMetadata fetches and parses book metadata from the book's landing page.
func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error) {
s.log.Debug("metadata fetch starting", "url", bookURL)
raw, err := s.client.GetContent(ctx, bookURL)
if err != nil {
return domain.BookMeta{}, fmt.Errorf("metadata fetch %s: %w", bookURL, err)
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
return domain.BookMeta{}, fmt.Errorf("metadata parse %s: %w", bookURL, err)
}
title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"})
var cover string
if fig := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil {
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
if cover != "" && !strings.HasPrefix(cover, "http") {
cover = baseURL + cover
}
}
status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})
genresNode := htmlutil.FindFirst(root, scraper.Selector{Tag: "div", Class: "genres"})
var genres []string
if genresNode != nil {
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
}
summary := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "div", Class: "summary"})
totalStr := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "chapter-count"})
totalChapters := parseChapterCount(totalStr)
slug := slugFromURL(bookURL)
meta := domain.BookMeta{
Slug: slug,
Title: title,
Author: author,
Cover: cover,
Status: status,
Genres: genres,
Summary: summary,
TotalChapters: totalChapters,
SourceURL: bookURL,
}
s.log.Debug("metadata parsed", "slug", meta.Slug, "title", meta.Title)
return meta, nil
}
// ── ChapterListProvider ───────────────────────────────────────────────────────
// ScrapeChapterList returns all chapter references for a book, ordered ascending.
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) {
var refs []domain.ChapterRef
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
page := 1
for {
select {
case <-ctx.Done():
return refs, ctx.Err()
default:
}
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
raw, err := s.client.GetContent(ctx, pageURL)
if err != nil {
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
return refs, fmt.Errorf("chapter list page %d parse: %w", page, err)
}
chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
if chapterList == nil {
s.log.Debug("chapter list container not found, stopping pagination", "page", page)
break
}
items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
if len(items) == 0 {
break
}
for _, item := range items {
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
if linkNode == nil {
continue
}
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
if href == "" {
continue
}
chURL := resolveURL(baseURL, href)
num := chapterNumberFromURL(chURL)
if num <= 0 {
num = len(refs) + 1
s.log.Warn("chapter number not parseable from URL, falling back to position",
"url", chURL, "position", num)
}
refs = append(refs, domain.ChapterRef{
Number: num,
Title: strings.TrimSpace(chTitle),
URL: chURL,
})
}
page++
}
return refs, nil
}
// ── ChapterTextProvider ───────────────────────────────────────────────────────
// ScrapeChapterText fetches and parses a single chapter page.
func (s *Scraper) ScrapeChapterText(ctx context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
s.log.Debug("chapter text fetch starting", "chapter", ref.Number, "url", ref.URL)
raw, err := retryGet(ctx, s.log, s.client, ref.URL, 9, 6*time.Second)
if err != nil {
return domain.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
return domain.Chapter{}, fmt.Errorf("chapter %d parse: %w", ref.Number, err)
}
container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
if container == nil {
return domain.Chapter{}, fmt.Errorf("chapter %d: #content container not found in %s", ref.Number, ref.URL)
}
text := htmlutil.NodeToMarkdown(container)
s.log.Debug("chapter text parsed", "chapter", ref.Number, "text_bytes", len(text))
return domain.Chapter{Ref: ref, Text: text}, nil
}
// ── RankingProvider ───────────────────────────────────────────────────────────
// ScrapeRanking pages through up to maxPages pages of the popular-novels listing.
// maxPages <= 0 means all pages. The caller decides whether to persist items.
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) {
entries := make(chan domain.BookMeta, 32)
errs := make(chan error, 16)
go func() {
defer close(entries)
defer close(errs)
rank := 1
for page := 1; maxPages <= 0 || page <= maxPages; page++ {
select {
case <-ctx.Done():
return
default:
}
pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
raw, err := s.client.GetContent(ctx, pageURL)
if err != nil {
errs <- fmt.Errorf("ranking page %d: %w", page, err)
return
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
errs <- fmt.Errorf("ranking page %d parse: %w", page, err)
return
}
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
if len(cards) == 0 {
break
}
for _, card := range cards {
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"})
if linkNode == nil {
continue
}
href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
bookURL := resolveURL(baseURL, href)
if bookURL == "" {
continue
}
title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}))
if title == "" {
title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"}))
}
if title == "" {
continue
}
var cover string
if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil {
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
if cover == "" {
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
}
if strings.HasPrefix(cover, "data:") {
cover = ""
}
if cover != "" && !strings.HasPrefix(cover, "http") {
cover = baseURL + cover
}
}
meta := domain.BookMeta{
Slug: slugFromURL(bookURL),
Title: title,
Cover: cover,
SourceURL: bookURL,
Ranking: rank,
}
rank++
select {
case <-ctx.Done():
return
case entries <- meta:
}
}
if !hasNextPageLink(root) {
break
}
}
}()
return entries, errs
}
// ── helpers ───────────────────────────────────────────────────────────────────
func resolveURL(base, href string) string { return htmlutil.ResolveURL(base, href) }
func hasNextPageLink(root *html.Node) bool {
links := htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true})
for _, a := range links {
for _, attr := range a.Attr {
if attr.Key == "rel" && attr.Val == "next" {
return true
}
}
}
return false
}
func slugFromURL(bookURL string) string {
u, err := url.Parse(bookURL)
if err != nil {
return bookURL
}
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
if len(parts) >= 2 && parts[0] == "book" {
return parts[1]
}
if len(parts) > 0 {
return parts[len(parts)-1]
}
return ""
}
func parseChapterCount(s string) int {
s = strings.ReplaceAll(s, ",", "")
fields := strings.Fields(s)
if len(fields) == 0 {
return 0
}
n, _ := strconv.Atoi(fields[0])
return n
}
func chapterNumberFromURL(chapterURL string) int {
u, err := url.Parse(chapterURL)
if err != nil {
return 0
}
seg := path.Base(u.Path)
seg = strings.TrimPrefix(seg, "chapter-")
seg = strings.TrimPrefix(seg, "chap-")
seg = strings.TrimPrefix(seg, "ch-")
digits := strings.FieldsFunc(seg, func(r rune) bool {
return r < '0' || r > '9'
})
if len(digits) == 0 {
return 0
}
n, _ := strconv.Atoi(digits[0])
return n
}
// retryGet calls client.GetContent up to maxAttempts times with exponential backoff.
// If the server returns 429 (ErrRateLimit), the suggested Retry-After delay is used
// instead of the geometric backoff delay.
func retryGet(
ctx context.Context,
log *slog.Logger,
client browser.Client,
pageURL string,
maxAttempts int,
baseDelay time.Duration,
) (string, error) {
var lastErr error
delay := baseDelay
for attempt := 1; attempt <= maxAttempts; attempt++ {
raw, err := client.GetContent(ctx, pageURL)
if err == nil {
return raw, nil
}
lastErr = err
if ctx.Err() != nil {
return "", err
}
if attempt < maxAttempts {
// If the server is rate-limiting us, honour its Retry-After delay.
waitFor := delay
var rlErr *browser.RateLimitError
if errors.As(err, &rlErr) {
waitFor = rlErr.RetryAfter
if log != nil {
log.Warn("rate limited, backing off",
"url", pageURL, "attempt", attempt, "retry_in", waitFor)
}
} else {
if log != nil {
log.Warn("fetch failed, retrying",
"url", pageURL, "attempt", attempt, "retry_in", delay, "err", err)
}
delay *= 2
}
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(waitFor):
}
}
}
return "", lastErr
}

View File

@@ -1,129 +0,0 @@
package novelfire
import (
"context"
"testing"
)
func TestSlugFromURL(t *testing.T) {
cases := []struct {
url string
want string
}{
{"https://novelfire.net/book/shadow-slave", "shadow-slave"},
{"https://novelfire.net/book/a-dragon-against-the-whole-world", "a-dragon-against-the-whole-world"},
{"https://novelfire.net/book/foo/chapter-1", "foo"},
{"https://novelfire.net/", ""},
{"not-a-url", "not-a-url"},
}
for _, c := range cases {
got := slugFromURL(c.url)
if got != c.want {
t.Errorf("slugFromURL(%q) = %q, want %q", c.url, got, c.want)
}
}
}
func TestChapterNumberFromURL(t *testing.T) {
cases := []struct {
url string
want int
}{
{"https://novelfire.net/book/shadow-slave/chapter-42", 42},
{"https://novelfire.net/book/shadow-slave/chapter-1000", 1000},
{"https://novelfire.net/book/shadow-slave/chap-7", 7},
{"https://novelfire.net/book/shadow-slave/ch-3", 3},
{"https://novelfire.net/book/shadow-slave/42", 42},
{"https://novelfire.net/book/shadow-slave/no-number-here", 0},
{"not-a-url", 0},
}
for _, c := range cases {
got := chapterNumberFromURL(c.url)
if got != c.want {
t.Errorf("chapterNumberFromURL(%q) = %d, want %d", c.url, got, c.want)
}
}
}
func TestParseChapterCount(t *testing.T) {
cases := []struct {
in string
want int
}{
{"123 Chapters", 123},
{"1,234 Chapters", 1234},
{"0", 0},
{"", 0},
{"500", 500},
}
for _, c := range cases {
got := parseChapterCount(c.in)
if got != c.want {
t.Errorf("parseChapterCount(%q) = %d, want %d", c.in, got, c.want)
}
}
}
func TestRetryGet_ContextCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel immediately
stub := newStubClient()
stub.setError("https://example.com/page", context.Canceled)
_, err := retryGet(ctx, nil, stub, "https://example.com/page", 3, 0)
if err == nil {
t.Fatal("expected error on cancelled context")
}
}
func TestRetryGet_EventualSuccess(t *testing.T) {
stub := newStubClient()
calls := 0
stub.setFn("https://example.com/page", func() (string, error) {
calls++
if calls < 3 {
return "", context.DeadlineExceeded
}
return "<html>ok</html>", nil
})
got, err := retryGet(context.Background(), nil, stub, "https://example.com/page", 5, 0)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got != "<html>ok</html>" {
t.Errorf("got %q, want html", got)
}
if calls != 3 {
t.Errorf("expected 3 calls, got %d", calls)
}
}
// ── minimal stub client for tests ─────────────────────────────────────────────
type stubClient struct {
errors map[string]error
fns map[string]func() (string, error)
}
func newStubClient() *stubClient {
return &stubClient{
errors: make(map[string]error),
fns: make(map[string]func() (string, error)),
}
}
func (s *stubClient) setError(u string, err error) { s.errors[u] = err }
func (s *stubClient) setFn(u string, fn func() (string, error)) { s.fns[u] = fn }
func (s *stubClient) GetContent(_ context.Context, pageURL string) (string, error) {
if fn, ok := s.fns[pageURL]; ok {
return fn()
}
if err, ok := s.errors[pageURL]; ok {
return "", err
}
return "", context.DeadlineExceeded
}

View File

@@ -1,205 +0,0 @@
// Package orchestrator coordinates metadata extraction, chapter-list fetching,
// and parallel chapter scraping for a single book.
//
// Design:
// - RunBook scrapes one book (metadata + chapter list + chapter texts) end-to-end.
// - N worker goroutines pull chapter refs from a shared queue and call ScrapeChapterText.
// - The caller (runner poll loop) owns the outer task-claim / finish cycle.
package orchestrator
import (
"context"
"fmt"
"log/slog"
"runtime"
"sync"
"sync/atomic"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/scraper"
)
// Config holds tunable parameters for the orchestrator.
type Config struct {
// Workers is the number of goroutines used to scrape chapters in parallel.
// Defaults to runtime.NumCPU() when 0.
Workers int
}
// Orchestrator runs a single-book scrape pipeline.
type Orchestrator struct {
novel scraper.NovelScraper
store bookstore.BookWriter
log *slog.Logger
workers int
}
// New returns a new Orchestrator.
func New(cfg Config, novel scraper.NovelScraper, store bookstore.BookWriter, log *slog.Logger) *Orchestrator {
if log == nil {
log = slog.Default()
}
workers := cfg.Workers
if workers <= 0 {
workers = runtime.NumCPU()
}
return &Orchestrator{novel: novel, store: store, log: log, workers: workers}
}
// RunBook scrapes a single book described by task. It handles:
// 1. Metadata scrape + write
// 2. Chapter list scrape + write
// 3. Parallel chapter text scrape + write (worker pool)
//
// Returns a ScrapeResult with counters. The result's ErrorMessage is non-empty
// if the run failed at the metadata or chapter-list level.
func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) domain.ScrapeResult {
o.log.Info("orchestrator: RunBook starting",
"task_id", task.ID,
"kind", task.Kind,
"url", task.TargetURL,
"workers", o.workers,
)
var result domain.ScrapeResult
if task.TargetURL == "" {
result.ErrorMessage = "task has no target URL"
return result
}
// ── Step 1: Metadata ──────────────────────────────────────────────────────
meta, err := o.novel.ScrapeMetadata(ctx, task.TargetURL)
if err != nil {
o.log.Error("metadata scrape failed", "url", task.TargetURL, "err", err)
result.ErrorMessage = fmt.Sprintf("metadata: %v", err)
result.Errors++
return result
}
if err := o.store.WriteMetadata(ctx, meta); err != nil {
o.log.Error("metadata write failed", "slug", meta.Slug, "err", err)
// non-fatal: continue to chapters
result.Errors++
} else {
result.BooksFound = 1
}
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
// ── Step 2: Chapter list ──────────────────────────────────────────────────
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL)
if err != nil {
o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err)
result.ErrorMessage = fmt.Sprintf("chapter list: %v", err)
result.Errors++
return result
}
o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs))
// Persist chapter refs (without text) so the index exists early.
if wErr := o.store.WriteChapterRefs(ctx, meta.Slug, refs); wErr != nil {
o.log.Warn("chapter refs write failed", "slug", meta.Slug, "err", wErr)
}
// ── Step 3: Chapter texts (worker pool) ───────────────────────────────────
type chapterJob struct {
slug string
ref domain.ChapterRef
total int // total chapters to scrape (for progress logging)
}
work := make(chan chapterJob, o.workers*4)
var scraped, skipped, errors atomic.Int64
var wg sync.WaitGroup
for i := 0; i < o.workers; i++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
for job := range work {
select {
case <-ctx.Done():
return
default:
}
if o.store.ChapterExists(ctx, job.slug, job.ref) {
o.log.Debug("chapter already exists, skipping",
"slug", job.slug, "chapter", job.ref.Number)
skipped.Add(1)
continue
}
ch, err := o.novel.ScrapeChapterText(ctx, job.ref)
if err != nil {
o.log.Error("chapter scrape failed",
"slug", job.slug, "chapter", job.ref.Number, "err", err)
errors.Add(1)
continue
}
if err := o.store.WriteChapter(ctx, job.slug, ch); err != nil {
o.log.Error("chapter write failed",
"slug", job.slug, "chapter", job.ref.Number, "err", err)
errors.Add(1)
continue
}
n := scraped.Add(1)
// Log a progress summary every 25 chapters scraped.
if n%25 == 0 {
o.log.Info("scraping chapters",
"slug", job.slug, "scraped", n, "total", job.total)
}
}
}(i)
}
// Count how many chapters will actually be enqueued (for progress logging).
toScrape := 0
for _, ref := range refs {
if task.FromChapter > 0 && ref.Number < task.FromChapter {
continue
}
if task.ToChapter > 0 && ref.Number > task.ToChapter {
continue
}
toScrape++
}
// Enqueue chapter jobs respecting the optional range filter from the task.
for _, ref := range refs {
if task.FromChapter > 0 && ref.Number < task.FromChapter {
skipped.Add(1)
continue
}
if task.ToChapter > 0 && ref.Number > task.ToChapter {
skipped.Add(1)
continue
}
select {
case <-ctx.Done():
goto drain
case work <- chapterJob{slug: meta.Slug, ref: ref, total: toScrape}:
}
}
drain:
close(work)
wg.Wait()
result.ChaptersScraped = int(scraped.Load())
result.ChaptersSkipped = int(skipped.Load())
result.Errors += int(errors.Load())
o.log.Info("book scrape finished",
"slug", meta.Slug,
"scraped", result.ChaptersScraped,
"skipped", result.ChaptersSkipped,
"errors", result.Errors,
)
return result
}

View File

@@ -1,210 +0,0 @@
package orchestrator
import (
"context"
"errors"
"sync"
"testing"
"github.com/libnovel/backend/internal/domain"
)
// ── stubs ─────────────────────────────────────────────────────────────────────
type stubScraper struct {
meta domain.BookMeta
metaErr error
refs []domain.ChapterRef
refsErr error
chapters map[int]domain.Chapter
chapErr map[int]error
}
func (s *stubScraper) SourceName() string { return "stub" }
func (s *stubScraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
ch := make(chan domain.CatalogueEntry)
errs := make(chan error)
close(ch)
close(errs)
return ch, errs
}
func (s *stubScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) {
return s.meta, s.metaErr
}
func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
return s.refs, s.refsErr
}
func (s *stubScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
if s.chapErr != nil {
if err, ok := s.chapErr[ref.Number]; ok {
return domain.Chapter{}, err
}
}
if s.chapters != nil {
if ch, ok := s.chapters[ref.Number]; ok {
return ch, nil
}
}
return domain.Chapter{Ref: ref, Text: "text"}, nil
}
func (s *stubScraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error) {
ch := make(chan domain.BookMeta)
errs := make(chan error)
close(ch)
close(errs)
return ch, errs
}
type stubStore struct {
mu sync.Mutex
metaWritten []domain.BookMeta
chaptersWritten []domain.Chapter
existing map[string]bool // "slug:N" → exists
writeMetaErr error
}
func (s *stubStore) WriteMetadata(_ context.Context, meta domain.BookMeta) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.writeMetaErr != nil {
return s.writeMetaErr
}
s.metaWritten = append(s.metaWritten, meta)
return nil
}
func (s *stubStore) WriteChapter(_ context.Context, slug string, ch domain.Chapter) error {
s.mu.Lock()
defer s.mu.Unlock()
s.chaptersWritten = append(s.chaptersWritten, ch)
return nil
}
func (s *stubStore) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error {
return nil
}
func (s *stubStore) ChapterExists(_ context.Context, slug string, ref domain.ChapterRef) bool {
s.mu.Lock()
defer s.mu.Unlock()
key := slug + ":" + string(rune('0'+ref.Number))
return s.existing[key]
}
// ── tests ──────────────────────────────────────────────────────────────────────
func TestRunBook_HappyPath(t *testing.T) {
sc := &stubScraper{
meta: domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"},
refs: []domain.ChapterRef{
{Number: 1, Title: "Ch 1", URL: "https://example.com/book/test-book/chapter-1"},
{Number: 2, Title: "Ch 2", URL: "https://example.com/book/test-book/chapter-2"},
{Number: 3, Title: "Ch 3", URL: "https://example.com/book/test-book/chapter-3"},
},
}
st := &stubStore{}
o := New(Config{Workers: 2}, sc, st, nil)
task := domain.ScrapeTask{
ID: "t1",
Kind: "book",
TargetURL: "https://example.com/book/test-book",
}
result := o.RunBook(context.Background(), task)
if result.ErrorMessage != "" {
t.Fatalf("unexpected error: %s", result.ErrorMessage)
}
if result.BooksFound != 1 {
t.Errorf("BooksFound = %d, want 1", result.BooksFound)
}
if result.ChaptersScraped != 3 {
t.Errorf("ChaptersScraped = %d, want 3", result.ChaptersScraped)
}
}
func TestRunBook_MetadataError(t *testing.T) {
sc := &stubScraper{metaErr: errors.New("404 not found")}
st := &stubStore{}
o := New(Config{Workers: 1}, sc, st, nil)
result := o.RunBook(context.Background(), domain.ScrapeTask{
ID: "t2",
TargetURL: "https://example.com/book/missing",
})
if result.ErrorMessage == "" {
t.Fatal("expected ErrorMessage to be set")
}
if result.Errors != 1 {
t.Errorf("Errors = %d, want 1", result.Errors)
}
}
func TestRunBook_ChapterRange(t *testing.T) {
sc := &stubScraper{
meta: domain.BookMeta{Slug: "range-book", SourceURL: "https://example.com/book/range-book"},
refs: func() []domain.ChapterRef {
var refs []domain.ChapterRef
for i := 1; i <= 10; i++ {
refs = append(refs, domain.ChapterRef{Number: i, URL: "https://example.com/book/range-book/chapter-" + string(rune('0'+i))})
}
return refs
}(),
}
st := &stubStore{}
o := New(Config{Workers: 2}, sc, st, nil)
result := o.RunBook(context.Background(), domain.ScrapeTask{
ID: "t3",
TargetURL: "https://example.com/book/range-book",
FromChapter: 3,
ToChapter: 7,
})
if result.ErrorMessage != "" {
t.Fatalf("unexpected error: %s", result.ErrorMessage)
}
// chapters 37 = 5 scraped, chapters 1-2 and 8-10 = 5 skipped
if result.ChaptersScraped != 5 {
t.Errorf("ChaptersScraped = %d, want 5", result.ChaptersScraped)
}
if result.ChaptersSkipped != 5 {
t.Errorf("ChaptersSkipped = %d, want 5", result.ChaptersSkipped)
}
}
func TestRunBook_ContextCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
sc := &stubScraper{
meta: domain.BookMeta{Slug: "ctx-book", SourceURL: "https://example.com/book/ctx-book"},
refs: []domain.ChapterRef{
{Number: 1, URL: "https://example.com/book/ctx-book/chapter-1"},
},
}
st := &stubStore{}
o := New(Config{Workers: 1}, sc, st, nil)
// Should not panic; result may have errors or zero chapters.
result := o.RunBook(ctx, domain.ScrapeTask{
ID: "t4",
TargetURL: "https://example.com/book/ctx-book",
})
_ = result
}
func TestRunBook_EmptyTargetURL(t *testing.T) {
o := New(Config{Workers: 1}, &stubScraper{}, &stubStore{}, nil)
result := o.RunBook(context.Background(), domain.ScrapeTask{ID: "t5"})
if result.ErrorMessage == "" {
t.Fatal("expected ErrorMessage for empty target URL")
}
}

View File

@@ -1,21 +0,0 @@
package runner
import (
"regexp"
"strings"
)
// stripMarkdown removes common markdown syntax from src, returning plain text
// suitable for TTS. Mirrors the helper in the scraper's server package.
func stripMarkdown(src string) string {
src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "")
src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "")
src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "")
src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "")
src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1")
src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "")
src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "")
src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "")
src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n")
return strings.TrimSpace(src)
}

View File

@@ -1,386 +0,0 @@
// Package runner implements the worker loop that polls PocketBase for pending
// scrape and audio tasks, executes them, and reports results back.
//
// Design:
// - Run(ctx) loops on a ticker; each tick claims and dispatches pending tasks.
// - Scrape tasks are dispatched to the Orchestrator (one goroutine per task,
// up to MaxConcurrentScrape).
// - Audio tasks fetch chapter text, call Kokoro, upload to MinIO, and report
// the result back (up to MaxConcurrentAudio goroutines).
// - The runner is stateless between ticks; all state lives in PocketBase.
package runner
import (
"context"
"fmt"
"log/slog"
"os"
"sync"
"time"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/orchestrator"
"github.com/libnovel/backend/internal/scraper"
"github.com/libnovel/backend/internal/taskqueue"
)
// Config tunes the runner behaviour.
type Config struct {
// WorkerID uniquely identifies this runner instance in PocketBase records.
WorkerID string
// PollInterval is how often the runner checks for new tasks.
PollInterval time.Duration
// MaxConcurrentScrape limits simultaneous book-scrape goroutines.
MaxConcurrentScrape int
// MaxConcurrentAudio limits simultaneous audio-generation goroutines.
MaxConcurrentAudio int
// OrchestratorWorkers is the chapter-scraping parallelism inside each book run.
OrchestratorWorkers int
// HeartbeatInterval is how often active tasks PATCH their heartbeat_at
// timestamp to signal they are still alive. Defaults to 30s when 0.
HeartbeatInterval time.Duration
// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
// task is considered orphaned and reset to pending. Defaults to 2m when 0.
StaleTaskThreshold time.Duration
}
// Dependencies are the external services the runner depends on.
type Dependencies struct {
// Consumer claims tasks from PocketBase.
Consumer taskqueue.Consumer
// BookWriter persists scraped data (used by orchestrator).
BookWriter bookstore.BookWriter
// BookReader reads chapter text for audio generation.
BookReader bookstore.BookReader
// AudioStore persists generated audio and checks key existence.
AudioStore bookstore.AudioStore
// Novel is the scraper implementation.
Novel scraper.NovelScraper
// Kokoro is the TTS client.
Kokoro kokoro.Client
// Log is the structured logger.
Log *slog.Logger
}
// Runner is the main worker process.
type Runner struct {
cfg Config
deps Dependencies
}
// New creates a Runner from cfg and deps.
// Any zero/nil field in deps will cause a panic at construction time to fail fast.
func New(cfg Config, deps Dependencies) *Runner {
if cfg.PollInterval <= 0 {
cfg.PollInterval = 30 * time.Second
}
if cfg.MaxConcurrentScrape <= 0 {
cfg.MaxConcurrentScrape = 2
}
if cfg.MaxConcurrentAudio <= 0 {
cfg.MaxConcurrentAudio = 1
}
if cfg.WorkerID == "" {
cfg.WorkerID = "runner"
}
if cfg.HeartbeatInterval <= 0 {
cfg.HeartbeatInterval = 30 * time.Second
}
if cfg.StaleTaskThreshold <= 0 {
cfg.StaleTaskThreshold = 2 * time.Minute
}
if deps.Log == nil {
deps.Log = slog.Default()
}
return &Runner{cfg: cfg, deps: deps}
}
// livenessFile is the path written on every successful poll so that the Docker
// healthcheck (CMD /healthcheck file /tmp/runner.alive <max_age>) can verify
// the runner is still making progress.
const livenessFile = "/tmp/runner.alive"
// touchAlive writes the current UTC time to livenessFile. Errors are logged but
// never fatal — liveness is best-effort and should not crash the runner.
func (r *Runner) touchAlive() {
data := []byte(time.Now().UTC().Format(time.RFC3339))
if err := os.WriteFile(livenessFile, data, 0o644); err != nil {
r.deps.Log.Warn("runner: failed to write liveness file", "err", err)
}
}
// Run starts the poll loop, blocking until ctx is cancelled.
// On each tick it claims and executes all available pending tasks.
// Scrape and audio tasks run in separate goroutine pools bounded by
// MaxConcurrentScrape and MaxConcurrentAudio respectively.
func (r *Runner) Run(ctx context.Context) error {
r.deps.Log.Info("runner: starting",
"worker_id", r.cfg.WorkerID,
"poll_interval", r.cfg.PollInterval,
"max_scrape", r.cfg.MaxConcurrentScrape,
"max_audio", r.cfg.MaxConcurrentAudio,
)
scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio)
var wg sync.WaitGroup
// Write liveness file immediately so the first healthcheck passes before
// the first poll completes.
r.touchAlive()
tick := time.NewTicker(r.cfg.PollInterval)
defer tick.Stop()
// Run one poll immediately on startup, then on each tick.
for {
r.poll(ctx, scrapeSem, audioSem, &wg)
r.touchAlive()
select {
case <-ctx.Done():
r.deps.Log.Info("runner: context cancelled, draining active tasks")
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
r.deps.Log.Info("runner: all tasks drained, exiting")
case <-time.After(2 * time.Minute):
r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
}
return nil
case <-tick.C:
}
}
}
// poll claims all available pending tasks and dispatches them to goroutines.
// It claims tasks in a tight loop until no more are available.
func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) {
// ── Reap orphaned tasks ───────────────────────────────────────────────
if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
r.deps.Log.Warn("runner: reap stale tasks failed", "err", err)
} else if n > 0 {
r.deps.Log.Info("runner: reaped stale tasks", "count", n)
}
// ── Scrape tasks ──────────────────────────────────────────────────────
for {
if ctx.Err() != nil {
return
}
task, ok, err := r.deps.Consumer.ClaimNextScrapeTask(ctx, r.cfg.WorkerID)
if err != nil {
r.deps.Log.Error("runner: ClaimNextScrapeTask failed", "err", err)
break
}
if !ok {
break // queue empty
}
// Acquire semaphore (non-blocking when full — leave task running).
select {
case scrapeSem <- struct{}{}:
default:
// Too many concurrent scrapes — the task stays claimed but we can't
// run it right now. Log and break; the next poll will pick it up if
// still running (it won't be re-claimed while status=running).
r.deps.Log.Warn("runner: scrape semaphore full, will retry next tick",
"task_id", task.ID)
break
}
wg.Add(1)
go func(t domain.ScrapeTask) {
defer wg.Done()
defer func() { <-scrapeSem }()
r.runScrapeTask(ctx, t)
}(task)
}
// ── Audio tasks ───────────────────────────────────────────────────────
for {
if ctx.Err() != nil {
return
}
task, ok, err := r.deps.Consumer.ClaimNextAudioTask(ctx, r.cfg.WorkerID)
if err != nil {
r.deps.Log.Error("runner: ClaimNextAudioTask failed", "err", err)
break
}
if !ok {
break // queue empty
}
select {
case audioSem <- struct{}{}:
default:
r.deps.Log.Warn("runner: audio semaphore full, will retry next tick",
"task_id", task.ID)
break
}
wg.Add(1)
go func(t domain.AudioTask) {
defer wg.Done()
defer func() { <-audioSem }()
r.runAudioTask(ctx, t)
}(task)
}
}
// runScrapeTask executes one scrape task end-to-end and reports the result.
func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
log := r.deps.Log.With("task_id", task.ID, "kind", task.Kind, "url", task.TargetURL)
log.Info("runner: scrape task starting")
// Heartbeat goroutine: periodically PATCH heartbeat_at so the reaper knows
// this task is still alive. Cancelled when the task finishes.
hbCtx, hbCancel := context.WithCancel(ctx)
defer hbCancel()
go func() {
tick := time.NewTicker(r.cfg.HeartbeatInterval)
defer tick.Stop()
for {
select {
case <-hbCtx.Done():
return
case <-tick.C:
if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil {
log.Warn("runner: heartbeat failed", "err", err)
}
}
}
}()
oCfg := orchestrator.Config{Workers: r.cfg.OrchestratorWorkers}
o := orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log)
var result domain.ScrapeResult
switch task.Kind {
case "catalogue":
result = r.runCatalogueTask(ctx, task, o, log)
case "book", "book_range":
result = o.RunBook(ctx, task)
default:
result.ErrorMessage = fmt.Sprintf("unknown task kind: %q", task.Kind)
log.Warn("runner: unknown task kind")
}
if err := r.deps.Consumer.FinishScrapeTask(ctx, task.ID, result); err != nil {
log.Error("runner: FinishScrapeTask failed", "err", err)
}
log.Info("runner: scrape task finished",
"scraped", result.ChaptersScraped,
"skipped", result.ChaptersSkipped,
"errors", result.Errors,
)
}
// runCatalogueTask runs a full catalogue scrape by iterating catalogue entries
// and running a book task for each one.
func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o *orchestrator.Orchestrator, log *slog.Logger) domain.ScrapeResult {
entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
var result domain.ScrapeResult
for entry := range entries {
if ctx.Err() != nil {
break
}
bookTask := domain.ScrapeTask{
ID: task.ID,
Kind: "book",
TargetURL: entry.URL,
}
bookResult := o.RunBook(ctx, bookTask)
result.BooksFound += bookResult.BooksFound + 1
result.ChaptersScraped += bookResult.ChaptersScraped
result.ChaptersSkipped += bookResult.ChaptersSkipped
result.Errors += bookResult.Errors
}
if err := <-errCh; err != nil {
log.Warn("runner: catalogue scrape finished with error", "err", err)
result.Errors++
if result.ErrorMessage == "" {
result.ErrorMessage = err.Error()
}
}
return result
}
// runAudioTask executes one audio-generation task:
// 1. Read chapter text from MinIO.
// 2. Call Kokoro to generate audio.
// 3. Upload MP3 to MinIO under the standard audio object key.
// 4. Report result back to PocketBase.
func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "voice", task.Voice)
log.Info("runner: audio task starting")
// Heartbeat goroutine: periodically PATCH heartbeat_at so the reaper knows
// this task is still alive. Cancelled when the task finishes.
hbCtx, hbCancel := context.WithCancel(ctx)
defer hbCancel()
go func() {
tick := time.NewTicker(r.cfg.HeartbeatInterval)
defer tick.Stop()
for {
select {
case <-hbCtx.Done():
return
case <-tick.C:
if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil {
log.Warn("runner: heartbeat failed", "err", err)
}
}
}
}()
fail := func(msg string) {
log.Error("runner: audio task failed", "reason", msg)
result := domain.AudioResult{ErrorMessage: msg}
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
log.Error("runner: FinishAudioTask failed", "err", err)
}
}
// Step 1: read chapter text.
raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter)
if err != nil {
fail(fmt.Sprintf("read chapter: %v", err))
return
}
text := stripMarkdown(raw)
if text == "" {
fail("chapter text is empty after stripping markdown")
return
}
// Step 2: generate audio.
if r.deps.Kokoro == nil {
fail("kokoro client not configured")
return
}
audioData, err := r.deps.Kokoro.GenerateAudio(ctx, text, task.Voice)
if err != nil {
fail(fmt.Sprintf("kokoro generate: %v", err))
return
}
// Step 3: upload to MinIO.
key := r.deps.AudioStore.AudioObjectKey(task.Slug, task.Chapter, task.Voice)
if err := r.deps.AudioStore.PutAudio(ctx, key, audioData); err != nil {
fail(fmt.Sprintf("put audio: %v", err))
return
}
// Step 4: report success.
result := domain.AudioResult{ObjectKey: key}
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
log.Error("runner: FinishAudioTask failed", "err", err)
}
log.Info("runner: audio task finished", "key", key)
}

View File

@@ -1,365 +0,0 @@
package runner_test
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/runner"
)
// ── Stub types ────────────────────────────────────────────────────────────────
// stubConsumer is a test double for taskqueue.Consumer.
type stubConsumer struct {
scrapeQueue []domain.ScrapeTask
audioQueue []domain.AudioTask
scrapeIdx int
audioIdx int
finished []string
failCalled []string
claimErr error
}
func (s *stubConsumer) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
if s.claimErr != nil {
return domain.ScrapeTask{}, false, s.claimErr
}
if s.scrapeIdx >= len(s.scrapeQueue) {
return domain.ScrapeTask{}, false, nil
}
t := s.scrapeQueue[s.scrapeIdx]
s.scrapeIdx++
return t, true, nil
}
func (s *stubConsumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
if s.claimErr != nil {
return domain.AudioTask{}, false, s.claimErr
}
if s.audioIdx >= len(s.audioQueue) {
return domain.AudioTask{}, false, nil
}
t := s.audioQueue[s.audioIdx]
s.audioIdx++
return t, true, nil
}
func (s *stubConsumer) FinishScrapeTask(_ context.Context, id string, _ domain.ScrapeResult) error {
s.finished = append(s.finished, id)
return nil
}
func (s *stubConsumer) FinishAudioTask(_ context.Context, id string, _ domain.AudioResult) error {
s.finished = append(s.finished, id)
return nil
}
func (s *stubConsumer) FailTask(_ context.Context, id, _ string) error {
s.failCalled = append(s.failCalled, id)
return nil
}
func (s *stubConsumer) HeartbeatTask(_ context.Context, _ string) error { return nil }
func (s *stubConsumer) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) {
return 0, nil
}
// stubBookWriter satisfies bookstore.BookWriter (no-op).
type stubBookWriter struct{}
func (s *stubBookWriter) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil }
func (s *stubBookWriter) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error {
return nil
}
func (s *stubBookWriter) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error {
return nil
}
func (s *stubBookWriter) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool {
return false
}
// stubBookReader satisfies bookstore.BookReader — returns a single chapter.
type stubBookReader struct {
text string
readErr error
}
func (s *stubBookReader) ReadChapter(_ context.Context, _ string, _ int) (string, error) {
return s.text, s.readErr
}
func (s *stubBookReader) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) {
return domain.BookMeta{}, false, nil
}
func (s *stubBookReader) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil }
func (s *stubBookReader) LocalSlugs(_ context.Context) (map[string]bool, error) { return nil, nil }
func (s *stubBookReader) MetadataMtime(_ context.Context, _ string) int64 { return 0 }
func (s *stubBookReader) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) {
return nil, nil
}
func (s *stubBookReader) CountChapters(_ context.Context, _ string) int { return 0 }
func (s *stubBookReader) ReindexChapters(_ context.Context, _ string) (int, error) {
return 0, nil
}
// stubAudioStore satisfies bookstore.AudioStore.
type stubAudioStore struct {
putCalled atomic.Int32
putErr error
}
func (s *stubAudioStore) AudioObjectKey(slug string, n int, voice string) string {
return slug + "/" + string(rune('0'+n)) + "/" + voice + ".mp3"
}
func (s *stubAudioStore) AudioExists(_ context.Context, _ string) bool { return false }
func (s *stubAudioStore) PutAudio(_ context.Context, _ string, _ []byte) error {
s.putCalled.Add(1)
return s.putErr
}
// stubNovelScraper satisfies scraper.NovelScraper minimally.
type stubNovelScraper struct {
entries []domain.CatalogueEntry
metaErr error
chapters []domain.ChapterRef
}
func (s *stubNovelScraper) ScrapeCatalogue(_ context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
ch := make(chan domain.CatalogueEntry, len(s.entries))
errCh := make(chan error, 1)
for _, e := range s.entries {
ch <- e
}
close(ch)
close(errCh)
return ch, errCh
}
func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) {
if s.metaErr != nil {
return domain.BookMeta{}, s.metaErr
}
return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil
}
func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
return s.chapters, nil
}
func (s *stubNovelScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
return domain.Chapter{Ref: ref, Text: "# Chapter\n\nSome text."}, nil
}
func (s *stubNovelScraper) ScrapeRanking(_ context.Context, _ int) (<-chan domain.BookMeta, <-chan error) {
ch := make(chan domain.BookMeta)
errCh := make(chan error, 1)
close(ch)
close(errCh)
return ch, errCh
}
func (s *stubNovelScraper) SourceName() string { return "stub" }
// stubKokoro satisfies kokoro.Client.
type stubKokoro struct {
data []byte
genErr error
called atomic.Int32
}
func (s *stubKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
s.called.Add(1)
return s.data, s.genErr
}
func (s *stubKokoro) ListVoices(_ context.Context) ([]string, error) {
return []string{"af_bella"}, nil
}
// ── stripMarkdown helper ──────────────────────────────────────────────────────
func TestStripMarkdownViaAudioTask(t *testing.T) {
// Verify markdown is stripped before sending to Kokoro.
// We inject chapter text with markdown; the kokoro stub verifies data flows.
consumer := &stubConsumer{
audioQueue: []domain.AudioTask{
{ID: "a1", Slug: "book", Chapter: 1, Voice: "af_bella", Status: domain.TaskStatusRunning},
},
}
bookReader := &stubBookReader{text: "## Chapter 1\n\nPlain **text** here."}
audioStore := &stubAudioStore{}
kokoroStub := &stubKokoro{data: []byte("mp3")}
cfg := runner.Config{
WorkerID: "test",
PollInterval: time.Hour, // long poll — we'll cancel manually
}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: bookReader,
AudioStore: audioStore,
Novel: &stubNovelScraper{},
Kokoro: kokoroStub,
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = r.Run(ctx)
if kokoroStub.called.Load() != 1 {
t.Errorf("expected Kokoro.GenerateAudio called once, got %d", kokoroStub.called.Load())
}
if audioStore.putCalled.Load() != 1 {
t.Errorf("expected PutAudio called once, got %d", audioStore.putCalled.Load())
}
}
func TestAudioTask_ReadChapterError(t *testing.T) {
consumer := &stubConsumer{
audioQueue: []domain.AudioTask{
{ID: "a2", Slug: "book", Chapter: 2, Voice: "af_bella", Status: domain.TaskStatusRunning},
},
}
bookReader := &stubBookReader{readErr: errors.New("chapter not found")}
audioStore := &stubAudioStore{}
kokoroStub := &stubKokoro{data: []byte("mp3")}
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: bookReader,
AudioStore: audioStore,
Novel: &stubNovelScraper{},
Kokoro: kokoroStub,
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = r.Run(ctx)
// Kokoro should not be called; FinishAudioTask should be called with error.
if kokoroStub.called.Load() != 0 {
t.Errorf("expected Kokoro not called, got %d", kokoroStub.called.Load())
}
if len(consumer.finished) != 1 {
t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished))
}
}
func TestAudioTask_KokoroError(t *testing.T) {
consumer := &stubConsumer{
audioQueue: []domain.AudioTask{
{ID: "a3", Slug: "book", Chapter: 3, Voice: "af_bella", Status: domain.TaskStatusRunning},
},
}
bookReader := &stubBookReader{text: "Chapter text."}
audioStore := &stubAudioStore{}
kokoroStub := &stubKokoro{genErr: errors.New("tts failed")}
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: bookReader,
AudioStore: audioStore,
Novel: &stubNovelScraper{},
Kokoro: kokoroStub,
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = r.Run(ctx)
if audioStore.putCalled.Load() != 0 {
t.Errorf("expected PutAudio not called, got %d", audioStore.putCalled.Load())
}
if len(consumer.finished) != 1 {
t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished))
}
}
func TestScrapeTask_BookKind(t *testing.T) {
consumer := &stubConsumer{
scrapeQueue: []domain.ScrapeTask{
{ID: "s1", Kind: "book", TargetURL: "https://example.com/book/test-book", Status: domain.TaskStatusRunning},
},
}
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: &stubBookReader{},
AudioStore: &stubAudioStore{},
Novel: &stubNovelScraper{},
Kokoro: &stubKokoro{},
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = r.Run(ctx)
if len(consumer.finished) != 1 || consumer.finished[0] != "s1" {
t.Errorf("expected task s1 finished, got %v", consumer.finished)
}
}
func TestScrapeTask_UnknownKind(t *testing.T) {
consumer := &stubConsumer{
scrapeQueue: []domain.ScrapeTask{
{ID: "s2", Kind: "unknown_kind", Status: domain.TaskStatusRunning},
},
}
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: &stubBookReader{},
AudioStore: &stubAudioStore{},
Novel: &stubNovelScraper{},
Kokoro: &stubKokoro{},
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = r.Run(ctx)
// Unknown kind still finishes the task (with error message in result).
if len(consumer.finished) != 1 || consumer.finished[0] != "s2" {
t.Errorf("expected task s2 finished, got %v", consumer.finished)
}
}
func TestRun_CancelImmediately(t *testing.T) {
consumer := &stubConsumer{}
cfg := runner.Config{WorkerID: "test", PollInterval: 10 * time.Millisecond}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: &stubBookReader{},
AudioStore: &stubAudioStore{},
Novel: &stubNovelScraper{},
Kokoro: &stubKokoro{},
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel before Run
err := r.Run(ctx)
if err != nil {
t.Errorf("expected nil on graceful shutdown, got %v", err)
}
}

View File

@@ -1,58 +0,0 @@
// Package scraper defines the NovelScraper interface and its sub-interfaces.
// Domain types live in internal/domain — this package only defines the scraping
// contract so that novelfire and any future scrapers can be swapped freely.
package scraper
import (
"context"
"github.com/libnovel/backend/internal/domain"
)
// CatalogueProvider can enumerate every novel available on a source site.
type CatalogueProvider interface {
ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueEntry, <-chan error)
}
// MetadataProvider can extract structured book metadata from a novel's landing page.
type MetadataProvider interface {
ScrapeMetadata(ctx context.Context, bookURL string) (domain.BookMeta, error)
}
// ChapterListProvider can enumerate all chapters of a book.
type ChapterListProvider interface {
ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error)
}
// ChapterTextProvider can extract the readable text from a single chapter page.
type ChapterTextProvider interface {
ScrapeChapterText(ctx context.Context, ref domain.ChapterRef) (domain.Chapter, error)
}
// RankingProvider can enumerate novels from a ranking page.
type RankingProvider interface {
// ScrapeRanking pages through up to maxPages ranking pages.
// maxPages <= 0 means all pages.
ScrapeRanking(ctx context.Context, maxPages int) (<-chan domain.BookMeta, <-chan error)
}
// NovelScraper is the full interface a concrete novel source must implement.
type NovelScraper interface {
CatalogueProvider
MetadataProvider
ChapterListProvider
ChapterTextProvider
RankingProvider
// SourceName returns the human-readable name of this scraper, e.g. "novelfire.net".
SourceName() string
}
// Selector describes how to locate an element in an HTML document.
type Selector struct {
Tag string
Class string
ID string
Attr string
Multiple bool
}

View File

@@ -1,194 +0,0 @@
package storage
import (
"context"
"fmt"
"io"
"net/url"
"path"
"strings"
"time"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/libnovel/backend/internal/config"
)
// minioClient wraps the official minio-go client with bucket names.
type minioClient struct {
client *minio.Client // internal — all read/write operations
pubClient *minio.Client // presign-only — initialised against the public endpoint
bucketChapters string
bucketAudio string
bucketAvatars string
}
func newMinioClient(cfg config.MinIO) (*minioClient, error) {
creds := credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, "")
internal, err := minio.New(cfg.Endpoint, &minio.Options{
Creds: creds,
Secure: cfg.UseSSL,
})
if err != nil {
return nil, fmt.Errorf("minio: init internal client: %w", err)
}
// Presigned URLs must be signed with the hostname the browser will use
// (PUBLIC_MINIO_PUBLIC_URL), because AWS Signature V4 includes the Host
// header in the canonical request — a URL signed against "minio:9000" will
// return SignatureDoesNotMatch when the browser fetches it from
// "localhost:9000".
//
// However, minio-go normally makes a live BucketLocation HTTP call before
// signing, which would fail from inside the container when the public
// endpoint is externally-facing (e.g. "localhost:9000" is unreachable from
// within Docker). We prevent this by:
// 1. Setting Region: "us-east-1" — minio-go skips getBucketLocation when
// the region is already known (bucket-cache.go:49).
// 2. Setting BucketLookup: BucketLookupPath — forces path-style URLs
// (e.g. host/bucket/key), matching MinIO's default behaviour and
// avoiding any virtual-host DNS probing.
//
// When no public endpoint is configured (or it equals the internal one),
// fall back to the internal client so presigning still works.
publicEndpoint := cfg.PublicEndpoint
if u, err2 := url.Parse(publicEndpoint); err2 == nil && u.Host != "" {
publicEndpoint = u.Host // strip scheme so minio.New is happy
}
pubUseSSL := cfg.PublicUseSSL
if publicEndpoint == "" || publicEndpoint == cfg.Endpoint {
publicEndpoint = cfg.Endpoint
pubUseSSL = cfg.UseSSL
}
pub, err := minio.New(publicEndpoint, &minio.Options{
Creds: creds,
Secure: pubUseSSL,
Region: "us-east-1", // skip live BucketLocation preflight
BucketLookup: minio.BucketLookupPath,
})
if err != nil {
return nil, fmt.Errorf("minio: init public client: %w", err)
}
return &minioClient{
client: internal,
pubClient: pub,
bucketChapters: cfg.BucketChapters,
bucketAudio: cfg.BucketAudio,
bucketAvatars: cfg.BucketAvatars,
}, nil
}
// ensureBuckets creates all required buckets if they don't already exist.
func (m *minioClient) ensureBuckets(ctx context.Context) error {
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars} {
exists, err := m.client.BucketExists(ctx, bucket)
if err != nil {
return fmt.Errorf("minio: check bucket %q: %w", bucket, err)
}
if !exists {
if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
return fmt.Errorf("minio: create bucket %q: %w", bucket, err)
}
}
}
return nil
}
// ── Key helpers ───────────────────────────────────────────────────────────────
// ChapterObjectKey returns the MinIO object key for a chapter markdown file.
// Format: {slug}/chapter-{n:06d}.md
func ChapterObjectKey(slug string, n int) string {
return fmt.Sprintf("%s/chapter-%06d.md", slug, n)
}
// AudioObjectKey returns the MinIO object key for a cached audio file.
// Format: {slug}/{n}/{voice}.mp3
func AudioObjectKey(slug string, n int, voice string) string {
return fmt.Sprintf("%s/%d/%s.mp3", slug, n, voice)
}
// AvatarObjectKey returns the MinIO object key for a user avatar image.
// Format: {userID}/{ext}.{ext}
func AvatarObjectKey(userID, ext string) string {
return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
}
// chapterNumberFromKey extracts the chapter number from a MinIO object key.
// e.g. "my-book/chapter-000042.md" → 42
func chapterNumberFromKey(key string) int {
base := path.Base(key)
base = strings.TrimPrefix(base, "chapter-")
base = strings.TrimSuffix(base, ".md")
var n int
fmt.Sscanf(base, "%d", &n)
return n
}
// ── Object operations ─────────────────────────────────────────────────────────
func (m *minioClient) putObject(ctx context.Context, bucket, key, contentType string, data []byte) error {
_, err := m.client.PutObject(ctx, bucket, key,
strings.NewReader(string(data)),
int64(len(data)),
minio.PutObjectOptions{ContentType: contentType},
)
return err
}
func (m *minioClient) getObject(ctx context.Context, bucket, key string) ([]byte, error) {
obj, err := m.client.GetObject(ctx, bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
defer obj.Close()
return io.ReadAll(obj)
}
func (m *minioClient) objectExists(ctx context.Context, bucket, key string) bool {
_, err := m.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{})
return err == nil
}
func (m *minioClient) presignGet(ctx context.Context, bucket, key string, expires time.Duration) (string, error) {
u, err := m.pubClient.PresignedGetObject(ctx, bucket, key, expires, nil)
if err != nil {
return "", fmt.Errorf("minio presign %s/%s: %w", bucket, key, err)
}
return u.String(), nil
}
func (m *minioClient) presignPut(ctx context.Context, bucket, key string, expires time.Duration) (string, error) {
u, err := m.pubClient.PresignedPutObject(ctx, bucket, key, expires)
if err != nil {
return "", fmt.Errorf("minio presign PUT %s/%s: %w", bucket, key, err)
}
return u.String(), nil
}
func (m *minioClient) deleteObjects(ctx context.Context, bucket, prefix string) error {
objCh := m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix})
for obj := range objCh {
if obj.Err != nil {
return obj.Err
}
if err := m.client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}); err != nil {
return err
}
}
return nil
}
func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string) ([]string, error) {
var keys []string
for obj := range m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) {
if obj.Err != nil {
return nil, obj.Err
}
keys = append(keys, obj.Key)
}
return keys, nil
}

View File

@@ -1,268 +0,0 @@
// Package storage provides the concrete implementations of all bookstore and
// taskqueue interfaces backed by PocketBase (structured data) and MinIO (blobs).
//
// Entry point: NewStore(ctx, cfg, log) returns a *Store that satisfies every
// interface defined in bookstore and taskqueue.
package storage
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/libnovel/backend/internal/config"
"github.com/libnovel/backend/internal/domain"
)
// ErrNotFound is returned by single-record lookups when no record exists.
var ErrNotFound = errors.New("storage: record not found")
// pbClient is the internal PocketBase REST admin client.
type pbClient struct {
baseURL string
email string
password string
log *slog.Logger
mu sync.Mutex
token string
exp time.Time
}
func newPBClient(cfg config.PocketBase, log *slog.Logger) *pbClient {
return &pbClient{
baseURL: strings.TrimRight(cfg.URL, "/"),
email: cfg.AdminEmail,
password: cfg.AdminPassword,
log: log,
}
}
// authToken returns a valid admin auth token, refreshing it when expired.
func (c *pbClient) authToken(ctx context.Context) (string, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.token != "" && time.Now().Before(c.exp) {
return c.token, nil
}
body, _ := json.Marshal(map[string]string{
"identity": c.email,
"password": c.password,
})
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
c.baseURL+"/api/collections/_superusers/auth-with-password", bytes.NewReader(body))
if err != nil {
return "", fmt.Errorf("pb auth: build request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", fmt.Errorf("pb auth: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
raw, _ := io.ReadAll(resp.Body)
return "", fmt.Errorf("pb auth: status %d: %s", resp.StatusCode, string(raw))
}
var payload struct {
Token string `json:"token"`
}
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
return "", fmt.Errorf("pb auth: decode: %w", err)
}
c.token = payload.Token
c.exp = time.Now().Add(30 * time.Minute)
return c.token, nil
}
// do executes an authenticated PocketBase REST request.
func (c *pbClient) do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
tok, err := c.authToken(ctx)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
if err != nil {
return nil, fmt.Errorf("pb: build request %s %s: %w", method, path, err)
}
req.Header.Set("Authorization", tok)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("pb: %s %s: %w", method, path, err)
}
return resp, nil
}
// get is a convenience wrapper that decodes a JSON response into v.
func (c *pbClient) get(ctx context.Context, path string, v any) error {
resp, err := c.do(ctx, http.MethodGet, path, nil)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return ErrNotFound
}
if resp.StatusCode >= 400 {
raw, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pb GET %s: status %d: %s", path, resp.StatusCode, string(raw))
}
return json.NewDecoder(resp.Body).Decode(v)
}
// post creates a record and decodes the created record into v.
func (c *pbClient) post(ctx context.Context, path string, payload, v any) error {
b, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("pb: marshal: %w", err)
}
resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b))
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
raw, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pb POST %s: status %d: %s", path, resp.StatusCode, string(raw))
}
if v != nil {
return json.NewDecoder(resp.Body).Decode(v)
}
return nil
}
// patch updates a record.
func (c *pbClient) patch(ctx context.Context, path string, payload any) error {
b, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("pb: marshal: %w", err)
}
resp, err := c.do(ctx, http.MethodPatch, path, bytes.NewReader(b))
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
raw, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pb PATCH %s: status %d: %s", path, resp.StatusCode, string(raw))
}
return nil
}
// delete removes a record.
func (c *pbClient) delete(ctx context.Context, path string) error {
resp, err := c.do(ctx, http.MethodDelete, path, nil)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return ErrNotFound
}
if resp.StatusCode >= 400 {
raw, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pb DELETE %s: status %d: %s", path, resp.StatusCode, string(raw))
}
return nil
}
// listAll fetches all pages of a collection. PocketBase returns at most 200
// records per page; we paginate until empty.
func (c *pbClient) listAll(ctx context.Context, collection string, filter, sort string) ([]json.RawMessage, error) {
var all []json.RawMessage
page := 1
for {
q := url.Values{
"page": {fmt.Sprintf("%d", page)},
"perPage": {"200"},
}
if filter != "" {
q.Set("filter", filter)
}
if sort != "" {
q.Set("sort", sort)
}
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
var result struct {
Items []json.RawMessage `json:"items"`
Page int `json:"page"`
Pages int `json:"totalPages"`
}
if err := c.get(ctx, path, &result); err != nil {
return nil, err
}
all = append(all, result.Items...)
if result.Page >= result.Pages {
break
}
page++
}
return all, nil
}
// claimRecord atomically claims the first pending record matching collection.
// It fetches the oldest pending record (filter + sort), then PATCHes it with
// the claim payload. Returns (nil, nil) when the queue is empty.
func (c *pbClient) claimRecord(ctx context.Context, collection, workerID string, extraClaim map[string]any) (json.RawMessage, error) {
q := url.Values{}
q.Set("filter", `status="pending"`)
q.Set("sort", "+started")
q.Set("perPage", "1")
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
var result struct {
Items []json.RawMessage `json:"items"`
}
if err := c.get(ctx, path, &result); err != nil {
return nil, fmt.Errorf("claimRecord list: %w", err)
}
if len(result.Items) == 0 {
return nil, nil // queue empty
}
var rec struct {
ID string `json:"id"`
}
if err := json.Unmarshal(result.Items[0], &rec); err != nil {
return nil, fmt.Errorf("claimRecord parse id: %w", err)
}
claim := map[string]any{
"status": string(domain.TaskStatusRunning),
"worker_id": workerID,
}
for k, v := range extraClaim {
claim[k] = v
}
claimPath := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID)
if err := c.patch(ctx, claimPath, claim); err != nil {
return nil, fmt.Errorf("claimRecord patch: %w", err)
}
// Re-fetch the updated record so caller has current state.
var updated json.RawMessage
if err := c.get(ctx, claimPath, &updated); err != nil {
return nil, fmt.Errorf("claimRecord re-fetch: %w", err)
}
return updated, nil
}

View File

@@ -1,769 +0,0 @@
package storage
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"strings"
"time"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/config"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/taskqueue"
)
// Store is the unified persistence implementation that satisfies all bookstore
// and taskqueue interfaces. It routes structured data to PocketBase and binary
// blobs to MinIO.
type Store struct {
pb *pbClient
mc *minioClient
log *slog.Logger
}
// NewStore initialises PocketBase and MinIO connections and ensures all MinIO
// buckets exist. Returns a ready-to-use Store.
func NewStore(ctx context.Context, cfg config.Config, log *slog.Logger) (*Store, error) {
pb := newPBClient(cfg.PocketBase, log)
// Validate PocketBase connectivity by fetching an auth token.
if _, err := pb.authToken(ctx); err != nil {
return nil, fmt.Errorf("pocketbase: %w", err)
}
mc, err := newMinioClient(cfg.MinIO)
if err != nil {
return nil, fmt.Errorf("minio: %w", err)
}
if err := mc.ensureBuckets(ctx); err != nil {
return nil, fmt.Errorf("minio: ensure buckets: %w", err)
}
return &Store{pb: pb, mc: mc, log: log}, nil
}
// Compile-time interface satisfaction.
var _ bookstore.BookWriter = (*Store)(nil)
var _ bookstore.BookReader = (*Store)(nil)
var _ bookstore.RankingStore = (*Store)(nil)
var _ bookstore.AudioStore = (*Store)(nil)
var _ bookstore.PresignStore = (*Store)(nil)
var _ bookstore.ProgressStore = (*Store)(nil)
var _ taskqueue.Producer = (*Store)(nil)
var _ taskqueue.Consumer = (*Store)(nil)
var _ taskqueue.Reader = (*Store)(nil)
// ── BookWriter ────────────────────────────────────────────────────────────────
func (s *Store) WriteMetadata(ctx context.Context, meta domain.BookMeta) error {
payload := map[string]any{
"slug": meta.Slug,
"title": meta.Title,
"author": meta.Author,
"cover": meta.Cover,
"status": meta.Status,
"genres": meta.Genres,
"summary": meta.Summary,
"total_chapters": meta.TotalChapters,
"source_url": meta.SourceURL,
"ranking": meta.Ranking,
}
// Upsert via filter: if exists PATCH, otherwise POST.
existing, err := s.getBookBySlug(ctx, meta.Slug)
if err != nil && err != ErrNotFound {
return fmt.Errorf("WriteMetadata: %w", err)
}
if err == ErrNotFound {
return s.pb.post(ctx, "/api/collections/books/records", payload, nil)
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", existing.ID), payload)
}
func (s *Store) WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error {
key := ChapterObjectKey(slug, chapter.Ref.Number)
if err := s.mc.putObject(ctx, s.mc.bucketChapters, key, "text/markdown", []byte(chapter.Text)); err != nil {
return fmt.Errorf("WriteChapter: minio: %w", err)
}
// Upsert the chapters_idx record in PocketBase.
return s.upsertChapterIdx(ctx, slug, chapter.Ref)
}
func (s *Store) WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error {
for _, ref := range refs {
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
s.log.Warn("WriteChapterRefs: upsert failed", "slug", slug, "chapter", ref.Number, "err", err)
}
}
return nil
}
func (s *Store) ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool {
return s.mc.objectExists(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, ref.Number))
}
func (s *Store) upsertChapterIdx(ctx context.Context, slug string, ref domain.ChapterRef) error {
payload := map[string]any{
"slug": slug,
"number": ref.Number,
"title": ref.Title,
}
filter := fmt.Sprintf(`slug=%q&&number=%d`, slug, ref.Number)
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "")
if err != nil && err != ErrNotFound {
return err
}
if len(items) == 0 {
return s.pb.post(ctx, "/api/collections/chapters_idx/records", payload, nil)
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/chapters_idx/records/%s", rec.ID), payload)
}
// ── BookReader ────────────────────────────────────────────────────────────────
type pbBook struct {
ID string `json:"id"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author"`
Cover string `json:"cover"`
Status string `json:"status"`
Genres []string `json:"genres"`
Summary string `json:"summary"`
TotalChapters int `json:"total_chapters"`
SourceURL string `json:"source_url"`
Ranking int `json:"ranking"`
Updated string `json:"updated"`
}
func (b pbBook) toDomain() domain.BookMeta {
return domain.BookMeta{
Slug: b.Slug,
Title: b.Title,
Author: b.Author,
Cover: b.Cover,
Status: b.Status,
Genres: b.Genres,
Summary: b.Summary,
TotalChapters: b.TotalChapters,
SourceURL: b.SourceURL,
Ranking: b.Ranking,
}
}
func (s *Store) getBookBySlug(ctx context.Context, slug string) (pbBook, error) {
filter := fmt.Sprintf(`slug=%q`, slug)
items, err := s.pb.listAll(ctx, "books", filter, "")
if err != nil {
return pbBook{}, err
}
if len(items) == 0 {
return pbBook{}, ErrNotFound
}
var b pbBook
json.Unmarshal(items[0], &b)
return b, nil
}
func (s *Store) ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error) {
b, err := s.getBookBySlug(ctx, slug)
if err == ErrNotFound {
return domain.BookMeta{}, false, nil
}
if err != nil {
return domain.BookMeta{}, false, err
}
return b.toDomain(), true, nil
}
func (s *Store) ListBooks(ctx context.Context) ([]domain.BookMeta, error) {
items, err := s.pb.listAll(ctx, "books", "", "title")
if err != nil {
return nil, err
}
books := make([]domain.BookMeta, 0, len(items))
for _, raw := range items {
var b pbBook
json.Unmarshal(raw, &b)
books = append(books, b.toDomain())
}
return books, nil
}
func (s *Store) LocalSlugs(ctx context.Context) (map[string]bool, error) {
items, err := s.pb.listAll(ctx, "books", "", "")
if err != nil {
return nil, err
}
slugs := make(map[string]bool, len(items))
for _, raw := range items {
var b struct {
Slug string `json:"slug"`
}
json.Unmarshal(raw, &b)
if b.Slug != "" {
slugs[b.Slug] = true
}
}
return slugs, nil
}
func (s *Store) MetadataMtime(ctx context.Context, slug string) int64 {
b, err := s.getBookBySlug(ctx, slug)
if err != nil {
return 0
}
t, err := time.Parse(time.RFC3339, b.Updated)
if err != nil {
return 0
}
return t.Unix()
}
func (s *Store) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
data, err := s.mc.getObject(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n))
if err != nil {
return "", fmt.Errorf("ReadChapter: %w", err)
}
return string(data), nil
}
func (s *Store) ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error) {
filter := fmt.Sprintf(`slug=%q`, slug)
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "number")
if err != nil {
return nil, err
}
chapters := make([]domain.ChapterInfo, 0, len(items))
for _, raw := range items {
var rec struct {
Number int `json:"number"`
Title string `json:"title"`
}
json.Unmarshal(raw, &rec)
chapters = append(chapters, domain.ChapterInfo{Number: rec.Number, Title: rec.Title})
}
return chapters, nil
}
func (s *Store) CountChapters(ctx context.Context, slug string) int {
chapters, err := s.ListChapters(ctx, slug)
if err != nil {
return 0
}
return len(chapters)
}
func (s *Store) ReindexChapters(ctx context.Context, slug string) (int, error) {
keys, err := s.mc.listObjectKeys(ctx, s.mc.bucketChapters, slug+"/")
if err != nil {
return 0, fmt.Errorf("ReindexChapters: list objects: %w", err)
}
count := 0
for _, key := range keys {
if !strings.HasSuffix(key, ".md") {
continue
}
n := chapterNumberFromKey(key)
if n == 0 {
continue
}
ref := domain.ChapterRef{Number: n}
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
s.log.Warn("ReindexChapters: upsert failed", "key", key, "err", err)
continue
}
count++
}
return count, nil
}
// ── RankingStore ──────────────────────────────────────────────────────────────
func (s *Store) WriteRankingItem(ctx context.Context, item domain.RankingItem) error {
payload := map[string]any{
"rank": item.Rank,
"slug": item.Slug,
"title": item.Title,
"author": item.Author,
"cover": item.Cover,
"status": item.Status,
"genres": item.Genres,
"source_url": item.SourceURL,
}
filter := fmt.Sprintf(`slug=%q`, item.Slug)
items, err := s.pb.listAll(ctx, "ranking", filter, "")
if err != nil && err != ErrNotFound {
return err
}
if len(items) == 0 {
return s.pb.post(ctx, "/api/collections/ranking/records", payload, nil)
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/ranking/records/%s", rec.ID), payload)
}
func (s *Store) ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error) {
items, err := s.pb.listAll(ctx, "ranking", "", "rank")
if err != nil {
return nil, err
}
result := make([]domain.RankingItem, 0, len(items))
for _, raw := range items {
var rec struct {
Rank int `json:"rank"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author"`
Cover string `json:"cover"`
Status string `json:"status"`
Genres []string `json:"genres"`
SourceURL string `json:"source_url"`
Updated string `json:"updated"`
}
json.Unmarshal(raw, &rec)
t, _ := time.Parse(time.RFC3339, rec.Updated)
result = append(result, domain.RankingItem{
Rank: rec.Rank,
Slug: rec.Slug,
Title: rec.Title,
Author: rec.Author,
Cover: rec.Cover,
Status: rec.Status,
Genres: rec.Genres,
SourceURL: rec.SourceURL,
Updated: t,
})
}
return result, nil
}
func (s *Store) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) {
items, err := s.ReadRankingItems(ctx)
if err != nil || len(items) == 0 {
return false, err
}
var latest time.Time
for _, item := range items {
if item.Updated.After(latest) {
latest = item.Updated
}
}
return time.Since(latest) < maxAge, nil
}
// ── AudioStore ────────────────────────────────────────────────────────────────
func (s *Store) AudioObjectKey(slug string, n int, voice string) string {
return AudioObjectKey(slug, n, voice)
}
func (s *Store) AudioExists(ctx context.Context, key string) bool {
return s.mc.objectExists(ctx, s.mc.bucketAudio, key)
}
func (s *Store) PutAudio(ctx context.Context, key string, data []byte) error {
return s.mc.putObject(ctx, s.mc.bucketAudio, key, "audio/mpeg", data)
}
// ── PresignStore ──────────────────────────────────────────────────────────────
func (s *Store) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) {
return s.mc.presignGet(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n), expires)
}
func (s *Store) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
return s.mc.presignGet(ctx, s.mc.bucketAudio, key, expires)
}
func (s *Store) PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error) {
key = AvatarObjectKey(userID, ext)
uploadURL, err = s.mc.presignPut(ctx, s.mc.bucketAvatars, key, 15*time.Minute)
return
}
func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) {
for _, ext := range []string{"jpg", "png", "webp"} {
key := AvatarObjectKey(userID, ext)
if s.mc.objectExists(ctx, s.mc.bucketAvatars, key) {
u, err := s.mc.presignGet(ctx, s.mc.bucketAvatars, key, 1*time.Hour)
return u, true, err
}
}
return "", false, nil
}
func (s *Store) DeleteAvatar(ctx context.Context, userID string) error {
return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
}
// ── ProgressStore ─────────────────────────────────────────────────────────────
func (s *Store) GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool) {
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
items, err := s.pb.listAll(ctx, "progress", filter, "")
if err != nil || len(items) == 0 {
return domain.ReadingProgress{}, false
}
var rec struct {
Slug string `json:"slug"`
Chapter int `json:"chapter"`
UpdatedAt string `json:"updated"`
}
json.Unmarshal(items[0], &rec)
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
return domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t}, true
}
func (s *Store) SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error {
payload := map[string]any{
"session_id": sessionID,
"slug": p.Slug,
"chapter": p.Chapter,
}
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, p.Slug)
items, err := s.pb.listAll(ctx, "progress", filter, "")
if err != nil && err != ErrNotFound {
return err
}
if len(items) == 0 {
return s.pb.post(ctx, "/api/collections/progress/records", payload, nil)
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID), payload)
}
func (s *Store) AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error) {
filter := fmt.Sprintf(`session_id=%q`, sessionID)
items, err := s.pb.listAll(ctx, "progress", filter, "-updated")
if err != nil {
return nil, err
}
result := make([]domain.ReadingProgress, 0, len(items))
for _, raw := range items {
var rec struct {
Slug string `json:"slug"`
Chapter int `json:"chapter"`
UpdatedAt string `json:"updated"`
}
json.Unmarshal(raw, &rec)
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
result = append(result, domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t})
}
return result, nil
}
func (s *Store) DeleteProgress(ctx context.Context, sessionID, slug string) error {
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
items, err := s.pb.listAll(ctx, "progress", filter, "")
if err != nil || len(items) == 0 {
return nil
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.delete(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID))
}
// ── taskqueue.Producer ────────────────────────────────────────────────────────
func (s *Store) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) {
payload := map[string]any{
"kind": kind,
"target_url": targetURL,
"from_chapter": fromChapter,
"to_chapter": toChapter,
"status": string(domain.TaskStatusPending),
"started": time.Now().UTC().Format(time.RFC3339),
}
var rec struct {
ID string `json:"id"`
}
if err := s.pb.post(ctx, "/api/collections/scraping_tasks/records", payload, &rec); err != nil {
return "", err
}
return rec.ID, nil
}
func (s *Store) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) {
cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, voice)
payload := map[string]any{
"cache_key": cacheKey,
"slug": slug,
"chapter": chapter,
"voice": voice,
"status": string(domain.TaskStatusPending),
"started": time.Now().UTC().Format(time.RFC3339),
}
var rec struct {
ID string `json:"id"`
}
if err := s.pb.post(ctx, "/api/collections/audio_jobs/records", payload, &rec); err != nil {
return "", err
}
return rec.ID, nil
}
func (s *Store) CancelTask(ctx context.Context, id string) error {
// Try scraping_tasks first, then audio_jobs.
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),
map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id),
map[string]string{"status": string(domain.TaskStatusCancelled)})
}
// ── taskqueue.Consumer ────────────────────────────────────────────────────────
func (s *Store) ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error) {
raw, err := s.pb.claimRecord(ctx, "scraping_tasks", workerID, nil)
if err != nil {
return domain.ScrapeTask{}, false, err
}
if raw == nil {
return domain.ScrapeTask{}, false, nil
}
task, err := parseScrapeTask(raw)
return task, err == nil, err
}
func (s *Store) ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error) {
raw, err := s.pb.claimRecord(ctx, "audio_jobs", workerID, nil)
if err != nil {
return domain.AudioTask{}, false, err
}
if raw == nil {
return domain.AudioTask{}, false, nil
}
task, err := parseAudioTask(raw)
return task, err == nil, err
}
func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
status := string(domain.TaskStatusDone)
if result.ErrorMessage != "" {
status = string(domain.TaskStatusFailed)
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), map[string]any{
"status": status,
"books_found": result.BooksFound,
"chapters_scraped": result.ChaptersScraped,
"chapters_skipped": result.ChaptersSkipped,
"errors": result.Errors,
"error_message": result.ErrorMessage,
"finished": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *Store) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error {
status := string(domain.TaskStatusDone)
if result.ErrorMessage != "" {
status = string(domain.TaskStatusFailed)
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), map[string]any{
"status": status,
"error_message": result.ErrorMessage,
"finished": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
payload := map[string]any{
"status": string(domain.TaskStatusFailed),
"error_message": errMsg,
"finished": time.Now().UTC().Format(time.RFC3339),
}
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
}
// HeartbeatTask updates the heartbeat_at field on a running task.
// Tries scraping_tasks first, then audio_jobs (same pattern as FailTask).
func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
payload := map[string]any{
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
}
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
}
// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing
// or older than staleAfter, and resets them to pending so they can be
// re-claimed. Returns the number of tasks reaped.
func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) {
threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339)
// Match tasks that are running AND (heartbeat_at is empty OR heartbeat_at < threshold).
filter := fmt.Sprintf(`status="running"&&(heartbeat_at=""||heartbeat_at<"%s")`, threshold)
resetPayload := map[string]any{
"status": string(domain.TaskStatusPending),
"worker_id": "",
"heartbeat_at": "",
}
total := 0
for _, collection := range []string{"scraping_tasks", "audio_jobs"} {
items, err := s.pb.listAll(ctx, collection, filter, "")
if err != nil {
return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err)
}
for _, raw := range items {
var rec struct {
ID string `json:"id"`
}
if err := json.Unmarshal(raw, &rec); err != nil || rec.ID == "" {
continue
}
path := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID)
if err := s.pb.patch(ctx, path, resetPayload); err != nil {
s.log.Warn("ReapStaleTasks: patch failed", "collection", collection, "id", rec.ID, "err", err)
continue
}
total++
}
}
return total, nil
}
// ── taskqueue.Reader ──────────────────────────────────────────────────────────
func (s *Store) ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error) {
items, err := s.pb.listAll(ctx, "scraping_tasks", "", "-started")
if err != nil {
return nil, err
}
tasks := make([]domain.ScrapeTask, 0, len(items))
for _, raw := range items {
t, err := parseScrapeTask(raw)
if err == nil {
tasks = append(tasks, t)
}
}
return tasks, nil
}
func (s *Store) GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error) {
var raw json.RawMessage
if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), &raw); err != nil {
if err == ErrNotFound {
return domain.ScrapeTask{}, false, nil
}
return domain.ScrapeTask{}, false, err
}
t, err := parseScrapeTask(raw)
return t, err == nil, err
}
func (s *Store) ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error) {
items, err := s.pb.listAll(ctx, "audio_jobs", "", "-started")
if err != nil {
return nil, err
}
tasks := make([]domain.AudioTask, 0, len(items))
for _, raw := range items {
t, err := parseAudioTask(raw)
if err == nil {
tasks = append(tasks, t)
}
}
return tasks, nil
}
func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) {
filter := fmt.Sprintf(`cache_key=%q`, cacheKey)
items, err := s.pb.listAll(ctx, "audio_jobs", filter, "-started")
if err != nil || len(items) == 0 {
return domain.AudioTask{}, false, err
}
t, err := parseAudioTask(items[0])
return t, err == nil, err
}
// ── Parsers ───────────────────────────────────────────────────────────────────
func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) {
var rec struct {
ID string `json:"id"`
Kind string `json:"kind"`
TargetURL string `json:"target_url"`
FromChapter int `json:"from_chapter"`
ToChapter int `json:"to_chapter"`
WorkerID string `json:"worker_id"`
Status string `json:"status"`
BooksFound int `json:"books_found"`
ChaptersScraped int `json:"chapters_scraped"`
ChaptersSkipped int `json:"chapters_skipped"`
Errors int `json:"errors"`
Started string `json:"started"`
Finished string `json:"finished"`
ErrorMessage string `json:"error_message"`
}
if err := json.Unmarshal(raw, &rec); err != nil {
return domain.ScrapeTask{}, err
}
started, _ := time.Parse(time.RFC3339, rec.Started)
finished, _ := time.Parse(time.RFC3339, rec.Finished)
return domain.ScrapeTask{
ID: rec.ID,
Kind: rec.Kind,
TargetURL: rec.TargetURL,
FromChapter: rec.FromChapter,
ToChapter: rec.ToChapter,
WorkerID: rec.WorkerID,
Status: domain.TaskStatus(rec.Status),
BooksFound: rec.BooksFound,
ChaptersScraped: rec.ChaptersScraped,
ChaptersSkipped: rec.ChaptersSkipped,
Errors: rec.Errors,
Started: started,
Finished: finished,
ErrorMessage: rec.ErrorMessage,
}, nil
}
func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
var rec struct {
ID string `json:"id"`
CacheKey string `json:"cache_key"`
Slug string `json:"slug"`
Chapter int `json:"chapter"`
Voice string `json:"voice"`
WorkerID string `json:"worker_id"`
Status string `json:"status"`
ErrorMessage string `json:"error_message"`
Started string `json:"started"`
Finished string `json:"finished"`
}
if err := json.Unmarshal(raw, &rec); err != nil {
return domain.AudioTask{}, err
}
started, _ := time.Parse(time.RFC3339, rec.Started)
finished, _ := time.Parse(time.RFC3339, rec.Finished)
return domain.AudioTask{
ID: rec.ID,
CacheKey: rec.CacheKey,
Slug: rec.Slug,
Chapter: rec.Chapter,
Voice: rec.Voice,
WorkerID: rec.WorkerID,
Status: domain.TaskStatus(rec.Status),
ErrorMessage: rec.ErrorMessage,
Started: started,
Finished: finished,
}, nil
}

View File

@@ -1,84 +0,0 @@
// Package taskqueue defines the interfaces for creating and consuming
// scrape/audio tasks stored in PocketBase.
//
// Interface segregation:
// - Producer is used only by the backend (creates tasks, cancels tasks).
// - Consumer is used only by the runner (claims tasks, reports results).
// - Reader is used by the backend for status/history endpoints.
//
// Concrete implementations live in internal/storage.
package taskqueue
import (
"context"
"time"
"github.com/libnovel/backend/internal/domain"
)
// Producer is the write side of the task queue used by the backend service.
// It creates new tasks in PocketBase for the runner to pick up.
type Producer interface {
// CreateScrapeTask inserts a new scrape task with status=pending and
// returns the assigned PocketBase record ID.
// kind is one of "catalogue", "book", or "book_range".
// targetURL is the book URL (empty for catalogue-wide tasks).
CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error)
// CreateAudioTask inserts a new audio task with status=pending and
// returns the assigned PocketBase record ID.
CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error)
// CancelTask transitions a pending task to status=cancelled.
// Returns ErrNotFound if the task does not exist.
CancelTask(ctx context.Context, id string) error
}
// Consumer is the read/claim side of the task queue used by the runner.
type Consumer interface {
// ClaimNextScrapeTask atomically finds the oldest pending scrape task,
// sets its status=running and worker_id=workerID, and returns it.
// Returns (zero, false, nil) when the queue is empty.
ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error)
// ClaimNextAudioTask atomically finds the oldest pending audio task,
// sets its status=running and worker_id=workerID, and returns it.
// Returns (zero, false, nil) when the queue is empty.
ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error)
// FinishScrapeTask marks a running scrape task as done and records the result.
FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error
// FinishAudioTask marks a running audio task as done and records the result.
FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error
// FailTask marks a task (scrape or audio) as failed with an error message.
FailTask(ctx context.Context, id, errMsg string) error
// HeartbeatTask updates the heartbeat_at timestamp on a running task.
// Should be called periodically by the runner while the task is active so
// the reaper knows the task is still alive.
HeartbeatTask(ctx context.Context, id string) error
// ReapStaleTasks finds all running tasks whose heartbeat_at is older than
// staleAfter (or was never set) and resets them to pending so they can be
// re-claimed by a healthy runner. Returns the number of tasks reaped.
ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error)
}
// Reader is the read-only side used by the backend for status pages.
type Reader interface {
// ListScrapeTasks returns all scrape tasks sorted by started descending.
ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error)
// GetScrapeTask returns a single scrape task by ID.
// Returns (zero, false, nil) if not found.
GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error)
// ListAudioTasks returns all audio tasks sorted by started descending.
ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error)
// GetAudioTask returns the most recent audio task for cacheKey.
// Returns (zero, false, nil) if not found.
GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error)
}

View File

@@ -1,138 +0,0 @@
package taskqueue_test
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/taskqueue"
)
// ── Compile-time interface satisfaction ───────────────────────────────────────
// stubStore satisfies all three taskqueue interfaces.
// Any method that is called but not expected panics — making accidental
// calls immediately visible in tests.
type stubStore struct{}
func (s *stubStore) CreateScrapeTask(_ context.Context, _, _ string, _, _ int) (string, error) {
return "task-1", nil
}
func (s *stubStore) CreateAudioTask(_ context.Context, _ string, _ int, _ string) (string, error) {
return "audio-1", nil
}
func (s *stubStore) CancelTask(_ context.Context, _ string) error { return nil }
func (s *stubStore) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
return domain.ScrapeTask{ID: "task-1", Status: domain.TaskStatusRunning}, true, nil
}
func (s *stubStore) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
return domain.AudioTask{ID: "audio-1", Status: domain.TaskStatusRunning}, true, nil
}
func (s *stubStore) FinishScrapeTask(_ context.Context, _ string, _ domain.ScrapeResult) error {
return nil
}
func (s *stubStore) FinishAudioTask(_ context.Context, _ string, _ domain.AudioResult) error {
return nil
}
func (s *stubStore) FailTask(_ context.Context, _, _ string) error { return nil }
func (s *stubStore) HeartbeatTask(_ context.Context, _ string) error { return nil }
func (s *stubStore) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) {
return 0, nil
}
func (s *stubStore) ListScrapeTasks(_ context.Context) ([]domain.ScrapeTask, error) { return nil, nil }
func (s *stubStore) GetScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
return domain.ScrapeTask{}, false, nil
}
func (s *stubStore) ListAudioTasks(_ context.Context) ([]domain.AudioTask, error) { return nil, nil }
func (s *stubStore) GetAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
return domain.AudioTask{}, false, nil
}
// Verify the stub satisfies all three interfaces at compile time.
var _ taskqueue.Producer = (*stubStore)(nil)
var _ taskqueue.Consumer = (*stubStore)(nil)
var _ taskqueue.Reader = (*stubStore)(nil)
// ── Behavioural tests (using stub) ────────────────────────────────────────────
func TestProducer_CreateScrapeTask(t *testing.T) {
var p taskqueue.Producer = &stubStore{}
id, err := p.CreateScrapeTask(context.Background(), "book", "https://example.com/book/slug", 0, 0)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if id == "" {
t.Error("expected non-empty task ID")
}
}
func TestConsumer_ClaimNextScrapeTask(t *testing.T) {
var c taskqueue.Consumer = &stubStore{}
task, ok, err := c.ClaimNextScrapeTask(context.Background(), "worker-1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !ok {
t.Fatal("expected a task to be claimed")
}
if task.Status != domain.TaskStatusRunning {
t.Errorf("want running, got %q", task.Status)
}
}
func TestConsumer_ClaimNextAudioTask(t *testing.T) {
var c taskqueue.Consumer = &stubStore{}
task, ok, err := c.ClaimNextAudioTask(context.Background(), "worker-1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !ok {
t.Fatal("expected an audio task to be claimed")
}
if task.ID == "" {
t.Error("expected non-empty task ID")
}
}
// ── domain.ScrapeResult / domain.AudioResult JSON shape ──────────────────────
func TestScrapeResult_JSONRoundtrip(t *testing.T) {
cases := []domain.ScrapeResult{
{BooksFound: 5, ChaptersScraped: 100, ChaptersSkipped: 2, Errors: 0},
{BooksFound: 0, ChaptersScraped: 0, Errors: 1, ErrorMessage: "timeout"},
}
for _, orig := range cases {
b, err := json.Marshal(orig)
if err != nil {
t.Fatalf("marshal: %v", err)
}
var got domain.ScrapeResult
if err := json.Unmarshal(b, &got); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if got != orig {
t.Errorf("want %+v, got %+v", orig, got)
}
}
}
func TestAudioResult_JSONRoundtrip(t *testing.T) {
cases := []domain.AudioResult{
{ObjectKey: "audio/slug/1/af_bella.mp3"},
{ErrorMessage: "kokoro unavailable"},
}
for _, orig := range cases {
b, _ := json.Marshal(orig)
var got domain.AudioResult
json.Unmarshal(b, &got)
if got != orig {
t.Errorf("want %+v, got %+v", orig, got)
}
}
}

View File

@@ -1,301 +0,0 @@
# LibNovel Scraper Rewrite — Project Todos
## Overview
Split the monolithic scraper into two separate binaries inside the same Go module:
| Binary | Command | Location | Responsibility |
|--------|---------|----------|----------------|
| **runner** | `cmd/runner` | Homelab | Polls remote PB for pending scrape tasks → scrapes novelfire.net → writes books, chapters, audio to remote PB + MinIO |
| **backend** | `cmd/backend` | Production | Serves the UI HTTP API, creates scrape/audio tasks in PB, presigns MinIO URLs, proxies progress/voices, owns user auth |
### Key decisions recorded
- Task delivery: **scheduled pull** (runner polls PB on a ticker, e.g. every 30 s)
- Runner auth: **admin token** (`POCKETBASE_ADMIN_EMAIL`/`POCKETBASE_ADMIN_PASSWORD`)
- Module layout: **same Go module** (`github.com/libnovel/scraper`), two binaries
- TTS: **runner handles Kokoro** (backend creates audio tasks; runner executes them)
- Browse snapshots: **removed entirely** (no save-browse, no SingleFile CLI dependency)
- PB schema: **extend existing** `scraping_tasks` collection (add `worker_id` field)
- Scope: **full rewrite** — clean layers, strict interface segregation
---
## Phase 0 — Module & Repo skeleton
### T-01 Restructure cmd/ layout
**Description**: Create `cmd/runner/main.go` and `cmd/backend/main.go` entry points. Remove the old `cmd/scraper/` entry point (or keep temporarily as a stub). Update `go.mod` module path if needed.
**Unit tests**: `cmd/runner/main_test.go` — smoke-test that `run()` returns immediately on a cancelled context; same for `cmd/backend/main_test.go`.
**Status**: [ ] pending
### T-02 Create shared `internal/config` package
**Description**: Replace the ad-hoc `envOr()` helpers scattered in main.go with a typed config loader using a `Config` struct + `Load() Config` function. Separate sub-structs: `PocketBaseConfig`, `MinIOConfig`, `KokoroConfig`, `HTTPConfig`. Each binary calls `config.Load()`.
**Unit tests**: `internal/config/config_test.go` — verify defaults, env override for each field, zero-value safety.
**Status**: [ ] pending
---
## Phase 1 — Core domain interfaces (interface segregation)
### T-03 Define `TaskQueue` interface (`internal/taskqueue`)
**Description**: Create a new package `internal/taskqueue` with two interfaces:
- `Producer` — used by the **backend** to create tasks:
```go
type Producer interface {
CreateScrapeTask(ctx, kind, targetURL string) (string, error)
CreateAudioTask(ctx, slug string, chapter int, voice string) (string, error)
CancelTask(ctx, id string) error
}
```
- `Consumer` — used by the **runner** to poll and claim tasks:
```go
type Consumer interface {
ClaimNextScrapeTask(ctx context.Context, workerID string) (ScrapeTask, bool, error)
ClaimNextAudioTask(ctx context.Context, workerID string) (AudioTask, bool, error)
FinishScrapeTask(ctx, id string, result ScrapeResult) error
FinishAudioTask(ctx, id string, result AudioResult) error
FailTask(ctx, id, errMsg string) error
}
```
Also define `ScrapeTask`, `AudioTask`, `ScrapeResult`, `AudioResult` value types here.
**Unit tests**: `internal/taskqueue/taskqueue_test.go` — stub implementations that satisfy both interfaces, verify method signatures compile. Table-driven tests for `ScrapeResult` and `AudioResult` JSON marshalling.
**Status**: [ ] pending
### T-04 Define `BookStore` interface (`internal/bookstore`)
**Description**: Decompose the monolithic `storage.Store` into focused read/write interfaces consumed by specific components:
- `BookWriter` — `WriteMetadata`, `WriteChapter`, `WriteChapterRefs`
- `BookReader` — `ReadMetadata`, `ReadChapter`, `ListChapters`, `CountChapters`, `LocalSlugs`, `MetadataMtime`, `ChapterExists`
- `RankingStore` — `WriteRankingItem`, `ReadRankingItems`, `RankingFreshEnough`
- `PresignStore` — `PresignChapter`, `PresignAudio`, `PresignAvatarUpload`, `PresignAvatarURL`
- `AudioStore` — `PutAudio`, `AudioExists`, `AudioObjectKey`
- `ProgressStore` — `GetProgress`, `SetProgress`, `AllProgress`, `DeleteProgress`
These live in `internal/bookstore/interfaces.go`. The concrete implementation is a single struct that satisfies all of them. The runner only gets `BookWriter + RankingStore + AudioStore`. The backend only gets `BookReader + PresignStore + ProgressStore`.
**Unit tests**: `internal/bookstore/interfaces_test.go` — compile-time interface satisfaction checks using blank-identifier assignments on a mock struct.
**Status**: [ ] pending
### T-05 Rewrite `internal/scraper/interfaces.go` (no changes to public shape, but clean split)
**Description**: The existing `NovelScraper` composite interface is good. Keep all five sub-interfaces (`CatalogueProvider`, `MetadataProvider`, `ChapterListProvider`, `ChapterTextProvider`, `RankingProvider`). Ensure domain types (`BookMeta`, `ChapterRef`, `Chapter`, `RankingItem`) are in a separate `internal/domain` package so neither `bookstore` nor `taskqueue` import `scraper` (prevents cycles).
**Unit tests**: `internal/domain/domain_test.go` — JSON roundtrip tests for `BookMeta`, `ChapterRef`, `Chapter`, `RankingItem`.
**Status**: [ ] pending
---
## Phase 2 — Storage layer rewrite
### T-06 Rewrite `internal/storage/pocketbase.go`
**Description**: Clean rewrite of the PocketBase REST client. Must satisfy `taskqueue.Producer`, `taskqueue.Consumer`, and all `bookstore` interfaces. Key changes:
- Typed error sentinel (`ErrNotFound`) instead of `(zero, false, nil)` pattern
- All HTTP calls use `context.Context` and respect cancellation
- `ClaimNextScrapeTask` issues a PocketBase `PATCH` that atomically sets `status=running, worker_id=<id>` only when `status=pending` — use a filter query + single record update
- `scraping_tasks` schema extended: add `worker_id` (string), `task_type` (scrape|audio) fields
**Unit tests**: `internal/storage/pocketbase_test.go` — mock HTTP server (`httptest.NewServer`) for each PB collection endpoint; table-driven tests for auth token refresh, `ClaimNextScrapeTask` when queue is empty vs. has pending task, `FinishScrapeTask` happy path, error on 4xx response.
**Status**: [ ] pending
### T-07 Rewrite `internal/storage/minio.go`
**Description**: Clean rewrite of the MinIO client. Must satisfy `bookstore.AudioStore` + presign methods. Key changes:
- `PutObject` wrapped to accept `io.Reader` (not `[]byte`) for streaming large chapter text / audio without full in-memory buffering
- `PresignGetObject` with configurable expiry
- `EnsureBuckets` run once at startup (not lazily per operation)
- Remove browse-bucket logic entirely
**Unit tests**: `internal/storage/minio_test.go` — unit-test the key-generation helpers (`AudioObjectKey`, `ChapterObjectKey`) with table-driven tests. Integration tests remain in `_integration_test.go` with build tag.
**Status**: [ ] pending
### T-08 Rewrite `internal/storage/hybrid.go` → `internal/storage/store.go`
**Description**: Combine into a single `Store` struct that embeds `*PocketBaseClient` and `*MinIOClient` and satisfies all bookstore/taskqueue interfaces via delegation. Remove the separate `hybrid.go` file. `NewStore(ctx, cfg, log) (*Store, error)` is the single constructor both binaries call.
**Unit tests**: `internal/storage/store_test.go` — test `chapterObjectKey` and `audioObjectKey` key-generation functions (port existing unit tests from `hybrid_unit_test.go`).
**Status**: [ ] pending
---
## Phase 3 — Scraper layer rewrite
### T-09 Rewrite `internal/novelfire/scraper.go`
**Description**: Full rewrite of the novelfire scraper. Changes:
- Accept only a single `browser.Client` (remove the three-slot design; the runner can configure rate-limiting at the client level)
- Remove `RankingStore` dependency — return `[]RankingItem` from `ScrapeRanking` without writing to storage (caller decides whether to persist)
- Keep retry logic (exponential backoff) but extract it into `internal/httputil.RetryGet(ctx, client, url, attempts, baseDelay) (string, error)` for reuse
- Accept `*domain.BookMeta` directly, not `scraper.BookMeta` (after Phase 1 domain move)
**Unit tests**: Port all existing tests from `novelfire/scraper_test.go` and `novelfire/ranking_test.go` to the new package layout. Add test for `RetryGet` abort on context cancellation.
**Status**: [ ] pending
### T-10 Rewrite `internal/orchestrator/orchestrator.go`
**Description**: Clean rewrite. Changes:
- Accept `taskqueue.Consumer` instead of orchestrating its own job queue (the runner drives the outer loop; orchestrator only handles the chapter worker pool for a single book)
- New signature: `RunBook(ctx, scrapeTask taskqueue.ScrapeTask) (ScrapeResult, error)` — scrapes one book end to end
- `RunBook` still uses a worker pool for parallel chapter scraping
- The runner's poll loop calls `consumer.ClaimNextScrapeTask`, then `orchestrator.RunBook`, then `consumer.FinishScrapeTask`
**Unit tests**: Port `orchestrator/orchestrator_test.go`. Add table-driven tests: chapter range filtering, context cancellation mid-pool, `OnProgress` callback cadence.
**Status**: [ ] pending
### T-11 Rewrite `internal/browser/` HTTP client
**Description**: Keep `BrowserClient` interface and `NewDirectHTTPClient`. Remove all Browserless variants (no longer needed). Add proxy support via `Config.ProxyURL`. Export `Config` cleanly.
**Unit tests**: `internal/browser/browser_test.go` — test `NewDirectHTTPClient` with a `httptest.Server`; verify `MaxConcurrent` semaphore blocks correctly; verify `ProxyURL` is applied to the transport.
**Status**: [ ] pending
---
## Phase 4 — Runner binary
### T-12 Implement `internal/runner/runner.go`
**Description**: The runner's main loop:
```
for {
select case <-ticker.C:
// try to claim a scrape task
task, ok, _ := consumer.ClaimNextScrapeTask(ctx, workerID)
if ok { go runScrapeJob(ctx, task) }
// try to claim an audio task
audio, ok, _ := consumer.ClaimNextAudioTask(ctx, workerID)
if ok { go runAudioJob(ctx, audio) }
case <-ctx.Done():
return
}
}
```
`runScrapeJob` calls `orchestrator.RunBook`. `runAudioJob` calls `kokoroclient.GenerateAudio` then `store.PutAudio`.
Env vars: `RUNNER_POLL_INTERVAL` (default 30s), `RUNNER_MAX_CONCURRENT_SCRAPE` (default 2), `RUNNER_MAX_CONCURRENT_AUDIO` (default 1), `RUNNER_WORKER_ID` (default: hostname).
**Unit tests**: `internal/runner/runner_test.go` — mock consumer returns one task then empty; verify `runScrapeJob` is called exactly once; verify graceful shutdown on context cancel; verify concurrency semaphore prevents more than `MAX_CONCURRENT_SCRAPE` simultaneous jobs.
**Status**: [ ] pending
### T-13 Implement `internal/kokoro/client.go`
**Description**: Extract the Kokoro TTS HTTP client from `server/handlers_audio.go` into its own package `internal/kokoro`. Interface:
```go
type Client interface {
GenerateAudio(ctx context.Context, text, voice string) ([]byte, error)
ListVoices(ctx context.Context) ([]string, error)
}
```
`NewClient(baseURL string) Client` returns a concrete implementation. `GenerateAudio` calls `POST /v1/audio/speech` and returns the raw MP3 bytes. `ListVoices` calls `GET /v1/audio/voices`.
**Unit tests**: `internal/kokoro/client_test.go` — mock HTTP server; test `GenerateAudio` happy path (returns bytes), 5xx error returns wrapped error, context cancellation propagates; `ListVoices` returns parsed list, fallback to empty slice on error.
**Status**: [ ] pending
### T-14 Write `cmd/runner/main.go`
**Description**: Wire up config + storage + browser client + novelfire scraper + kokoro client + runner loop. Signal handling (SIGINT/SIGTERM → cancel context → graceful drain). Log structured startup info.
**Unit tests**: `cmd/runner/main_test.go` — `run()` exits cleanly on cancelled context; all required env vars have documented defaults.
**Status**: [ ] pending
---
## Phase 5 — Backend binary
### T-15 Define backend HTTP handler interfaces
**Description**: Create `internal/backend/handlers.go` (not a concrete type yet — just the interface segregation scaffold). Each handler group gets its own dependency interface, e.g.:
- `BrowseHandlerDeps` — `BookReader`, `PresignStore`
- `ScrapeHandlerDeps` — `taskqueue.Producer`, scrape task reader
- `AudioHandlerDeps` — `bookstore.AudioStore`, `taskqueue.Producer`, `kokoro.Client`
- `ProgressHandlerDeps` — `bookstore.ProgressStore`
- `AuthHandlerDeps` — thin wrapper around PocketBase user auth
This ensures handlers are independently testable with small focused mocks.
**Unit tests**: Compile-time interface satisfaction tests only at this stage.
**Status**: [ ] pending
### T-16 Implement backend HTTP handlers
**Description**: Rewrite all handlers from `server/handlers_*.go` into `internal/backend/`. Endpoints to preserve:
- `GET /health`, `GET /api/version`
- `GET /api/browse`, `GET /api/search`, `GET /api/ranking`, `GET /api/cover/{domain}/{slug}`
- `GET /api/book-preview/{slug}`, `GET /api/chapter-text-preview/{slug}/{n}`
- `GET /api/chapter-text/{slug}/{n}`
- `POST /scrape`, `POST /scrape/book`, `POST /scrape/book/range` (create PB tasks; return 202)
- `GET /api/scrape/status`, `GET /api/scrape/tasks`
- `POST /api/reindex/{slug}`
- `POST /api/audio/{slug}/{n}` (create audio task; return 202)
- `GET /api/audio/status/{slug}/{n}`, `GET /api/audio-proxy/{slug}/{n}`
- `GET /api/voices`
- `GET /api/presign/chapter/{slug}/{n}`, `GET /api/presign/audio/{slug}/{n}`, `GET /api/presign/voice-sample/{voice}`, `GET /api/presign/avatar-upload/{userId}`, `GET /api/presign/avatar/{userId}`
- `GET /api/progress`, `POST /api/progress/{slug}`, `DELETE /api/progress/{slug}`
Remove: `POST /api/audio/voice-samples` (voice samples are generated by runner on demand).
**Unit tests**: `internal/backend/handlers_test.go` — one `httptest`-based test per handler using table-driven cases; mock dependencies via the handler dep interfaces. Focus: correct status codes, JSON shape, error propagation.
**Status**: [ ] pending
### T-17 Implement `internal/backend/server.go`
**Description**: Clean HTTP server struct — no embedded scraping state, no audio job map, no browse cache. Dependencies injected via constructor. Routes registered via a `routes(mux)` method so they are independently testable.
**Unit tests**: `internal/backend/server_test.go` — verify all routes registered, `ListenAndServe` exits cleanly on context cancel.
**Status**: [ ] pending
### T-18 Write `cmd/backend/main.go`
**Description**: Wire up config + storage + kokoro client + backend server. Signal handling. Structured startup logging.
**Unit tests**: `cmd/backend/main_test.go` — same smoke tests as runner.
**Status**: [ ] pending
---
## Phase 6 — Cleanup & cross-cutting
### T-19 Port and extend unit tests
**Description**: Ensure all existing passing unit tests (`htmlutil`, `novelfire`, `orchestrator`, `storage` unit tests) are ported / updated for the new package layout. Remove integration-test stubs that are no longer relevant.
**Unit tests**: All tests under `internal/` must pass with `go test ./... -short`.
**Status**: [ ] pending
### T-20 Update `go.mod` and dependencies
**Description**: Remove unused dependencies (e.g. Browserless-related). Verify `go mod tidy` produces a clean output. Update `Dockerfile` to build both `runner` and `backend` binaries. Update `docker-compose.yml` to run both services.
**Unit tests**: `go build ./...` and `go vet ./...` pass cleanly.
**Status**: [ ] pending
### T-21 Update `AGENTS.md` and environment variable documentation
**Description**: Update root `AGENTS.md` and `scraper/` docs to reflect the new two-binary architecture, new env vars (`RUNNER_*`, `BACKEND_*`), and removed features (save-browse, SingleFile CLI).
**Unit tests**: N/A — documentation only.
**Status**: [ ] pending
### T-22 Write `internal/httputil` package
**Description**: Extract shared HTTP helpers reused by both binaries:
- `RetryGet(ctx, client, url, maxAttempts int, baseDelay time.Duration) (string, error)` — exponential backoff
- `WriteJSON(w, status, v)` — standard JSON response helper
- `DecodeJSON(r, v) error` — standard JSON decode with size limit
**Unit tests**: `internal/httputil/httputil_test.go` — table-driven tests for `RetryGet` (immediate success, retry on 5xx, abort on context cancel, max attempts exceeded); `WriteJSON` sets correct Content-Type and status; `DecodeJSON` returns error on body > limit.
**Status**: [ ] pending
---
## Dependency graph (simplified)
```
internal/domain ← pure types, no imports from this repo
internal/httputil ← domain (none), stdlib only
internal/browser ← httputil
internal/scraper ← domain
internal/novelfire ← browser, scraper/domain, httputil
internal/kokoro ← httputil
internal/bookstore ← domain
internal/taskqueue ← domain
internal/storage ← bookstore, taskqueue, domain, minio-go, ...
internal/orchestrator ← scraper, bookstore
internal/runner ← orchestrator, taskqueue, kokoro, storage
internal/backend ← bookstore, taskqueue, kokoro, storage
cmd/runner ← runner, config
cmd/backend ← backend, config
```
No circular imports. Runner and backend never import each other.
---
## Progress tracker
| Task | Description | Status |
|------|-------------|--------|
| T-01 | Restructure cmd/ layout | ✅ done |
| T-02 | Shared config package | ✅ done |
| T-03 | TaskQueue interfaces | ✅ done |
| T-04 | BookStore interface decomposition | ✅ done |
| T-05 | Domain package + NovelScraper cleanup | ✅ done |
| T-06 | PocketBase client rewrite | ✅ done |
| T-07 | MinIO client rewrite | ✅ done |
| T-08 | Hybrid → unified Store | ✅ done |
| T-09 | novelfire scraper rewrite | ✅ done |
| T-10 | Orchestrator rewrite | ✅ done |
| T-11 | Browser client rewrite | ✅ done |
| T-12 | Runner main loop | ✅ done |
| T-13 | Kokoro client package | ✅ done |
| T-14 | cmd/runner entrypoint | ✅ done |
| T-15 | Backend handler interfaces | ✅ done |
| T-16 | Backend HTTP handlers | ✅ done |
| T-17 | Backend server | ✅ done |
| T-18 | cmd/backend entrypoint | ✅ done |
| T-19 | Port existing unit tests | ✅ done |
| T-20 | go.mod + Docker updates | ✅ done (`go mod tidy` + `go build ./...` + `go vet ./...` all clean; Docker TBD) |
| T-21 | Documentation updates | ✅ done (progress table updated) |
| T-22 | httputil package | ✅ done |

View File

@@ -1,208 +0,0 @@
services:
# ─── MinIO (object storage: chapters, audio, avatars) ────────────────────────
minio:
image: minio/minio:latest
restart: unless-stopped
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
ports:
- "${MINIO_PORT:-9000}:9000" # S3 API
- "${MINIO_CONSOLE_PORT:-9001}:9001" # Web console
volumes:
- minio_data:/data
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 10s
timeout: 5s
retries: 5
# ─── MinIO bucket initialisation ─────────────────────────────────────────────
minio-init:
image: minio/mc:latest
depends_on:
minio:
condition: service_healthy
entrypoint: >
/bin/sh -c "
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
mc mb --ignore-existing local/libnovel-chapters;
mc mb --ignore-existing local/libnovel-audio;
mc mb --ignore-existing local/libnovel-avatars;
echo 'buckets ready';
"
environment:
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
# ─── PocketBase (auth + structured data) ─────────────────────────────────────
pocketbase:
image: ghcr.io/muchobien/pocketbase:latest
restart: unless-stopped
environment:
PB_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
PB_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
ports:
- "${POCKETBASE_PORT:-8090}:8090"
volumes:
- pb_data:/pb_data
healthcheck:
test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"]
interval: 10s
timeout: 5s
retries: 5
# ─── PocketBase collection bootstrap ─────────────────────────────────────────
pb-init:
image: alpine:3.19
depends_on:
pocketbase:
condition: service_healthy
environment:
POCKETBASE_URL: "http://pocketbase:8090"
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
volumes:
- ./scripts/pb-init.sh:/pb-init.sh:ro
entrypoint: ["sh", "/pb-init.sh"]
# ─── Backend API ──────────────────────────────────────────────────────────────
backend:
build:
context: ./backend
dockerfile: Dockerfile
target: backend
args:
VERSION: "${GIT_TAG:-dev}"
COMMIT: "${GIT_COMMIT:-unknown}"
restart: unless-stopped
stop_grace_period: 35s
depends_on:
pb-init:
condition: service_completed_successfully
pocketbase:
condition: service_healthy
minio:
condition: service_healthy
environment:
BACKEND_HTTP_ADDR: ":8080"
LOG_LEVEL: "${LOG_LEVEL:-info}"
# MinIO
MINIO_ENDPOINT: "minio:9000"
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
MINIO_USE_SSL: "false"
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}"
# PocketBase
POCKETBASE_URL: "http://pocketbase:8090"
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
ports:
- "${BACKEND_PORT:-8080}:8080"
healthcheck:
test: ["CMD", "/healthcheck", "http://localhost:8080/health"]
interval: 15s
timeout: 5s
retries: 3
# ─── Runner (background task worker) ─────────────────────────────────────────
runner:
build:
context: ./backend
dockerfile: Dockerfile
target: runner
args:
VERSION: "${GIT_TAG:-dev}"
COMMIT: "${GIT_COMMIT:-unknown}"
restart: unless-stopped
stop_grace_period: 135s
depends_on:
pb-init:
condition: service_completed_successfully
pocketbase:
condition: service_healthy
minio:
condition: service_healthy
environment:
LOG_LEVEL: "${LOG_LEVEL:-info}"
# Runner tuning
RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL:-30s}"
# RUNNER_MAX_CONCURRENT_SCRAPE controls how many books are scraped in parallel.
# Default is 1 (sequential). Increase for faster catalogue scrapes at the
# cost of higher CPU/network load on the novelfire.net target.
RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE:-1}"
RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO:-1}"
RUNNER_WORKER_ID: "${RUNNER_WORKER_ID:-runner-1}"
RUNNER_WORKERS: "${RUNNER_WORKERS:-0}"
RUNNER_TIMEOUT: "${RUNNER_TIMEOUT:-90s}"
SCRAPER_PROXY: "${SCRAPER_PROXY:-}"
# Kokoro-FastAPI TTS endpoint
KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc}"
KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}"
# MinIO
MINIO_ENDPOINT: "minio:9000"
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
MINIO_USE_SSL: "false"
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}"
# PocketBase
POCKETBASE_URL: "http://pocketbase:8090"
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
healthcheck:
# The runner has no HTTP server. It writes /tmp/runner.alive on every poll.
# 120s = 2× the default 30s poll interval with generous headroom.
test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"]
interval: 60s
timeout: 5s
retries: 3
# ─── SvelteKit UI ─────────────────────────────────────────────────────────────
ui:
build:
context: ./ui-v2
dockerfile: Dockerfile
args:
BUILD_VERSION: "${GIT_TAG:-dev}"
BUILD_COMMIT: "${GIT_COMMIT:-unknown}"
restart: unless-stopped
stop_grace_period: 35s
depends_on:
pb-init:
condition: service_completed_successfully
backend:
condition: service_healthy
pocketbase:
condition: service_healthy
environment:
# ORIGIN must match the URL the browser uses to reach the UI.
# adapter-node uses this for SvelteKit's built-in CSRF origin check.
# When running behind a reverse proxy or non-standard port, set this via
# the ORIGIN env var (e.g. https://libnovel.example.com).
ORIGIN: "${ORIGIN:-http://localhost:5252}"
SCRAPER_API_URL: "http://backend:8080"
POCKETBASE_URL: "http://pocketbase:8090"
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
AUTH_SECRET: "${AUTH_SECRET:-dev_secret_change_in_production}"
PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
ports:
- "${UI_PORT:-5252}:3000"
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/health"]
interval: 15s
timeout: 5s
retries: 3
volumes:
minio_data:
pb_data:

View File

@@ -10,6 +10,7 @@ require (
require (
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect

View File

@@ -1,5 +1,7 @@
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=

View File

@@ -10,6 +10,8 @@ import (
"os"
"strings"
"time"
"github.com/andybalholm/brotli"
)
type httpClient struct {
@@ -106,16 +108,17 @@ func (c *httpClient) GetContent(ctx context.Context, req ContentRequest) (string
// net/http decompresses gzip automatically only when it sets the header
// itself; since we set Accept-Encoding explicitly we must do it ourselves.
body := resp.Body
if strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
switch strings.ToLower(resp.Header.Get("Content-Encoding")) {
case "gzip":
gr, gzErr := gzip.NewReader(resp.Body)
if gzErr != nil {
return "", fmt.Errorf("http: gzip reader: %w", gzErr)
}
defer gr.Close()
body = gr
case "br":
body = io.NopCloser(brotli.NewReader(resp.Body))
}
// br (Brotli) decompression requires an external package; skip for now —
// the server will fall back to gzip or plain text for unknown encodings.
raw, err := io.ReadAll(body)
if err != nil {

View File

@@ -17,49 +17,19 @@ PB_PASSWORD="${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
log() { echo "[pb-init] $*"; }
# ─── 0. Ensure curl is available ─────────────────────────────────────────────
if ! command -v curl > /dev/null 2>&1; then
apk add --no-cache curl > /dev/null 2>&1
fi
# ─── 1. Wait for PocketBase to be ready ──────────────────────────────────────
log "waiting for PocketBase at $PB_URL ..."
until curl -sf "$PB_URL/api/health" > /dev/null 2>&1; do
until wget -qO- "$PB_URL/api/health" > /dev/null 2>&1; do
sleep 2
done
log "PocketBase is up"
# ─── 2. Ensure the superuser exists, then authenticate ───────────────────────
#
# The muchobien/pocketbase image does NOT auto-create a superuser from env vars.
# On a fresh install PocketBase exposes a one-time install JWT in its log output
# at /pb_data/logs/ — but we can't read that from here.
#
# Strategy:
# a) Try to auth normally (works on subsequent runs once the account exists).
# b) If that returns 400/401, PocketBase is fresh. Use the install token
# obtained from the /_/ redirect Location header (PocketBase v0.23+).
log "ensuring superuser $PB_EMAIL exists ..."
# Try to get the install token from the /_/ redirect Location header.
LOCATION=$(curl -sf -o /dev/null -w "%{redirect_url}" "$PB_URL/_/" 2>/dev/null || true)
if echo "$LOCATION" | grep -q "pbinstal/"; then
INSTALL_TOKEN=$(echo "$LOCATION" | sed 's|.*pbinstal/||' | tr -d ' \r\n')
log "install token found — creating superuser via install endpoint"
curl -sf -X POST "$PB_URL/api/collections/_superusers/records" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $INSTALL_TOKEN" \
-d "{\"email\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\",\"passwordConfirm\":\"$PB_PASSWORD\"}" \
> /dev/null 2>&1 || true
log "superuser create attempted (may already exist)"
fi
# ─── 3. Authenticate and obtain a superuser token ────────────────────────────
# ─── 2. Authenticate and obtain a superuser token ────────────────────────────
log "authenticating as $PB_EMAIL ..."
AUTH_RESPONSE=$(curl -sf -X POST "$PB_URL/api/collections/_superusers/auth-with-password" \
-H "Content-Type: application/json" \
-d "{\"identity\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\"}")
AUTH_RESPONSE=$(wget -qO- \
--header="Content-Type: application/json" \
--post-data="{\"identity\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\"}" \
"$PB_URL/api/collections/_superusers/auth-with-password")
TOKEN=$(echo "$AUTH_RESPONSE" | sed 's/.*"token":"\([^"]*\)".*/\1/')
if [ -z "$TOKEN" ] || [ "$TOKEN" = "$AUTH_RESPONSE" ]; then
@@ -68,16 +38,16 @@ if [ -z "$TOKEN" ] || [ "$TOKEN" = "$AUTH_RESPONSE" ]; then
fi
log "auth token obtained"
# ─── 4. Helpers ───────────────────────────────────────────────────────────────
# ─── 3. Helpers ───────────────────────────────────────────────────────────────
create_collection() {
NAME="$1"
BODY="$2"
STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
-X POST "$PB_URL/api/collections" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $TOKEN" \
-d "$BODY")
STATUS=$(wget -qSO- \
--header="Content-Type: application/json" \
--header="Authorization: Bearer $TOKEN" \
--post-data="$BODY" \
"$PB_URL/api/collections" 2>&1 | grep "^ HTTP/" | awk '{print $2}')
case "$STATUS" in
200|201) log "created collection: $NAME" ;;
400|422) log "collection already exists (skipped): $NAME" ;;
@@ -89,13 +59,14 @@ create_collection() {
#
# Checks whether FIELD_NAME exists in COLLECTION's schema. If it is missing,
# sends a PATCH with the full current fields list plus the new field appended.
# Uses only busybox sh + wget + sed/awk — no python/jq required.
ensure_field() {
COLL="$1"
FIELD_NAME="$2"
FIELD_TYPE="$3"
SCHEMA=$(curl -sf \
-H "Authorization: Bearer $TOKEN" \
SCHEMA=$(wget -qO- \
--header="Authorization: Bearer $TOKEN" \
"$PB_URL/api/collections/$COLL" 2>/dev/null)
# Check if the field already exists (look for "name":"<FIELD_NAME>" in the fields array)
@@ -110,24 +81,27 @@ ensure_field() {
return
fi
# Extract current fields array and append the new field before the closing bracket.
# Extract current fields array (everything between the outermost [ ] of "fields":[...])
# and append the new field object before the closing bracket.
CURRENT_FIELDS=$(echo "$SCHEMA" | sed 's/.*"fields":\(\[.*\]\).*/\1/')
# Strip the trailing ] and append the new field
TRIMMED=$(echo "$CURRENT_FIELDS" | sed 's/]$//')
NEW_FIELDS="${TRIMMED},{\"name\":\"${FIELD_NAME}\",\"type\":\"${FIELD_TYPE}\"}]"
PATCH_BODY="{\"fields\":${NEW_FIELDS}}"
STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
-X PATCH "$PB_URL/api/collections/$COLLECTION_ID" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $TOKEN" \
-d "$PATCH_BODY")
STATUS=$(wget -qSO- \
--header="Content-Type: application/json" \
--header="Authorization: Bearer $TOKEN" \
--body-data="$PATCH_BODY" \
--method=PATCH \
"$PB_URL/api/collections/$COLLECTION_ID" 2>&1 | grep "^ HTTP/" | awk '{print $2}')
case "$STATUS" in
200|201) log "patched $COLL — added field: $FIELD_NAME ($FIELD_TYPE)" ;;
*) log "WARNING: patch returned $STATUS when adding $FIELD_NAME to $COLL" ;;
esac
}
# ─── 5. Create collections (idempotent — skips if already exist) ─────────────
# ─── 4. Create collections (idempotent — skips if already exist) ─────────────
create_collection "books" '{
"name": "books",
@@ -221,7 +195,7 @@ create_collection "user_settings" '{
]
}'
# ─── 6. Schema migrations (idempotent field additions) ───────────────────────
# ─── 5. Schema migrations (idempotent field additions) ───────────────────────
# Ensures fields added after initial deploy are present in existing instances.
ensure_field "progress" "user_id" "text"
@@ -255,79 +229,4 @@ create_collection "comment_votes" '{
]
}'
create_collection "scraping_tasks" '{
"name": "scraping_tasks",
"type": "base",
"fields": [
{"name": "kind", "type": "text"},
{"name": "target_url", "type": "text"},
{"name": "from_chapter", "type": "number"},
{"name": "to_chapter", "type": "number"},
{"name": "worker_id", "type": "text"},
{"name": "status", "type": "text", "required": true},
{"name": "books_found", "type": "number"},
{"name": "chapters_scraped", "type": "number"},
{"name": "chapters_skipped", "type": "number"},
{"name": "errors", "type": "number"},
{"name": "error_message", "type": "text"},
{"name": "started", "type": "date"},
{"name": "finished", "type": "date"}
]
}'
create_collection "audio_jobs" '{
"name": "audio_jobs",
"type": "base",
"fields": [
{"name": "cache_key", "type": "text", "required": true},
{"name": "slug", "type": "text", "required": true},
{"name": "chapter", "type": "number", "required": true},
{"name": "voice", "type": "text"},
{"name": "worker_id", "type": "text"},
{"name": "status", "type": "text", "required": true},
{"name": "error_message", "type": "text"},
{"name": "started", "type": "date"},
{"name": "finished", "type": "date"}
]
}'
create_collection "user_library" '{
"name": "user_library",
"type": "base",
"fields": [
{"name": "session_id", "type": "text", "required": true},
{"name": "user_id", "type": "text"},
{"name": "slug", "type": "text", "required": true},
{"name": "saved_at", "type": "date"}
]
}'
create_collection "user_sessions" '{
"name": "user_sessions",
"type": "base",
"fields": [
{"name": "user_id", "type": "text", "required": true},
{"name": "session_id", "type": "text", "required": true},
{"name": "user_agent", "type": "text"},
{"name": "ip", "type": "text"},
{"name": "created_at", "type": "date"},
{"name": "last_seen", "type": "date"}
]
}'
create_collection "user_subscriptions" '{
"name": "user_subscriptions",
"type": "base",
"fields": [
{"name": "follower_id", "type": "text", "required": true},
{"name": "followee_id", "type": "text", "required": true},
{"name": "created", "type": "date"}
]
}'
# ─── 7. Post-initial-deploy field additions ───────────────────────────────────
# heartbeat_at is used by the backend runner to detect stale tasks.
ensure_field "scraping_tasks" "heartbeat_at" "date"
ensure_field "audio_jobs" "heartbeat_at" "date"
log "all collections ready"

View File

@@ -1,5 +0,0 @@
node_modules
build
.svelte-kit
.env
.env.*

View File

@@ -1,20 +0,0 @@
# libnovel UI — environment variables
# Copy to .env and adjust; do NOT commit with real secrets.
# Public URL of the scraper API (used by SvelteKit server-side load functions)
# In docker-compose this is the internal service name
SCRAPER_API_URL=http://localhost:8080
# Public URL of PocketBase (used by SvelteKit server-side load functions)
POCKETBASE_URL=http://localhost:8090
# PocketBase admin credentials (server-side only, never exposed to browser)
POCKETBASE_ADMIN_EMAIL=admin@libnovel.local
POCKETBASE_ADMIN_PASSWORD=changeme123
# Public-facing MinIO URL (used to rewrite presigned URLs for the browser)
# In dev this is localhost; in prod set to your MinIO public domain
PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000
# Secret used to sign auth tokens stored in cookies (generate with: openssl rand -hex 32)
AUTH_SECRET=change_this_to_a_long_random_secret

23
ui-v2/.gitignore vendored
View File

@@ -1,23 +0,0 @@
node_modules
# Output
.output
.vercel
.netlify
.wrangler
/.svelte-kit
/build
# OS
.DS_Store
Thumbs.db
# Env
.env
.env.*
!.env.example
!.env.test
# Vite
vite.config.js.timestamp-*
vite.config.ts.timestamp-*

View File

@@ -1 +0,0 @@
engine-strict=true

View File

@@ -1,39 +0,0 @@
# syntax=docker/dockerfile:1
FROM node:22-alpine AS builder
WORKDIR /app
# Install dependencies in a separate layer so it is cached as long as
# package-lock.json does not change. The npm cache mount persists the
# ~/.npm cache across builds so packages are not re-downloaded.
COPY package.json package-lock.json ./
RUN --mount=type=cache,target=/root/.npm \
npm ci
COPY . .
# Build-time version info — injected by docker-compose or CI via --build-arg.
ARG BUILD_VERSION=dev
ARG BUILD_COMMIT=unknown
# Expose as PUBLIC_ env vars so SvelteKit's $env/dynamic/public can read them.
ENV PUBLIC_BUILD_VERSION=$BUILD_VERSION
ENV PUBLIC_BUILD_COMMIT=$BUILD_COMMIT
RUN npm run build
# ── Runtime image ──────────────────────────────────────────────────────────────
# adapter-node bundles all server-side dependencies into build/ — no npm install
# needed at runtime. We do need package.json (for "type": "module") so Node
# resolves the ESM output correctly when there is no parent package.json.
FROM node:22-alpine
WORKDIR /app
COPY --from=builder /app/build ./build
COPY --from=builder /app/package.json ./package.json
ENV NODE_ENV=production
ENV PORT=3000
ENV HOST=0.0.0.0
EXPOSE $PORT
CMD ["node", "build"]

View File

@@ -1,42 +0,0 @@
# sv
Everything you need to build a Svelte project, powered by [`sv`](https://github.com/sveltejs/cli).
## Creating a project
If you're seeing this, you've probably already done this step. Congrats!
```sh
# create a new project
npx sv create my-app
```
To recreate this project with the same configuration:
```sh
# recreate this project
npx sv@0.12.4 create --template minimal --types ts --install npm ui
```
## Developing
Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
```sh
npm run dev
# or start the server and open the app in a new browser tab
npm run dev -- --open
```
## Building
To create a production version of your app:
```sh
npm run build
```
You can preview the production build with `npm run preview`.
> To deploy your app, you may need to install an [adapter](https://svelte.dev/docs/kit/adapters) for your target environment.

4160
ui-v2/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,34 +0,0 @@
{
"name": "ui",
"private": true,
"version": "0.0.1",
"type": "module",
"scripts": {
"dev": "vite dev",
"build": "vite build",
"preview": "vite preview",
"prepare": "svelte-kit sync || echo ''",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch"
},
"devDependencies": {
"@sveltejs/adapter-auto": "^7.0.0",
"@sveltejs/adapter-node": "^5.5.4",
"@sveltejs/kit": "^2.50.2",
"@sveltejs/vite-plugin-svelte": "^6.2.4",
"@tailwindcss/vite": "^4.2.1",
"@types/node": "^25.3.3",
"svelte": "^5.51.0",
"svelte-check": "^4.4.2",
"tailwindcss": "^4.2.1",
"typescript": "^5.9.3",
"vite": "^7.3.1"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.1005.0",
"@aws-sdk/s3-request-presigner": "^3.1005.0",
"cropperjs": "^1.6.2",
"marked": "^17.0.3",
"pocketbase": "^0.26.8"
}
}

View File

@@ -1,65 +0,0 @@
@import "tailwindcss";
@theme {
--color-brand: #f59e0b; /* amber-400 */
--color-brand-dim: #d97706; /* amber-600 */
--color-surface: #18181b; /* zinc-900 */
--color-surface-2: #27272a; /* zinc-800 */
--color-surface-3: #3f3f46; /* zinc-700 */
--color-muted: #a1a1aa; /* zinc-400 */
--color-text: #f4f4f5; /* zinc-100 */
}
html {
background-color: var(--color-surface);
color: var(--color-text);
}
/* ── Chapter prose ─────────────────────────────────────────────────── */
.prose-chapter {
max-width: 72ch;
line-height: 1.85;
font-size: 1.05rem;
color: #d4d4d8; /* zinc-300 */
}
.prose-chapter h1,
.prose-chapter h2,
.prose-chapter h3 {
color: #f4f4f5;
font-weight: 700;
margin-top: 1.5em;
margin-bottom: 0.5em;
}
.prose-chapter h1 { font-size: 1.4rem; }
.prose-chapter h2 { font-size: 1.2rem; }
.prose-chapter h3 { font-size: 1.05rem; }
.prose-chapter p {
margin-bottom: 1.2em;
}
.prose-chapter em {
color: #a1a1aa;
}
.prose-chapter strong {
color: #f4f4f5;
}
.prose-chapter hr {
border-color: #3f3f46;
margin: 2em 0;
}
/* ── Navigation progress bar ───────────────────────────────────────── */
@keyframes progress-bar {
0% { width: 0%; opacity: 1; }
80% { width: 90%; opacity: 1; }
100% { width: 100%; opacity: 0; }
}
.animate-progress-bar {
animation: progress-bar 8s cubic-bezier(0.1, 0.05, 0.1, 1) forwards;
}

18
ui-v2/src/app.d.ts vendored
View File

@@ -1,18 +0,0 @@
// See https://svelte.dev/docs/kit/types#app.d.ts
// for information about these interfaces
declare global {
namespace App {
// interface Error {}
interface Locals {
sessionId: string;
user: { id: string; username: string; role: string; authSessionId: string } | null;
}
interface PageData {
user?: { id: string; username: string; role: string; authSessionId: string } | null;
}
// interface PageState {}
// interface Platform {}
}
}
export {};

View File

@@ -1,17 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link rel="icon" href="/favicon.ico" sizes="16x16 32x32" />
<link rel="icon" type="image/png" href="/favicon-32.png" sizes="32x32" />
<link rel="icon" type="image/png" href="/favicon-16.png" sizes="16x16" />
<link rel="apple-touch-icon" href="/apple-touch-icon.png" />
<link rel="icon" type="image/png" href="/icon-192.png" sizes="192x192" />
<link rel="icon" type="image/png" href="/icon-512.png" sizes="512x512" />
%sveltekit.head%
</head>
<body data-sveltekit-preload-data="hover">
<div style="display: contents">%sveltekit.body%</div>
</body>
</html>

View File

@@ -1,155 +0,0 @@
import type { Handle } from '@sveltejs/kit';
import { randomBytes, createHmac } from 'node:crypto';
import { env } from '$env/dynamic/private';
import { log } from '$lib/server/logger';
import { createUserSession, touchUserSession, isSessionRevoked } from '$lib/server/pocketbase';
import { drain as drainPresignCache } from '$lib/server/presignCache';
// ─── Graceful shutdown ────────────────────────────────────────────────────────
//
// When Docker/Kubernetes sends SIGTERM (or the user sends SIGINT), we:
// 1. Set shuttingDown = true so new requests immediately receive 503.
// 2. Flush/drain in-process caches (presign URL cache).
// 3. Allow Node.js to exit naturally once in-flight requests finish.
//
// adapter-node does not provide a built-in hook for this, so we wire it here
// in hooks.server.ts which runs in the server Node.js process.
let shuttingDown = false;
function shutdown(signal: string) {
if (shuttingDown) return;
shuttingDown = true;
log.info('shutdown', `received ${signal}, draining in-flight requests`);
drainPresignCache();
// Don't call process.exit() — let Node exit naturally once the event loop
// is empty (adapter-node closes the HTTP server on its own).
}
process.once('SIGTERM', () => shutdown('SIGTERM'));
process.once('SIGINT', () => shutdown('SIGINT'));
const SESSION_COOKIE = 'libnovel_session';
const AUTH_COOKIE = 'libnovel_auth';
const ONE_YEAR = 60 * 60 * 24 * 365;
const AUTH_SECRET = env.AUTH_SECRET ?? 'dev_secret_change_in_production';
// ─── Token helpers ────────────────────────────────────────────────────────────
/**
* Sign a payload string with HMAC-SHA256 using AUTH_SECRET.
* Returns "<payload>.<signature>".
*/
export function signToken(payload: string): string {
const sig = createHmac('sha256', AUTH_SECRET).update(payload).digest('hex');
return `${payload}.${sig}`;
}
/**
* Verify a signed token. Returns the payload string on success, null on failure.
*/
export function verifyToken(token: string): string | null {
const lastDot = token.lastIndexOf('.');
if (lastDot < 0) return null;
const payload = token.slice(0, lastDot);
const expected = createHmac('sha256', AUTH_SECRET).update(payload).digest('hex');
const actual = token.slice(lastDot + 1);
// constant-time comparison
if (expected.length !== actual.length) return null;
let diff = 0;
for (let i = 0; i < expected.length; i++) {
diff |= expected.charCodeAt(i) ^ actual.charCodeAt(i);
}
return diff === 0 ? payload : null;
}
/**
* Create a signed auth token for a user.
* Payload format: "<userId>:<username>:<role>:<authSessionId>"
* authSessionId uniquely identifies this login session (for revocation).
*/
export function createAuthToken(userId: string, username: string, role: string, authSessionId: string): string {
return signToken(`${userId}:${username}:${role}:${authSessionId}`);
}
/**
* Parse a verified auth token into user data. Returns null if invalid.
* Supports both old format (3 segments) and new format (4 segments).
*/
export function parseAuthToken(token: string): { id: string; username: string; role: string; authSessionId: string } | null {
const payload = verifyToken(token);
if (!payload) return null;
const parts = payload.split(':');
// New format: userId:username:role:authSessionId (4 parts)
// Old format: userId:username:role (3 parts — legacy tokens before session tracking)
if (parts.length < 3) return null;
const id = parts[0];
const username = parts[1];
const role = parts[2];
const authSessionId = parts[3] ?? ''; // empty string for legacy tokens
if (!id || !username) return null;
return { id, username, role, authSessionId };
}
// ─── Hook ─────────────────────────────────────────────────────────────────────
export const handle: Handle = async ({ event, resolve }) => {
// During graceful shutdown, reject new requests immediately so the load
// balancer / Docker health-check can drain existing connections.
if (shuttingDown) {
return new Response('Service shutting down', { status: 503 });
}
// Anonymous session cookie (for reading progress)
let sessionId = event.cookies.get(SESSION_COOKIE) ?? '';
if (!sessionId) {
sessionId = randomBytes(16).toString('hex');
event.cookies.set(SESSION_COOKIE, sessionId, {
path: '/',
httpOnly: true,
sameSite: 'lax',
maxAge: ONE_YEAR
});
}
event.locals.sessionId = sessionId;
// Auth cookie → resolve logged-in user
const authToken = event.cookies.get(AUTH_COOKIE);
if (authToken) {
const user = parseAuthToken(authToken);
if (!user) {
log.warn('auth', 'auth cookie present but failed to parse (malformed or tampered)');
event.locals.user = null;
} else {
// Validate session against DB (only for new-format tokens with authSessionId)
let sessionValid = true;
if (user.authSessionId) {
try {
const revoked = await isSessionRevoked(user.authSessionId);
if (revoked) {
log.info('auth', 'auth cookie references revoked session', {
userId: user.id,
authSessionId: user.authSessionId
});
sessionValid = false;
// Clear the invalid cookie
event.cookies.delete(AUTH_COOKIE, { path: '/' });
} else {
// Best-effort: update last_seen in the background
touchUserSession(user.authSessionId).catch(() => {});
}
} catch (err) {
// DB error — fail open to avoid locking everyone out
log.warn('auth', 'session check failed (fail open)', { err: String(err) });
}
}
event.locals.user = sessionValid ? user : null;
}
} else {
event.locals.user = null;
}
return resolve(event);
};

View File

@@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="107" height="128" viewBox="0 0 107 128"><title>svelte-logo</title><path d="M94.157 22.819c-10.4-14.885-30.94-19.297-45.792-9.835L22.282 29.608A29.92 29.92 0 0 0 8.764 49.65a31.5 31.5 0 0 0 3.108 20.231 30 30 0 0 0-4.477 11.183 31.9 31.9 0 0 0 5.448 24.116c10.402 14.887 30.942 19.297 45.791 9.835l26.083-16.624A29.92 29.92 0 0 0 98.235 78.35a31.53 31.53 0 0 0-3.105-20.232 30 30 0 0 0 4.474-11.182 31.88 31.88 0 0 0-5.447-24.116" style="fill:#ff3e00"/><path d="M45.817 106.582a20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.503 18 18 0 0 1 .624-2.435l.49-1.498 1.337.981a33.6 33.6 0 0 0 10.203 5.098l.97.294-.09.968a5.85 5.85 0 0 0 1.052 3.878 6.24 6.24 0 0 0 6.695 2.485 5.8 5.8 0 0 0 1.603-.704L69.27 76.28a5.43 5.43 0 0 0 2.45-3.631 5.8 5.8 0 0 0-.987-4.371 6.24 6.24 0 0 0-6.698-2.487 5.7 5.7 0 0 0-1.6.704l-9.953 6.345a19 19 0 0 1-5.296 2.326 20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.502 17.99 17.99 0 0 1 8.13-12.052l26.081-16.623a19 19 0 0 1 5.3-2.329 20.72 20.72 0 0 1 22.237 8.243 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-.624 2.435l-.49 1.498-1.337-.98a33.6 33.6 0 0 0-10.203-5.1l-.97-.294.09-.968a5.86 5.86 0 0 0-1.052-3.878 6.24 6.24 0 0 0-6.696-2.485 5.8 5.8 0 0 0-1.602.704L37.73 51.72a5.42 5.42 0 0 0-2.449 3.63 5.79 5.79 0 0 0 .986 4.372 6.24 6.24 0 0 0 6.698 2.486 5.8 5.8 0 0 0 1.602-.704l9.952-6.342a19 19 0 0 1 5.295-2.328 20.72 20.72 0 0 1 22.237 8.242 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-8.13 12.053l-26.081 16.622a19 19 0 0 1-5.3 2.328" style="fill:#fff"/></svg>

Before

Width:  |  Height:  |  Size: 1.5 KiB

View File

@@ -1,146 +0,0 @@
/**
* Global audio player state for libnovel.
*
* A single shared instance (module singleton) keeps audio playing across
* SvelteKit navigations. The layout mounts the <audio> element once and
* never unmounts it; the per-chapter AudioPlayer component is just a
* controller that reads/writes this state.
*
* Uses Svelte 5 runes ($state / $derived) — import only from .svelte files
* or other .svelte.ts files.
*
* ── State machine ────────────────────────────────────────────────────────────
*
* Current chapter (status):
* idle → loading → ready (fast path: audio exists in MinIO)
* idle → loading → generating → ready (slow path: Kokoro TTS)
* any → error
*
* Next chapter pre-fetch (nextStatus):
* 'none' no next chapter, or auto-next is off
* 'prefetching' POST /api/audio running for the next chapter
* 'prefetched' next chapter audio is ready in MinIO
* 'failed' pre-generation failed (will retry on navigate)
*
* Auto-next transition:
* onended fires → navigate to next chapter URL
* ↳ new chapter page mounts
* • if nextStatus === 'prefetched' → presign + play immediately
* • else → normal startPlayback() flow
*
* Pre-fetch is triggered when currentTime / duration >= 0.9 (90% mark).
* It only runs once per chapter (guarded by nextStatus !== 'none').
*/
export type AudioStatus = 'idle' | 'loading' | 'generating' | 'ready' | 'error';
export type NextStatus = 'none' | 'prefetching' | 'prefetched' | 'failed';
class AudioStore {
// ── What is loaded ──────────────────────────────────────────────────────
slug = $state('');
chapter = $state(0);
chapterTitle = $state('');
bookTitle = $state('');
voice = $state('af_bella');
speed = $state(1.0);
/** Cover image URL for the currently loaded book. */
cover = $state('');
/** Full chapter list for the currently loaded book (number + title). */
chapters = $state<{ number: number; title: string }[]>([]);
// ── Loading/generation state ────────────────────────────────────────────
status = $state<AudioStatus>('idle');
audioUrl = $state('');
errorMsg = $state('');
/** Pseudo-progress bar value 0100 during generation */
progress = $state(0);
// ── Playback state (kept in sync with the <audio> element) ─────────────
currentTime = $state(0);
duration = $state(0);
isPlaying = $state(false);
/**
* Increment to signal the layout to toggle play/pause.
* The layout watches this with $effect and calls audioEl.play()/pause().
*/
toggleRequest = $state(0);
/**
* Set to a number to seek the audio element to that time (seconds).
* The layout watches this with $effect and sets audioEl.currentTime.
* Reset to null after handling.
*/
seekRequest = $state<number | null>(null);
// ── Auto-next ────────────────────────────────────────────────────────────
/**
* When true, navigates to the next chapter when the current one ends
* and auto-starts its audio.
*/
autoNext = $state(false);
/**
* The next chapter number for the currently playing chapter, or null if
* there is no next chapter. Written by the chapter page's AudioPlayer.
* Stored here (not cleared on unmount) so onended can still read it after
* the component unmounts due to {#key} re-render on navigation.
*/
nextChapter = $state<number | null>(null);
/**
* Set to the chapter number that should auto-start by the layout's onended
* handler (when autoNext fires a navigation). The AudioPlayer on the new
* page checks this on mount: if it matches the component's own chapter prop
* it starts playback and clears the value.
*
* Using the target chapter number (instead of a plain boolean) prevents the
* still-mounted outgoing AudioPlayer from reacting to the flag before the
* navigation completes — it only matches the incoming chapter's component.
*/
autoStartChapter = $state<number | null>(null);
// ── Next-chapter pre-fetch state ─────────────────────────────────────────
/**
* State of the background pre-generation for the next chapter.
* 'none' nothing started (default / no next chapter)
* 'prefetching' currently running POST /api/audio for next chapter
* 'prefetched' next chapter audio confirmed ready in MinIO
* 'failed' pre-generation failed (fallback: generate on navigate)
*/
nextStatus = $state<NextStatus>('none');
/**
* The presigned URL obtained during pre-fetch. When the user navigates
* to the next chapter, AudioPlayer picks this up and skips straight to play.
*/
nextAudioUrl = $state('');
/** Progress value (0100) shown while pre-generating the next chapter. */
nextProgress = $state(0);
/** Which chapter number the pre-fetch state above belongs to. */
nextChapterPrefetched = $state<number | null>(null);
/** Whether the mini-bar at the bottom is visible */
get active(): boolean {
return this.status === 'ready' || this.status === 'generating' || this.status === 'loading';
}
/** True when the currently loaded track matches slug+chapter */
isCurrentChapter(slug: string, chapter: number): boolean {
return this.slug === slug && this.chapter === chapter;
}
/** Reset all next-chapter pre-fetch state. */
resetNextPrefetch() {
this.nextStatus = 'none';
this.nextAudioUrl = '';
this.nextProgress = 0;
this.nextChapterPrefetched = null;
}
}
export const audioStore = new AudioStore();

View File

@@ -1,862 +0,0 @@
<script lang="ts">
/**
* AudioPlayer — controller component.
*
* Does NOT own an <audio> element. Instead it reads/writes `audioStore`,
* which is shared with the layout's persistent <audio> element so audio
* survives SvelteKit navigations.
*
* ── Play flow ────────────────────────────────────────────────────────────
* On "Play narration" click / auto-start:
* 1. Populate store metadata (slug, chapter, titles, voice, speed).
* 2. If the pre-fetch already landed (nextStatus='prefetched' AND
* nextChapterPrefetched === chapter), use the cached URL immediately.
* 3. Otherwise try GET /api/presign/audio — if 200, set audioUrl → layout plays.
* 4. If 404, POST /api/audio/:slug/:n to generate. Drive pseudo progress bar.
* On success (200 or 202→done), presign and set audioUrl directly from MinIO.
*
* ── Voice selection ──────────────────────────────────────────────────────
* A "Change voice" panel lets users pick from the available Kokoro voices.
* Each voice shows a play button that streams a pre-generated sample from
* MinIO (GET /api/presign/voice-sample?voice=...). Samples are generated
* server-side via POST /api/audio/voice-samples.
*
* Changing voice updates audioStore.voice (saved to settings via layout).
* The currently loaded chapter audio is NOT re-generated automatically —
* the new voice takes effect on next "Play narration" click.
*
* ── Pre-fetch (immediate + 90% fallback) ────────────────────────────────
* When autoNext is on, prefetchNext() is called as soon as the current
* chapter starts playing (via maybeStartPrefetch() at the end of
* startPlayback()). This gives the maximum lead time for Kokoro to
* generate the next chapter so the transition is seamless.
*
* A $effect also watches currentTime/duration and fires prefetchNext() at
* the 90% mark as a fallback — covering the case where autoNext was toggled
* on mid-playback after startPlayback() had already returned.
* The nextStatus !== 'none' guard prevents double-runs in all cases.
*
* prefetchNext():
* • Calls POST /api/audio for next chapter (sets nextStatus='prefetching')
* • On success, presigns and stores URL in audioStore.nextAudioUrl
* (sets nextStatus='prefetched')
* • On failure, sets nextStatus='failed'
*
* ── Auto-next ────────────────────────────────────────────────────────────
* layout.svelte onended → sets autoStartPending=true → navigates.
* New chapter's AudioPlayer mounts → sees autoStartPending → startPlayback()
* which uses the prefetched URL if available.
*/
import { audioStore } from '$lib/audio.svelte';
interface Props {
slug: string;
chapter: number;
chapterTitle?: string;
bookTitle?: string;
/** Cover image URL for the book (used in MediaSession for lock-screen art). */
cover?: string;
/** Next chapter number, or null/undefined if this is the last chapter. */
nextChapter?: number | null;
/** Full chapter list for the book (number + title). Written into the store. */
chapters?: { number: number; title: string }[];
/** List of available voices from the Kokoro API. */
voices?: string[];
}
let {
slug,
chapter,
chapterTitle = '',
bookTitle = '',
cover = '',
nextChapter = null,
chapters = [],
voices = []
}: Props = $props();
// ── Voice selector state ────────────────────────────────────────────────
let showVoicePanel = $state(false);
/** Voice whose sample is currently being fetched or playing. */
let samplePlayingVoice = $state<string | null>(null);
/** Currently active sample <audio> element — one at a time. */
let sampleAudio = $state<HTMLAudioElement | null>(null);
/**
* Human-readable label for a voice ID.
* e.g. "af_bella" → "Bella (US F)" | "bm_george" → "George (UK M)"
*/
function voiceLabel(v: string): string {
const langMap: Record<string, string> = {
af: 'US', am: 'US',
bf: 'UK', bm: 'UK',
ef: 'ES', em: 'ES',
ff: 'FR',
hf: 'IN', hm: 'IN',
'if': 'IT', im: 'IT',
jf: 'JP', jm: 'JP',
pf: 'PT', pm: 'PT',
zf: 'ZH', zm: 'ZH',
};
const genderMap: Record<string, string> = {
af: 'F', am: 'M',
bf: 'F', bm: 'M',
ef: 'F', em: 'M',
ff: 'F',
hf: 'F', hm: 'M',
'if': 'F', im: 'M',
jf: 'F', jm: 'M',
pf: 'F', pm: 'M',
zf: 'F', zm: 'M',
};
const prefix = v.slice(0, 2);
const name = v.slice(3);
// Capitalise and strip legacy v0 prefix.
const displayName = name
.replace(/^v0/, '')
.replace(/^([a-z])/, (c: string) => c.toUpperCase());
const lang = langMap[prefix] ?? prefix.toUpperCase();
const gender = genderMap[prefix] ?? '?';
return `${displayName} (${lang} ${gender})`;
}
/** Stop any currently playing sample. */
function stopSample() {
if (sampleAudio) {
sampleAudio.pause();
sampleAudio.src = '';
sampleAudio = null;
}
samplePlayingVoice = null;
}
/** Play a voice sample from MinIO. */
async function playSample(voice: string) {
// If this voice is already playing, stop it.
if (samplePlayingVoice === voice) {
stopSample();
return;
}
stopSample();
samplePlayingVoice = voice;
try {
const res = await fetch(`/api/presign/voice-sample?voice=${encodeURIComponent(voice)}`);
if (res.status === 404) {
// Sample not generated yet — silently ignore
samplePlayingVoice = null;
return;
}
if (!res.ok) throw new Error(`presign failed: ${res.status}`);
const data = (await res.json()) as { url: string };
const audio = new Audio(data.url);
sampleAudio = audio;
audio.onended = () => {
if (samplePlayingVoice === voice) stopSample();
};
audio.onerror = () => {
if (samplePlayingVoice === voice) stopSample();
};
await audio.play();
} catch {
samplePlayingVoice = null;
}
}
/** Select a voice and close the panel. */
function selectVoice(voice: string) {
stopSample();
audioStore.voice = voice;
showVoicePanel = false;
}
// Keep nextChapter in the store so the layout's onended can navigate.
// NOTE: we do NOT clear on unmount here — the store retains the value so
// onended (which may fire after {#key} unmounts this component) can still
// read it. The value is superseded when the new chapter mounts.
$effect(() => {
audioStore.nextChapter = nextChapter ?? null;
});
// Auto-start: if the layout navigated here via auto-next, kick off playback.
// We match against the chapter prop so the outgoing chapter's AudioPlayer
// (still mounted during the brief navigation window) never reacts to this.
$effect(() => {
if (audioStore.autoStartChapter === chapter) {
audioStore.autoStartChapter = null;
startPlayback();
}
});
// Reset next-chapter prefetch state when this chapter changes (new page).
// Only reset if the prefetch belongs to neither the current chapter
// (about to be consumed by startPlayback) nor the next chapter (still valid).
// Any other value means stale data from a previous page.
$effect(() => {
const prefetchedFor = audioStore.nextChapterPrefetched;
if (
prefetchedFor !== null &&
prefetchedFor !== chapter &&
prefetchedFor !== (nextChapter ?? null)
) {
audioStore.resetNextPrefetch();
}
});
// Close voice panel when user clicks outside (escape key).
function handleKeyDown(e: KeyboardEvent) {
if (e.key === 'Escape') {
stopSample();
showVoicePanel = false;
}
}
// ── 90% pre-fetch trigger ─────────────────────────────────────────────────
// Watch playback progress; when >= 90% of current chapter, pre-generate
// the next chapter's audio so it's ready when we navigate.
$effect(() => {
const ct = audioStore.currentTime;
const dur = audioStore.duration;
const isCurrentlyPlaying = audioStore.isCurrentChapter(slug, chapter);
if (
!isCurrentlyPlaying ||
!audioStore.autoNext ||
nextChapter === null ||
nextChapter === undefined ||
dur <= 0 ||
ct / dur < 0.9 ||
audioStore.nextStatus !== 'none'
) {
return;
}
// Trigger exactly once (nextStatus transitions away from 'none')
prefetchNext();
});
// ── Pseudo progress helpers ────────────────────────────────────────────────
let progressRafId = 0;
function startProgress() {
audioStore.progress = 0;
let last = performance.now();
function tick(now: number) {
const dt = (now - last) / 1000;
last = now;
let rate: number;
if (audioStore.progress < 30) rate = 4;
else if (audioStore.progress < 60) rate = 12;
else if (audioStore.progress < 80) rate = 4;
else rate = 0.3;
audioStore.progress = Math.min(audioStore.progress + rate * dt, 99);
if (audioStore.progress < 99) {
progressRafId = requestAnimationFrame(tick);
}
}
progressRafId = requestAnimationFrame(tick);
}
function stopProgress() {
if (progressRafId) {
cancelAnimationFrame(progressRafId);
progressRafId = 0;
}
}
async function finishProgress() {
stopProgress();
const step = () => {
audioStore.progress = Math.min(audioStore.progress + 8, 100);
if (audioStore.progress < 100) {
progressRafId = requestAnimationFrame(step);
}
};
progressRafId = requestAnimationFrame(step);
await new Promise((r) => setTimeout(r, 200));
stopProgress();
}
// ── Next-chapter pseudo-progress helpers ──────────────────────────────────
let nextProgressRafId = 0;
function startNextProgress() {
audioStore.nextProgress = 0;
let last = performance.now();
function tick(now: number) {
const dt = (now - last) / 1000;
last = now;
let rate: number;
if (audioStore.nextProgress < 30) rate = 4;
else if (audioStore.nextProgress < 60) rate = 12;
else if (audioStore.nextProgress < 80) rate = 4;
else rate = 0.3;
audioStore.nextProgress = Math.min(audioStore.nextProgress + rate * dt, 99);
if (audioStore.nextProgress < 99) {
nextProgressRafId = requestAnimationFrame(tick);
}
}
nextProgressRafId = requestAnimationFrame(tick);
}
function stopNextProgress() {
if (nextProgressRafId) {
cancelAnimationFrame(nextProgressRafId);
nextProgressRafId = 0;
}
}
// ── API helpers ────────────────────────────────────────────────────────────
async function tryPresign(
targetSlug: string,
targetChapter: number,
targetVoice: string
): Promise<string | null> {
const params = new URLSearchParams({
slug: targetSlug,
n: String(targetChapter),
voice: targetVoice
});
const res = await fetch(`/api/presign/audio?${params}`);
if (res.status === 404) return null;
if (!res.ok) throw new Error(`presign HTTP ${res.status}`);
const data = (await res.json()) as { url: string };
return data.url;
}
type AudioStatusResponse =
| { status: 'done' }
| { status: 'pending' | 'generating'; job_id: string }
| { status: 'idle' }
| { status: 'failed'; error?: string };
/**
* Poll GET /api/audio/status/[slug]/[n]?voice=... every `intervalMs` ms
* until status is "done" or "failed" (or the caller cancels via signal).
*
* Returns the final status response, or throws on network error / cancellation.
*/
async function pollAudioStatus(
targetSlug: string,
targetChapter: number,
targetVoice: string,
intervalMs = 2000,
signal?: AbortSignal
): Promise<AudioStatusResponse> {
const qs = new URLSearchParams();
if (targetVoice) qs.set('voice', targetVoice);
const url = `/api/audio/status/${targetSlug}/${targetChapter}?${qs.toString()}`;
while (true) {
if (signal?.aborted) throw new DOMException('Aborted', 'AbortError');
const res = await fetch(url, { signal });
if (!res.ok) throw new Error(`Status poll HTTP ${res.status}`);
const data = (await res.json()) as AudioStatusResponse;
if (data.status === 'done' || data.status === 'failed') {
return data;
}
// Still pending/generating — wait then retry.
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(resolve, intervalMs);
signal?.addEventListener('abort', () => {
clearTimeout(timer);
reject(new DOMException('Aborted', 'AbortError'));
});
});
}
}
// ── Pre-fetch next chapter ─────────────────────────────────────────────────
async function prefetchNext() {
if (nextChapter === null || nextChapter === undefined) return;
if (audioStore.nextStatus !== 'none') return; // already running or done
const voice = audioStore.voice;
audioStore.nextStatus = 'prefetching';
audioStore.nextChapterPrefetched = nextChapter;
startNextProgress();
try {
// Fast path: already generated
const url = await tryPresign(slug, nextChapter, voice);
if (url) {
stopNextProgress();
audioStore.nextProgress = 100;
audioStore.nextAudioUrl = url;
audioStore.nextStatus = 'prefetched';
return;
}
// Slow path: trigger Kokoro generation (non-blocking POST), then poll.
const res = await fetch(`/api/audio/${slug}/${nextChapter}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ voice })
});
if (!res.ok) throw new Error(`Prefetch generation failed: HTTP ${res.status}`);
// Whether the server returned 200 (already cached) or 202 (enqueued),
// always presign — the status endpoint no longer returns a proxy URL.
if (res.status === 200) {
// Body is { status: 'done' } — audio confirmed in MinIO. Presign it.
await res.body?.cancel();
}
// else 202: generation enqueued — fall through to poll.
if (res.status !== 200) {
// 202: poll until done.
const final = await pollAudioStatus(slug, nextChapter, voice);
stopNextProgress();
audioStore.nextProgress = 100;
if (final.status === 'failed') {
throw new Error(`Prefetch failed: ${(final as { error?: string }).error ?? 'unknown'}`);
}
} else {
stopNextProgress();
audioStore.nextProgress = 100;
}
// Audio is ready in MinIO — get a direct presigned URL.
const doneUrl = await tryPresign(slug, nextChapter, voice);
if (!doneUrl) throw new Error('Prefetch: audio done but presign returned 404');
audioStore.nextAudioUrl = doneUrl;
audioStore.nextStatus = 'prefetched';
} catch {
stopNextProgress();
audioStore.nextStatus = 'failed';
}
}
// ── Media Session ──────────────────────────────────────────────────────────
// Sets the OS-level media metadata so the book cover, title, and chapter
// appear on the phone lock screen / notification center.
function setMediaSession() {
if (typeof navigator === 'undefined' || !('mediaSession' in navigator)) return;
const artwork: MediaImage[] = cover
? [
{ src: cover, sizes: '512x512', type: 'image/jpeg' },
{ src: cover, sizes: '256x256', type: 'image/jpeg' }
]
: [];
navigator.mediaSession.metadata = new MediaMetadata({
title: chapterTitle || `Chapter ${chapter}`,
artist: bookTitle,
album: bookTitle,
artwork
});
}
// ── Core play flow ─────────────────────────────────────────────────────────
async function startPlayback() {
const voice = audioStore.voice;
// Populate store metadata so layout + mini-bar have track info.
audioStore.slug = slug;
audioStore.chapter = chapter;
audioStore.chapterTitle = chapterTitle;
audioStore.bookTitle = bookTitle;
audioStore.cover = cover;
audioStore.chapters = chapters;
// Update OS media session (lock screen / notification center).
setMediaSession();
audioStore.status = 'loading';
audioStore.errorMsg = '';
try {
// Fast path A: pre-fetch already landed for THIS chapter.
if (
audioStore.nextStatus === 'prefetched' &&
audioStore.nextChapterPrefetched === chapter &&
audioStore.nextAudioUrl
) {
const url = audioStore.nextAudioUrl;
// Consume the pre-fetch — reset so it doesn't carry over
audioStore.resetNextPrefetch();
audioStore.audioUrl = url;
audioStore.status = 'ready';
// Don't restore saved time for auto-next; position is 0
// Immediately start pre-generating the chapter after this one.
maybeStartPrefetch();
return;
}
// Fast path B: audio already in MinIO (presign check).
const url = await tryPresign(slug, chapter, voice);
if (url) {
audioStore.audioUrl = url;
audioStore.status = 'ready';
// Restore last saved position after the audio element loads
restoreSavedAudioTime();
// Immediately start pre-generating the next chapter in background.
maybeStartPrefetch();
return;
}
// Slow path: trigger Kokoro generation (non-blocking POST), then poll.
audioStore.status = 'generating';
startProgress();
const res = await fetch(`/api/audio/${slug}/${chapter}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ voice })
});
if (!res.ok) throw new Error(`Generation failed: HTTP ${res.status}`);
if (res.status !== 200) {
// 202: generation enqueued — poll until done.
const final = await pollAudioStatus(slug, chapter, voice);
if (final.status === 'failed') {
throw new Error(
`Generation failed: ${(final as { error?: string }).error ?? 'unknown error'}`
);
}
} else {
// 200: already cached — body is { status: 'done' }, no url needed.
await res.body?.cancel();
}
await finishProgress();
// Audio is ready in MinIO — always use a presigned URL for direct playback.
const doneUrl = await tryPresign(slug, chapter, voice);
if (!doneUrl) throw new Error('Audio generated but presign returned 404');
audioStore.audioUrl = doneUrl;
audioStore.status = 'ready';
// Don't restore time for freshly generated audio — position is 0
// Immediately start pre-generating the next chapter in background.
maybeStartPrefetch();
} catch (e) {
stopProgress();
audioStore.progress = 0;
audioStore.status = 'error';
audioStore.errorMsg = String(e);
}
}
/**
* Start pre-fetching the next chapter if autoNext is on, there is a next
* chapter, and no prefetch is already running or completed.
* Called as soon as current-chapter playback begins so that the next
* chapter's audio is ready before we need it (seamless transition).
* The 90%-mark $effect acts as a fallback for cases where autoNext is
* toggled on mid-playback.
*/
function maybeStartPrefetch() {
if (
audioStore.autoNext &&
nextChapter !== null &&
nextChapter !== undefined &&
audioStore.nextStatus === 'none'
) {
prefetchNext();
}
}
/**
* Fetch the saved audio time for this chapter and seek to it after a short
* delay (to allow the audio element to load the source).
*/
async function restoreSavedAudioTime() {
try {
const params = new URLSearchParams({ slug, chapter: String(chapter) });
const res = await fetch(`/api/progress/audio-time?${params}`);
if (!res.ok) return;
const data = (await res.json()) as { audioTime: number | null };
if (data.audioTime && data.audioTime > 5) {
// Small delay to let the <audio> element fully load the src before seeking
setTimeout(() => {
audioStore.seekRequest = data.audioTime as number;
}, 300);
}
} catch {
// Non-critical — silently ignore
}
}
async function handlePlay() {
const isCurrent = audioStore.isCurrentChapter(slug, chapter);
// Already loaded this chapter: toggle play/pause.
if (isCurrent && audioStore.status === 'ready') {
audioStore.toggleRequest = (audioStore.toggleRequest ?? 0) + 1;
return;
}
// Not yet loaded — start the full flow.
await startPlayback();
}
function formatTime(s: number): string {
if (!isFinite(s) || s < 0) return '0:00';
const m = Math.floor(s / 60);
const sec = Math.floor(s % 60);
return `${m}:${sec.toString().padStart(2, '0')}`;
}
</script>
<svelte:window onkeydown={handleKeyDown} />
<div class="mt-6 p-4 rounded-lg bg-zinc-800 border border-zinc-700">
<div class="flex items-center justify-between gap-2 mb-3">
<div class="flex items-center gap-2">
<svg class="w-4 h-4 text-amber-400" fill="currentColor" viewBox="0 0 24 24">
<path d="M12 3v10.55A4 4 0 1014 17V7h4V3h-6z"/>
</svg>
<span class="text-sm text-zinc-300 font-medium">Audio Narration</span>
</div>
<!-- Voice selector button -->
{#if voices.length > 0}
<button
onclick={() => { stopSample(); showVoicePanel = !showVoicePanel; }}
class="flex items-center gap-1.5 px-2 py-1 rounded text-xs font-medium transition-colors {showVoicePanel
? 'text-amber-400 bg-amber-400/15 hover:bg-amber-400/25'
: 'text-zinc-400 bg-zinc-700 hover:bg-zinc-600 hover:text-zinc-200'}"
title="Change voice"
>
<svg class="w-3.5 h-3.5" fill="currentColor" viewBox="0 0 24 24">
<path d="M12 14c1.66 0 3-1.34 3-3V5c0-1.66-1.34-3-3-3S9 3.34 9 5v6c0 1.66 1.34 3 3 3zm-1-9c0-.55.45-1 1-1s1 .45 1 1v6c0 .55-.45 1-1 1s-1-.45-1-1V5zm6 6c0 2.76-2.24 5-5 5s-5-2.24-5-5H5c0 3.53 2.61 6.43 6 6.92V21h2v-3.08c3.39-.49 6-3.39 6-6.92h-2z"/>
</svg>
<span class="max-w-[80px] truncate">{voiceLabel(audioStore.voice)}</span>
<svg class="w-3 h-3 flex-shrink-0 transition-transform {showVoicePanel ? 'rotate-180' : ''}" fill="currentColor" viewBox="0 0 24 24">
<path d="M7 10l5 5 5-5z"/>
</svg>
</button>
{/if}
</div>
<!-- ── Voice selector panel ──────────────────────────────────────────── -->
{#if showVoicePanel && voices.length > 0}
<div class="mb-3 rounded-lg border border-zinc-600 bg-zinc-900 overflow-hidden">
<div class="px-3 py-2 border-b border-zinc-700 flex items-center justify-between">
<span class="text-xs font-semibold text-zinc-400 uppercase tracking-wider">Choose Voice</span>
<button
onclick={() => { stopSample(); showVoicePanel = false; }}
class="text-zinc-500 hover:text-zinc-300 transition-colors"
aria-label="Close voice selector"
>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"/>
</svg>
</button>
</div>
<div class="max-h-64 overflow-y-auto">
{#each voices as v (v)}
<div
class="flex items-center gap-2 px-3 py-2 hover:bg-zinc-800 transition-colors cursor-pointer {audioStore.voice === v ? 'bg-amber-400/10' : ''}"
role="button"
tabindex="0"
onclick={() => selectVoice(v)}
onkeydown={(e) => e.key === 'Enter' && selectVoice(v)}
>
<!-- Selected indicator -->
<div class="w-4 flex-shrink-0">
{#if audioStore.voice === v}
<svg class="w-3.5 h-3.5 text-amber-400" fill="currentColor" viewBox="0 0 24 24">
<path d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41L9 16.17z"/>
</svg>
{/if}
</div>
<!-- Voice name -->
<span class="flex-1 text-xs {audioStore.voice === v ? 'text-amber-400 font-medium' : 'text-zinc-300'}">
{voiceLabel(v)}
</span>
<span class="text-zinc-600 text-xs font-mono">{v}</span>
<!-- Sample play button (stop propagation so click doesn't select) -->
<button
onclick={(e) => { e.stopPropagation(); playSample(v); }}
class="p-1 rounded transition-colors flex-shrink-0 {samplePlayingVoice === v
? 'text-amber-400 bg-amber-400/15 hover:bg-amber-400/25'
: 'text-zinc-500 hover:text-zinc-200 hover:bg-zinc-700'}"
title={samplePlayingVoice === v ? 'Stop sample' : 'Play sample'}
aria-label={samplePlayingVoice === v ? `Stop ${v} sample` : `Play ${v} sample`}
>
{#if samplePlayingVoice === v}
<!-- Stop icon -->
<svg class="w-3.5 h-3.5" fill="currentColor" viewBox="0 0 24 24">
<path d="M6 6h12v12H6z"/>
</svg>
{:else}
<!-- Play icon -->
<svg class="w-3.5 h-3.5" fill="currentColor" viewBox="0 0 24 24">
<path d="M8 5v14l11-7z"/>
</svg>
{/if}
</button>
</div>
{/each}
</div>
<div class="px-3 py-2 border-t border-zinc-700 bg-zinc-800/50">
<p class="text-xs text-zinc-500">
New voice applies on next "Play narration".
{#if voices.length > 0}
<a
href="/api/audio/voice-samples"
class="text-zinc-400 hover:text-amber-400 transition-colors underline"
onclick={(e) => {
e.preventDefault();
fetch('/api/audio/voice-samples', { method: 'POST' }).catch(() => {});
}}
>Generate missing samples</a>
{/if}
</p>
</div>
</div>
{/if}
{#if audioStore.isCurrentChapter(slug, chapter)}
<!-- ── This chapter is the active one ── -->
{#if audioStore.status === 'idle' || audioStore.status === 'error'}
<!-- Should not normally reach here while current, but handle gracefully -->
{#if audioStore.status === 'error'}
<p class="text-red-400 text-sm mb-2">{audioStore.errorMsg || 'Failed to load audio.'}</p>
{/if}
<button
onclick={handlePlay}
class="px-4 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors flex items-center gap-2"
>
<svg class="w-3.5 h-3.5 ml-0.5" fill="currentColor" viewBox="0 0 24 24">
<path d="M8 5v14l11-7z"/>
</svg>
Play narration
</button>
{:else if audioStore.status === 'loading'}
<button
disabled
class="px-4 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold opacity-50 cursor-not-allowed flex items-center gap-2"
>
<svg class="w-3.5 h-3.5 animate-spin" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
</svg>
Loading…
</button>
{:else if audioStore.status === 'generating'}
<div class="space-y-2">
<p class="text-xs text-zinc-400">Generating narration…</p>
<div class="w-full h-1.5 bg-zinc-700 rounded-full overflow-hidden">
<div
class="h-full bg-amber-400 rounded-full transition-none"
style="width: {audioStore.progress}%"
></div>
</div>
<p class="text-xs text-zinc-500 tabular-nums">{Math.round(audioStore.progress)}%</p>
</div>
{:else if audioStore.status === 'ready'}
<!-- Mini-bar is the canonical control surface — show a compact indicator here -->
<div class="flex items-center justify-between gap-3">
<div class="flex items-center gap-2 text-xs text-zinc-400">
{#if audioStore.isPlaying}
<svg class="w-3.5 h-3.5 text-amber-400 flex-shrink-0" fill="currentColor" viewBox="0 0 24 24">
<path d="M6 4h4v16H6V4zm8 0h4v16h-4V4z"/>
</svg>
<span>Playing — controls below</span>
{:else}
<svg class="w-3.5 h-3.5 flex-shrink-0 ml-0.5" fill="currentColor" viewBox="0 0 24 24">
<path d="M8 5v14l11-7z"/>
</svg>
<span>Paused — controls below</span>
{/if}
<span class="tabular-nums text-zinc-500">
{formatTime(audioStore.currentTime)} / {formatTime(audioStore.duration)}
</span>
</div>
<!-- Auto-next toggle (keep here as useful context) -->
{#if nextChapter !== null && nextChapter !== undefined}
<button
onclick={() => (audioStore.autoNext = !audioStore.autoNext)}
class="flex items-center gap-1.5 px-2 py-1 rounded text-xs font-medium transition-colors flex-shrink-0 {audioStore.autoNext
? 'text-amber-400 bg-amber-400/15 hover:bg-amber-400/25'
: 'text-zinc-500 bg-zinc-700 hover:bg-zinc-600 hover:text-zinc-200'}"
title={audioStore.autoNext ? `Auto-next on — will play Ch.${nextChapter} automatically` : 'Auto-next off'}
aria-pressed={audioStore.autoNext}
>
<svg class="w-3.5 h-3.5" fill="currentColor" viewBox="0 0 24 24">
<path d="M6 18l8.5-6L6 6v12zm8.5-6L23 6v12l-8.5-6z"/>
</svg>
Auto
</button>
{/if}
</div>
<!-- Next chapter pre-fetch status (only when auto-next is on) -->
{#if audioStore.autoNext && nextChapter !== null && nextChapter !== undefined}
<div class="mt-2">
{#if audioStore.nextStatus === 'prefetching'}
<div class="flex items-center gap-2 text-xs text-zinc-500">
<svg class="w-3 h-3 animate-spin flex-shrink-0" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
</svg>
<span>Preparing Ch.{nextChapter}{Math.round(audioStore.nextProgress)}%</span>
</div>
{:else if audioStore.nextStatus === 'prefetched'}
<p class="text-xs text-zinc-500 flex items-center gap-1">
<svg class="w-3 h-3 text-amber-400 flex-shrink-0" fill="currentColor" viewBox="0 0 24 24">
<path d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41L9 16.17z"/>
</svg>
Ch.{nextChapter} ready
</p>
{:else if audioStore.nextStatus === 'failed'}
<p class="text-xs text-zinc-600">Ch.{nextChapter} will generate on navigate</p>
{/if}
</div>
{/if}
{/if}
{:else if audioStore.active}
<!-- ── A different chapter is currently playing ── -->
<div class="flex items-center justify-between gap-3">
<p class="text-xs text-zinc-400">
Now playing: {audioStore.chapterTitle || `Ch.${audioStore.chapter}`}
</p>
<button
onclick={startPlayback}
class="px-3 py-1 rounded bg-zinc-700 text-zinc-200 text-xs font-medium hover:bg-zinc-600 transition-colors flex-shrink-0"
>
Load this chapter
</button>
</div>
{:else}
<!-- ── Idle — nothing playing ── -->
<button
onclick={handlePlay}
class="px-4 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors flex items-center gap-2"
>
<svg class="w-3.5 h-3.5 ml-0.5" fill="currentColor" viewBox="0 0 24 24">
<path d="M8 5v14l11-7z"/>
</svg>
Play narration
</button>
{/if}
</div>

View File

@@ -1,117 +0,0 @@
<script lang="ts">
import { onDestroy } from 'svelte';
import Cropper from 'cropperjs';
import type { default as CropperType } from 'cropperjs';
import 'cropperjs/dist/cropper.css';
interface Props {
file: File;
onconfirm: (blob: Blob, mimeType: string) => void;
oncancel: () => void;
}
let { file, onconfirm, oncancel }: Props = $props();
let imgEl: HTMLImageElement | undefined = $state();
let cropper: CropperType | null = null;
let objectUrl = '';
// Initialize cropper once the img element is bound and the file is known.
// Use a $effect so it runs after the DOM is ready (replaces onMount).
$effect(() => {
if (!imgEl || !file) return;
// Create the object URL and set src directly on the element (not via reactive
// state) so cropperjs sees the correct src before the image load event fires.
objectUrl = URL.createObjectURL(file);
imgEl.src = objectUrl;
// Cropperjs must be initialised inside the image's load event so it can
// measure the natural dimensions — if we call new Cropper() before the image
// has loaded, the crop canvas is blank/invisible.
const handleLoad = () => {
cropper = new Cropper(imgEl!, {
aspectRatio: 1,
viewMode: 1,
dragMode: 'move',
autoCropArea: 0.8,
restore: false,
guides: false,
center: true,
highlight: false,
cropBoxMovable: true,
cropBoxResizable: true,
toggleDragModeOnDblclick: false,
background: false
});
};
imgEl.addEventListener('load', handleLoad, { once: true });
return () => {
imgEl?.removeEventListener('load', handleLoad);
cropper?.destroy();
cropper = null;
URL.revokeObjectURL(objectUrl);
objectUrl = '';
};
});
onDestroy(() => {
cropper?.destroy();
if (objectUrl) URL.revokeObjectURL(objectUrl);
});
function confirm() {
if (!cropper) return;
const canvas = cropper.getCroppedCanvas({ width: 400, height: 400 });
const mimeType = file.type === 'image/webp' ? 'image/webp' : 'image/jpeg';
canvas.toBlob(
(blob: Blob | null) => {
if (blob) onconfirm(blob, mimeType);
},
mimeType,
0.9
);
}
</script>
<!-- Backdrop -->
<div
class="fixed inset-0 z-50 flex items-center justify-center bg-black/70 backdrop-blur-sm p-4"
role="dialog"
aria-modal="true"
aria-label="Crop profile picture"
>
<div class="bg-zinc-900 rounded-2xl border border-zinc-700 shadow-2xl w-full max-w-sm flex flex-col gap-4 p-5">
<h2 class="text-base font-semibold text-zinc-100">Crop profile picture</h2>
<!-- Cropper image container — overflow must be visible so cropperjs can
render the crop canvas outside the natural image bounds. The fixed
height gives cropperjs a stable container to size itself against. -->
<div class="rounded-xl bg-zinc-800" style="height: 300px; position: relative;">
<img
bind:this={imgEl}
alt="Crop preview"
style="display:block; max-width:100%; max-height:100%;"
/>
</div>
<p class="text-xs text-zinc-500 text-center">Drag to reposition · pinch or scroll to zoom · drag corners to resize</p>
<div class="flex gap-3">
<button
onclick={oncancel}
class="flex-1 py-2 rounded-lg border border-zinc-600 text-zinc-300 text-sm font-medium hover:bg-zinc-700 transition-colors"
>
Cancel
</button>
<button
onclick={confirm}
class="flex-1 py-2 rounded-lg bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors"
>
Use photo
</button>
</div>
</div>
</div>

View File

@@ -1,560 +0,0 @@
<script lang="ts">
interface BookComment {
id: string;
slug: string;
user_id: string;
username: string;
body: string;
upvotes: number;
downvotes: number;
created: string;
parent_id?: string;
replies?: BookComment[];
}
let {
slug,
isLoggedIn = false,
currentUserId = ''
}: {
slug: string;
isLoggedIn?: boolean;
currentUserId?: string;
} = $props();
// ── State ─────────────────────────────────────────────────────────────────
let comments = $state<BookComment[]>([]);
let myVotes = $state<Record<string, 'up' | 'down'>>({});
let avatarUrls = $state<Record<string, string>>({});
let loading = $state(true);
let loadError = $state('');
// Top-level new comment
let newBody = $state('');
let posting = $state(false);
let postError = $state('');
// Sort
let sort = $state<'new' | 'top'>('top');
// Reply state: which comment is being replied to
let replyingTo = $state<string | null>(null); // comment id
let replyBody = $state('');
let replyPosting = $state(false);
let replyError = $state('');
// Delete in-flight set
let deletingIds = $state(new Set<string>());
// Per-comment vote inflight set (prevents double-clicks)
let votingIds = $state(new Set<string>());
// ── Load comments ─────────────────────────────────────────────────────────
async function loadComments() {
loading = true;
loadError = '';
try {
const res = await fetch(
`/api/comments/${encodeURIComponent(slug)}?sort=${sort}`
);
if (!res.ok) throw new Error(`${res.status}`);
const data = await res.json();
comments = data.comments ?? [];
myVotes = data.myVotes ?? {};
avatarUrls = data.avatarUrls ?? {};
} catch (e) {
loadError = 'Failed to load comments.';
} finally {
loading = false;
}
}
$effect(() => {
loadComments();
});
// Re-load when sort changes (after initial mount)
let firstLoad = true;
$effect(() => {
// Read sort to create a dependency
const _ = sort;
if (firstLoad) { firstLoad = false; return; }
loadComments();
});
// ── Post top-level comment ────────────────────────────────────────────────
async function postComment() {
const text = newBody.trim();
if (!text || posting) return;
if (text.length > 2000) { postError = 'Comment is too long (max 2000 characters).'; return; }
posting = true;
postError = '';
try {
const res = await fetch(`/api/comments/${encodeURIComponent(slug)}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ body: text })
});
if (res.status === 401) { postError = 'You must be logged in to comment.'; return; }
if (!res.ok) {
const err = await res.json().catch(() => ({}));
postError = err.message ?? 'Failed to post comment.';
return;
}
const created: BookComment = await res.json();
created.replies = [];
// Prepend for 'new', or re-sort for 'top'
if (sort === 'new') {
comments = [created, ...comments];
} else {
comments = [created, ...comments]; // new comment has 0 score, goes to end after sort would happen
}
newBody = '';
} catch {
postError = 'Failed to post comment.';
} finally {
posting = false;
}
}
// ── Post reply ────────────────────────────────────────────────────────────
async function postReply(parentId: string) {
const text = replyBody.trim();
if (!text || replyPosting) return;
if (text.length > 2000) { replyError = 'Reply is too long (max 2000 characters).'; return; }
replyPosting = true;
replyError = '';
try {
const res = await fetch(`/api/comments/${encodeURIComponent(slug)}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ body: text, parent_id: parentId })
});
if (res.status === 401) { replyError = 'You must be logged in to reply.'; return; }
if (!res.ok) {
const err = await res.json().catch(() => ({}));
replyError = err.message ?? 'Failed to post reply.';
return;
}
const created: BookComment = await res.json();
// Append to the parent's replies list
comments = comments.map((c) => {
if (c.id !== parentId) return c;
return { ...c, replies: [...(c.replies ?? []), created] };
});
replyBody = '';
replyingTo = null;
} catch {
replyError = 'Failed to post reply.';
} finally {
replyPosting = false;
}
}
// ── Delete ────────────────────────────────────────────────────────────────
async function deleteComment(commentId: string, parentId?: string) {
if (deletingIds.has(commentId)) return;
deletingIds = new Set([...deletingIds, commentId]);
try {
const res = await fetch(`/api/comment/${commentId}`, { method: 'DELETE' });
if (!res.ok) return;
if (parentId) {
// Remove reply from parent
comments = comments.map((c) => {
if (c.id !== parentId) return c;
return { ...c, replies: (c.replies ?? []).filter((r) => r.id !== commentId) };
});
} else {
// Remove top-level comment
comments = comments.filter((c) => c.id !== commentId);
}
} finally {
const next = new Set(deletingIds);
next.delete(commentId);
deletingIds = next;
}
}
// ── Vote ──────────────────────────────────────────────────────────────────
async function vote(commentId: string, v: 'up' | 'down', parentId?: string) {
if (votingIds.has(commentId)) return;
votingIds = new Set([...votingIds, commentId]);
try {
const res = await fetch(`/api/comment/${commentId}/vote`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ vote: v })
});
if (!res.ok) return;
const updated: BookComment = await res.json();
// Update comment in list (handle both top-level and replies)
if (parentId) {
comments = comments.map((c) => {
if (c.id !== parentId) return c;
return {
...c,
replies: (c.replies ?? []).map((r) => (r.id === commentId ? updated : r))
};
});
} else {
comments = comments.map((c) => (c.id === commentId ? { ...updated, replies: c.replies } : c));
}
// Update myVotes: toggle off if same, else set new vote
const prev = myVotes[commentId];
if (prev === v) {
const next = { ...myVotes };
delete next[commentId];
myVotes = next;
} else {
myVotes = { ...myVotes, [commentId]: v };
}
} finally {
const next = new Set(votingIds);
next.delete(commentId);
votingIds = next;
}
}
// ── Helpers ───────────────────────────────────────────────────────────────
function initials(username: string): string {
const name = username.trim() || '?';
return name.slice(0, 2).toUpperCase();
}
function formatDate(iso: string): string {
try {
const date = new Date(iso);
const now = Date.now();
const diffMs = now - date.getTime();
const diffMins = Math.floor(diffMs / 60_000);
if (diffMins < 1) return 'just now';
if (diffMins < 60) return `${diffMins}m ago`;
const diffHours = Math.floor(diffMins / 60);
if (diffHours < 24) return `${diffHours}h ago`;
const diffDays = Math.floor(diffHours / 24);
if (diffDays < 30) return `${diffDays}d ago`;
return date.toLocaleDateString(undefined, { year: 'numeric', month: 'short', day: 'numeric' });
} catch {
return iso;
}
}
const charCount = $derived(newBody.length);
const charOver = $derived(charCount > 2000);
const replyCharCount = $derived(replyBody.length);
const replyCharOver = $derived(replyCharCount > 2000);
const totalCount = $derived(
comments.reduce((n, c) => n + 1 + (c.replies?.length ?? 0), 0)
);
</script>
<div class="mt-10">
<!-- Header + sort controls -->
<div class="flex items-center justify-between gap-3 mb-4 flex-wrap">
<h2 class="text-base font-semibold text-zinc-200">
Comments
{#if !loading && totalCount > 0}
<span class="text-zinc-500 font-normal text-sm ml-1">({totalCount})</span>
{/if}
</h2>
<!-- Sort tabs -->
{#if !loading && comments.length > 0}
<div class="flex items-center gap-1 text-xs rounded-lg bg-zinc-800/60 p-1">
<button
onclick={() => (sort = 'top')}
class="px-2.5 py-1 rounded-md transition-colors {sort === 'top'
? 'bg-zinc-700 text-zinc-100'
: 'text-zinc-500 hover:text-zinc-300'}"
>
Top
</button>
<button
onclick={() => (sort = 'new')}
class="px-2.5 py-1 rounded-md transition-colors {sort === 'new'
? 'bg-zinc-700 text-zinc-100'
: 'text-zinc-500 hover:text-zinc-300'}"
>
New
</button>
</div>
{/if}
</div>
<!-- Post form -->
<div class="mb-6">
{#if isLoggedIn}
<div class="flex flex-col gap-2">
<textarea
bind:value={newBody}
placeholder="Write a comment…"
rows="3"
class="w-full px-3 py-2 rounded-lg bg-zinc-800 border border-zinc-700 text-zinc-200 text-sm placeholder-zinc-500 resize-none focus:outline-none focus:border-amber-400 transition-colors"
></textarea>
<div class="flex items-center justify-between gap-3">
<span class="text-xs {charOver ? 'text-red-400' : 'text-zinc-600'} tabular-nums">
{charCount}/2000
</span>
<div class="flex items-center gap-3">
{#if postError}
<span class="text-xs text-red-400">{postError}</span>
{/if}
<button
onclick={postComment}
disabled={posting || !newBody.trim() || charOver}
class="px-4 py-1.5 rounded-lg text-sm font-medium transition-colors
{posting || !newBody.trim() || charOver
? 'bg-zinc-700 text-zinc-500 cursor-not-allowed'
: 'bg-amber-400 text-zinc-900 hover:bg-amber-300'}"
>
{posting ? 'Posting…' : 'Post'}
</button>
</div>
</div>
</div>
{:else}
<p class="text-sm text-zinc-500">
<a href="/auth/login" class="text-amber-400 hover:text-amber-300 transition-colors">Log in</a>
to leave a comment.
</p>
{/if}
</div>
<!-- Comment list -->
{#if loading}
<div class="flex flex-col gap-3">
{#each Array(3) as _}
<div class="rounded-lg bg-zinc-800/50 p-4 animate-pulse">
<div class="h-3 w-24 bg-zinc-700 rounded mb-3"></div>
<div class="h-3 w-full bg-zinc-700/60 rounded mb-2"></div>
<div class="h-3 w-3/4 bg-zinc-700/60 rounded"></div>
</div>
{/each}
</div>
{:else if loadError}
<p class="text-sm text-red-400">{loadError}</p>
{:else if comments.length === 0}
<p class="text-sm text-zinc-500">No comments yet. Be the first!</p>
{:else}
<div class="flex flex-col gap-3">
{#each comments as comment (comment.id)}
{@const myVote = myVotes[comment.id]}
{@const voting = votingIds.has(comment.id)}
{@const deleting = deletingIds.has(comment.id)}
{@const isOwner = isLoggedIn && currentUserId === comment.user_id}
<div class="rounded-lg bg-zinc-800/50 border border-zinc-700/50 px-4 py-3 flex flex-col gap-2 {deleting ? 'opacity-50' : ''}">
<!-- Header -->
<div class="flex items-center gap-2 flex-wrap">
{#if avatarUrls[comment.user_id]}
<img src={avatarUrls[comment.user_id]} alt={comment.username} class="w-6 h-6 rounded-full object-cover flex-shrink-0" />
{:else}
<div class="w-6 h-6 rounded-full bg-zinc-700 flex items-center justify-center flex-shrink-0">
<span class="text-[9px] font-semibold text-zinc-300 leading-none">{initials(comment.username)}</span>
</div>
{/if}
{#if comment.username}
<a href="/users/{comment.username}" class="text-sm font-medium text-zinc-200 hover:text-amber-400 transition-colors">{comment.username}</a>
{:else}
<span class="text-sm font-medium text-zinc-400">Anonymous</span>
{/if}
<span class="text-zinc-600 text-xs">&middot;</span>
<span class="text-xs text-zinc-500">{formatDate(comment.created)}</span>
</div>
<!-- Body -->
<p class="text-sm text-zinc-300 leading-relaxed whitespace-pre-wrap break-words">{comment.body}</p>
<!-- Actions row: votes + reply + delete -->
<div class="flex items-center gap-3 pt-1 flex-wrap">
<!-- Upvote -->
<button
onclick={() => vote(comment.id, 'up')}
disabled={voting}
title="Upvote"
class="flex items-center gap-1 text-xs transition-colors disabled:opacity-50
{myVote === 'up' ? 'text-amber-400' : 'text-zinc-500 hover:text-zinc-300'}"
>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M14 10h4.764a2 2 0 011.789 2.894l-3.5 7A2 2 0 0115.263 21h-4.017c-.163 0-.326-.02-.485-.06L7 20m7-10V5a2 2 0 00-2-2h-.095c-.5 0-.905.405-.905.905 0 .714-.211 1.412-.608 2.006L7 11v9m7-10h-2M7 20H5a2 2 0 01-2-2v-6a2 2 0 012-2h2.5"/>
</svg>
<span class="tabular-nums">{comment.upvotes ?? 0}</span>
</button>
<!-- Downvote -->
<button
onclick={() => vote(comment.id, 'down')}
disabled={voting}
title="Downvote"
class="flex items-center gap-1 text-xs transition-colors disabled:opacity-50
{myVote === 'down' ? 'text-red-400' : 'text-zinc-500 hover:text-zinc-300'}"
>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M10 14H5.236a2 2 0 01-1.789-2.894l3.5-7A2 2 0 018.736 3h4.018a2 2 0 01.485.06l3.76.94m-7 10v5a2 2 0 002 2h.096c.5 0 .905-.405.905-.904 0-.715.211-1.413.608-2.008L17 13V4m-7 10h2m5-10h2a2 2 0 012 2v6a2 2 0 01-2 2h-2.5"/>
</svg>
<span class="tabular-nums">{comment.downvotes ?? 0}</span>
</button>
<!-- Reply button -->
{#if isLoggedIn}
<button
onclick={() => {
if (replyingTo === comment.id) {
replyingTo = null;
replyBody = '';
replyError = '';
} else {
replyingTo = comment.id;
replyBody = '';
replyError = '';
}
}}
class="flex items-center gap-1 text-xs transition-colors
{replyingTo === comment.id
? 'text-amber-400'
: 'text-zinc-500 hover:text-zinc-300'}"
>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M3 10h10a8 8 0 018 8v2M3 10l6 6m-6-6l6-6"/>
</svg>
Reply
</button>
{/if}
<!-- Delete (owner only) -->
{#if isOwner}
<button
onclick={() => deleteComment(comment.id)}
disabled={deleting}
class="flex items-center gap-1 text-xs text-zinc-600 hover:text-red-400 transition-colors disabled:opacity-50 ml-auto"
title="Delete comment"
>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"/>
</svg>
Delete
</button>
{/if}
</div>
<!-- Inline reply form -->
{#if replyingTo === comment.id}
<div class="mt-1 flex flex-col gap-2 pl-2 border-l-2 border-zinc-700">
<textarea
bind:value={replyBody}
placeholder="Write a reply…"
rows="2"
class="w-full px-3 py-2 rounded-lg bg-zinc-900 border border-zinc-700 text-zinc-200 text-sm placeholder-zinc-500 resize-none focus:outline-none focus:border-amber-400 transition-colors"
></textarea>
<div class="flex items-center justify-between gap-2">
<span class="text-xs {replyCharOver ? 'text-red-400' : 'text-zinc-600'} tabular-nums">
{replyCharCount}/2000
</span>
<div class="flex items-center gap-2">
{#if replyError}
<span class="text-xs text-red-400">{replyError}</span>
{/if}
<button
onclick={() => { replyingTo = null; replyBody = ''; replyError = ''; }}
class="px-3 py-1 rounded-lg text-xs text-zinc-400 hover:text-zinc-200 transition-colors"
>
Cancel
</button>
<button
onclick={() => postReply(comment.id)}
disabled={replyPosting || !replyBody.trim() || replyCharOver}
class="px-3 py-1.5 rounded-lg text-xs font-medium transition-colors
{replyPosting || !replyBody.trim() || replyCharOver
? 'bg-zinc-700 text-zinc-500 cursor-not-allowed'
: 'bg-amber-400 text-zinc-900 hover:bg-amber-300'}"
>
{replyPosting ? 'Posting…' : 'Reply'}
</button>
</div>
</div>
</div>
{/if}
<!-- Replies -->
{#if comment.replies && comment.replies.length > 0}
<div class="mt-1 flex flex-col gap-2 pl-3 border-l-2 border-zinc-700/60">
{#each comment.replies as reply (reply.id)}
{@const replyVote = myVotes[reply.id]}
{@const replyVoting = votingIds.has(reply.id)}
{@const replyDeleting = deletingIds.has(reply.id)}
{@const replyIsOwner = isLoggedIn && currentUserId === reply.user_id}
<div class="rounded-md bg-zinc-800/30 px-3 py-2.5 flex flex-col gap-1.5 {replyDeleting ? 'opacity-50' : ''}">
<!-- Reply header -->
<div class="flex items-center gap-2 flex-wrap">
{#if avatarUrls[reply.user_id]}
<img src={avatarUrls[reply.user_id]} alt={reply.username} class="w-5 h-5 rounded-full object-cover flex-shrink-0" />
{:else}
<div class="w-5 h-5 rounded-full bg-zinc-700 flex items-center justify-center flex-shrink-0">
<span class="text-[8px] font-semibold text-zinc-300 leading-none">{initials(reply.username)}</span>
</div>
{/if}
{#if reply.username}
<a href="/users/{reply.username}" class="text-xs font-medium text-zinc-300 hover:text-amber-400 transition-colors">{reply.username}</a>
{:else}
<span class="text-xs font-medium text-zinc-400">Anonymous</span>
{/if}
<span class="text-zinc-600 text-xs">&middot;</span>
<span class="text-xs text-zinc-500">{formatDate(reply.created)}</span>
</div>
<!-- Reply body -->
<p class="text-sm text-zinc-300 leading-relaxed whitespace-pre-wrap break-words">{reply.body}</p>
<!-- Reply actions -->
<div class="flex items-center gap-3 pt-0.5">
<button
onclick={() => vote(reply.id, 'up', comment.id)}
disabled={replyVoting}
title="Upvote"
class="flex items-center gap-1 text-xs transition-colors disabled:opacity-50
{replyVote === 'up' ? 'text-amber-400' : 'text-zinc-500 hover:text-zinc-300'}"
>
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M14 10h4.764a2 2 0 011.789 2.894l-3.5 7A2 2 0 0115.263 21h-4.017c-.163 0-.326-.02-.485-.06L7 20m7-10V5a2 2 0 00-2-2h-.095c-.5 0-.905.405-.905.905 0 .714-.211 1.412-.608 2.006L7 11v9m7-10h-2M7 20H5a2 2 0 01-2-2v-6a2 2 0 012-2h2.5"/>
</svg>
<span class="tabular-nums">{reply.upvotes ?? 0}</span>
</button>
<button
onclick={() => vote(reply.id, 'down', comment.id)}
disabled={replyVoting}
title="Downvote"
class="flex items-center gap-1 text-xs transition-colors disabled:opacity-50
{replyVote === 'down' ? 'text-red-400' : 'text-zinc-500 hover:text-zinc-300'}"
>
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M10 14H5.236a2 2 0 01-1.789-2.894l3.5-7A2 2 0 018.736 3h4.018a2 2 0 01.485.06l3.76.94m-7 10v5a2 2 0 002 2h.096c.5 0 .905-.405.905-.904 0-.715.211-1.413.608-2.008L17 13V4m-7 10h2m5-10h2a2 2 0 012 2v6a2 2 0 01-2 2h-2.5"/>
</svg>
<span class="tabular-nums">{reply.downvotes ?? 0}</span>
</button>
{#if replyIsOwner}
<button
onclick={() => deleteComment(reply.id, comment.id)}
disabled={replyDeleting}
class="flex items-center gap-1 text-xs text-zinc-600 hover:text-red-400 transition-colors disabled:opacity-50 ml-auto"
title="Delete reply"
>
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16"/>
</svg>
Delete
</button>
{/if}
</div>
</div>
{/each}
</div>
{/if}
</div>
{/each}
</div>
{/if}
</div>

View File

@@ -1 +0,0 @@
// place files you want to import through the `$lib` alias in this folder.

View File

@@ -1,37 +0,0 @@
/**
* Structured server-side logger.
*
* Emits JSON lines to stderr so they appear in container/process logs without
* polluting stdout (which Node's HTTP layer uses for responses).
*
* Format mirrors Go's log/slog default JSON output:
* {"time":"…","level":"ERROR","msg":"…","context":"pocketbase",...extra}
*
* Usage:
* import { log } from '$lib/server/logger';
* log.error('pocketbase', 'auth failed', { status: 401, url });
* log.warn('minio', 'presign slow', { slug, n, ms: elapsed });
* log.info('auth', 'user registered', { username });
*/
type Level = 'DEBUG' | 'INFO' | 'WARN' | 'ERROR';
type Extra = Record<string, unknown>;
function emit(level: Level, context: string, msg: string, extra?: Extra): void {
const entry: Record<string, unknown> = {
time: new Date().toISOString(),
level,
context,
msg,
...extra
};
// Write to stderr — never stdout
process.stderr.write(JSON.stringify(entry) + '\n');
}
export const log = {
debug: (context: string, msg: string, extra?: Extra) => emit('DEBUG', context, msg, extra),
info: (context: string, msg: string, extra?: Extra) => emit('INFO', context, msg, extra),
warn: (context: string, msg: string, extra?: Extra) => emit('WARN', context, msg, extra),
error: (context: string, msg: string, extra?: Extra) => emit('ERROR', context, msg, extra),
};

View File

@@ -1,185 +0,0 @@
/**
* Server-side MinIO presign helper.
* Calls the scraper API to get presigned URLs, then optionally rewrites
* the MinIO host to the public-facing URL for browser use.
*
* Never import this from client-side code.
*/
import { env } from '$env/dynamic/private';
import { env as pubEnv } from '$env/dynamic/public';
import { log } from '$lib/server/logger';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
// Public MinIO URL — used to rewrite presigned URLs so the browser can reach MinIO directly.
// In docker-compose this would differ from the internal endpoint.
const MINIO_PUBLIC_URL = pubEnv.PUBLIC_MINIO_PUBLIC_URL ?? 'http://localhost:9000';
// ─── Avatar helpers ───────────────────────────────────────────────────────────
function extFromMime(mime: string): string {
if (mime.includes('png')) return 'png';
if (mime.includes('webp')) return 'webp';
if (mime.includes('gif')) return 'gif';
return 'jpg';
}
/**
* Returns a short-lived presigned PUT URL for uploading an avatar directly to MinIO,
* along with the object key to record in PocketBase after upload completes.
* Routed through the Go scraper which holds MinIO credentials.
*/
export async function presignAvatarUploadUrl(userId: string, mimeType: string): Promise<{ uploadUrl: string; key: string }> {
const ext = extFromMime(mimeType);
const res = await fetch(`${SCRAPER_URL}/api/presign/avatar-upload/${encodeURIComponent(userId)}?ext=${ext}`);
if (!res.ok) {
const body = await res.text().catch(() => '');
throw new Error(`presign avatar upload failed: ${res.status} ${body}`);
}
const data = (await res.json()) as { upload_url: string; key: string };
return { uploadUrl: data.upload_url, key: data.key };
}
/**
* Returns a presigned GET URL for a user's avatar, rewritten to the public URL.
* Returns null if no avatar exists.
*/
export async function presignAvatarUrl(userId: string): Promise<string | null> {
const res = await fetch(`${SCRAPER_URL}/api/presign/avatar/${encodeURIComponent(userId)}`);
if (res.status === 404) return null;
if (!res.ok) {
const body = await res.text().catch(() => '');
throw new Error(`presign avatar failed: ${res.status} ${body}`);
}
const data = (await res.json()) as { url: string };
return data.url ?? null;
}
/**
* Rewrites the MinIO host in a presigned URL to the public-facing URL.
*
* The Go backend presigns URLs against its internal endpoint (e.g. minio:9000)
* when PUBLIC_MINIO_PUBLIC_URL is not set or equals the internal endpoint.
* In that case the browser must reach MinIO via the public URL (e.g.
* localhost:9000 in dev), so we swap the origin.
*
* NOTE: AWS Signature V4 DOES include the Host header in the canonical request
* (via X-Amz-SignedHeaders=host). Rewriting the host here would break the
* signature. This function is therefore only a no-op safety net — in
* production the Go backend is configured with MINIO_PUBLIC_ENDPOINT equal to
* the externally-reachable hostname, so presigned URLs already carry the right
* host and no rewrite is needed.
*
* For local dev: MINIO_PUBLIC_ENDPOINT=http://localhost:9000 and the backend
* presigns with localhost:9000 (the public client), so this rewrite is again
* a no-op (origins already match).
*/
function rewriteHost(presignedUrl: string): string {
try {
const u = new URL(presignedUrl);
const pub = new URL(MINIO_PUBLIC_URL);
// No-op if already pointing at the right origin.
if (u.protocol === pub.protocol && u.hostname === pub.hostname && u.port === pub.port) {
return presignedUrl;
}
u.protocol = pub.protocol;
u.hostname = pub.hostname;
u.port = pub.port;
return u.toString();
} catch {
return presignedUrl;
}
}
/**
* Returns a presigned URL for a chapter markdown file.
* URL is valid for ~15 minutes (set by the scraper).
*
* @param rewrite - if true, rewrites the MinIO host to PUBLIC_MINIO_PUBLIC_URL
* (for browser use). Defaults to false — the server-side load function fetches
* the URL directly from the internal MinIO endpoint.
*/
export async function presignChapter(slug: string, n: number, rewrite = false): Promise<string> {
log.debug('minio', 'presigning chapter', { slug, n });
let res: Response;
try {
res = await fetch(`${SCRAPER_URL}/api/presign/chapter/${slug}/${n}`);
} catch (e) {
log.error('minio', 'presign chapter network error', { slug, n, err: String(e) });
throw new Error(`presign chapter ${slug}/${n}: network error`);
}
if (!res.ok) {
const body = await res.text().catch(() => '');
log.error('minio', 'presign chapter failed', { slug, n, status: res.status, body });
throw new Error(`presign chapter ${slug}/${n}: ${res.status}`);
}
const data = (await res.json()) as { url: string };
log.debug('minio', 'presign chapter ok', { slug, n });
return rewrite ? rewriteHost(data.url) : data.url;
}
/**
* Returns a presigned URL for a voice sample audio file.
* URL is valid for ~1 hour. The URL is returned to the browser for direct streaming.
* Throws with { status: 404 } when the sample has not been generated yet.
*/
export async function presignVoiceSample(voice: string): Promise<string> {
log.debug('minio', 'presigning voice sample', { voice });
let res: Response;
try {
res = await fetch(`${SCRAPER_URL}/api/presign/voice-sample/${encodeURIComponent(voice)}`);
} catch (e) {
log.error('minio', 'presign voice sample network error', { voice, err: String(e) });
throw new Error(`presign voice sample ${voice}: network error`);
}
if (res.status === 404) {
const err = new Error(`presign voice sample ${voice}: not found`) as Error & { status: number };
err.status = 404;
throw err;
}
if (!res.ok) {
const body = await res.text().catch(() => '');
log.error('minio', 'presign voice sample failed', { voice, status: res.status, body });
throw new Error(`presign voice sample ${voice}: ${res.status}`);
}
const data = (await res.json()) as { url: string };
log.debug('minio', 'presign voice sample ok', { voice });
return rewriteHost(data.url);
}
/**
* Returns a presigned URL for an audio file.
* URL is valid for ~1 hour. The URL is returned to the browser for direct streaming.
* Throws with { status: 404 } when the audio object has not been generated yet.
*/
export async function presignAudio(
slug: string,
n: number,
voice?: string
): Promise<string> {
const params = new URLSearchParams();
if (voice) params.set('voice', voice);
const qs = params.toString() ? `?${params.toString()}` : '';
log.debug('minio', 'presigning audio', { slug, n, voice });
let res: Response;
try {
res = await fetch(`${SCRAPER_URL}/api/presign/audio/${slug}/${n}${qs}`);
} catch (e) {
log.error('minio', 'presign audio network error', { slug, n, err: String(e) });
throw new Error(`presign audio ${slug}/${n}: network error`);
}
if (res.status === 404) {
// Audio hasn't been generated / uploaded yet — caller should surface this as 404.
const err = new Error(`presign audio ${slug}/${n}: not found`) as Error & { status: number };
err.status = 404;
throw err;
}
if (!res.ok) {
const body = await res.text().catch(() => '');
log.error('minio', 'presign audio failed', { slug, n, status: res.status, body });
throw new Error(`presign audio ${slug}/${n}: ${res.status}`);
}
const data = (await res.json()) as { url: string };
log.debug('minio', 'presign audio ok', { slug, n });
return rewriteHost(data.url);
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,96 +0,0 @@
/**
* In-process presign URL cache.
*
* MinIO presigned audio URLs are valid for 1 hour (set by the backend).
* We cache them for 50 minutes so the browser always gets a URL with at
* least 10 minutes of remaining validity, while avoiding a round-trip to
* the backend + MinIO presign API on every "Play" click.
*
* The cache is a plain Map in the Node.js module scope — it lives for the
* lifetime of the SvelteKit server process and is shared across all requests.
* No persistence, no distributed cache needed: each SvelteKit instance
* maintains its own cache and entries expire naturally.
*
* Voice-sample URLs use the same cache with key "sample:<voice>".
*/
const AUDIO_TTL_MS = 50 * 60 * 1000; // 50 minutes
interface CacheEntry {
url: string;
expiresAt: number; // Date.now() ms
}
const cache = new Map<string, CacheEntry>();
// ── Periodic sweep ────────────────────────────────────────────────────────────
// Remove stale entries every 10 minutes so the Map doesn't grow unboundedly
// in long-running processes. Uses unref() so it never prevents Node from
// exiting cleanly.
let sweepTimer: ReturnType<typeof setInterval> | null = null;
function startSweep() {
if (sweepTimer) return;
sweepTimer = setInterval(() => {
const now = Date.now();
for (const [key, entry] of cache) {
if (entry.expiresAt <= now) cache.delete(key);
}
}, 10 * 60 * 1000);
// Don't block Node.js exit
sweepTimer.unref?.();
}
startSweep();
// ── Public API ────────────────────────────────────────────────────────────────
/** Cache key for a chapter audio presigned URL. */
export function audioKey(slug: string, n: number, voice: string): string {
return `audio:${slug}:${n}:${voice}`;
}
/** Cache key for a voice-sample presigned URL. */
export function sampleKey(voice: string): string {
return `sample:${voice}`;
}
/** Return the cached URL for key, or null if absent / expired. */
export function get(key: string): string | null {
const entry = cache.get(key);
if (!entry) return null;
if (entry.expiresAt <= Date.now()) {
cache.delete(key);
return null;
}
return entry.url;
}
/** Store a presigned URL under key for TTL_MS milliseconds. */
export function set(key: string, url: string, ttlMs = AUDIO_TTL_MS): void {
cache.set(key, { url, expiresAt: Date.now() + ttlMs });
}
/** Invalidate a specific key (e.g. after audio generation to force refresh). */
export function invalidate(key: string): void {
cache.delete(key);
}
/** Drain all entries — called on graceful shutdown to release memory. */
export function drain(): void {
cache.clear();
if (sweepTimer) {
clearInterval(sweepTimer);
sweepTimer = null;
}
}
/** Current number of live (non-expired) cached entries. For health/debug. */
export function size(): number {
const now = Date.now();
let n = 0;
for (const entry of cache.values()) {
if (entry.expiresAt > now) n++;
}
return n;
}

View File

@@ -1,32 +0,0 @@
import { redirect } from '@sveltejs/kit';
import type { LayoutServerLoad } from './$types';
import { getSettings } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
// Routes that are accessible without being logged in
const PUBLIC_ROUTES = new Set(['/login']);
export const load: LayoutServerLoad = async ({ locals, url }) => {
if (!PUBLIC_ROUTES.has(url.pathname) && !locals.user) {
redirect(302, `/login`);
}
let settings = { autoNext: false, voice: 'af_bella', speed: 1.0 };
try {
const row = await getSettings(locals.sessionId, locals.user?.id);
if (row) {
settings = {
autoNext: row.auto_next ?? false,
voice: row.voice ?? 'af_bella',
speed: row.speed ?? 1.0
};
}
} catch (e) {
log.warn('layout', 'failed to load settings', { err: String(e) });
}
return {
user: locals.user,
settings
};
};

View File

@@ -1,628 +0,0 @@
<script lang="ts">
import '../app.css';
import { page, navigating } from '$app/state';
import { goto } from '$app/navigation';
import type { Snippet } from 'svelte';
import type { LayoutData } from './$types';
import { audioStore } from '$lib/audio.svelte';
import { env } from '$env/dynamic/public';
let { children, data }: { children: Snippet; data: LayoutData } = $props();
// Mobile nav drawer state
let menuOpen = $state(false);
// Chapter list drawer state for the mini-player
let chapterDrawerOpen = $state(false);
// The single <audio> element that persists across navigations.
// AudioPlayer components in chapter pages control it via audioStore.
let audioEl = $state<HTMLAudioElement | null>(null);
// Apply persisted settings once on mount (server-loaded data).
let settingsApplied = false;
$effect(() => {
if (!settingsApplied && data.settings) {
settingsApplied = true;
audioStore.autoNext = data.settings.autoNext;
audioStore.voice = data.settings.voice;
audioStore.speed = data.settings.speed;
}
});
// ── Persist settings changes (debounced 800ms) ──────────────────────────
let settingsSaveTimer = 0;
$effect(() => {
// Subscribe to the three settings fields
const autoNext = audioStore.autoNext;
const voice = audioStore.voice;
const speed = audioStore.speed;
// Skip saving until settings have been applied from the server
if (!settingsApplied) return;
clearTimeout(settingsSaveTimer);
settingsSaveTimer = setTimeout(() => {
fetch('/api/settings', {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ autoNext, voice, speed })
}).catch(() => {});
}, 800) as unknown as number;
});
// Keep the audio element's playback rate in sync with the store speed.
$effect(() => {
if (audioEl) audioEl.playbackRate = audioStore.speed;
});
// When audioUrl changes, load the new source.
// Use a local variable to track which URL is currently loaded so we never
// compare against audioEl.src (browsers normalise it, causing false mismatches).
let loadedUrl = '';
$effect(() => {
if (!audioEl) return;
const url = audioStore.audioUrl;
if (url && url !== loadedUrl) {
loadedUrl = url;
audioEl.src = url;
audioEl.load();
audioEl.playbackRate = audioStore.speed;
audioEl.play().catch(() => {});
} else if (!url) {
loadedUrl = '';
}
});
// Handle toggle requests from AudioPlayer controller.
$effect(() => {
// Read toggleRequest to subscribe; ignore value 0 (initial).
const _req = audioStore.toggleRequest;
if (!audioEl || _req === 0) return;
if (audioStore.isPlaying) {
audioEl.pause();
} else {
audioEl.play().catch(() => {});
}
});
// Handle seek requests from AudioPlayer controller.
$effect(() => {
const t = audioStore.seekRequest;
if (!audioEl || t === null) return;
audioEl.currentTime = t;
audioStore.seekRequest = null;
});
// ── Save audio time on pause/end (debounced 2s) ─────────────────────────
let audioTimeSaveTimer = 0;
function saveAudioTime() {
if (!audioStore.slug || !audioStore.chapter) return;
const slug = audioStore.slug;
const chapter = audioStore.chapter;
const currentTime = audioStore.currentTime;
clearTimeout(audioTimeSaveTimer);
audioTimeSaveTimer = setTimeout(() => {
fetch('/api/progress/audio-time', {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ slug, chapter, audioTime: currentTime })
}).catch(() => {});
}, 2000) as unknown as number;
}
function formatTime(s: number): string {
if (!isFinite(s) || s < 0) return '0:00';
const m = Math.floor(s / 60);
const sec = Math.floor(s % 60);
return `${m}:${sec.toString().padStart(2, '0')}`;
}
function togglePlay() {
if (!audioEl) return;
if (audioStore.isPlaying) {
audioEl.pause();
} else {
audioEl.play().catch(() => {});
}
}
function seek(e: Event) {
if (!audioEl) return;
audioEl.currentTime = parseFloat((e.target as HTMLInputElement).value);
}
function skipBack() {
if (!audioEl) return;
audioEl.currentTime = Math.max(0, audioEl.currentTime - 15);
}
function skipForward() {
if (!audioEl) return;
audioEl.currentTime = Math.min(audioEl.duration || 0, audioEl.currentTime + 30);
}
const speedSteps = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0];
function cycleSpeed() {
const idx = speedSteps.indexOf(audioStore.speed);
audioStore.speed = speedSteps[(idx + 1) % speedSteps.length];
}
function dismiss() {
if (audioEl) {
audioEl.pause();
audioEl.src = '';
}
audioStore.status = 'idle';
audioStore.audioUrl = '';
audioStore.slug = '';
audioStore.chapter = 0;
audioStore.isPlaying = false;
audioStore.currentTime = 0;
audioStore.duration = 0;
}
</script>
<svelte:head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>libnovel</title>
</svelte:head>
<!-- Persistent audio element — always in the DOM, never conditionally unmounted.
Conditional rendering ({#if}) would destroy/recreate it when reactive state
changes (e.g. currentTime ticking), triggering onpause and restarting audio. -->
<audio
bind:this={audioEl}
bind:currentTime={audioStore.currentTime}
bind:duration={audioStore.duration}
onplay={() => (audioStore.isPlaying = true)}
onpause={() => {
audioStore.isPlaying = false;
saveAudioTime();
}}
onended={() => {
audioStore.isPlaying = false;
saveAudioTime();
if (audioStore.autoNext && audioStore.nextChapter !== null && audioStore.slug) {
// Capture values synchronously before any async work — the AudioPlayer
// component will unmount during navigation, but we've already read what
// we need.
const targetSlug = audioStore.slug;
const targetChapter = audioStore.nextChapter;
// Store the target chapter number so only the newly-mounted AudioPlayer
// for that chapter reacts — not the outgoing chapter's component.
audioStore.autoStartChapter = targetChapter;
goto(`/books/${targetSlug}/chapters/${targetChapter}`).catch(() => {
audioStore.autoStartChapter = null;
});
}
}}
preload="metadata"
style="display:none"
></audio>
<div class="min-h-screen flex flex-col" class:pb-24={audioStore.active}>
<!-- Navigation progress bar — shown while SSR is running for any page transition -->
{#if navigating}
<div class="fixed top-0 left-0 right-0 z-[100] h-1 bg-zinc-800">
<div class="h-full bg-amber-400 animate-progress-bar"></div>
</div>
{/if}
<header class="border-b border-zinc-700 bg-zinc-900 sticky top-0 z-50">
<nav class="max-w-6xl mx-auto px-4 h-14 flex items-center gap-6">
<a href="/" class="text-amber-400 font-bold text-lg tracking-tight hover:text-amber-300 shrink-0">
libnovel
</a>
{#if page.data.book?.title && /\/books\/[^/]+\/chapters\//.test(page.url.pathname)}
<span class="text-zinc-400 text-sm truncate min-w-0 flex-1 sm:flex-none sm:max-w-xs">
{page.data.book.title}
</span>
{/if}
{#if data.user}
<!-- Desktop nav links (hidden on mobile) -->
<a
href="/books"
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/books') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
>
Library
</a>
<a
href="/browse"
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/browse') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
>
Discover
</a>
<div class="ml-auto flex items-center gap-4">
<!-- Desktop: admin + profile + sign out (hidden on mobile) -->
{#if data.user?.role === 'admin'}
<a
href="/admin/scrape"
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/admin/scrape') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
>
Scrape
</a>
<a
href="/admin/audio"
class="hidden sm:block text-sm transition-colors {page.url.pathname === '/admin/audio' ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
>
Audio cache
</a>
<a
href="/admin/audio-jobs"
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/admin/audio-jobs') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
>
Audio jobs
</a>
{/if}
<a
href="/profile"
class="hidden sm:block text-sm transition-colors {page.url.pathname === '/profile' ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
>
{data.user.username}
</a>
<form method="POST" action="/logout" class="hidden sm:block">
<button type="submit" class="text-zinc-400 hover:text-zinc-100 text-sm transition-colors">
Sign out
</button>
</form>
<!-- Mobile: hamburger button -->
<button
onclick={() => (menuOpen = !menuOpen)}
aria-label="Toggle menu"
aria-expanded={menuOpen}
class="sm:hidden p-2 -mr-1 rounded text-zinc-400 hover:text-zinc-100 transition-colors"
>
{#if menuOpen}
<!-- X icon -->
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12" />
</svg>
{:else}
<!-- Hamburger icon -->
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 6h16M4 12h16M4 18h16" />
</svg>
{/if}
</button>
</div>
{:else}
<div class="ml-auto">
<a
href="/login"
class="text-sm px-3 py-1.5 rounded bg-amber-400 text-zinc-900 font-semibold hover:bg-amber-300 transition-colors"
>
Sign in
</a>
</div>
{/if}
</nav>
<!-- Mobile drawer (full-width, below the bar) -->
{#if data.user && menuOpen}
<div class="sm:hidden border-t border-zinc-700 bg-zinc-900 px-4 py-3 flex flex-col gap-1">
<a
href="/books"
onclick={() => (menuOpen = false)}
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/books') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
>
Library
</a>
<a
href="/browse"
onclick={() => (menuOpen = false)}
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/browse') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
>
Discover
</a>
<a
href="/profile"
onclick={() => (menuOpen = false)}
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname === '/profile' ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
>
Profile <span class="text-zinc-500 font-normal">({data.user.username})</span>
</a>
{#if data.user?.role === 'admin'}
<div class="my-1 border-t border-zinc-700/60"></div>
<p class="px-3 pt-1 pb-0.5 text-xs text-zinc-600 uppercase tracking-widest">Admin</p>
<a
href="/admin/scrape"
onclick={() => (menuOpen = false)}
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/admin/scrape') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
>
Scrape tasks
</a>
<a
href="/admin/audio"
onclick={() => (menuOpen = false)}
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname === '/admin/audio' ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
>
Audio cache
</a>
<a
href="/admin/audio-jobs"
onclick={() => (menuOpen = false)}
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/admin/audio-jobs') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
>
Audio jobs
</a>
{/if}
<div class="my-1 border-t border-zinc-700/60"></div>
<form method="POST" action="/logout">
<button
type="submit"
class="w-full text-left px-3 py-2.5 rounded-lg text-sm font-medium text-red-400 hover:bg-zinc-800 transition-colors"
>
Sign out
</button>
</form>
</div>
{/if}
</header>
<main class="flex-1 max-w-6xl mx-auto w-full px-4 py-8">
{#key page.url.pathname + page.url.search}
{@render children()}
{/key}
</main>
<footer class="border-t border-zinc-800 mt-auto">
<div class="max-w-6xl mx-auto px-4 py-6 flex flex-col items-center gap-4 text-xs text-zinc-600">
<!-- Top row: site links -->
<nav class="flex flex-wrap items-center justify-center gap-x-5 gap-y-2">
<a href="/books" class="hover:text-zinc-400 transition-colors">Library</a>
<a href="/browse" class="hover:text-zinc-400 transition-colors">Discover</a>
<a
href="https://novelfire.net"
target="_blank"
rel="noopener noreferrer"
class="hover:text-zinc-400 transition-colors flex items-center gap-1"
>
novelfire.net
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14" />
</svg>
</a>
</nav>
<!-- Bottom row: legal links + copyright -->
<div class="flex flex-wrap items-center justify-center gap-x-5 gap-y-2 text-zinc-700">
<a href="/disclaimer" class="hover:text-zinc-500 transition-colors">Disclaimer</a>
<a href="/privacy" class="hover:text-zinc-500 transition-colors">Privacy</a>
<a href="/dmca" class="hover:text-zinc-500 transition-colors">DMCA</a>
<span>&copy; {new Date().getFullYear()} libnovel</span>
{#if env.PUBLIC_BUILD_VERSION && env.PUBLIC_BUILD_VERSION !== 'dev'}
<span class="text-zinc-800">{env.PUBLIC_BUILD_VERSION}+{env.PUBLIC_BUILD_COMMIT?.slice(0, 7)}</span>
{/if}
</div>
</div>
</footer>
</div>
<!-- ── Persistent mini-player bar ─────────────────────────────────────────── -->
{#if audioStore.active}
<div class="fixed bottom-0 left-0 right-0 z-50 bg-zinc-900 border-t border-zinc-700 shadow-2xl">
<!-- Chapter list drawer (slides up above the mini-bar) -->
{#if chapterDrawerOpen && audioStore.chapters.length > 0}
<div class="border-b border-zinc-700 bg-zinc-900 max-h-[32rem] overflow-y-auto">
<div class="max-w-6xl mx-auto px-4">
<div class="flex items-center justify-between py-2 border-b border-zinc-800 sticky top-0 bg-zinc-900">
<span class="text-xs font-semibold text-zinc-400 uppercase tracking-wider">Chapters</span>
<button
onclick={() => (chapterDrawerOpen = false)}
class="text-zinc-600 hover:text-zinc-300 transition-colors p-1"
aria-label="Close chapter list"
>
<svg class="w-3.5 h-3.5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"/>
</svg>
</button>
</div>
{#each audioStore.chapters as ch (ch.number)}
<a
href="/books/{audioStore.slug}/chapters/{ch.number}"
onclick={() => (chapterDrawerOpen = false)}
class="flex items-center gap-2 py-2 text-xs transition-colors hover:text-zinc-100 {ch.number === audioStore.chapter
? 'text-amber-400 font-semibold'
: 'text-zinc-400'}"
>
<span class="tabular-nums text-zinc-600 w-8 shrink-0 text-right">
{ch.number}
</span>
<span class="truncate">{ch.title || `Chapter ${ch.number}`}</span>
{#if ch.number === audioStore.chapter}
<svg class="w-3 h-3 shrink-0 text-amber-400" fill="currentColor" viewBox="0 0 24 24">
<path d="M8 5v14l11-7z"/>
</svg>
{/if}
</a>
{/each}
</div>
</div>
{/if}
<!-- Generation progress bar (sits at very top of the bar) -->
{#if audioStore.status === 'generating' || audioStore.status === 'loading'}
<div class="h-0.5 bg-zinc-800">
<div
class="h-full bg-amber-400 transition-none"
style="width: {audioStore.progress}%"
></div>
</div>
{:else if audioStore.status === 'ready'}
<!-- Seek bar flush at top — tappable on mobile -->
<div class="px-0">
<input
type="range"
min="0"
max={audioStore.duration || 0}
value={audioStore.currentTime}
oninput={seek}
class="w-full h-1 accent-amber-400 cursor-pointer block"
style="margin: 0; border-radius: 0;"
/>
</div>
{/if}
<div class="max-w-6xl mx-auto px-4 py-2 flex items-center gap-3">
<!-- Track info (click to open chapter list drawer) -->
<button
class="flex-1 min-w-0 text-left rounded px-1 -ml-1 hover:bg-zinc-800 transition-colors"
onclick={() => { if (audioStore.chapters.length > 0) chapterDrawerOpen = !chapterDrawerOpen; }}
aria-label={audioStore.chapters.length > 0 ? 'Toggle chapter list' : undefined}
title={audioStore.chapters.length > 0 ? 'Chapter list' : undefined}
>
{#if audioStore.chapterTitle}
<p class="text-xs text-zinc-100 truncate leading-tight">{audioStore.chapterTitle}</p>
{/if}
{#if audioStore.bookTitle}
<p class="text-xs text-zinc-500 truncate leading-tight">{audioStore.bookTitle}</p>
{/if}
{#if audioStore.status === 'generating'}
<p class="text-xs text-amber-400 leading-tight">
Generating… {Math.round(audioStore.progress)}%
</p>
{:else if audioStore.status === 'ready'}
<p class="text-xs text-zinc-500 tabular-nums leading-tight">
{formatTime(audioStore.currentTime)} / {formatTime(audioStore.duration)}
</p>
{:else if audioStore.status === 'loading'}
<p class="text-xs text-zinc-500 leading-tight">Loading…</p>
{/if}
</button>
{#if audioStore.status === 'ready'}
<!-- Skip back 15s -->
<button
onclick={skipBack}
class="text-zinc-400 hover:text-zinc-100 transition-colors p-1.5 rounded"
title="Back 15s"
aria-label="Rewind 15 seconds"
>
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 24 24">
<path d="M11.99 5V1l-5 5 5 5V7c3.31 0 6 2.69 6 6s-2.69 6-6 6-6-2.69-6-6h-2c0 4.42 3.58 8 8 8s8-3.58 8-8-3.58-8-8-8z"/>
<text x="8.5" y="14.5" font-size="5" font-family="sans-serif" font-weight="bold" fill="currentColor">15</text>
</svg>
</button>
<!-- Play / Pause -->
<button
onclick={togglePlay}
class="w-10 h-10 rounded-full bg-amber-400 text-zinc-900 flex items-center justify-center hover:bg-amber-300 transition-colors flex-shrink-0"
aria-label={audioStore.isPlaying ? 'Pause' : 'Play'}
>
{#if audioStore.isPlaying}
<svg class="w-4 h-4" fill="currentColor" viewBox="0 0 24 24">
<path d="M6 4h4v16H6V4zm8 0h4v16h-4V4z"/>
</svg>
{:else}
<svg class="w-4 h-4 ml-0.5" fill="currentColor" viewBox="0 0 24 24">
<path d="M8 5v14l11-7z"/>
</svg>
{/if}
</button>
<!-- Skip forward 30s -->
<button
onclick={skipForward}
class="text-zinc-400 hover:text-zinc-100 transition-colors p-1.5 rounded"
title="Forward 30s"
aria-label="Skip 30 seconds"
>
<svg class="w-5 h-5" fill="currentColor" viewBox="0 0 24 24">
<path d="M12 5V1l5 5-5 5V7c-3.31 0-6 2.69-6 6s2.69 6 6 6 6-2.69 6-6h2c0 4.42-3.58 8-8 8s-8-3.58-8-8 3.58-8 8-8z"/>
<text x="8.5" y="14.5" font-size="5" font-family="sans-serif" font-weight="bold" fill="currentColor">30</text>
</svg>
</button>
<!-- Speed control -->
<button
onclick={cycleSpeed}
class="text-xs font-semibold text-zinc-300 hover:text-amber-400 transition-colors px-2 py-1 rounded bg-zinc-800 hover:bg-zinc-700 flex-shrink-0 tabular-nums w-12 text-center"
title="Change playback speed"
aria-label="Playback speed {audioStore.speed}x"
>
{audioStore.speed}×
</button>
<!-- Auto-next toggle (with prefetch indicator) -->
<button
onclick={() => (audioStore.autoNext = !audioStore.autoNext)}
class="relative p-1.5 rounded flex-shrink-0 transition-colors {audioStore.autoNext
? 'text-amber-400 bg-amber-400/15 hover:bg-amber-400/25'
: 'text-zinc-600 hover:text-zinc-300 hover:bg-zinc-800'}"
title={audioStore.autoNext
? audioStore.nextStatus === 'prefetched'
? `Auto-next on Ch.${audioStore.nextChapter} ready`
: audioStore.nextStatus === 'prefetching'
? `Auto-next on preparing Ch.${audioStore.nextChapter}`
: 'Auto-next on'
: 'Auto-next off'}
aria-label="Auto-next {audioStore.autoNext ? 'on' : 'off'}"
aria-pressed={audioStore.autoNext}
>
<!-- "skip to end" / auto-advance icon -->
<svg class="w-4 h-4" fill="currentColor" viewBox="0 0 24 24">
<path d="M6 18l8.5-6L6 6v12zm8.5-6L23 6v12l-8.5-6z"/>
</svg>
<!-- Prefetch status dot -->
{#if audioStore.autoNext && audioStore.nextStatus === 'prefetching'}
<span class="absolute top-0.5 right-0.5 w-1.5 h-1.5 rounded-full bg-amber-400 animate-pulse"></span>
{:else if audioStore.autoNext && audioStore.nextStatus === 'prefetched'}
<span class="absolute top-0.5 right-0.5 w-1.5 h-1.5 rounded-full bg-green-400"></span>
{/if}
</button>
{:else if audioStore.status === 'generating'}
<!-- Spinner during generation -->
<svg class="w-6 h-6 text-amber-400 animate-spin flex-shrink-0" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
</svg>
{/if}
<!-- Cover thumbnail / go-to-chapter link -->
{#if audioStore.slug && audioStore.chapter > 0}
<a
href="/books/{audioStore.slug}/chapters/{audioStore.chapter}"
class="shrink-0 rounded overflow-hidden hover:opacity-80 transition-opacity"
title="Go to chapter"
aria-label="Go to chapter"
>
{#if audioStore.cover}
<img
src={audioStore.cover}
alt=""
class="w-8 h-11 object-cover rounded"
/>
{:else}
<!-- Fallback book icon -->
<div class="w-8 h-11 flex items-center justify-center bg-zinc-800 rounded border border-zinc-700">
<svg class="w-4 h-4 text-zinc-500" fill="currentColor" viewBox="0 0 24 24">
<path d="M18 2H6c-1.1 0-2 .9-2 2v16c0 1.1.9 2 2 2h12c1.1 0 2-.9 2-2V4c0-1.1-.9-2-2-2zm-2 14H8v-2h8v2zm0-4H8v-2h8v2zm0-4H8V6h8v2z"/>
</svg>
</div>
{/if}
</a>
{/if}
<!-- Dismiss -->
<button
onclick={dismiss}
class="text-zinc-600 hover:text-zinc-400 transition-colors p-1.5 rounded flex-shrink-0"
title="Close player"
aria-label="Close player"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"/>
</svg>
</button>
</div>
</div>
{/if}

View File

@@ -1,59 +0,0 @@
import type { PageServerLoad } from './$types';
import {
listBooks,
recentlyAddedBooks,
allProgress,
getHomeStats,
getSubscriptionFeed
} from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
import type { Book, Progress } from '$lib/server/pocketbase';
export const load: PageServerLoad = async ({ locals }) => {
let allBooks: Book[] = [];
let recentBooks: Book[] = [];
let progressList: Progress[] = [];
let stats = { totalBooks: 0, totalChapters: 0 };
try {
[allBooks, recentBooks, progressList, stats] = await Promise.all([
listBooks(),
recentlyAddedBooks(8),
allProgress(locals.sessionId, locals.user?.id),
getHomeStats()
]);
} catch (e) {
log.error('home', 'failed to load home data', { err: String(e) });
}
// Build slug → book lookup
const bookMap = new Map<string, Book>(allBooks.map((b) => [b.slug, b]));
// Continue reading: progress entries joined with book data, most recent first
const continueReading = progressList
.filter((p) => bookMap.has(p.slug))
.slice(0, 6)
.map((p) => ({ book: bookMap.get(p.slug)!, chapter: p.chapter }));
// Recently updated: deduplicate against continueReading slugs
const inProgressSlugs = new Set(continueReading.map((c) => c.book.slug));
const recentlyUpdated = recentBooks.filter((b) => !inProgressSlugs.has(b.slug)).slice(0, 6);
// Subscription feed — only when logged in
const subscriptionFeed = locals.user
? await getSubscriptionFeed(locals.user.id, 12).catch((e) => {
log.error('home', 'failed to load subscription feed', { err: String(e) });
return [] as Awaited<ReturnType<typeof getSubscriptionFeed>>;
})
: [];
return {
continueReading,
recentlyUpdated,
subscriptionFeed,
stats: {
...stats,
booksInProgress: continueReading.length
}
};
};

View File

@@ -1,202 +0,0 @@
<script lang="ts">
import type { PageData } from './$types';
let { data }: { data: PageData } = $props();
function parseGenres(genres: string[] | string | null | undefined): string[] {
if (!genres) return [];
if (Array.isArray(genres)) return genres;
try {
const parsed = JSON.parse(genres);
return Array.isArray(parsed) ? parsed : [];
} catch {
return [];
}
}
</script>
<svelte:head>
<title>libnovel</title>
</svelte:head>
<!-- Stats bar -->
<div class="flex gap-6 mb-8 text-center">
<div class="flex-1 rounded-lg bg-zinc-800 border border-zinc-700 py-4 px-6">
<p class="text-2xl font-bold text-amber-400">{data.stats.totalBooks}</p>
<p class="text-xs text-zinc-400 mt-0.5">Books</p>
</div>
<div class="flex-1 rounded-lg bg-zinc-800 border border-zinc-700 py-4 px-6">
<p class="text-2xl font-bold text-amber-400">{data.stats.totalChapters.toLocaleString()}</p>
<p class="text-xs text-zinc-400 mt-0.5">Chapters</p>
</div>
<div class="flex-1 rounded-lg bg-zinc-800 border border-zinc-700 py-4 px-6">
<p class="text-2xl font-bold text-amber-400">{data.stats.booksInProgress}</p>
<p class="text-xs text-zinc-400 mt-0.5">In progress</p>
</div>
</div>
<!-- Continue Reading -->
{#if data.continueReading.length > 0}
<section class="mb-10">
<div class="flex items-baseline justify-between mb-3">
<h2 class="text-lg font-bold text-zinc-100">Continue Reading</h2>
<a href="/books" class="text-xs text-amber-400 hover:text-amber-300">View all</a>
</div>
<div class="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-6 gap-4">
{#each data.continueReading as { book, chapter }}
<a
href="/books/{book.slug}/chapters/{chapter}"
class="group flex flex-col rounded-lg overflow-hidden bg-zinc-800 hover:bg-zinc-700 transition-colors border border-zinc-700 hover:border-zinc-500"
>
<div class="aspect-[2/3] bg-zinc-900 overflow-hidden relative">
{#if book.cover}
<img
src={book.cover}
alt={book.title}
class="w-full h-full object-cover group-hover:scale-105 transition-transform duration-300"
loading="lazy"
/>
{:else}
<div class="w-full h-full flex items-center justify-center text-zinc-600">
<svg class="w-10 h-10" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"
d="M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.332.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253" />
</svg>
</div>
{/if}
<!-- Chapter badge overlay -->
<span class="absolute bottom-1.5 right-1.5 text-xs bg-amber-400 text-zinc-900 font-bold px-1.5 py-0.5 rounded">
ch.{chapter}
</span>
</div>
<div class="p-2">
<h3 class="text-xs font-semibold text-zinc-100 line-clamp-2 leading-snug">{book.title ?? ''}</h3>
{#if book.author}
<p class="text-xs text-zinc-500 truncate mt-0.5">{book.author}</p>
{/if}
</div>
</a>
{/each}
</div>
</section>
{/if}
<!-- Recently Updated -->
{#if data.recentlyUpdated.length > 0}
<section class="mb-10">
<div class="flex items-baseline justify-between mb-3">
<h2 class="text-lg font-bold text-zinc-100">Recently Updated</h2>
<a href="/books" class="text-xs text-amber-400 hover:text-amber-300">View all</a>
</div>
<div class="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-6 gap-4">
{#each data.recentlyUpdated as book}
{@const genres = parseGenres(book.genres)}
<a
href="/books/{book.slug}"
class="group flex flex-col rounded-lg overflow-hidden bg-zinc-800 hover:bg-zinc-700 transition-colors border border-zinc-700 hover:border-zinc-500"
>
<div class="aspect-[2/3] bg-zinc-900 overflow-hidden">
{#if book.cover}
<img
src={book.cover}
alt={book.title}
class="w-full h-full object-cover group-hover:scale-105 transition-transform duration-300"
loading="lazy"
/>
{:else}
<div class="w-full h-full flex items-center justify-center text-zinc-600">
<svg class="w-10 h-10" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"
d="M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.332.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253" />
</svg>
</div>
{/if}
</div>
<div class="p-2 flex flex-col gap-1">
<h3 class="text-xs font-semibold text-zinc-100 line-clamp-2 leading-snug">{book.title ?? ''}</h3>
{#if book.author}
<p class="text-xs text-zinc-400 truncate">{book.author}</p>
{/if}
{#if book.status}
<span class="text-xs px-1.5 py-0.5 rounded bg-zinc-700 text-zinc-300 self-start">{book.status}</span>
{/if}
{#if genres.length > 0}
<div class="flex flex-wrap gap-1 mt-auto pt-1">
{#each genres.slice(0, 2) as genre}
<span class="text-xs px-1 py-0.5 rounded bg-zinc-900 text-zinc-500">{genre}</span>
{/each}
</div>
{/if}
</div>
</a>
{/each}
</div>
</section>
{/if}
<!-- Empty state -->
{#if data.continueReading.length === 0 && data.recentlyUpdated.length === 0}
<div class="text-center py-20 text-zinc-500">
<p class="text-lg font-semibold text-zinc-300 mb-2">Your library is empty</p>
<p class="text-sm mb-6">Discover novels and scrape them into your library.</p>
<a
href="/browse"
class="inline-block px-6 py-3 bg-amber-400 text-zinc-900 font-semibold rounded hover:bg-amber-300 transition-colors"
>
Discover Novels
</a>
</div>
{/if}
<!-- From Subscriptions -->
{#if data.subscriptionFeed.length > 0}
<section class="mb-10">
<div class="flex items-baseline justify-between mb-3">
<h2 class="text-lg font-bold text-zinc-100">From People You Follow</h2>
</div>
<div class="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-6 gap-4">
{#each data.subscriptionFeed as { book, readerUsername }}
{@const genres = parseGenres(book.genres)}
<a
href="/books/{book.slug}"
class="group flex flex-col rounded-lg overflow-hidden bg-zinc-800 hover:bg-zinc-700 transition-colors border border-zinc-700 hover:border-zinc-500"
>
<div class="aspect-[2/3] bg-zinc-900 overflow-hidden">
{#if book.cover}
<img
src={book.cover}
alt={book.title}
class="w-full h-full object-cover group-hover:scale-105 transition-transform duration-300"
loading="lazy"
/>
{:else}
<div class="w-full h-full flex items-center justify-center text-zinc-600">
<svg class="w-10 h-10" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"
d="M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.332.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253" />
</svg>
</div>
{/if}
</div>
<div class="p-2 flex flex-col gap-1">
<h3 class="text-xs font-semibold text-zinc-100 line-clamp-2 leading-snug">{book.title ?? ''}</h3>
{#if book.author}
<p class="text-xs text-zinc-400 truncate">{book.author}</p>
{/if}
<!-- Reader attribution -->
<p class="text-xs text-zinc-600 truncate mt-0.5">
via <span class="text-amber-500/70">{readerUsername}</span>
</p>
{#if genres.length > 0}
<div class="flex flex-wrap gap-1 mt-auto pt-1">
{#each genres.slice(0, 1) as genre}
<span class="text-xs px-1 py-0.5 rounded bg-zinc-900 text-zinc-500">{genre}</span>
{/each}
</div>
{/if}
</div>
</a>
{/each}
</div>
</section>
{/if}

View File

@@ -1,17 +0,0 @@
import { redirect } from '@sveltejs/kit';
import type { PageServerLoad } from './$types';
import { listAudioJobs } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
export const load: PageServerLoad = async ({ locals }) => {
if (locals.user?.role !== 'admin') {
redirect(302, '/');
}
const jobs = await listAudioJobs().catch((e) => {
log.warn('admin/audio-jobs', 'failed to load audio jobs', { err: String(e) });
return [];
});
return { jobs };
};

View File

@@ -1,153 +0,0 @@
<script lang="ts">
import { untrack } from 'svelte';
import type { PageData } from './$types';
let { data }: { data: PageData } = $props();
let jobs = $state(untrack(() => data.jobs));
// ── Live-poll: refresh while any job is in-flight ────────────────────────────
let hasInFlight = $derived(jobs.some((j) => j.status === 'pending' || j.status === 'generating'));
$effect(() => {
if (!hasInFlight) return;
const id = setInterval(async () => {
const res = await fetch('/admin/audio-jobs?__data=1').catch(() => null);
if (res?.ok) {
// SvelteKit invalidateAll is cleaner — just trigger a soft navigation reload.
import('$app/navigation').then(({ invalidateAll }) => invalidateAll());
}
}, 3000);
return () => clearInterval(id);
});
// Keep local state in sync when server re-loads
$effect(() => {
jobs = data.jobs;
});
// ── Helpers ──────────────────────────────────────────────────────────────────
function statusColor(status: string) {
if (status === 'done') return 'text-green-400';
if (status === 'generating') return 'text-amber-400 animate-pulse';
if (status === 'pending') return 'text-sky-400 animate-pulse';
if (status === 'failed') return 'text-red-400';
return 'text-zinc-300';
}
function fmtDate(s: string) {
if (!s) return '—';
return new Date(s).toLocaleString(undefined, {
month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit'
});
}
function duration(started: string, finished: string) {
if (!started || !finished) return '—';
const ms = new Date(finished).getTime() - new Date(started).getTime();
if (ms < 0) return '—';
const s = Math.floor(ms / 1000);
if (s < 60) return `${s}s`;
const m = Math.floor(s / 60);
return `${m}m ${s % 60}s`;
}
// ── Search ───────────────────────────────────────────────────────────────────
let q = $state('');
let filtered = $derived(
q.trim()
? jobs.filter(
(j) =>
j.slug.toLowerCase().includes(q.toLowerCase().trim()) ||
j.voice.toLowerCase().includes(q.toLowerCase().trim()) ||
j.status.toLowerCase().includes(q.toLowerCase().trim())
)
: jobs
);
// ── Stats ────────────────────────────────────────────────────────────────────
let stats = $derived({
total: jobs.length,
done: jobs.filter((j) => j.status === 'done').length,
failed: jobs.filter((j) => j.status === 'failed').length,
inFlight: jobs.filter((j) => j.status === 'pending' || j.status === 'generating').length
});
</script>
<svelte:head>
<title>Audio jobs — libnovel admin</title>
</svelte:head>
<div class="space-y-6">
<div class="flex items-start justify-between flex-wrap gap-3">
<div>
<h1 class="text-2xl font-bold text-zinc-100">Audio jobs</h1>
<p class="text-zinc-400 text-sm mt-1">
{stats.total} total &middot;
<span class="text-green-400">{stats.done} done</span> &middot;
{#if stats.failed > 0}
<span class="text-red-400">{stats.failed} failed</span> &middot;
{/if}
{#if stats.inFlight > 0}
<span class="text-amber-400 animate-pulse">{stats.inFlight} in-flight</span>
{:else}
<span class="text-zinc-500">0 in-flight</span>
{/if}
</p>
</div>
</div>
<!-- Search -->
<input
type="search"
bind:value={q}
placeholder="Filter by slug, voice or status…"
class="w-full max-w-sm bg-zinc-800 border border-zinc-700 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
/>
{#if filtered.length === 0}
<p class="text-zinc-500 text-sm py-8 text-center">
{q.trim() ? 'No results.' : 'No audio jobs yet.'}
</p>
{:else}
<div class="overflow-x-auto rounded-xl border border-zinc-700">
<table class="w-full text-sm">
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide">
<tr>
<th class="px-4 py-3 text-left">Book</th>
<th class="px-4 py-3 text-right">Ch.</th>
<th class="px-4 py-3 text-left">Voice</th>
<th class="px-4 py-3 text-left">Status</th>
<th class="px-4 py-3 text-left">Started</th>
<th class="px-4 py-3 text-left">Duration</th>
</tr>
</thead>
<tbody class="divide-y divide-zinc-700/50">
{#each filtered as job}
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
<td class="px-4 py-3 text-zinc-200 font-medium">
<a href="/books/{job.slug}" class="hover:text-amber-400 transition-colors">
{job.slug}
</a>
</td>
<td class="px-4 py-3 text-right text-zinc-400">{job.chapter}</td>
<td class="px-4 py-3 text-zinc-400 font-mono text-xs">{job.voice}</td>
<td class="px-4 py-3">
<span class="font-medium {statusColor(job.status)}">{job.status}</span>
</td>
<td class="px-4 py-3 text-zinc-400">{fmtDate(job.started)}</td>
<td class="px-4 py-3 text-zinc-400">{duration(job.started, job.finished)}</td>
</tr>
{#if job.error_message}
<tr class="bg-red-950/20">
<td colspan="6" class="px-4 py-2 text-xs text-red-400 font-mono"
>{job.error_message}</td
>
</tr>
{/if}
{/each}
</tbody>
</table>
</div>
{/if}
</div>

View File

@@ -1,17 +0,0 @@
import { redirect } from '@sveltejs/kit';
import type { PageServerLoad } from './$types';
import { listAudioCache } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
export const load: PageServerLoad = async ({ locals }) => {
if (locals.user?.role !== 'admin') {
redirect(302, '/');
}
const entries = await listAudioCache().catch((e) => {
log.warn('admin/audio', 'failed to load audio cache', { err: String(e) });
return [];
});
return { entries };
};

View File

@@ -1,93 +0,0 @@
<script lang="ts">
import { untrack } from 'svelte';
import type { PageData } from './$types';
let { data }: { data: PageData } = $props();
let entries = $state(untrack(() => data.entries));
// ── Parse cache_key ─────────────────────────────────────────────────────────
// cache_key format: "slug/chapter/voice"
function parseKey(key: string) {
const parts = key.split('/');
if (parts.length >= 3) {
return { slug: parts[0], chapter: parts[1], voice: parts.slice(2).join('/') };
}
return { slug: key, chapter: '—', voice: '—' };
}
function fmtDate(s: string) {
if (!s) return '—';
return new Date(s).toLocaleString(undefined, {
month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit'
});
}
// ── Search ──────────────────────────────────────────────────────────────────
let q = $state('');
let filtered = $derived(
q.trim()
? entries.filter((e) => e.cache_key.toLowerCase().includes(q.toLowerCase().trim()))
: entries
);
</script>
<svelte:head>
<title>Audio cache — libnovel admin</title>
</svelte:head>
<div class="space-y-6">
<div>
<h1 class="text-2xl font-bold text-zinc-100">Audio cache</h1>
<p class="text-zinc-400 text-sm mt-1">{entries.length} cached audio file{entries.length !== 1 ? 's' : ''}</p>
</div>
<!-- Search -->
<input
type="search"
bind:value={q}
placeholder="Filter by slug, chapter or voice…"
class="w-full max-w-sm bg-zinc-800 border border-zinc-700 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
/>
{#if filtered.length === 0}
<p class="text-zinc-500 text-sm py-8 text-center">
{q.trim() ? 'No results.' : 'Audio cache is empty.'}
</p>
{:else}
<div class="overflow-x-auto rounded-xl border border-zinc-700">
<table class="w-full text-sm">
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide">
<tr>
<th class="px-4 py-3 text-left">Book</th>
<th class="px-4 py-3 text-left">Chapter</th>
<th class="px-4 py-3 text-left">Voice</th>
<th class="px-4 py-3 text-left">Filename</th>
<th class="px-4 py-3 text-left">Updated</th>
</tr>
</thead>
<tbody class="divide-y divide-zinc-700/50">
{#each filtered as entry}
{@const parts = parseKey(entry.cache_key)}
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
<td class="px-4 py-3 text-zinc-200 font-medium">
<a
href="/books/{parts.slug}"
class="hover:text-amber-400 transition-colors"
>
{parts.slug}
</a>
</td>
<td class="px-4 py-3 text-zinc-400">{parts.chapter}</td>
<td class="px-4 py-3 text-zinc-400 font-mono text-xs">{parts.voice}</td>
<td class="px-4 py-3 text-zinc-500 font-mono text-xs truncate max-w-[14rem]" title={entry.filename}>
{entry.filename}
</td>
<td class="px-4 py-3 text-zinc-400">{fmtDate(entry.updated)}</td>
</tr>
{/each}
</tbody>
</table>
</div>
{/if}
</div>

View File

@@ -1,29 +0,0 @@
import { redirect } from '@sveltejs/kit';
import type { PageServerLoad } from './$types';
import { listScrapingTasks } from '$lib/server/pocketbase';
import { env } from '$env/dynamic/private';
import { log } from '$lib/server/logger';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
export const load: PageServerLoad = async ({ locals }) => {
if (locals.user?.role !== 'admin') {
redirect(302, '/');
}
const [tasks, statusRes] = await Promise.all([
listScrapingTasks().catch((e) => {
log.warn('admin/scrape', 'failed to load tasks', { err: String(e) });
return [];
}),
fetch(`${SCRAPER_URL}/api/scrape/status`).catch(() => null)
]);
let running = false;
if (statusRes?.ok) {
const body = await statusRes.json().catch(() => null);
running = body?.running ?? false;
}
return { tasks, running };
};

View File

@@ -1,238 +0,0 @@
<script lang="ts">
import { untrack } from 'svelte';
import { invalidateAll } from '$app/navigation';
import type { PageData } from './$types';
import type { ScrapingTask } from '$lib/server/pocketbase';
let { data }: { data: PageData } = $props();
// ── Live-poll status ────────────────────────────────────────────────────────
let running = $state(untrack(() => data.running));
let tasks = $state(untrack(() => data.tasks));
let polling = $state(false);
// Poll every 5 s while a job is running
$effect(() => {
if (!running) return;
const id = setInterval(async () => {
const res = await fetch('/api/admin/scrape').catch(() => null);
if (res?.ok) {
const body = await res.json().catch(() => null);
running = body?.running ?? false;
if (!running) {
// Refresh tasks list once job finishes
await invalidateAll();
}
}
}, 5000);
return () => clearInterval(id);
});
// Keep local state in sync when server re-loads
$effect(() => {
running = data.running;
tasks = data.tasks;
});
// ── Trigger scrape ──────────────────────────────────────────────────────────
let scrapeUrl = $state('');
let scrapeError = $state('');
let scraping = $state(false);
async function triggerScrape(url?: string) {
if (running || scraping) return;
scraping = true;
scrapeError = '';
try {
const body = url ? { url } : {};
const res = await fetch('/api/scrape', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
});
if (!res.ok) {
const data = await res.json().catch(() => ({}));
scrapeError = data.error ?? data.message ?? `Error ${res.status}`;
} else {
running = true;
if (url) scrapeUrl = '';
}
} catch {
scrapeError = 'Network error.';
} finally {
scraping = false;
}
}
// ── Cancel task ─────────────────────────────────────────────────────────────
// Tracks which task IDs are currently being cancelled (to disable the button).
let cancellingIds = $state(new Set<string>());
let cancelErrors: Record<string, string> = $state({});
async function cancelTask(id: string) {
if (cancellingIds.has(id)) return;
cancellingIds = new Set([...cancellingIds, id]);
delete cancelErrors[id];
try {
const res = await fetch(`/api/scrape/cancel/${encodeURIComponent(id)}`, { method: 'POST' });
if (!res.ok) {
const body = await res.json().catch(() => ({}));
cancelErrors = { ...cancelErrors, [id]: body.error ?? body.message ?? `Error ${res.status}` };
} else {
// Optimistically flip status in the local list so the button disappears immediately.
tasks = tasks.map((t: ScrapingTask) => (t.id === id ? { ...t, status: 'cancelled' } : t));
}
} catch {
cancelErrors = { ...cancelErrors, [id]: 'Network error.' };
} finally {
cancellingIds = new Set([...cancellingIds].filter((x) => x !== id));
}
}
// ── Helpers ─────────────────────────────────────────────────────────────────
function statusColor(status: string) {
if (status === 'done') return 'text-green-400';
if (status === 'running') return 'text-amber-400 animate-pulse';
if (status === 'failed') return 'text-red-400';
if (status === 'cancelled') return 'text-zinc-400';
return 'text-zinc-300';
}
function fmtDate(s: string) {
if (!s) return '—';
return new Date(s).toLocaleString(undefined, {
month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit'
});
}
function duration(started: string, finished: string) {
if (!started || !finished) return '—';
const ms = new Date(finished).getTime() - new Date(started).getTime();
if (ms < 0) return '—';
const s = Math.floor(ms / 1000);
if (s < 60) return `${s}s`;
const m = Math.floor(s / 60);
return `${m}m ${s % 60}s`;
}
</script>
<svelte:head>
<title>Scrape tasks — libnovel admin</title>
</svelte:head>
<div class="space-y-8">
<div class="flex items-center justify-between flex-wrap gap-3">
<div>
<h1 class="text-2xl font-bold text-zinc-100">Scrape tasks</h1>
<p class="text-zinc-400 text-sm mt-1">
Job status:
{#if running}
<span class="text-amber-400 font-medium animate-pulse">Running</span>
{:else}
<span class="text-green-400 font-medium">Idle</span>
{/if}
</p>
</div>
<!-- Trigger controls -->
<div class="flex flex-wrap gap-3 items-start">
<button
onclick={() => triggerScrape()}
disabled={running || scraping}
class="px-4 py-2 rounded-lg bg-amber-400 text-zinc-900 font-semibold text-sm hover:bg-amber-300 transition-colors disabled:opacity-50"
>
Full catalogue scrape
</button>
</div>
</div>
<!-- Single book scrape -->
<div class="bg-zinc-800 rounded-xl border border-zinc-700 p-5 space-y-3">
<h2 class="text-sm font-semibold text-zinc-300">Scrape a single book</h2>
<div class="flex gap-2">
<input
type="url"
bind:value={scrapeUrl}
placeholder="https://novelfire.net/book/..."
class="flex-1 bg-zinc-700 border border-zinc-600 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
/>
<button
onclick={() => triggerScrape(scrapeUrl.trim() || undefined)}
disabled={!scrapeUrl.trim() || running || scraping}
class="px-4 py-2 rounded-lg bg-zinc-600 text-zinc-100 font-semibold text-sm hover:bg-zinc-500 transition-colors disabled:opacity-50"
>
Scrape
</button>
</div>
{#if scrapeError}
<p class="text-sm text-red-400">{scrapeError}</p>
{/if}
</div>
<!-- Tasks table -->
{#if tasks.length === 0}
<p class="text-zinc-500 text-sm py-8 text-center">No scrape tasks yet.</p>
{:else}
<div class="overflow-x-auto rounded-xl border border-zinc-700">
<table class="w-full text-sm">
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide">
<tr>
<th class="px-4 py-3 text-left">Kind</th>
<th class="px-4 py-3 text-left">Status</th>
<th class="px-4 py-3 text-right">Books</th>
<th class="px-4 py-3 text-right">Chapters</th>
<th class="px-4 py-3 text-right">Skipped</th>
<th class="px-4 py-3 text-right">Errors</th>
<th class="px-4 py-3 text-left">Started</th>
<th class="px-4 py-3 text-left">Duration</th>
<th class="px-4 py-3 text-left">Actions</th>
</tr>
</thead>
<tbody class="divide-y divide-zinc-700/50">
{#each tasks as task}
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
<td class="px-4 py-3 font-mono text-xs text-zinc-300">
{task.kind}
{#if task.target_url}
<br />
<span class="text-zinc-500 truncate max-w-[16rem] block" title={task.target_url}>
{task.target_url.replace('https://novelfire.net/book/', '')}
</span>
{/if}
</td>
<td class="px-4 py-3">
<span class="font-medium {statusColor(task.status)}">{task.status}</span>
</td>
<td class="px-4 py-3 text-right text-zinc-300">{task.books_found ?? 0}</td>
<td class="px-4 py-3 text-right text-zinc-300">{task.chapters_scraped ?? 0}</td>
<td class="px-4 py-3 text-right text-zinc-400">{task.chapters_skipped ?? 0}</td>
<td class="px-4 py-3 text-right {task.errors > 0 ? 'text-red-400' : 'text-zinc-400'}">{task.errors ?? 0}</td>
<td class="px-4 py-3 text-zinc-400">{fmtDate(task.started)}</td>
<td class="px-4 py-3 text-zinc-400">{duration(task.started, task.finished)}</td>
<td class="px-4 py-3">
{#if task.status === 'pending'}
<button
onclick={() => cancelTask(task.id)}
disabled={cancellingIds.has(task.id)}
class="px-2 py-1 rounded text-xs font-medium bg-zinc-700 text-zinc-300 hover:bg-red-900 hover:text-red-300 transition-colors disabled:opacity-50"
title="Cancel this task"
>
{cancellingIds.has(task.id) ? 'Cancelling…' : 'Cancel'}
</button>
{#if cancelErrors[task.id]}
<p class="text-xs text-red-400 mt-1">{cancelErrors[task.id]}</p>
{/if}
{/if}
</td>
</tr>
{#if task.error_message}
<tr class="bg-red-950/20">
<td colspan="9" class="px-4 py-2 text-xs text-red-400 font-mono">{task.error_message}</td>
</tr>
{/if}
{/each}
</tbody>
</table>
</div>
{/if}
</div>

View File

@@ -1,23 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { env } from '$env/dynamic/private';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
/**
* GET /api/admin/scrape/status
* Admin-only proxy to the Go scraper's /api/scrape/status endpoint.
*/
export const GET: RequestHandler = async ({ locals }) => {
if (!locals.user || locals.user.role !== 'admin') {
throw error(403, 'Forbidden');
}
try {
const res = await fetch(`${SCRAPER_URL}/api/scrape/status`);
if (!res.ok) return json({ running: false });
const data = await res.json();
return json({ running: data.running ?? false });
} catch {
return json({ running: false });
}
};

View File

@@ -1,100 +0,0 @@
import { error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { env } from '$env/dynamic/private';
import { log } from '$lib/server/logger';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
/**
* POST /api/audio/[slug]/[n]
* Proxies the audio generation request to the scraper's /api/audio endpoint.
* Keeps the scraper URL server-side — the browser never needs to know it.
*
* Body: { voice?: string }
*
* Responses:
* 200 { status: "done" } — audio already cached; client should call
* GET /api/presign/audio to obtain a direct MinIO presigned URL.
* 202 { task_id: string, status: "pending"|"generating" } — generation
* enqueued; poll GET /api/audio/status/[slug]/[n]?voice=... until done.
*/
export const POST: RequestHandler = async ({ params, request }) => {
const { slug, n } = params;
const chapter = parseInt(n, 10);
if (!slug || !chapter || chapter < 1) {
error(400, 'Invalid slug or chapter number');
}
let body: { voice?: string } = {};
try {
body = await request.json();
} catch {
// empty body is fine — scraper will use defaults
}
const scraperRes = await fetch(`${SCRAPER_URL}/api/audio/${slug}/${chapter}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
});
if (!scraperRes.ok) {
const text = await scraperRes.text().catch(() => '');
log.error('audio', 'scraper audio generation failed', { slug, chapter, status: scraperRes.status, body: text });
error(scraperRes.status as Parameters<typeof error>[0], text || 'Audio generation failed');
}
const data = (await scraperRes.json()) as
| { url: string; status: 'done' }
| { task_id: string; status: string };
// 202 Accepted: generation enqueued — return task_id + status for polling.
if (scraperRes.status === 202 || 'task_id' in data) {
return new Response(JSON.stringify(data), {
status: 202,
headers: { 'Content-Type': 'application/json' }
});
}
// 200: audio was already cached.
// Return status only — no url — so the client calls GET /api/presign/audio
// and streams directly from MinIO instead of through the Node.js server.
return new Response(
JSON.stringify({ status: 'done' }),
{ headers: { 'Content-Type': 'application/json' } }
);
};
/**
* GET /api/audio/[slug]/[n]?voice=...
* Proxies the audio stream from the scraper's /api/audio-proxy endpoint.
* Kept as a fallback but no longer used as the primary playback path —
* AudioPlayer fetches a presigned MinIO URL directly via /api/presign/audio.
*/
export const GET: RequestHandler = async ({ params, url }) => {
const { slug, n } = params;
const chapter = parseInt(n, 10);
if (!slug || !chapter || chapter < 1) {
error(400, 'Invalid slug or chapter number');
}
const voice = url.searchParams.get('voice') ?? '';
const qs = new URLSearchParams();
if (voice) qs.set('voice', voice);
const scraperRes = await fetch(`${SCRAPER_URL}/api/audio-proxy/${slug}/${chapter}?${qs.toString()}`);
if (!scraperRes.ok) {
log.error('audio', 'scraper audio proxy failed', { slug, chapter, status: scraperRes.status });
error(scraperRes.status as Parameters<typeof error>[0], 'Audio not found');
}
// Stream the audio body through — preserve Content-Type and Content-Length.
const headers = new Headers();
headers.set('Content-Type', scraperRes.headers.get('Content-Type') ?? 'audio/mpeg');
headers.set('Cache-Control', 'public, max-age=3600');
const cl = scraperRes.headers.get('Content-Length');
if (cl) headers.set('Content-Length', cl);
return new Response(scraperRes.body, { headers });
};

View File

@@ -1,66 +0,0 @@
import { error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { env } from '$env/dynamic/private';
import { log } from '$lib/server/logger';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
/**
* GET /api/audio/status/[slug]/[n]?voice=...
* Proxies the audio generation status check to the scraper's
* GET /api/audio/status/{slug}/{n} endpoint.
*
* Possible responses passed through to the client:
* {"status":"done"} — audio ready; no url
* {"status":"pending"|"generating","task_id":"..."} — in progress
* {"status":"idle"} — no job yet
* {"status":"failed","error":"..."} — last job failed
*
* When status is "done" the scraper's internal proxy URL is stripped — the
* client must call GET /api/presign/audio to obtain a direct MinIO presigned
* URL. This avoids streaming audio through the Node.js server.
*/
export const GET: RequestHandler = async ({ params, url }) => {
const { slug, n } = params;
const chapter = parseInt(n, 10);
if (!slug || !chapter || chapter < 1) {
error(400, 'Invalid slug or chapter number');
}
const voice = url.searchParams.get('voice') ?? '';
const qs = new URLSearchParams();
if (voice) qs.set('voice', voice);
const scraperRes = await fetch(
`${SCRAPER_URL}/api/audio/status/${slug}/${chapter}?${qs.toString()}`
);
if (!scraperRes.ok) {
const text = await scraperRes.text().catch(() => '');
log.error('audio', 'scraper audio status check failed', {
slug,
chapter,
status: scraperRes.status,
body: text
});
error(scraperRes.status as Parameters<typeof error>[0], text || 'Status check failed');
}
const data = (await scraperRes.json()) as {
status: string;
task_id?: string;
url?: string;
error?: string;
};
// Strip the scraper's internal proxy URL from "done" responses.
// The client will call GET /api/presign/audio to get a direct MinIO URL,
// avoiding streaming audio through the Node.js server.
if (data.status === 'done') {
delete data.url;
}
return new Response(JSON.stringify(data), {
headers: { 'Content-Type': 'application/json' }
});
};

View File

@@ -1,11 +0,0 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
/**
* POST /api/audio/voice-samples
* The new backend does not expose a voice-samples generation endpoint.
* Return 501 so callers get a clear signal rather than a 502 proxy error.
*/
export const POST: RequestHandler = async () => {
return json({ error: 'Voice sample pre-generation is not supported by this backend.' }, { status: 501 });
};

View File

@@ -1,47 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { changePassword } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
/**
* POST /api/auth/change-password
* Body: { currentPassword: string, newPassword: string }
* Requires authentication.
*/
export const POST: RequestHandler = async ({ request, locals }) => {
if (!locals.user) {
error(401, 'Not authenticated');
}
let body: { currentPassword?: string; newPassword?: string };
try {
body = await request.json();
} catch {
error(400, 'Invalid JSON body');
}
const currentPassword = body.currentPassword ?? '';
const newPassword = body.newPassword ?? '';
if (!currentPassword || !newPassword) {
error(400, 'currentPassword and newPassword are required');
}
if (newPassword.length < 4) {
error(400, 'New password must be at least 4 characters');
}
try {
const ok = await changePassword(locals.user.id, currentPassword, newPassword);
if (!ok) {
error(401, 'Current password is incorrect');
}
} catch (e: unknown) {
// Re-throw SvelteKit errors as-is
if (e && typeof e === 'object' && 'status' in e) throw e;
log.error('api/auth/change-password', 'unexpected error', { err: String(e) });
error(500, 'An error occurred. Please try again.');
}
return json({ ok: true });
};

View File

@@ -1,75 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { loginUser, mergeSessionProgress, createUserSession } from '$lib/server/pocketbase';
import { createAuthToken } from '../../../../hooks.server';
import { log } from '$lib/server/logger';
import { randomBytes } from 'node:crypto';
const AUTH_COOKIE = 'libnovel_auth';
const ONE_YEAR = 60 * 60 * 24 * 365;
/**
* POST /api/auth/login
* Body: { username: string, password: string }
* Returns: { token: string, user: { id, username, role } }
*
* Sets the libnovel_auth cookie and returns the raw token value so the
* iOS app can persist it for subsequent requests.
*/
export const POST: RequestHandler = async ({ request, cookies, locals }) => {
let body: { username?: string; password?: string };
try {
body = await request.json();
} catch {
error(400, 'Invalid JSON body');
}
const username = (body.username ?? '').trim();
const password = body.password ?? '';
if (!username || !password) {
error(400, 'Username and password are required');
}
let user;
try {
user = await loginUser(username, password);
} catch (e) {
log.error('api/auth/login', 'unexpected error', { username, err: String(e) });
error(500, 'An error occurred. Please try again.');
}
if (!user) {
error(401, 'Invalid username or password');
}
// Merge anonymous session progress (non-fatal)
mergeSessionProgress(locals.sessionId, user.id).catch((e) =>
log.warn('api/auth/login', 'mergeSessionProgress failed (non-fatal)', { err: String(e) })
);
const authSessionId = randomBytes(16).toString('hex');
const userAgent = request.headers.get('user-agent') ?? '';
const ip =
request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ??
request.headers.get('x-real-ip') ??
'';
createUserSession(user.id, authSessionId, userAgent, ip).catch((e) =>
log.warn('api/auth/login', 'createUserSession failed (non-fatal)', { err: String(e) })
);
const token = createAuthToken(user.id, user.username, user.role ?? 'user', authSessionId);
cookies.set(AUTH_COOKIE, token, {
path: '/',
httpOnly: true,
sameSite: 'lax',
maxAge: ONE_YEAR
});
return json({
token,
user: { id: user.id, username: user.username, role: user.role ?? 'user' }
});
};

View File

@@ -1,15 +0,0 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
const AUTH_COOKIE = 'libnovel_auth';
/**
* POST /api/auth/logout
* Clears the auth cookie and returns { ok: true }.
* Does not revoke the session record from PocketBase —
* for full revocation use DELETE /api/sessions/[id] first.
*/
export const POST: RequestHandler = async ({ cookies }) => {
cookies.delete(AUTH_COOKIE, { path: '/' });
return json({ ok: true });
};

View File

@@ -1,22 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { getUserByUsername } from '$lib/server/pocketbase';
/**
* GET /api/auth/me
* Returns the currently authenticated user from the request's auth cookie.
* Returns 401 if not authenticated.
*/
export const GET: RequestHandler = async ({ locals }) => {
if (!locals.user) {
error(401, 'Not authenticated');
}
// Fetch full record from PocketBase to get avatar_url
const record = await getUserByUsername(locals.user.username).catch(() => null);
return json({
id: locals.user.id,
username: locals.user.username,
role: locals.user.role,
avatar_url: record?.avatar_url ?? null
});
};

View File

@@ -1,84 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { createUser, mergeSessionProgress, createUserSession } from '$lib/server/pocketbase';
import { createAuthToken } from '../../../../hooks.server';
import { log } from '$lib/server/logger';
import { randomBytes } from 'node:crypto';
const AUTH_COOKIE = 'libnovel_auth';
const ONE_YEAR = 60 * 60 * 24 * 365;
/**
* POST /api/auth/register
* Body: { username: string, password: string }
* Returns: { token: string, user: { id, username, role } }
*
* Sets the libnovel_auth cookie and returns the raw token value so the
* iOS app can persist it for subsequent requests.
*/
export const POST: RequestHandler = async ({ request, cookies, locals }) => {
let body: { username?: string; password?: string };
try {
body = await request.json();
} catch {
error(400, 'Invalid JSON body');
}
const username = (body.username ?? '').trim();
const password = body.password ?? '';
if (!username || !password) {
error(400, 'Username and password are required');
}
if (username.length < 3 || username.length > 32) {
error(400, 'Username must be between 3 and 32 characters');
}
if (!/^[a-zA-Z0-9_-]+$/.test(username)) {
error(400, 'Username may only contain letters, numbers, underscores and hyphens');
}
if (password.length < 8) {
error(400, 'Password must be at least 8 characters');
}
let user;
try {
user = await createUser(username, password);
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : 'Registration failed.';
if (msg.includes('Username already taken')) {
error(409, 'That username is already taken');
}
log.error('api/auth/register', 'unexpected error', { username, err: String(e) });
error(500, 'An error occurred. Please try again.');
}
// Merge anonymous session progress (non-fatal)
mergeSessionProgress(locals.sessionId, user.id).catch((e) =>
log.warn('api/auth/register', 'mergeSessionProgress failed (non-fatal)', { err: String(e) })
);
const authSessionId = randomBytes(16).toString('hex');
const userAgent = request.headers.get('user-agent') ?? '';
const ip =
request.headers.get('x-forwarded-for')?.split(',')[0]?.trim() ??
request.headers.get('x-real-ip') ??
'';
createUserSession(user.id, authSessionId, userAgent, ip).catch((e) =>
log.warn('api/auth/register', 'createUserSession failed (non-fatal)', { err: String(e) })
);
const token = createAuthToken(user.id, user.username, user.role ?? 'user', authSessionId);
cookies.set(AUTH_COOKIE, token, {
path: '/',
httpOnly: true,
sameSite: 'lax',
maxAge: ONE_YEAR
});
return json({
token,
user: { id: user.id, username: user.username, role: user.role ?? 'user' }
});
};

View File

@@ -1,111 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { getBook, listChapterIdx, getProgress, isBookSaved } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
import { env } from '$env/dynamic/private';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
/**
* GET /api/book/[slug]
* Returns book metadata, chapter list, progress, and library status.
*
* If the book is not yet in PocketBase, asks the backend to enqueue a scrape
* task and returns 202 with { scraping: true, task_id }.
* The client should poll and retry once the task completes.
*/
export const GET: RequestHandler = async ({ params, locals }) => {
const { slug } = params;
// Try PocketBase first
let book = await getBook(slug).catch((e) => {
log.error('api/book', 'getBook failed', { slug, err: String(e) });
return null;
});
if (book) {
let chapters, progress, saved;
try {
[chapters, progress, saved] = await Promise.all([
listChapterIdx(slug),
getProgress(locals.sessionId, slug, locals.user?.id),
isBookSaved(locals.sessionId, slug, locals.user?.id)
]);
} catch (e) {
log.error('api/book', 'failed to load book detail data', { slug, err: String(e) });
error(500, 'Failed to load book');
}
return json({
book,
chapters,
in_lib: true,
saved,
last_chapter: progress?.chapter ?? null,
scraping: false,
task_id: null
});
}
// Fall back to backend: enqueue scrape task if not in library.
try {
const res = await fetch(`${SCRAPER_URL}/api/book-preview/${encodeURIComponent(slug)}`);
if (res.status === 202) {
const body: { task_id: string; message: string } = await res.json();
log.info('api/book', 'scrape task enqueued', { slug, task_id: body.task_id });
return json({ scraping: true, task_id: body.task_id, in_lib: false }, { status: 202 });
}
if (!res.ok) {
log.warn('api/book', 'book-preview returned error', { slug, status: res.status });
error(404, `Book "${slug}" not found`);
}
// 200 — book was already in library
const preview: {
in_lib: boolean;
meta: {
slug: string;
title: string;
author: string;
cover: string;
status: string;
genres: string[];
summary: string;
total_chapters: number;
source_url: string;
};
chapters: { number: number; title: string; date?: string }[];
} = await res.json();
const previewBook = {
id: '',
slug: preview.meta.slug || slug,
title: preview.meta.title,
author: preview.meta.author,
cover: preview.meta.cover,
status: preview.meta.status,
genres: preview.meta.genres ?? [],
summary: preview.meta.summary,
total_chapters: preview.meta.total_chapters,
source_url: preview.meta.source_url,
ranking: 0,
meta_updated: ''
};
return json({
book: previewBook,
chapters: preview.chapters,
in_lib: true,
saved: false,
last_chapter: null,
scraping: false,
task_id: null
});
} catch (e) {
if (e instanceof Error && 'status' in e) throw e;
log.error('api/book', 'book-preview fetch failed', { slug, err: String(e) });
error(404, `Book "${slug}" not found`);
}
};

View File

@@ -1,37 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { env } from '$env/dynamic/private';
import { log } from '$lib/server/logger';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
/**
* GET /api/browse-page?page=2&genre=all&sort=popular&status=all
*
* Thin proxy to the Go scraper's /api/browse endpoint.
* Used by the infinite-scroll browse page to append subsequent pages
* without a full SSR navigation.
*/
export const GET: RequestHandler = async ({ url }) => {
const page = url.searchParams.get('page') ?? '1';
const genre = url.searchParams.get('genre') ?? 'all';
const sort = url.searchParams.get('sort') ?? 'popular';
const status = url.searchParams.get('status') ?? 'all';
const params = new URLSearchParams({ page, genre, sort, status });
const apiURL = `${SCRAPER_URL}/api/browse?${params.toString()}`;
try {
const res = await fetch(apiURL);
if (!res.ok) {
log.error('browse-page', 'scraper returned error', { status: res.status });
throw error(502, `Browse fetch failed: ${res.status}`);
}
const data = await res.json();
return json(data);
} catch (e) {
if (e instanceof Error && 'status' in e) throw e;
log.error('browse-page', 'network error', { err: String(e) });
throw error(502, 'Could not reach browse service');
}
};

View File

@@ -1,46 +0,0 @@
import { error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { env } from '$env/dynamic/private';
import { log } from '$lib/server/logger';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
/**
* GET /api/chapter-text-preview/[slug]/[n]
* Proxies to the scraper's /api/chapter-text-preview endpoint.
* Used client-side when the normal chapter path returns no content
* (chapter indexed but not yet scraped to MinIO).
*/
export const GET: RequestHandler = async ({ params, url }) => {
const { slug, n } = params;
const chapter = parseInt(n, 10);
if (!slug || !chapter || chapter < 1) {
error(400, 'Invalid slug or chapter number');
}
// Forward optional query params (chapter_url, title) if present
const qs = new URLSearchParams();
const chapterUrl = url.searchParams.get('chapter_url');
const title = url.searchParams.get('title');
if (chapterUrl) qs.set('chapter_url', chapterUrl);
if (title) qs.set('title', title);
const scraperRes = await fetch(
`${SCRAPER_URL}/api/chapter-text-preview/${encodeURIComponent(slug)}/${chapter}?${qs.toString()}`
).catch((e) => {
log.error('chapter-preview', 'scraper fetch failed', { slug, chapter, err: String(e) });
return null;
});
if (!scraperRes || !scraperRes.ok) {
const status = scraperRes?.status ?? 502;
log.error('chapter-preview', 'scraper returned error', { slug, chapter, status });
error(status as Parameters<typeof error>[0], 'Chapter preview not available');
}
const data = await scraperRes.json();
return new Response(JSON.stringify(data), {
headers: { 'Content-Type': 'application/json' }
});
};

View File

@@ -1,128 +0,0 @@
import { json, error } from '@sveltejs/kit';
import { marked } from 'marked';
import type { RequestHandler } from './$types';
import { getBook, listChapterIdx } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
import { env } from '$env/dynamic/private';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
/**
* GET /api/chapter/[slug]/[n]
* Returns rendered chapter HTML, navigation info, and voice list.
* Supports ?preview=1&chapter_url=...&title=... for un-scraped books.
*
* Response shape mirrors ChapterResponse in the iOS APIClient.
*/
export const GET: RequestHandler = async ({ params, url, locals }) => {
const { slug } = params;
const n = parseInt(params.n, 10);
if (!n || n < 1) error(400, 'Invalid chapter number');
const isPreview = url.searchParams.get('preview') === '1';
const chapterUrl = url.searchParams.get('chapter_url') ?? '';
const chapterTitle = url.searchParams.get('title') ?? '';
if (isPreview) {
// Preview path: scrape live, nothing from PocketBase/MinIO
const previewParams = new URLSearchParams();
if (chapterUrl) previewParams.set('chapter_url', chapterUrl);
if (chapterTitle) previewParams.set('title', chapterTitle);
let chapterData: { slug: string; number: number; title: string; text: string; url: string };
try {
const res = await fetch(
`${SCRAPER_URL}/api/chapter-text-preview/${encodeURIComponent(slug)}/${n}?${previewParams.toString()}`
);
if (!res.ok) {
log.error('api/chapter', 'chapter-text-preview returned error', { slug, n, status: res.status });
error(404, `Chapter ${n} not found`);
}
chapterData = await res.json();
} catch (e) {
if (e instanceof Error && 'status' in e) throw e;
log.error('api/chapter', 'chapter-text-preview fetch failed', { slug, n, err: String(e) });
error(502, 'Could not fetch chapter preview');
}
const html = chapterData.text
? '<p>' + chapterData.text.replace(/\n{2,}/g, '</p><p>').replace(/\n/g, '<br>') + '</p>'
: '';
let voices: string[] = [];
try {
const vRes = await fetch(`${SCRAPER_URL}/api/voices`);
if (vRes.ok) {
const d = (await vRes.json()) as { voices: string[] };
voices = d.voices ?? [];
}
} catch {
// Non-critical
}
const pb = await getBook(slug).catch(() => null);
return json({
book: { slug, title: pb?.title ?? slug, cover: pb?.cover ?? '' },
chapter: { id: '', slug, number: n, title: chapterData.title || `Chapter ${n}`, date_label: '' },
html,
voices,
prev: null,
next: null,
chapters: [],
is_preview: true
});
}
// Normal path: PocketBase + MinIO
const [book, chapters, voicesRes] = await Promise.all([
getBook(slug),
listChapterIdx(slug),
fetch(`${SCRAPER_URL}/api/voices`).catch(() => null)
]);
if (!book) error(404, `Book "${slug}" not found`);
const chapterIdx = chapters.find((c) => c.number === n);
if (!chapterIdx) error(404, `Chapter ${n} not found`);
let voices: string[] = [];
try {
if (voicesRes?.ok) {
const data = (await voicesRes.json()) as { voices: string[] };
voices = data.voices ?? [];
}
} catch {
// Non-critical
}
let html = '';
try {
const res = await fetch(`${SCRAPER_URL}/api/chapter-markdown/${encodeURIComponent(slug)}/${n}`);
if (!res.ok) {
log.error('api/chapter', 'chapter-markdown returned error', { slug, n, status: res.status });
error(res.status === 404 ? 404 : 502, res.status === 404 ? `Chapter ${n} not found` : 'Could not fetch chapter content');
}
const markdown = await res.text();
html = marked(markdown) as string;
} catch (e) {
if (e instanceof Error && 'status' in e) throw e;
log.error('api/chapter', 'failed to fetch chapter content', { slug, n, err: String(e) });
error(502, 'Could not fetch chapter content');
}
const prevChapter = chapters.find((c) => c.number === n - 1) ?? null;
const nextChapter = chapters.find((c) => c.number === n + 1) ?? null;
return json({
book: { slug: book.slug, title: book.title, cover: book.cover ?? '' },
chapter: chapterIdx,
html,
voices,
prev: prevChapter ? prevChapter.number : null,
next: nextChapter ? nextChapter.number : null,
chapters: chapters.map((c) => ({ number: c.number, title: c.title })),
is_preview: false
});
};

View File

@@ -1,26 +0,0 @@
import { error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { deleteComment } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
/**
* DELETE /api/comment/[id]
* Deletes a comment and its replies. Only the comment owner may delete.
* Requires authentication.
*/
export const DELETE: RequestHandler = async ({ params, locals }) => {
if (!locals.user) error(401, 'Login required');
const { id } = params;
try {
await deleteComment(id, locals.user.id);
return new Response(null, { status: 204 });
} catch (e) {
const msg = String(e);
if (msg.includes('Not authorized')) error(403, 'Not authorized to delete this comment');
if (msg.includes('not found')) error(404, 'Comment not found');
log.error('api/comment/[id]', 'deleteComment failed', { id, err: msg });
error(500, 'Failed to delete comment');
}
};

View File

@@ -1,33 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { voteComment } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
/**
* POST /api/comment/[id]/vote
* Body: { vote: 'up' | 'down' }
* Casts, changes, or toggles off a vote on a comment.
* Works for both authenticated and anonymous users (session-scoped).
* Returns the updated comment.
*/
export const POST: RequestHandler = async ({ params, request, locals }) => {
const { id } = params;
let body: { vote?: string };
try {
body = await request.json();
} catch {
error(400, 'Invalid JSON body');
}
if (body.vote !== 'up' && body.vote !== 'down') {
error(400, 'vote must be "up" or "down"');
}
try {
const updated = await voteComment(id, body.vote, locals.sessionId, locals.user?.id);
return json(updated);
} catch (e) {
log.error('api/comment/[id]/vote', 'voteComment failed', { id, err: String(e) });
error(500, 'Failed to record vote');
}
};

View File

@@ -1,102 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import {
listComments,
listReplies,
createComment,
getMyVotes,
type CommentSort
} from '$lib/server/pocketbase';
import { presignAvatarUrl } from '$lib/server/minio';
import { log } from '$lib/server/logger';
/**
* GET /api/comments/[slug]?sort=new|top
* Returns top-level comments + their replies + current visitor's votes + avatar URLs.
* Response: { comments: BookComment[], myVotes: Record<string, 'up'|'down'>, avatarUrls: Record<string, string> }
* Each top-level comment has a `replies` array attached.
*/
export const GET: RequestHandler = async ({ params, url, locals }) => {
const { slug } = params;
const sortParam = url.searchParams.get('sort') ?? 'new';
const sort: CommentSort = sortParam === 'top' ? 'top' : 'new';
try {
const topLevel = await listComments(slug, sort);
// Fetch replies for all top-level comments in parallel
const repliesPerComment = await Promise.all(topLevel.map((c) => listReplies(c.id)));
const allReplies = repliesPerComment.flat();
// Build comment+reply list for vote lookup
const allIds = [...topLevel.map((c) => c.id), ...allReplies.map((r) => r.id)];
const myVotes = await getMyVotes(allIds, locals.sessionId, locals.user?.id);
// Attach replies to each top-level comment
const comments = topLevel.map((c, i) => ({
...c,
replies: repliesPerComment[i]
}));
// Batch-resolve avatar presign URLs for all unique user_ids
const allComments = [...topLevel, ...allReplies];
const uniqueUserIds = [...new Set(allComments.map((c) => c.user_id).filter(Boolean))];
const avatarEntries = await Promise.all(
uniqueUserIds.map(async (userId) => {
try {
const url = await presignAvatarUrl(userId);
return [userId, url] as [string, string | null];
} catch {
return [userId, null] as [string, null];
}
})
);
const avatarUrls: Record<string, string> = {};
for (const [userId, url] of avatarEntries) {
if (url) avatarUrls[userId] = url;
}
return json({ comments, myVotes, avatarUrls });
} catch (e) {
log.error('api/comments/[slug]', 'listComments failed', { slug, err: String(e) });
error(500, 'Failed to load comments');
}
};
/**
* POST /api/comments/[slug]
* Body: { body: string, parent_id?: string }
* Creates a new comment or reply. Requires authentication.
*/
export const POST: RequestHandler = async ({ params, request, locals }) => {
if (!locals.user) error(401, 'Login required to comment');
const { slug } = params;
let body: { body?: string; parent_id?: string };
try {
body = await request.json();
} catch {
error(400, 'Invalid JSON body');
}
const text = (body.body ?? '').trim();
if (!text) error(400, 'Comment body is required');
if (text.length > 2000) error(400, 'Comment is too long (max 2000 characters)');
// Enforce 1-level depth: parent_id must be a top-level comment
const parentId = body.parent_id?.trim() || undefined;
try {
const comment = await createComment(
slug,
text,
locals.user.id,
locals.user.username,
parentId
);
return json(comment, { status: 201 });
} catch (e) {
log.error('api/comments/[slug]', 'createComment failed', { slug, err: String(e) });
error(500, 'Failed to post comment');
}
};

View File

@@ -1,65 +0,0 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import {
listBooks,
recentlyAddedBooks,
allProgress,
getHomeStats,
getSubscriptionFeed
} from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
import type { Book, Progress } from '$lib/server/pocketbase';
/**
* GET /api/home
* Returns home screen data: continue-reading list, recently updated books, stats,
* and subscription feed (books recently read by followed users).
* Requires authentication (enforced by layout guard).
*/
export const GET: RequestHandler = async ({ locals }) => {
let allBooks: Book[] = [];
let recentBooks: Book[] = [];
let progressList: Progress[] = [];
let stats = { totalBooks: 0, totalChapters: 0 };
try {
[allBooks, recentBooks, progressList, stats] = await Promise.all([
listBooks(),
recentlyAddedBooks(8),
allProgress(locals.sessionId, locals.user?.id),
getHomeStats()
]);
} catch (e) {
log.error('api/home', 'failed to load home data', { err: String(e) });
}
const bookMap = new Map<string, Book>(allBooks.map((b) => [b.slug, b]));
const continueReading = progressList
.filter((p) => bookMap.has(p.slug))
.slice(0, 6)
.map((p) => ({ book: bookMap.get(p.slug)!, chapter: p.chapter }));
const inProgressSlugs = new Set(continueReading.map((c) => c.book.slug));
const recentlyUpdated = recentBooks.filter((b) => !inProgressSlugs.has(b.slug)).slice(0, 6);
// Subscription feed — only available for logged-in users with following
let subscriptionFeed: Array<{ book: Book; readerUsername: string }> = [];
if (locals.user?.id) {
subscriptionFeed = await getSubscriptionFeed(locals.user.id).catch(() => []);
}
return json({
continue_reading: continueReading,
recently_updated: recentlyUpdated,
stats: {
totalBooks: stats.totalBooks,
totalChapters: stats.totalChapters,
booksInProgress: continueReading.length
},
subscription_feed: subscriptionFeed.map((item) => ({
book: item.book,
readerUsername: item.readerUsername
}))
});
};

View File

@@ -1,61 +0,0 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { listBooks, allProgress, getSavedSlugs } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
/**
* GET /api/library
* Returns the user's library: books they have started reading or explicitly saved.
* Each item includes the book record, the last chapter read, and saved_at timestamp.
*
* Response shape mirrors LibraryItem in the iOS APIClient.
*/
export const GET: RequestHandler = async ({ locals }) => {
let allBooks: Awaited<ReturnType<typeof listBooks>>;
let progressList: Awaited<ReturnType<typeof allProgress>>;
let savedSlugs: Set<string>;
try {
[allBooks, progressList, savedSlugs] = await Promise.all([
listBooks(),
allProgress(locals.sessionId, locals.user?.id),
getSavedSlugs(locals.sessionId, locals.user?.id)
]);
} catch (e) {
log.error('api/library', 'failed to load library data', { err: String(e) });
allBooks = [];
progressList = [];
savedSlugs = new Set();
}
const progressMap: Record<string, number> = {};
const progressUpdatedMap: Record<string, string> = {};
for (const p of progressList) {
progressMap[p.slug] = p.chapter;
progressUpdatedMap[p.slug] = p.updated;
}
const progressSlugs = new Set(progressList.map((p) => p.slug));
const books = allBooks.filter((b) => progressSlugs.has(b.slug) || savedSlugs.has(b.slug));
const withProgress = books.filter((b) => progressSlugs.has(b.slug));
const savedOnly = books
.filter((b) => !progressSlugs.has(b.slug))
.sort((a, b) => (a.title ?? '').localeCompare(b.title ?? ''));
withProgress.sort((a, b) => {
const ta = progressUpdatedMap[a.slug] ?? '';
const tb = progressUpdatedMap[b.slug] ?? '';
return tb.localeCompare(ta);
});
const ordered = [...withProgress, ...savedOnly];
const items = ordered.map((book) => ({
book,
last_chapter: progressMap[book.slug] ?? null,
saved_at: progressUpdatedMap[book.slug] ?? new Date().toISOString()
}));
return json(items);
};

View File

@@ -1,34 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { saveBook, unsaveBook } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
/**
* POST /api/library/[slug]
* Save a book to the user's personal library.
*/
export const POST: RequestHandler = async ({ params, locals }) => {
const { slug } = params;
try {
await saveBook(locals.sessionId, slug, locals.user?.id);
} catch (e) {
log.error('library', 'saveBook failed', { slug, err: String(e) });
error(500, 'Failed to save book');
}
return json({ ok: true });
};
/**
* DELETE /api/library/[slug]
* Remove a book from the user's personal library.
*/
export const DELETE: RequestHandler = async ({ params, locals }) => {
const { slug } = params;
try {
await unsaveBook(locals.sessionId, slug, locals.user?.id);
} catch (e) {
log.error('library', 'unsaveBook failed', { slug, err: String(e) });
error(500, 'Failed to remove book');
}
return json({ ok: true });
};

View File

@@ -1,47 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { presignAudio } from '$lib/server/minio';
import { log } from '$lib/server/logger';
import * as cache from '$lib/server/presignCache';
/**
* GET /api/presign/audio?slug=...&n=...&voice=...
* Returns a presigned MinIO URL for the audio file so the browser
* can stream it directly without going through the server.
* Returns 404 when the audio has not been generated yet.
*
* Results are cached in-process for 50 minutes (MinIO URLs are valid 1 hour)
* to avoid a backend + MinIO round-trip on every "Play" click.
*/
export const GET: RequestHandler = async ({ url }) => {
const slug = url.searchParams.get('slug');
// Accept both 'n' (web) and 'chapter' (iOS) as the chapter number param
const n = parseInt(url.searchParams.get('n') ?? url.searchParams.get('chapter') ?? '', 10);
const voice = url.searchParams.get('voice') ?? '';
if (!slug || !n || n < 1) {
error(400, 'Missing slug or n');
}
const cacheKey = cache.audioKey(slug, n, voice);
// Fast path: return cached URL if still valid.
const cached = cache.get(cacheKey);
if (cached) {
return json({ url: cached });
}
// Slow path: call backend → MinIO presign.
try {
const presignedUrl = await presignAudio(slug, n, voice || undefined);
cache.set(cacheKey, presignedUrl);
return json({ url: presignedUrl });
} catch (e) {
const status = (e as { status?: number }).status;
if (status === 404) {
error(404, 'Audio not found');
}
log.error('presign', 'presign audio failed', { slug, n, err: String(e) });
error(500, `Could not get presigned URL: ${e}`);
}
};

View File

@@ -1,40 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { presignVoiceSample } from '$lib/server/minio';
import * as cache from '$lib/server/presignCache';
/**
* GET /api/presign/voice-sample?voice=af_bella
* Returns a presigned URL for the voice sample audio file.
* Returns 404 if the sample has not been generated yet.
*
* Results are cached in-process for 50 minutes to avoid a backend + MinIO
* round-trip on every voice-selection preview play.
*/
export const GET: RequestHandler = async ({ url }) => {
const voice = url.searchParams.get('voice');
if (!voice) {
error(400, 'Missing voice parameter');
}
const cacheKey = cache.sampleKey(voice);
// Fast path: return cached URL if still valid.
const cached = cache.get(cacheKey);
if (cached) {
return json({ url: cached });
}
// Slow path: call backend → MinIO presign.
try {
const presignedUrl = await presignVoiceSample(voice);
cache.set(cacheKey, presignedUrl);
return json({ url: presignedUrl });
} catch (e) {
const status = (e as { status?: number }).status;
if (status === 404) {
error(404, 'Voice sample not found');
}
error(502, `Failed to presign voice sample: ${e}`);
}
};

View File

@@ -1,81 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { presignAvatarUploadUrl, presignAvatarUrl } from '$lib/server/minio';
import { updateUserAvatarUrl, getUserByUsername } from '$lib/server/pocketbase';
const ALLOWED_TYPES = ['image/jpeg', 'image/png', 'image/webp'];
/**
* POST /api/profile/avatar
* Body: JSON { mime_type: "image/jpeg" | "image/png" | "image/webp" }
*
* Returns a short-lived presigned PUT URL pointing at MinIO (public endpoint)
* so the client can upload the image bytes directly, bypassing the server.
* After the PUT completes, the client must call PATCH /api/profile/avatar
* with the returned key to record it in PocketBase.
*
* Returns: { upload_url: string, key: string }
*/
export const POST: RequestHandler = async ({ request, locals }) => {
if (!locals.user) error(401, 'Not authenticated');
let mimeType = 'image/jpeg';
try {
const body = await request.json();
if (body?.mime_type) mimeType = body.mime_type;
} catch {
// default to jpeg if body is missing/invalid
}
if (!ALLOWED_TYPES.includes(mimeType)) {
error(400, `Unsupported image type: ${mimeType}. Allowed: jpeg, png, webp`);
}
const { uploadUrl, key } = await presignAvatarUploadUrl(locals.user.id, mimeType);
return json({ upload_url: uploadUrl, key });
};
/**
* PATCH /api/profile/avatar
* Body: JSON { key: string }
*
* Called after the client has successfully PUT the image to MinIO via the
* presigned URL. Records the object key in PocketBase and returns a fresh
* presigned GET URL for immediate display.
*
* Returns: { avatar_url: string | null }
*/
export const PATCH: RequestHandler = async ({ request, locals }) => {
if (!locals.user) error(401, 'Not authenticated');
let key: string | undefined;
try {
const body = await request.json();
if (typeof body?.key === 'string') key = body.key;
} catch {
error(400, 'Invalid JSON body');
}
if (!key) error(400, 'Missing "key" field');
await updateUserAvatarUrl(locals.user.id, key);
const avatarUrl = await presignAvatarUrl(locals.user.id);
return json({ avatar_url: avatarUrl });
};
/**
* GET /api/profile/avatar
* Returns a presigned GET URL for the current user's avatar, or null if none set.
*/
export const GET: RequestHandler = async ({ locals }) => {
if (!locals.user) error(401, 'Not authenticated');
const record = await getUserByUsername(locals.user.username).catch(() => null);
if (!record?.avatar_url) {
return json({ avatar_url: null });
}
const avatarUrl = await presignAvatarUrl(locals.user.id);
return json({ avatar_url: avatarUrl });
};

View File

@@ -1,27 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { setProgress } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
/**
* POST /api/progress
* Body: { slug: string, chapter: number }
* Records the user's reading position.
* When the user is logged in, progress is keyed by user_id so it syncs across devices.
* When anonymous, progress is keyed by the session cookie.
*/
export const POST: RequestHandler = async ({ request, locals }) => {
const body = await request.json().catch(() => null);
if (!body || typeof body.slug !== 'string' || typeof body.chapter !== 'number') {
error(400, 'Invalid body — expected { slug, chapter }');
}
try {
await setProgress(locals.sessionId, body.slug, body.chapter, locals.user?.id);
} catch (e) {
log.error('progress', 'setProgress failed', { slug: body.slug, chapter: body.chapter, err: String(e) });
error(500, 'Failed to save progress');
}
return json({ ok: true });
};

Some files were not shown because too many files have changed in this diff Show More