Compare commits

...

21 Commits

Author SHA1 Message Date
Admin
38e400a4c7 feat(scraper): remove HTMX UI — delete ui.go, extract helpers.go, drop goldmark dependency 2026-03-02 22:03:36 +05:00
Admin
cb90771248 refactor(scraper): rename /ui/audio*, /ui/chapter-text routes to /api/ namespace 2026-03-02 22:01:52 +05:00
Admin
59b1cfab1d chore: add all missing port and URL env vars to .env.example 2026-03-02 22:00:45 +05:00
Admin
f95ad3ed29 feat(ui): proxy audio generation and streaming through SvelteKit, fix hardcoded scraper URL in AudioPlayer 2026-03-02 22:00:13 +05:00
Admin
e4c4f8de66 feat(ui): wire SvelteKit into docker-compose with ui service and env vars 2026-03-02 21:43:32 +05:00
Admin
4f84bd29c9 feat(ui): add audio player component, presign API route, and reading progress tracking 2026-03-02 21:41:20 +05:00
Admin
6bf79ab392 feat(ui): add chapter reader page with SSR markdown rendering from MinIO 2026-03-02 21:39:11 +05:00
Admin
4ae6f0ab42 feat(ui): add book list page and book detail page with chapter index 2026-03-02 21:37:32 +05:00
Admin
33e2a4dc01 feat: add MinIO presign endpoints to scraper API and SvelteKit presign helper 2026-03-02 21:35:27 +05:00
Admin
cb4be0848f feat(ui): add server-side PocketBase client and session cookie hook 2026-03-02 21:32:20 +05:00
Admin
2f948f2a50 feat(ui): scaffold SvelteKit app with Tailwind v4, adapter-node, zinc/amber theme 2026-03-02 21:30:50 +05:00
Admin
baab66823d fix: remove container names for name collisions 2026-03-02 20:08:46 +05:00
Admin
11d2eaa0e5 chore: trigger deploy 2026-03-02 19:56:09 +05:00
Admin
9c115f00c4 chore: remove kokoro from compose, make host ports env-configurable
- Drop kokoro service (deployed separately); update default KOKORO_URL to kokoro.kalekber.cc
- Expose host ports via env vars (MINIO_PORT, MINIO_CONSOLE_PORT, POCKETBASE_PORT, BROWSERLESS_PORT, SCRAPER_PORT) for preview deployments
2026-03-02 15:13:26 +05:00
Admin
5ac89da513 Steps 11-12: migrate chapter + book page reading-progress JS from localStorage to /api/progress 2026-03-02 14:52:01 +05:00
Admin
af86c6f96f Step 10: migrate home page reading-progress JS from localStorage to /api/progress 2026-03-02 14:51:07 +05:00
Admin
da4a182f85 Step 9: add session cookie + /api/progress GET/POST/DELETE endpoints 2026-03-02 14:47:58 +05:00
Admin
18e76c9668 steps 6-8: wire HybridStore into orchestrator, server, and main
- Add storage/hybrid.go: HybridStore composing PocketBase + MinIO backends
- Rewrite orchestrator to accept storage.Store instead of *writer.Writer
- Replace *writer.Writer with storage.Store in server.go and ui.go
- Wire audio cache, reading progress, chapter reads/writes through store
- Add rankingCacheAdapter in main.go to bridge context-free RankingPageCacher
  interface to HybridStore's context-aware methods
2026-03-02 14:44:02 +05:00
Admin
9add9033b9 feat(v2): add internal/storage package with MinIO + PocketBase clients and Store interface 2026-03-02 14:34:25 +05:00
Admin
66d8481637 chore(v2): add minio-go/v7 dependency 2026-03-02 14:31:36 +05:00
Admin
7f92a58fd7 chore(v2): add PocketBase and MinIO services to docker-compose 2026-03-02 14:30:47 +05:00
44 changed files with 5530 additions and 3380 deletions

View File

@@ -1,6 +1,26 @@
# libnovel scraper — environment overrides
# Copy to .env and adjust values; do NOT commit this file with real secrets.
# ── Service ports (host-side) ─────────────────────────────────────────────────
# Port the scraper HTTP API listens on (default 8080)
SCRAPER_PORT=8080
# Port PocketBase listens on (default 8090)
POCKETBASE_PORT=8090
# Port MinIO S3 API listens on (default 9000)
MINIO_PORT=9000
# Port MinIO web console listens on (default 9001)
MINIO_CONSOLE_PORT=9001
# Port Browserless Chrome listens on (default 3030)
BROWSERLESS_PORT=3030
# Port the SvelteKit UI listens on (default 3000)
UI_PORT=3000
# ── Browserless ───────────────────────────────────────────────────────────────
# Browserless API token (leave empty to disable auth)
BROWSERLESS_TOKEN=
@@ -23,6 +43,7 @@ BROWSERLESS_STRATEGY=direct
# Set to direct to use plain HTTP, or content/scrape/cdp for browserless.
BROWSERLESS_URL_STRATEGY=content
# ── Scraper ───────────────────────────────────────────────────────────────────
# Chapter worker goroutines (0 = NumCPU inside the container)
SCRAPER_WORKERS=0
@@ -39,3 +60,28 @@ KOKORO_URL=http://kokoro:8880
# Single voices: af_bella, af_sky, af_heart, am_adam, …
# Mixed voices: af_bella+af_sky or af_bella(2)+af_sky(1) (weighted blend)
KOKORO_VOICE=af_bella
# ── MinIO / S3 object storage ─────────────────────────────────────────────────
MINIO_ROOT_USER=admin
MINIO_ROOT_PASSWORD=changeme123
MINIO_BUCKET_CHAPTERS=libnovel-chapters
MINIO_BUCKET_AUDIO=libnovel-audio
# ── PocketBase ────────────────────────────────────────────────────────────────
# Admin credentials (used by scraper + UI server-side)
POCKETBASE_ADMIN_EMAIL=admin@libnovel.local
POCKETBASE_ADMIN_PASSWORD=changeme123
# ── SvelteKit UI ─────────────────────────────────────────────────────────────
# Internal URL the SvelteKit server uses to reach the scraper API.
# In docker-compose this is http://scraper:8080 (wired automatically).
# Override here only if running the UI outside of docker-compose.
SCRAPER_API_URL=http://localhost:8080
# Internal URL the SvelteKit server uses to reach PocketBase.
# In docker-compose this is http://pocketbase:8090 (wired automatically).
POCKETBASE_URL=http://localhost:8090
# Public MinIO URL reachable from the browser (for audio/presigned URLs).
# In production, point this at your MinIO reverse-proxy or CDN domain.
PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000

View File

@@ -1,10 +1,64 @@
version: "3.9"
services:
# ─── MinIO (object storage for chapter .md files + audio cache) ─────────────
minio:
image: minio/minio:latest
#container_name: libnovel-minio
restart: unless-stopped
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
ports:
- "${MINIO_PORT:-9000}:9000" # S3 API
- "${MINIO_CONSOLE_PORT:-9001}:9001" # Web console
volumes:
- minio_data:/data
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 10s
timeout: 5s
retries: 5
# ─── MinIO bucket initialisation ─────────────────────────────────────────────
# Runs once to create the default buckets and then exits.
minio-init:
image: minio/mc:latest
#container_name: libnovel-minio-init
depends_on:
minio:
condition: service_healthy
entrypoint: >
/bin/sh -c "
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
mc mb --ignore-existing local/libnovel-chapters;
mc mb --ignore-existing local/libnovel-audio;
echo 'buckets ready';
"
environment:
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
# ─── PocketBase (auth + structured data: books, chapters index, ranking, progress) ──
pocketbase:
image: ghcr.io/muchobien/pocketbase:latest
#container_name: libnovel-pocketbase
restart: unless-stopped
ports:
- "${POCKETBASE_PORT:-8090}:8090"
volumes:
- pb_data:/pb/pb_data
healthcheck:
test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"]
interval: 10s
timeout: 5s
retries: 5
# ─── Browserless ────────────────────────────────────────────────────────────
browserless:
image: ghcr.io/browserless/chromium:latest
container_name: libnovel-browserless
#container_name: libnovel-browserless
restart: unless-stopped
environment:
# Set a token to lock down the endpoint; the scraper reads it via
@@ -19,7 +73,7 @@ services:
# Optional webhook URL for Browserless error alerts.
ERROR_ALERT_URL: "${ERROR_ALERT_URL:-}"
ports:
- "3030:3000"
- "${BROWSERLESS_PORT:-3030}:3000"
# Shared memory is required for Chrome.
shm_size: "2gb"
healthcheck:
@@ -28,30 +82,17 @@ services:
timeout: 5s
retries: 5
# ─── Kokoro-FastAPI (TTS) ────────────────────────────────────────────────────
# CPU image; swap for ghcr.io/remsky/kokoro-fastapi-gpu:latest on NVIDIA hosts.
# Models are baked in — no volume mount required for the default voice set.
kokoro:
image: ghcr.io/remsky/kokoro-fastapi-cpu:latest
container_name: libnovel-kokoro
restart: unless-stopped
ports:
- "8880:8880"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8880/health"]
interval: 15s
timeout: 5s
retries: 5
# ─── Scraper ─────────────────────────────────────────────────────────────────
scraper:
build:
context: ./scraper
dockerfile: Dockerfile
container_name: libnovel-scraper
#container_name: libnovel-scraper
restart: unless-stopped
depends_on:
kokoro:
pocketbase:
condition: service_healthy
minio:
condition: service_healthy
environment:
BROWSERLESS_URL: "http://browserless:3000"
@@ -66,10 +107,21 @@ services:
SCRAPER_HTTP_ADDR: ":8080"
LOG_LEVEL: "debug"
# Kokoro-FastAPI TTS endpoint.
KOKORO_URL: "${KOKORO_URL:-http://localhost:8880}"
KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc}"
KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}"
# MinIO / S3 object storage
MINIO_ENDPOINT: "minio:9000"
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
MINIO_USE_SSL: "false"
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
# PocketBase
POCKETBASE_URL: "http://pocketbase:8090"
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
ports:
- "8080:8080"
- "${SCRAPER_PORT:-8080}:8080"
volumes:
- static_books:/app/static/books
healthcheck:
@@ -78,5 +130,33 @@ services:
timeout: 5s
retries: 3
# ─── SvelteKit UI ────────────────────────────────────────────────────────────
ui:
build:
context: ./ui
dockerfile: Dockerfile
container_name: libnovel-ui
restart: unless-stopped
depends_on:
scraper:
condition: service_healthy
pocketbase:
condition: service_healthy
environment:
SCRAPER_API_URL: "http://scraper:8080"
POCKETBASE_URL: "http://pocketbase:8090"
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
PUBLIC_MINIO_PUBLIC_URL: "${PUBLIC_MINIO_PUBLIC_URL:-http://localhost:9000}"
ports:
- "${UI_PORT:-3000}:3000"
healthcheck:
test: ["CMD", "wget", "-qO-", "http://localhost:3000/"]
interval: 15s
timeout: 5s
retries: 3
volumes:
static_books:
minio_data:
pb_data:

View File

@@ -10,16 +10,24 @@
//
// Environment variables:
//
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
// KOKORO_VOICE Default TTS voice (default: af_bella)
// LOG_LEVEL debug | info | warn | error (default: info)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
// KOKORO_VOICE Default TTS voice (default: af_bella)
// POCKETBASE_URL PocketBase API base URL (default: http://localhost:8090)
// POCKETBASE_EMAIL PocketBase admin email (default: admin@libnovel.local)
// POCKETBASE_PASSWORD PocketBase admin password (default: adminpassword)
// MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000)
// MINIO_ACCESS_KEY MinIO access key (default: minioadmin)
// MINIO_SECRET_KEY MinIO secret key (default: minioadmin)
// MINIO_USE_SSL Use TLS for MinIO (default: false)
// MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters)
// MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio)
// LOG_LEVEL debug | info | warn | error (default: info)
package main
import (
@@ -38,7 +46,7 @@ import (
"github.com/libnovel/scraper/internal/novelfire"
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/server"
"github.com/libnovel/scraper/internal/writer"
"github.com/libnovel/scraper/internal/storage"
)
func main() {
@@ -88,9 +96,30 @@ func run(log *slog.Logger) error {
bc := newBrowserClient(strategy, browserCfg)
urlClient := newBrowserClient(urlStrategy, browserCfg)
staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books")
w := writer.New(staticRoot)
nf := novelfire.New(bc, log, urlClient, w)
// ── Storage backends ────────────────────────────────────────────────────
minioCfg := storage.MinioConfig{
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
AccessKey: envOr("MINIO_ACCESS_KEY", "minioadmin"),
SecretKey: envOr("MINIO_SECRET_KEY", "minioadmin"),
UseSSL: strings.ToLower(os.Getenv("MINIO_USE_SSL")) == "true",
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
}
pbCfg := storage.PocketBaseConfig{
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
AdminEmail: envOr("POCKETBASE_EMAIL", "admin@libnovel.local"),
AdminPassword: envOr("POCKETBASE_PASSWORD", "adminpassword"),
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
store, err := storage.NewHybridStore(ctx, pbCfg, minioCfg)
if err != nil {
return fmt.Errorf("storage init failed: %w", err)
}
nf := novelfire.New(bc, log, urlClient, &rankingCacheAdapter{store: store})
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -104,13 +133,9 @@ func run(log *slog.Logger) error {
}
oCfg := orchestrator.Config{
Workers: workers,
StaticRoot: staticRoot,
Workers: workers,
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
switch cmd {
case "run":
// Optional --url flag.
@@ -121,10 +146,9 @@ func run(log *slog.Logger) error {
"strategy", strategy,
"workers", workers,
"max_concurrent", browserCfg.MaxConcurrent,
"static_root", oCfg.StaticRoot,
"single_book", oCfg.SingleBookURL,
)
o := orchestrator.New(oCfg, nf, log)
o := orchestrator.New(oCfg, nf, log, store)
return o.Run(ctx)
case "refresh":
@@ -133,13 +157,12 @@ func run(log *slog.Logger) error {
return fmt.Errorf("refresh command requires a book slug argument")
}
slug := args[1]
w := writer.New(oCfg.StaticRoot)
meta, ok, err := w.ReadMetadata(slug)
meta, ok, err := store.ReadMetadata(ctx, slug)
if err != nil {
return fmt.Errorf("failed to read metadata for %s: %w", slug, err)
}
if !ok {
return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot)
return fmt.Errorf("book %q not found in store", slug)
}
if meta.SourceURL == "" {
return fmt.Errorf("book %q has no source_url in metadata", slug)
@@ -149,12 +172,12 @@ func run(log *slog.Logger) error {
"slug", slug,
"source_url", meta.SourceURL,
)
o := orchestrator.New(oCfg, nf, log)
o := orchestrator.New(oCfg, nf, log, store)
return o.Run(ctx)
case "serve":
addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
kokoroURL := envOr("KOKORO_URL", "")
kokoroURL := envOr("KOKORO_URL", "https://kokoro.kalekber.cc")
kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
log.Info("starting HTTP server",
"addr", addr,
@@ -164,7 +187,7 @@ func run(log *slog.Logger) error {
"kokoro_url", kokoroURL,
"kokoro_voice", kokoroVoice,
)
srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice)
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
return srv.ListenAndServe(ctx)
default:
@@ -192,6 +215,20 @@ func envOr(key, fallback string) string {
return fallback
}
// rankingCacheAdapter bridges storage.HybridStore (context-aware) to the
// context-free scraper.RankingPageCacher interface expected by novelfire.New.
type rankingCacheAdapter struct {
store *storage.HybridStore
}
func (a *rankingCacheAdapter) WriteRankingPageCache(page int, html string) error {
return a.store.WriteRankingPageCache(context.Background(), page, html)
}
func (a *rankingCacheAdapter) ReadRankingPageCache(page int) (string, error) {
return a.store.ReadRankingPageCache(context.Background(), page)
}
func printUsage() {
fmt.Fprintf(os.Stderr, `libnovel scraper

View File

@@ -3,8 +3,28 @@ module github.com/libnovel/scraper
go 1.25.0
require (
github.com/gorilla/websocket v1.5.3 // indirect
github.com/yuin/goldmark v1.7.16 // indirect
golang.org/x/net v0.51.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
github.com/gorilla/websocket v1.5.3
github.com/minio/minio-go/v7 v7.0.98
golang.org/x/net v0.51.0
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
)

View File

@@ -1,9 +1,47 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE=
github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -19,7 +19,7 @@ import (
"sync"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/writer"
"github.com/libnovel/scraper/internal/storage"
)
// Config holds tunable parameters for the orchestrator.
@@ -28,7 +28,8 @@ type Config struct {
// Defaults to runtime.NumCPU() when 0.
Workers int
// StaticRoot is the path to the static/books output directory.
// StaticRoot is kept for backwards-compatibility but is no longer used
// when a Store is provided.
StaticRoot string
// SingleBookURL when non-empty causes the orchestrator to scrape only
@@ -40,13 +41,13 @@ type Config struct {
type Orchestrator struct {
cfg Config
novel scraper.NovelScraper
writer *writer.Writer
store storage.Store
log *slog.Logger
workers int
}
// New returns a new Orchestrator.
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator {
// New returns a new Orchestrator backed by the provided Store.
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store) *Orchestrator {
workers := cfg.Workers
if workers <= 0 {
workers = runtime.NumCPU()
@@ -54,7 +55,7 @@ func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator
return &Orchestrator{
cfg: cfg,
novel: novel,
writer: writer.New(cfg.StaticRoot),
store: store,
log: log,
workers: workers,
}
@@ -66,7 +67,6 @@ func (o *Orchestrator) Run(ctx context.Context) error {
o.log.Info("orchestrator starting",
"source", o.novel.SourceName(),
"workers", o.workers,
"static_root", o.cfg.StaticRoot,
)
// chapterWork is the shared queue consumed by chapter worker goroutines.
@@ -89,8 +89,8 @@ func (o *Orchestrator) Run(ctx context.Context) error {
default:
}
// Skip if already on disk.
if o.writer.ChapterExists(job.slug, job.ref) {
// Skip if already stored.
if o.store.ChapterExists(ctx, job.slug, job.ref) {
o.log.Debug("chapter already exists, skipping",
"book", job.slug, "chapter", job.ref.Number)
continue
@@ -107,7 +107,7 @@ func (o *Orchestrator) Run(ctx context.Context) error {
continue
}
if err := o.writer.WriteChapter(job.slug, chapter); err != nil {
if err := o.store.WriteChapter(ctx, job.slug, chapter); err != nil {
o.log.Error("chapter write failed",
"book", job.slug,
"chapter", job.ref.Number,
@@ -135,8 +135,8 @@ func (o *Orchestrator) Run(ctx context.Context) error {
return
}
// Persist / update metadata.yaml.
if err := o.writer.WriteMetadata(meta); err != nil {
// Persist / update metadata.
if err := o.store.WriteMetadata(ctx, meta); err != nil {
o.log.Error("metadata write failed", "slug", meta.Slug, "err", err)
// Continue — chapters can still be scraped.
}

View File

@@ -0,0 +1,62 @@
package server
import (
"regexp"
"strings"
)
// kokoroVoices is the built-in fallback list of voices shipped with Kokoro-FastAPI.
// Used when the live GET /v1/audio/voices request to Kokoro fails.
// Grouped by language prefix:
//
// af_ / am_ American English female / male
// bf_ / bm_ British English female / male
// ef_ / em_ Spanish female / male
// ff_ French female
// hf_ / hm_ Hindi female / male
// if_ / im_ Italian female / male
// jf_ / jm_ Japanese female / male
// pf_ / pm_ Portuguese female / male
// zf_ / zm_ Chinese female / male
var kokoroVoices = []string{
// American English
"af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia",
"af_jessica", "af_kore", "af_nicole", "af_nova", "af_river",
"af_sarah", "af_sky",
"am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam",
"am_michael", "am_onyx", "am_puck",
// British English
"bf_alice", "bf_emma", "bf_lily",
"bm_daniel", "bm_fable", "bm_george", "bm_lewis",
// Spanish
"ef_dora", "em_alex",
// French
"ff_siwis",
// Hindi
"hf_alpha", "hf_beta", "hm_omega", "hm_psi",
// Italian
"if_sara", "im_nicola",
// Japanese
"jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo",
// Portuguese
"pf_dora", "pm_alex",
// Chinese
"zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi",
"zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang",
}
// stripMarkdown removes common markdown syntax from src, returning plain text
// suitable for TTS or display. Not a full markdown parser — handles the most
// common constructs (headings, bold/italic, code blocks, links, blockquotes).
func stripMarkdown(src string) string {
src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "")
src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "")
src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "")
src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "")
src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1")
src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "")
src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "")
src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "")
src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n")
return strings.TrimSpace(src)
}

View File

@@ -1,15 +1,25 @@
// Package server exposes the scraper as an HTTP service.
// Package server exposes the scraper as an HTTP API service.
//
// Endpoints:
//
// POST /scrape — enqueue a full catalogue scrape
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
// GET /health — liveness probe
// POST /scrape — enqueue a full catalogue scrape
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
// GET /health — liveness probe
// GET /api/progress — get reading progress map (session-scoped)
// POST /api/progress/{slug} — set reading progress
// DELETE /api/progress/{slug} — delete reading progress
// GET /api/presign/chapter/{slug}/{n} — presigned MinIO URL for chapter markdown
// GET /api/presign/audio/{slug}/{n} — presigned MinIO URL for chapter audio
// GET /api/chapter-text/{slug}/{n} — plain text of chapter (markdown stripped)
// POST /api/audio/{slug}/{n} — trigger Kokoro audio generation
// GET /api/audio-proxy/{slug}/{n} — proxy generated audio from Kokoro
package server
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"io"
@@ -22,7 +32,7 @@ import (
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/writer"
"github.com/libnovel/scraper/internal/storage"
)
// Server wraps an HTTP mux with the scraping endpoints.
@@ -31,7 +41,7 @@ type Server struct {
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
writer *writer.Writer
store storage.Store
mu sync.Mutex
running bool
rankingRunning bool
@@ -42,26 +52,23 @@ type Server struct {
voiceMu sync.RWMutex
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
// audioMu guards audioCache and audioInFlight.
// audioCache maps a cache key to the Kokoro download filename returned by
// POST /v1/audio/speech with return_download_link=true.
// audioMu guards audioInFlight only.
// Completed audio filenames are persisted to the Store (PocketBase).
// audioInFlight deduplicates concurrent generation requests for the same key.
audioMu sync.Mutex
audioCache map[string]string // cacheKey → kokoro download filename
audioInFlight map[string]chan struct{} // cacheKey → closed when done
}
// New creates a new Server.
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server {
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
return &Server{
addr: addr,
oCfg: oCfg,
novel: novel,
log: log,
writer: writer.New(oCfg.StaticRoot),
store: store,
kokoroURL: kokoroURL,
kokoroVoice: kokoroVoice,
audioCache: make(map[string]string),
audioInFlight: make(map[string]chan struct{}),
}
}
@@ -112,20 +119,15 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /health", s.handleHealth)
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
// UI routes
mux.HandleFunc("GET /", s.handleHome)
mux.HandleFunc("GET /scrape", s.handleScrape)
mux.HandleFunc("GET /ranking", s.handleRanking)
mux.HandleFunc("POST /ranking/refresh", s.handleRankingRefresh)
mux.HandleFunc("GET /ranking/view", s.handleRankingView)
mux.HandleFunc("GET /books/{slug}", s.handleBook)
mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter)
mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage)
mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus)
// Plain-text chapter content for browser-side TTS
mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)
// Progress API
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
// Presigned URL API (for SvelteKit UI)
mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter)
mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio)
// Plain-text chapter content (used server-side for audio generation)
mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText)
// Server-side audio generation via Kokoro /v1/audio/speech.
// Generation can take several minutes, so wrap in its own timeout handler.
audioGenHandler := http.TimeoutHandler(
@@ -133,9 +135,9 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
10*time.Minute,
`{"error":"audio generation timed out"}`,
)
mux.Handle("POST /ui/audio/{slug}/{n}", audioGenHandler)
mux.Handle("POST /api/audio/{slug}/{n}", audioGenHandler)
// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
mux.HandleFunc("GET /ui/audio-proxy/{slug}/{n}", s.handleAudioProxy)
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
srv := &http.Server{
Addr: s.addr,
@@ -165,8 +167,125 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}
// ─── Session cookie helpers ───────────────────────────────────────────────────
const sessionCookieName = "libnovel_session"
// sessionID returns the session ID from the request cookie, or "" if absent.
func sessionID(r *http.Request) string {
c, err := r.Cookie(sessionCookieName)
if err != nil {
return ""
}
return c.Value
}
// newSessionID generates a random 16-byte hex session ID.
func newSessionID() (string, error) {
b := make([]byte, 16)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
// ensureSession issues a new session cookie if the request does not already
// carry one, and returns the session ID (either existing or newly issued).
func ensureSession(w http.ResponseWriter, r *http.Request) string {
if id := sessionID(r); id != "" {
return id
}
id, err := newSessionID()
if err != nil {
// Very unlikely, but fall back to a timestamp-based ID.
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
}
http.SetCookie(w, &http.Cookie{
Name: sessionCookieName,
Value: id,
Path: "/",
HttpOnly: true,
SameSite: http.SameSiteLaxMode,
MaxAge: 365 * 24 * 60 * 60, // 1 year
})
return id
}
// ─── Reading progress API ─────────────────────────────────────────────────────
// handleGetProgress handles GET /api/progress.
// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...}
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
sid := ensureSession(w, r)
entries, err := s.store.AllProgress(r.Context(), sid)
if err != nil {
s.log.Error("AllProgress failed", "err", err)
entries = nil
}
progress := make(map[string]interface{}, len(entries)*2)
for _, p := range entries {
progress[p.Slug] = p.Chapter
progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(progress)
}
// handleSetProgress handles POST /api/progress/{slug}.
// Body: {"chapter": N}
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
sid := ensureSession(w, r)
slug := r.PathValue("slug")
if slug == "" {
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
return
}
var body struct {
Chapter int `json:"chapter"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest)
return
}
p := storage.ReadingProgress{
Slug: slug,
Chapter: body.Chapter,
UpdatedAt: time.Now(),
}
if err := s.store.SetProgress(r.Context(), sid, p); err != nil {
s.log.Error("SetProgress failed", "slug", slug, "err", err)
http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{})
}
// handleDeleteProgress handles DELETE /api/progress/{slug}.
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
sid := ensureSession(w, r)
slug := r.PathValue("slug")
if slug == "" {
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
return
}
if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil {
s.log.Error("DeleteProgress failed", "slug", slug, "err", err)
// Non-fatal — treat as success.
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{})
}
// handleChapterText returns the plain text of a chapter (markdown stripped)
// for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI.
// for server-side audio generation. Called by handleAudioGenerate internally.
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
@@ -174,7 +293,7 @@ func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r)
return
}
raw, err := s.writer.ReadChapter(slug, n)
raw, err := s.store.ReadChapter(r.Context(), slug, n)
if err != nil {
http.NotFound(w, r)
return
@@ -186,7 +305,7 @@ func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
// ─── Audio generation via Kokoro /v1/audio/speech ────────────────────────────
//
// handleAudioGenerate handles POST /ui/audio/{slug}/{n}.
// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
//
// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
// Kokoro generates the audio, saves it to its own temp storage, and returns
@@ -223,15 +342,14 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
// Fast path: already generated this session.
s.audioMu.Lock()
if filename, ok := s.audioCache[cacheKey]; ok {
s.audioMu.Unlock()
// Fast path: already generated (check persistent store first).
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
s.writeAudioResponse(w, slug, n, voice, speed, filename)
return
}
// Deduplicate concurrent generation for the same key.
s.audioMu.Lock()
if ch, ok := s.audioInFlight[cacheKey]; ok {
s.audioMu.Unlock()
select {
@@ -240,10 +358,8 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
return
}
s.audioMu.Lock()
filename, ok := s.audioCache[cacheKey]
s.audioMu.Unlock()
if ok {
// Check store again after waiting.
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
s.writeAudioResponse(w, slug, n, voice, speed, filename)
} else {
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
@@ -262,7 +378,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
}()
// Load and validate chapter text.
raw, err := s.writer.ReadChapter(slug, n)
raw, err := s.store.ReadChapter(r.Context(), slug, n)
if err != nil {
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
return
@@ -287,9 +403,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
return
}
s.audioMu.Lock()
s.audioCache[cacheKey] = filename
s.audioMu.Unlock()
_ = s.store.SetAudioCache(r.Context(), cacheKey, filename)
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
s.writeAudioResponse(w, slug, n, voice, speed, filename)
@@ -345,9 +459,9 @@ func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed f
}
// writeAudioResponse writes the JSON response for a generated audio chapter.
// The URL points to our proxy handler which fetches from Kokoro on demand.
// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) {
proxyURL := fmt.Sprintf("/ui/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed)
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"url": proxyURL,
@@ -355,7 +469,7 @@ func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, v
})
}
// handleAudioProxy handles GET /ui/audio-proxy/{slug}/{n}.
// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}.
// It looks up the Kokoro download filename for this chapter (voice/speed) and
// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
@@ -378,10 +492,7 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
}
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
s.audioMu.Lock()
filename, ok := s.audioCache[cacheKey]
s.audioMu.Unlock()
filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
if !ok {
http.Error(w, "audio not generated yet", http.StatusNotFound)
return
@@ -414,6 +525,64 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(w, resp.Body)
}
// ─── Presigned URL handlers ───────────────────────────────────────────────────
// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}.
// Returns a short-lived presigned MinIO URL for the chapter markdown object.
// The SvelteKit server uses this to fetch chapter content server-side.
func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
return
}
url, err := s.store.PresignChapter(r.Context(), slug, n, 15*time.Minute)
if err != nil {
s.log.Error("presign chapter failed", "slug", slug, "n", n, "err", err)
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
}
// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}.
// Returns a presigned MinIO URL for the audio object (if it has been generated).
// Query params: voice, speed (optional, defaults to server defaults).
func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
return
}
voice := r.URL.Query().Get("voice")
if voice == "" {
voice = s.kokoroVoice
}
speed := 1.0
if sv := r.URL.Query().Get("speed"); sv != "" {
if v, err := strconv.ParseFloat(sv, 64); err == nil && v > 0 {
speed = v
}
}
key := s.store.AudioObjectKey(slug, n, voice, speed)
url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
if err != nil {
s.log.Error("presign audio failed", "slug", slug, "n", n, "err", err)
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
}
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
cfg := s.oCfg
cfg.SingleBookURL = "" // full catalogue
@@ -462,7 +631,7 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
defer cancel()
o := orchestrator.New(cfg, s.novel, s.log)
o := orchestrator.New(cfg, s.novel, s.log, s.store)
if err := o.Run(ctx); err != nil {
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err))
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,310 @@
// hybrid.go implements the Store interface using PocketBase for structured data
// and MinIO for binary chapter/audio blobs.
package storage
import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"strconv"
"strings"
"time"
"github.com/libnovel/scraper/internal/scraper"
)
// HybridStore satisfies Store by routing structured data to PocketBase and
// binary objects (chapters, audio) to MinIO.
type HybridStore struct {
pb *PocketBaseStore
minio *MinioClient
}
// NewHybridStore constructs a HybridStore. It connects to both backends and
// calls EnsureCollections to bootstrap any missing PocketBase collections.
func NewHybridStore(ctx context.Context, pbCfg PocketBaseConfig, minioCfg MinioConfig) (*HybridStore, error) {
mc, err := NewMinioClient(ctx, minioCfg)
if err != nil {
return nil, fmt.Errorf("storage: minio: %w", err)
}
pb := NewPocketBaseStore(pbCfg)
if err := pb.EnsureCollections(ctx); err != nil {
// Log but don't fail — collection creation errors are often "already exists"
_ = err
}
return &HybridStore{pb: pb, minio: mc}, nil
}
// ─── Book metadata ────────────────────────────────────────────────────────────
func (h *HybridStore) WriteMetadata(ctx context.Context, meta scraper.BookMeta) error {
return h.pb.UpsertBook(ctx,
meta.Slug, meta.Title, meta.Author, meta.Cover,
meta.Status, meta.Summary, meta.SourceURL,
meta.Genres, meta.TotalChapters, meta.Ranking,
)
}
func (h *HybridStore) ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) {
rec, found, err := h.pb.GetBook(ctx, slug)
if err != nil || !found {
return scraper.BookMeta{}, found, err
}
return recToBookMeta(rec), true, nil
}
func (h *HybridStore) ListBooks(ctx context.Context) ([]scraper.BookMeta, error) {
rows, err := h.pb.ListBooks(ctx)
if err != nil {
return nil, err
}
books := make([]scraper.BookMeta, 0, len(rows))
for _, r := range rows {
books = append(books, recToBookMeta(r))
}
return books, nil
}
func (h *HybridStore) LocalSlugs(ctx context.Context) (map[string]bool, error) {
books, err := h.ListBooks(ctx)
if err != nil {
return nil, err
}
slugs := make(map[string]bool, len(books))
for _, b := range books {
slugs[b.Slug] = true
}
return slugs, nil
}
func (h *HybridStore) MetadataMtime(ctx context.Context, slug string) int64 {
t, err := h.pb.BookMetaUpdated(ctx, slug)
if err != nil || t.IsZero() {
return 0
}
return t.Unix()
}
// ─── Chapters ─────────────────────────────────────────────────────────────────
func (h *HybridStore) ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool {
return h.minio.ChapterExists(ctx, slug, ref.Volume, ref.Number)
}
func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error {
content := "# " + chapter.Ref.Title + "\n\n" + chapter.Text + "\n"
if err := h.minio.PutChapter(ctx, slug, chapter.Ref.Volume, chapter.Ref.Number, content); err != nil {
return err
}
// Update chapter index in PocketBase.
title, dateLabel := splitChapterTitle(chapter.Ref.Title)
_ = h.pb.UpsertChapterIdx(ctx, slug, chapter.Ref.Number, title, dateLabel)
return nil
}
func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
return h.minio.GetChapter(ctx, slug, 0, n)
}
func (h *HybridStore) ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) {
rows, err := h.pb.ListChapterIdx(ctx, slug)
if err != nil {
return nil, err
}
infos := make([]ChapterInfo, 0, len(rows))
for _, r := range rows {
n := int(floatVal(r, "number"))
title, _ := r["title"].(string)
date, _ := r["date_label"].(string)
infos = append(infos, ChapterInfo{Number: n, Title: title, Date: date})
}
sort.Slice(infos, func(i, j int) bool { return infos[i].Number < infos[j].Number })
return infos, nil
}
func (h *HybridStore) CountChapters(ctx context.Context, slug string) int {
return h.pb.CountChapterIdx(ctx, slug)
}
// ─── Ranking ─────────────────────────────────────────────────────────────────
func (h *HybridStore) WriteRanking(ctx context.Context, items []RankingItem) error {
data, err := json.Marshal(items)
if err != nil {
return fmt.Errorf("storage: marshal ranking: %w", err)
}
return h.pb.SetRanking(ctx, string(data))
}
func (h *HybridStore) ReadRankingItems(ctx context.Context) ([]RankingItem, error) {
dataStr, _, err := h.pb.GetRanking(ctx)
if err != nil || dataStr == "" {
return nil, err
}
var items []RankingItem
if err := json.Unmarshal([]byte(dataStr), &items); err != nil {
return nil, fmt.Errorf("storage: unmarshal ranking: %w", err)
}
return items, nil
}
func (h *HybridStore) RankingFileInfo(ctx context.Context) (os.FileInfo, error) {
return h.pb.RankingModTime(ctx)
}
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
func (h *HybridStore) WriteRankingPageCache(ctx context.Context, page int, html string) error {
return h.pb.SetRankingPageHTML(ctx, page, html)
}
func (h *HybridStore) ReadRankingPageCache(ctx context.Context, page int) (string, error) {
html, _, err := h.pb.GetRankingPageHTML(ctx, page)
return html, err
}
func (h *HybridStore) RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error) {
return h.pb.RankingPageCacheModTime(ctx, page)
}
// ─── Audio cache ──────────────────────────────────────────────────────────────
func (h *HybridStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool) {
filename, ok, _ := h.pb.GetAudioCache(ctx, cacheKey)
return filename, ok
}
func (h *HybridStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
return h.pb.SetAudioCache(ctx, cacheKey, filename)
}
// ─── Reading progress ─────────────────────────────────────────────────────────
func (h *HybridStore) GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) {
ch, updated, ok, err := h.pb.GetProgress(ctx, sessionID, slug)
if err != nil || !ok {
return ReadingProgress{}, false
}
return ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}, true
}
func (h *HybridStore) SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error {
return h.pb.SetProgress(ctx, sessionID, p.Slug, p.Chapter)
}
func (h *HybridStore) AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) {
rows, err := h.pb.AllProgress(ctx, sessionID)
if err != nil {
return nil, err
}
out := make([]ReadingProgress, 0, len(rows))
for _, r := range rows {
slug, _ := r["slug"].(string)
ch := int(floatVal(r, "chapter"))
var updated time.Time
if ts, ok := r["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
}
out = append(out, ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated})
}
return out, nil
}
func (h *HybridStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
return h.pb.DeleteProgress(ctx, sessionID, slug)
}
// ─── AudioObjectKey ───────────────────────────────────────────────────────────
func (h *HybridStore) AudioObjectKey(slug string, n int, voice string, speed float64) string {
return AudioObjectKey(slug, n, voice, speed)
}
// ─── Presigned URLs ───────────────────────────────────────────────────────────
func (h *HybridStore) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) {
return h.minio.PresignChapter(ctx, slug, 0, n, expires)
}
func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
return h.minio.PresignAudio(ctx, key, expires)
}
// ─── helpers ──────────────────────────────────────────────────────────────────
func recToBookMeta(rec map[string]interface{}) scraper.BookMeta {
m := scraper.BookMeta{
Slug: strVal(rec, "slug"),
Title: strVal(rec, "title"),
Author: strVal(rec, "author"),
Cover: strVal(rec, "cover"),
Status: strVal(rec, "status"),
Summary: strVal(rec, "summary"),
SourceURL: strVal(rec, "source_url"),
}
if tc := floatVal(rec, "total_chapters"); tc > 0 {
m.TotalChapters = int(tc)
}
if rk := floatVal(rec, "ranking"); rk > 0 {
m.Ranking = int(rk)
}
// Genres stored as JSON string or array.
switch v := rec["genres"].(type) {
case string:
_ = json.Unmarshal([]byte(v), &m.Genres)
case []interface{}:
for _, g := range v {
if s, ok := g.(string); ok {
m.Genres = append(m.Genres, s)
}
}
}
return m
}
func strVal(m map[string]interface{}, key string) string {
if v, ok := m[key].(string); ok {
return v
}
return ""
}
// splitChapterTitle mirrors writer.SplitChapterTitle logic (simplified).
func splitChapterTitle(raw string) (title, date string) {
raw = strings.TrimSpace(raw)
// Strip leading numeric index.
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
prefix := raw[:idx]
allDigit := true
for _, c := range prefix {
if c < '0' || c > '9' {
allDigit = false
break
}
}
if allDigit {
raw = strings.TrimSpace(raw[idx:])
}
}
// Detect trailing relative date.
units := []string{"second", "minute", "hour", "day", "week", "month", "year"}
lower := strings.ToLower(raw)
for _, u := range units {
for _, suffix := range []string{u + "s ago", u + " ago"} {
if idx := strings.LastIndex(lower, suffix); idx > 0 {
// Find start of date token (digit before the unit).
start := strings.LastIndex(raw[:idx], " ")
if start < 0 {
start = 0
}
numPart := strings.TrimSpace(raw[start:idx])
if _, err := strconv.Atoi(strings.Fields(numPart)[0]); err == nil {
return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)])
}
}
}
}
return raw, ""
}

View File

@@ -0,0 +1,203 @@
package storage
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
// MinioConfig holds connection parameters for MinIO.
type MinioConfig struct {
Endpoint string // e.g. "minio:9000"
AccessKey string
SecretKey string
UseSSL bool
BucketChapters string // e.g. "libnovel-chapters"
BucketAudio string // e.g. "libnovel-audio"
}
// MinioClient wraps a minio.Client and exposes object operations for
// chapters and audio files.
type MinioClient struct {
c *minio.Client
cfg MinioConfig
}
// NewMinioClient creates a connected MinIO client and ensures the required
// buckets exist.
func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error) {
c, err := minio.New(cfg.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: cfg.UseSSL,
})
if err != nil {
return nil, fmt.Errorf("minio: new client: %w", err)
}
mc := &MinioClient{c: c, cfg: cfg}
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio} {
if err := mc.ensureBucket(ctx, bucket); err != nil {
return nil, err
}
}
return mc, nil
}
// ensureBucket creates a bucket if it does not exist.
func (m *MinioClient) ensureBucket(ctx context.Context, bucket string) error {
exists, err := m.c.BucketExists(ctx, bucket)
if err != nil {
return fmt.Errorf("minio: bucket exists %q: %w", bucket, err)
}
if !exists {
if err := m.c.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
return fmt.Errorf("minio: make bucket %q: %w", bucket, err)
}
}
return nil
}
// ─── Chapter objects ──────────────────────────────────────────────────────────
// chapterKey returns the MinIO object key for a chapter.
// Layout: {slug}/vol-{vol}/{lo}-{hi}/chapter-{n}.md
func chapterKey(slug string, vol, n int) string {
const chaptersPerFolder = 50
lo := ((n-1)/chaptersPerFolder)*chaptersPerFolder + 1
hi := lo + chaptersPerFolder - 1
return fmt.Sprintf("%s/vol-%d/%d-%d/chapter-%d.md", slug, vol, lo, hi, n)
}
// PutChapter stores chapter markdown in MinIO.
func (m *MinioClient) PutChapter(ctx context.Context, slug string, vol, n int, content string) error {
key := chapterKey(slug, vol, n)
data := []byte(content)
_, err := m.c.PutObject(ctx, m.cfg.BucketChapters, key,
bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: "text/markdown; charset=utf-8"})
if err != nil {
return fmt.Errorf("minio: put chapter %s: %w", key, err)
}
return nil
}
// GetChapter retrieves chapter markdown from MinIO.
func (m *MinioClient) GetChapter(ctx context.Context, slug string, vol, n int) (string, error) {
key := chapterKey(slug, vol, n)
obj, err := m.c.GetObject(ctx, m.cfg.BucketChapters, key, minio.GetObjectOptions{})
if err != nil {
return "", fmt.Errorf("minio: get chapter %s: %w", key, err)
}
defer obj.Close()
data, err := io.ReadAll(obj)
if err != nil {
return "", fmt.Errorf("minio: read chapter %s: %w", key, err)
}
return string(data), nil
}
// ChapterExists returns true if the object for this chapter is present.
func (m *MinioClient) ChapterExists(ctx context.Context, slug string, vol, n int) bool {
key := chapterKey(slug, vol, n)
_, err := m.c.StatObject(ctx, m.cfg.BucketChapters, key, minio.StatObjectOptions{})
return err == nil
}
// ListChapterKeys returns all object keys under slug/ in the chapters bucket,
// sorted lexicographically (MinIO returns them in order).
func (m *MinioClient) ListChapterKeys(ctx context.Context, slug string) ([]string, error) {
prefix := slug + "/"
var keys []string
for obj := range m.c.ListObjects(ctx, m.cfg.BucketChapters,
minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
if obj.Err != nil {
return nil, fmt.Errorf("minio: list chapters %s: %w", slug, obj.Err)
}
keys = append(keys, obj.Key)
}
return keys, nil
}
// CountChapters returns the number of chapter objects for a slug.
func (m *MinioClient) CountChapters(ctx context.Context, slug string) int {
keys, _ := m.ListChapterKeys(ctx, slug)
return len(keys)
}
// ─── Audio objects ────────────────────────────────────────────────────────────
// AudioObjectKey returns the MinIO key for a cached audio file.
// Key: {slug}/ch{n}-{voice}-{speed:.1f}.mp3
func AudioObjectKey(slug string, n int, voice string, speed float64) string {
safe := sanitiseVoice(voice)
return fmt.Sprintf("%s/ch%d-%s-%.1f.mp3", slug, n, safe, speed)
}
// PutAudio stores an audio file in the audio bucket.
func (m *MinioClient) PutAudio(ctx context.Context, key string, data []byte) error {
_, err := m.c.PutObject(ctx, m.cfg.BucketAudio, key,
bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: "audio/mpeg"})
if err != nil {
return fmt.Errorf("minio: put audio %s: %w", key, err)
}
return nil
}
// GetAudio retrieves audio bytes from the audio bucket.
func (m *MinioClient) GetAudio(ctx context.Context, key string) ([]byte, error) {
obj, err := m.c.GetObject(ctx, m.cfg.BucketAudio, key, minio.GetObjectOptions{})
if err != nil {
return nil, fmt.Errorf("minio: get audio %s: %w", key, err)
}
defer obj.Close()
return io.ReadAll(obj)
}
// AudioExists returns true if the audio object is present in the bucket.
func (m *MinioClient) AudioExists(ctx context.Context, key string) bool {
_, err := m.c.StatObject(ctx, m.cfg.BucketAudio, key, minio.StatObjectOptions{})
return err == nil
}
// ─── Presigned URLs ───────────────────────────────────────────────────────────
// PresignChapter returns a presigned GET URL for a chapter object, valid for
// the given duration. The URL is signed with the MinIO credentials and can be
// fetched directly by the browser without authentication.
func (m *MinioClient) PresignChapter(ctx context.Context, slug string, vol, n int, expires time.Duration) (string, error) {
key := chapterKey(slug, vol, n)
u, err := m.c.PresignedGetObject(ctx, m.cfg.BucketChapters, key, expires, nil)
if err != nil {
return "", fmt.Errorf("minio: presign chapter %s: %w", key, err)
}
return u.String(), nil
}
// PresignAudio returns a presigned GET URL for an audio object.
func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
u, err := m.c.PresignedGetObject(ctx, m.cfg.BucketAudio, key, expires, nil)
if err != nil {
return "", fmt.Errorf("minio: presign audio %s: %w", key, err)
}
return u.String(), nil
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// sanitiseVoice converts a voice name to a filename-safe string.
func sanitiseVoice(voice string) string {
return strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
return r
}
return '_'
}, voice)
}

View File

@@ -0,0 +1,560 @@
// Package storage — PocketBase REST client.
//
// Collections expected in PocketBase:
//
// books — slug(text,unique), title, author, cover, status, genres(json),
// summary, total_chapters(number), source_url, ranking(number), updated(date)
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
// ranking — data(json), updated(date) [single row, upserted by slug="_ranking_"]
// ranking_html — page(number,unique), html(text), updated(date)
// progress — session_id(text), slug(text), chapter(number), updated(date)
// audio_cache — cache_key(text,unique), filename(text), updated(date)
package storage
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
)
// PocketBaseConfig holds PocketBase connection settings.
type PocketBaseConfig struct {
BaseURL string // e.g. "http://pocketbase:8090"
AdminEmail string
AdminPassword string
}
// pbClient is a minimal PocketBase admin REST client.
type pbClient struct {
cfg PocketBaseConfig
httpClient *http.Client
tokenMu sync.RWMutex
token string
tokenExp time.Time
}
// newPBClient creates a new PocketBase client. It does not authenticate yet;
// authentication happens lazily on the first API call.
func newPBClient(cfg PocketBaseConfig) *pbClient {
return &pbClient{
cfg: cfg,
httpClient: &http.Client{Timeout: 15 * time.Second},
}
}
// ─── Auth ─────────────────────────────────────────────────────────────────────
func (p *pbClient) authenticate(ctx context.Context) error {
body, _ := json.Marshal(map[string]string{
"identity": p.cfg.AdminEmail,
"password": p.cfg.AdminPassword,
})
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
p.cfg.BaseURL+"/api/admins/auth-with-password", bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := p.httpClient.Do(req)
if err != nil {
return fmt.Errorf("pocketbase: auth: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pocketbase: auth status %d: %s", resp.StatusCode, b)
}
var result struct {
Token string `json:"token"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return fmt.Errorf("pocketbase: decode auth: %w", err)
}
p.tokenMu.Lock()
p.token = result.Token
p.tokenExp = time.Now().Add(12 * time.Hour)
p.tokenMu.Unlock()
return nil
}
func (p *pbClient) authToken(ctx context.Context) (string, error) {
p.tokenMu.RLock()
tok, exp := p.token, p.tokenExp
p.tokenMu.RUnlock()
if tok != "" && time.Now().Before(exp) {
return tok, nil
}
if err := p.authenticate(ctx); err != nil {
return "", err
}
p.tokenMu.RLock()
defer p.tokenMu.RUnlock()
return p.token, nil
}
// ─── Generic CRUD helpers ──────────────────────────────────────────────────────
func (p *pbClient) do(ctx context.Context, method, path string, body interface{}) (*http.Response, error) {
tok, err := p.authToken(ctx)
if err != nil {
return nil, err
}
var bodyReader io.Reader
if body != nil {
b, _ := json.Marshal(body)
bodyReader = bytes.NewReader(b)
}
req, err := http.NewRequestWithContext(ctx, method, p.cfg.BaseURL+path, bodyReader)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", tok)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
return p.httpClient.Do(req)
}
// listOne fetches the first matching record from a collection.
func (p *pbClient) listOne(ctx context.Context, collection, filter string) (map[string]interface{}, error) {
q := url.Values{}
q.Set("filter", filter)
q.Set("perPage", "1")
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
resp, err := p.do(ctx, http.MethodGet, path, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, nil
}
var result struct {
Items []map[string]interface{} `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
if len(result.Items) == 0 {
return nil, nil
}
return result.Items[0], nil
}
// listAll returns all records (up to 500) from a collection matching filter.
func (p *pbClient) listAll(ctx context.Context, collection, filter, sort string) ([]map[string]interface{}, error) {
q := url.Values{}
if filter != "" {
q.Set("filter", filter)
}
if sort != "" {
q.Set("sort", sort)
}
q.Set("perPage", "500")
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
resp, err := p.do(ctx, http.MethodGet, path, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var result struct {
Items []map[string]interface{} `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
return result.Items, nil
}
// upsert creates a record; if one matching filter already exists it updates it.
func (p *pbClient) upsert(ctx context.Context, collection, filter string, data map[string]interface{}) error {
existing, err := p.listOne(ctx, collection, filter)
if err != nil {
return err
}
if existing != nil {
id := existing["id"].(string)
resp, err := p.do(ctx, http.MethodPatch,
fmt.Sprintf("/api/collections/%s/records/%s", collection, id), data)
if err != nil {
return err
}
resp.Body.Close()
return nil
}
resp, err := p.do(ctx, http.MethodPost,
fmt.Sprintf("/api/collections/%s/records", collection), data)
if err != nil {
return err
}
resp.Body.Close()
return nil
}
// deleteWhere deletes all records matching filter in collection.
func (p *pbClient) deleteWhere(ctx context.Context, collection, filter string) error {
items, err := p.listAll(ctx, collection, filter, "")
if err != nil {
return err
}
for _, item := range items {
id, _ := item["id"].(string)
resp, err := p.do(ctx, http.MethodDelete,
fmt.Sprintf("/api/collections/%s/records/%s", collection, id), nil)
if err != nil {
return err
}
resp.Body.Close()
}
return nil
}
// ─── PocketBaseStore ──────────────────────────────────────────────────────────
// PocketBaseStore implements the structured-data portion of the Store interface
// backed by PocketBase REST API.
type PocketBaseStore struct {
pb *pbClient
}
// NewPocketBaseStore returns a connected PocketBaseStore.
func NewPocketBaseStore(cfg PocketBaseConfig) *PocketBaseStore {
return &PocketBaseStore{pb: newPBClient(cfg)}
}
// Ping verifies connectivity by authenticating.
func (s *PocketBaseStore) Ping(ctx context.Context) error {
_, err := s.pb.authToken(ctx)
return err
}
// ─── Collections schema bootstrap ────────────────────────────────────────────
// CollectionDef maps a collection name to its fields for auto-creation.
// EnsureCollections creates missing collections via the PocketBase API.
// Safe to call on every startup — existing collections are skipped.
func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
// We just attempt to create each collection; 400/422 errors for "already
// exists" are silently ignored.
collections := []map[string]interface{}{
{
"name": "books",
"type": "base",
"schema": []map[string]interface{}{
{"name": "slug", "type": "text", "required": true, "options": map[string]interface{}{"min": 1}},
{"name": "title", "type": "text", "required": true},
{"name": "author", "type": "text"},
{"name": "cover", "type": "url"},
{"name": "status", "type": "text"},
{"name": "genres", "type": "json"},
{"name": "summary", "type": "text"},
{"name": "total_chapters", "type": "number"},
{"name": "source_url", "type": "url"},
{"name": "ranking", "type": "number"},
{"name": "meta_updated", "type": "date"},
},
},
{
"name": "chapters_idx",
"type": "base",
"schema": []map[string]interface{}{
{"name": "slug", "type": "text", "required": true},
{"name": "number", "type": "number", "required": true},
{"name": "title", "type": "text"},
{"name": "date_label", "type": "text"},
},
},
{
"name": "ranking",
"type": "base",
"schema": []map[string]interface{}{
{"name": "key", "type": "text", "required": true},
{"name": "data", "type": "json"},
{"name": "updated", "type": "date"},
},
},
{
"name": "ranking_html",
"type": "base",
"schema": []map[string]interface{}{
{"name": "page", "type": "number", "required": true},
{"name": "html", "type": "text"},
{"name": "updated", "type": "date"},
},
},
{
"name": "progress",
"type": "base",
"schema": []map[string]interface{}{
{"name": "session_id", "type": "text", "required": true},
{"name": "slug", "type": "text", "required": true},
{"name": "chapter", "type": "number"},
{"name": "updated", "type": "date"},
},
},
{
"name": "audio_cache",
"type": "base",
"schema": []map[string]interface{}{
{"name": "cache_key", "type": "text", "required": true},
{"name": "filename", "type": "text"},
{"name": "updated", "type": "date"},
},
},
}
for _, col := range collections {
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections", col)
if err != nil {
return fmt.Errorf("pocketbase: ensure collection %v: %w", col["name"], err)
}
resp.Body.Close()
// 400/422 = already exists or schema mismatch — ignore
}
return nil
}
// ─── Book metadata ────────────────────────────────────────────────────────────
func (s *PocketBaseStore) UpsertBook(ctx context.Context, slug, title, author, cover, status, summary, sourceURL string, genres []string, totalChapters, ranking int) error {
genresJSON, _ := json.Marshal(genres)
return s.pb.upsert(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)), map[string]interface{}{
"slug": slug,
"title": title,
"author": author,
"cover": cover,
"status": status,
"genres": string(genresJSON),
"summary": summary,
"total_chapters": totalChapters,
"source_url": sourceURL,
"ranking": ranking,
"meta_updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetBook(ctx context.Context, slug string) (map[string]interface{}, bool, error) {
rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)))
if err != nil {
return nil, false, err
}
if rec == nil {
return nil, false, nil
}
return rec, true, nil
}
func (s *PocketBaseStore) ListBooks(ctx context.Context) ([]map[string]interface{}, error) {
return s.pb.listAll(ctx, "books", "", "+title")
}
func (s *PocketBaseStore) BookMetaUpdated(ctx context.Context, slug string) (time.Time, error) {
rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)))
if err != nil || rec == nil {
return time.Time{}, err
}
if ts, ok := rec["meta_updated"].(string); ok {
t, err := time.Parse(time.RFC3339, ts)
if err == nil {
return t, nil
}
}
return time.Time{}, nil
}
// ─── Chapter index ────────────────────────────────────────────────────────────
func (s *PocketBaseStore) UpsertChapterIdx(ctx context.Context, slug string, number int, title, dateLabel string) error {
return s.pb.upsert(ctx, "chapters_idx",
fmt.Sprintf(`slug="%s"&&number=%d`, pbEsc(slug), number),
map[string]interface{}{
"slug": slug,
"number": number,
"title": title,
"date_label": dateLabel,
})
}
func (s *PocketBaseStore) ListChapterIdx(ctx context.Context, slug string) ([]map[string]interface{}, error) {
return s.pb.listAll(ctx, "chapters_idx",
fmt.Sprintf(`slug="%s"`, pbEsc(slug)), "+number")
}
func (s *PocketBaseStore) CountChapterIdx(ctx context.Context, slug string) int {
rows, _ := s.ListChapterIdx(ctx, slug)
return len(rows)
}
// ─── Ranking ──────────────────────────────────────────────────────────────────
func (s *PocketBaseStore) SetRanking(ctx context.Context, dataJSON string) error {
return s.pb.upsert(ctx, "ranking", `key="_ranking_"`, map[string]interface{}{
"key": "_ranking_",
"data": dataJSON,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetRanking(ctx context.Context) (string, time.Time, error) {
rec, err := s.pb.listOne(ctx, "ranking", `key="_ranking_"`)
if err != nil || rec == nil {
return "", time.Time{}, err
}
data, _ := rec["data"].(string)
var updated time.Time
if ts, ok := rec["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
}
return data, updated, nil
}
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
func (s *PocketBaseStore) SetRankingPageHTML(ctx context.Context, page int, html string) error {
return s.pb.upsert(ctx, "ranking_html",
fmt.Sprintf(`page=%d`, page),
map[string]interface{}{
"page": page,
"html": html,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetRankingPageHTML(ctx context.Context, page int) (string, time.Time, error) {
rec, err := s.pb.listOne(ctx, "ranking_html", fmt.Sprintf(`page=%d`, page))
if err != nil || rec == nil {
return "", time.Time{}, err
}
html, _ := rec["html"].(string)
var updated time.Time
if ts, ok := rec["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
}
return html, updated, nil
}
// ─── Reading progress ─────────────────────────────────────────────────────────
func (s *PocketBaseStore) SetProgress(ctx context.Context, sessionID, slug string, chapter int) error {
return s.pb.upsert(ctx, "progress",
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)),
map[string]interface{}{
"session_id": sessionID,
"slug": slug,
"chapter": chapter,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetProgress(ctx context.Context, sessionID, slug string) (int, time.Time, bool, error) {
rec, err := s.pb.listOne(ctx, "progress",
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)))
if err != nil {
return 0, time.Time{}, false, err
}
if rec == nil {
return 0, time.Time{}, false, nil
}
ch := int(floatVal(rec, "chapter"))
var updated time.Time
if ts, ok := rec["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
}
return ch, updated, true, nil
}
func (s *PocketBaseStore) AllProgress(ctx context.Context, sessionID string) ([]map[string]interface{}, error) {
return s.pb.listAll(ctx, "progress",
fmt.Sprintf(`session_id="%s"`, pbEsc(sessionID)), "-updated")
}
func (s *PocketBaseStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
return s.pb.deleteWhere(ctx, "progress",
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)))
}
// ─── Audio cache ──────────────────────────────────────────────────────────────
func (s *PocketBaseStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
return s.pb.upsert(ctx, "audio_cache",
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)),
map[string]interface{}{
"cache_key": cacheKey,
"filename": filename,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool, error) {
rec, err := s.pb.listOne(ctx, "audio_cache",
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)))
if err != nil || rec == nil {
return "", false, err
}
filename, _ := rec["filename"].(string)
return filename, filename != "", nil
}
// ─── rankingFileInfo is a minimal os.FileInfo implementation ─────────────────
type rankingFileInfo struct {
modTime time.Time
}
func (r rankingFileInfo) Name() string { return "ranking" }
func (r rankingFileInfo) Size() int64 { return 0 }
func (r rankingFileInfo) Mode() os.FileMode { return 0o444 }
func (r rankingFileInfo) ModTime() time.Time { return r.modTime }
func (r rankingFileInfo) IsDir() bool { return false }
func (r rankingFileInfo) Sys() interface{} { return nil }
var _ os.FileInfo = rankingFileInfo{}
// RankingModTime returns file-info-compatible data for the ranking record.
func (s *PocketBaseStore) RankingModTime(ctx context.Context) (os.FileInfo, error) {
_, updated, err := s.GetRanking(ctx)
if err != nil {
return nil, err
}
if updated.IsZero() {
return nil, nil
}
return rankingFileInfo{modTime: updated}, nil
}
// RankingPageCacheModTime returns file-info for a cached ranking page.
func (s *PocketBaseStore) RankingPageCacheModTime(ctx context.Context, page int) (os.FileInfo, error) {
_, updated, err := s.GetRankingPageHTML(ctx, page)
if err != nil {
return nil, err
}
if updated.IsZero() {
return nil, nil
}
return rankingFileInfo{modTime: updated}, nil
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// pbEsc escapes a string for use in a PocketBase filter expression.
// Only escapes double-quotes to prevent injection.
func pbEsc(s string) string {
return strings.ReplaceAll(s, `"`, `\"`)
}
func floatVal(m map[string]interface{}, key string) float64 {
if v, ok := m[key].(float64); ok {
return v
}
return 0
}

View File

@@ -0,0 +1,132 @@
// Package storage defines the unified Store interface and helper types used by
// the server and orchestrator. Concrete implementations back the interface
// with PocketBase (structured data) and MinIO (binary objects).
package storage
import (
"context"
"os"
"time"
"github.com/libnovel/scraper/internal/scraper"
)
// ─── Shared types ─────────────────────────────────────────────────────────────
// ChapterInfo is a lightweight chapter descriptor (mirrors writer.ChapterInfo).
type ChapterInfo struct {
Number int
Title string
Date string
}
// RankingItem represents a single entry in the novel ranking list.
type RankingItem struct {
Rank int `json:"rank"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author,omitempty"`
Cover string `json:"cover,omitempty"`
Status string `json:"status,omitempty"`
Genres []string `json:"genres,omitempty"`
SourceURL string `json:"source_url,omitempty"`
}
// ReadingProgress holds a single user's reading position for one book.
type ReadingProgress struct {
Slug string `json:"slug"`
Chapter int `json:"chapter"`
UpdatedAt time.Time `json:"updated_at"`
}
// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro
// download filename so audio is not re-generated after a server restart.
type AudioCacheEntry struct {
CacheKey string `json:"cache_key"`
Filename string `json:"filename"`
}
// ─── Store interface ──────────────────────────────────────────────────────────
// Store is the single persistence abstraction consumed by the server and the
// orchestrator. Implementations may route calls to different backends
// (PocketBase for structured records, MinIO for binary blobs).
type Store interface {
// ── Book metadata ──────────────────────────────────────────────────────
// WriteMetadata upserts book metadata.
WriteMetadata(ctx context.Context, meta scraper.BookMeta) error
// ReadMetadata returns the metadata for slug. Returns (zero, false, nil)
// when the book is not found.
ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error)
// ListBooks returns all books, sorted alphabetically by title.
ListBooks(ctx context.Context) ([]scraper.BookMeta, error)
// LocalSlugs returns the set of slugs that have metadata stored.
LocalSlugs(ctx context.Context) (map[string]bool, error)
// MetadataMtime returns the Unix-second mtime of the metadata record, or 0.
MetadataMtime(ctx context.Context, slug string) int64
// ── Chapters (binary blobs in MinIO) ───────────────────────────────────
// ChapterExists returns true if the markdown file for the given ref exists.
ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool
// WriteChapter stores the chapter markdown.
WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error
// ReadChapter returns the raw markdown for chapter number n.
ReadChapter(ctx context.Context, slug string, n int) (string, error)
// ListChapters returns all stored chapters for slug, sorted by number.
ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error)
// CountChapters returns the number of stored chapters for slug.
CountChapters(ctx context.Context, slug string) int
// ── Ranking ────────────────────────────────────────────────────────────
// WriteRanking persists the ranking list.
WriteRanking(ctx context.Context, items []RankingItem) error
// ReadRankingItems returns the stored ranking items.
ReadRankingItems(ctx context.Context) ([]RankingItem, error)
// RankingFileInfo returns os.FileInfo-like data for the ranking record.
// Returns (nil, nil) when no ranking has been stored yet.
RankingFileInfo(ctx context.Context) (os.FileInfo, error)
// ── Ranking page HTML cache ────────────────────────────────────────────
// WriteRankingPageCache stores raw HTML for a ranking page.
WriteRankingPageCache(ctx context.Context, page int, html string) error
// ReadRankingPageCache returns cached HTML for a ranking page, or "" on miss.
ReadRankingPageCache(ctx context.Context, page int) (string, error)
// RankingPageCacheInfo returns file-like info for a cached ranking page.
RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error)
// ── Audio cache ────────────────────────────────────────────────────────
// GetAudioCache returns the Kokoro filename for cacheKey, or ("", false).
GetAudioCache(ctx context.Context, cacheKey string) (string, bool)
// SetAudioCache persists a Kokoro filename for cacheKey.
SetAudioCache(ctx context.Context, cacheKey, filename string) error
// ── Reading progress ───────────────────────────────────────────────────
// GetProgress returns the reading progress for the given session ID and slug.
// Returns (zero, false) if no progress is recorded.
GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool)
// SetProgress saves or updates reading progress.
SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error
// AllProgress returns all progress entries for a session.
AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error)
// DeleteProgress removes progress for a specific slug.
DeleteProgress(ctx context.Context, sessionID, slug string) error
// ── Audio object paths (MinIO) ─────────────────────────────────────────
// AudioObjectKey returns the MinIO object key for a cached audio file.
AudioObjectKey(slug string, n int, voice string, speed float64) string
// ── Presigned URLs ─────────────────────────────────────────────────────
// PresignChapter returns a presigned GET URL for a chapter markdown object.
PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error)
// PresignAudio returns a presigned GET URL for an audio object.
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
}

13
ui/.env.example Normal file
View File

@@ -0,0 +1,13 @@
# libnovel UI — environment variables
# Copy to .env and adjust; do NOT commit with real secrets.
# Public URL of the scraper API (used by SvelteKit server-side load functions)
# In docker-compose this is the internal service name
SCRAPER_API_URL=http://localhost:8080
# Public URL of PocketBase (used by SvelteKit server-side load functions)
POCKETBASE_URL=http://localhost:8090
# Public-facing MinIO URL (used to rewrite presigned URLs for the browser)
# In dev this is localhost; in prod set to your MinIO public domain
PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000

23
ui/.gitignore vendored Normal file
View File

@@ -0,0 +1,23 @@
node_modules
# Output
.output
.vercel
.netlify
.wrangler
/.svelte-kit
/build
# OS
.DS_Store
Thumbs.db
# Env
.env
.env.*
!.env.example
!.env.test
# Vite
vite.config.js.timestamp-*
vite.config.ts.timestamp-*

1
ui/.npmrc Normal file
View File

@@ -0,0 +1 @@
engine-strict=true

23
ui/Dockerfile Normal file
View File

@@ -0,0 +1,23 @@
FROM node:22-alpine AS builder
WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci
COPY . .
RUN npm run build
# ── Runtime image ──────────────────────────────────────────────────────────────
FROM node:22-alpine
WORKDIR /app
# adapter-node produces a standalone build/
COPY --from=builder /app/build ./build
COPY --from=builder /app/package.json ./
ENV NODE_ENV=production
ENV PORT=3000
ENV HOST=0.0.0.0
EXPOSE 3000
CMD ["node", "build"]

42
ui/README.md Normal file
View File

@@ -0,0 +1,42 @@
# sv
Everything you need to build a Svelte project, powered by [`sv`](https://github.com/sveltejs/cli).
## Creating a project
If you're seeing this, you've probably already done this step. Congrats!
```sh
# create a new project
npx sv create my-app
```
To recreate this project with the same configuration:
```sh
# recreate this project
npx sv@0.12.4 create --template minimal --types ts --install npm ui
```
## Developing
Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
```sh
npm run dev
# or start the server and open the app in a new browser tab
npm run dev -- --open
```
## Building
To create a production version of your app:
```sh
npm run build
```
You can preview the production build with `npm run preview`.
> To deploy your app, you may need to install an [adapter](https://svelte.dev/docs/kit/adapters) for your target environment.

2462
ui/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

30
ui/package.json Normal file
View File

@@ -0,0 +1,30 @@
{
"name": "ui",
"private": true,
"version": "0.0.1",
"type": "module",
"scripts": {
"dev": "vite dev",
"build": "vite build",
"preview": "vite preview",
"prepare": "svelte-kit sync || echo ''",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch"
},
"devDependencies": {
"@sveltejs/adapter-auto": "^7.0.0",
"@sveltejs/adapter-node": "^5.5.4",
"@sveltejs/kit": "^2.50.2",
"@sveltejs/vite-plugin-svelte": "^6.2.4",
"@tailwindcss/vite": "^4.2.1",
"svelte": "^5.51.0",
"svelte-check": "^4.4.2",
"tailwindcss": "^4.2.1",
"typescript": "^5.9.3",
"vite": "^7.3.1"
},
"dependencies": {
"marked": "^17.0.3",
"pocketbase": "^0.26.8"
}
}

55
ui/src/app.css Normal file
View File

@@ -0,0 +1,55 @@
@import "tailwindcss";
@theme {
--color-brand: #f59e0b; /* amber-400 */
--color-brand-dim: #d97706; /* amber-600 */
--color-surface: #18181b; /* zinc-900 */
--color-surface-2: #27272a; /* zinc-800 */
--color-surface-3: #3f3f46; /* zinc-700 */
--color-muted: #a1a1aa; /* zinc-400 */
--color-text: #f4f4f5; /* zinc-100 */
}
html {
background-color: var(--color-surface);
color: var(--color-text);
}
/* ── Chapter prose ─────────────────────────────────────────────────── */
.prose-chapter {
max-width: 72ch;
line-height: 1.85;
font-size: 1.05rem;
color: #d4d4d8; /* zinc-300 */
}
.prose-chapter h1,
.prose-chapter h2,
.prose-chapter h3 {
color: #f4f4f5;
font-weight: 700;
margin-top: 1.5em;
margin-bottom: 0.5em;
}
.prose-chapter h1 { font-size: 1.4rem; }
.prose-chapter h2 { font-size: 1.2rem; }
.prose-chapter h3 { font-size: 1.05rem; }
.prose-chapter p {
margin-bottom: 1.2em;
}
.prose-chapter em {
color: #a1a1aa;
}
.prose-chapter strong {
color: #f4f4f5;
}
.prose-chapter hr {
border-color: #3f3f46;
margin: 2em 0;
}

15
ui/src/app.d.ts vendored Normal file
View File

@@ -0,0 +1,15 @@
// See https://svelte.dev/docs/kit/types#app.d.ts
// for information about these interfaces
declare global {
namespace App {
// interface Error {}
interface Locals {
sessionId: string;
}
// interface PageData {}
// interface PageState {}
// interface Platform {}
}
}
export {};

11
ui/src/app.html Normal file
View File

@@ -0,0 +1,11 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
%sveltekit.head%
</head>
<body data-sveltekit-preload-data="hover">
<div style="display: contents">%sveltekit.body%</div>
</body>
</html>

23
ui/src/hooks.server.ts Normal file
View File

@@ -0,0 +1,23 @@
import type { Handle } from '@sveltejs/kit';
import { randomBytes } from 'node:crypto';
const SESSION_COOKIE = 'libnovel_session';
const ONE_YEAR = 60 * 60 * 24 * 365;
export const handle: Handle = async ({ event, resolve }) => {
let sessionId = event.cookies.get(SESSION_COOKIE);
if (!sessionId) {
sessionId = randomBytes(16).toString('hex');
event.cookies.set(SESSION_COOKIE, sessionId, {
path: '/',
httpOnly: true,
sameSite: 'lax',
maxAge: ONE_YEAR
});
}
event.locals.sessionId = sessionId;
return resolve(event);
};

View File

@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="107" height="128" viewBox="0 0 107 128"><title>svelte-logo</title><path d="M94.157 22.819c-10.4-14.885-30.94-19.297-45.792-9.835L22.282 29.608A29.92 29.92 0 0 0 8.764 49.65a31.5 31.5 0 0 0 3.108 20.231 30 30 0 0 0-4.477 11.183 31.9 31.9 0 0 0 5.448 24.116c10.402 14.887 30.942 19.297 45.791 9.835l26.083-16.624A29.92 29.92 0 0 0 98.235 78.35a31.53 31.53 0 0 0-3.105-20.232 30 30 0 0 0 4.474-11.182 31.88 31.88 0 0 0-5.447-24.116" style="fill:#ff3e00"/><path d="M45.817 106.582a20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.503 18 18 0 0 1 .624-2.435l.49-1.498 1.337.981a33.6 33.6 0 0 0 10.203 5.098l.97.294-.09.968a5.85 5.85 0 0 0 1.052 3.878 6.24 6.24 0 0 0 6.695 2.485 5.8 5.8 0 0 0 1.603-.704L69.27 76.28a5.43 5.43 0 0 0 2.45-3.631 5.8 5.8 0 0 0-.987-4.371 6.24 6.24 0 0 0-6.698-2.487 5.7 5.7 0 0 0-1.6.704l-9.953 6.345a19 19 0 0 1-5.296 2.326 20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.502 17.99 17.99 0 0 1 8.13-12.052l26.081-16.623a19 19 0 0 1 5.3-2.329 20.72 20.72 0 0 1 22.237 8.243 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-.624 2.435l-.49 1.498-1.337-.98a33.6 33.6 0 0 0-10.203-5.1l-.97-.294.09-.968a5.86 5.86 0 0 0-1.052-3.878 6.24 6.24 0 0 0-6.696-2.485 5.8 5.8 0 0 0-1.602.704L37.73 51.72a5.42 5.42 0 0 0-2.449 3.63 5.79 5.79 0 0 0 .986 4.372 6.24 6.24 0 0 0 6.698 2.486 5.8 5.8 0 0 0 1.602-.704l9.952-6.342a19 19 0 0 1 5.295-2.328 20.72 20.72 0 0 1 22.237 8.242 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-8.13 12.053l-26.081 16.622a19 19 0 0 1-5.3 2.328" style="fill:#fff"/></svg>

After

Width:  |  Height:  |  Size: 1.5 KiB

View File

@@ -0,0 +1,187 @@
<script lang="ts">
/**
* AudioPlayer — fetches a presigned MinIO URL for the chapter audio
* and renders a native <audio> element with custom controls.
*
* The audio is generated server-side by Kokoro and cached.
* If no audio exists yet, the user can trigger generation via the scraper API.
*/
interface Props {
slug: string;
chapter: number;
voice?: string;
speed?: number;
}
let { slug, chapter, voice = 'af_bella', speed = 1.0 }: Props = $props();
type Status = 'idle' | 'loading' | 'ready' | 'error' | 'generating';
let status = $state<Status>('idle');
let audioUrl = $state('');
let errorMsg = $state('');
let audioEl = $state<HTMLAudioElement | null>(null);
let currentTime = $state(0);
let duration = $state(0);
let isPlaying = $state(false);
async function loadAudio() {
status = 'loading';
errorMsg = '';
try {
const params = new URLSearchParams({
slug,
n: String(chapter),
voice,
speed: String(speed)
});
const res = await fetch(`/api/presign/audio?${params.toString()}`);
if (res.status === 404) {
status = 'idle';
return;
}
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const data = (await res.json()) as { url: string };
audioUrl = data.url;
status = 'ready';
} catch (e) {
status = 'error';
errorMsg = String(e);
}
}
async function generateAudio() {
status = 'generating';
errorMsg = '';
try {
const res = await fetch(`/api/audio/${slug}/${chapter}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ voice, speed })
});
if (!res.ok) throw new Error(`Generation failed: ${res.status}`);
// After generation, fetch the presigned URL
await loadAudio();
} catch (e) {
status = 'error';
errorMsg = `Generation failed: ${e}`;
}
}
function formatTime(s: number): string {
const m = Math.floor(s / 60);
const sec = Math.floor(s % 60);
return `${m}:${sec.toString().padStart(2, '0')}`;
}
function togglePlay() {
if (!audioEl) return;
if (isPlaying) {
audioEl.pause();
} else {
audioEl.play();
}
}
function seek(e: Event) {
if (!audioEl) return;
const input = e.target as HTMLInputElement;
audioEl.currentTime = parseFloat(input.value);
}
</script>
<div class="mt-6 p-4 rounded-lg bg-zinc-800 border border-zinc-700">
<div class="flex items-center gap-2 mb-3">
<svg class="w-4 h-4 text-amber-400" fill="currentColor" viewBox="0 0 24 24">
<path d="M11.536 3.464a5 5 0 010 7.072L8 14H5v3H2V7h3l3.536-3.536a5 5 0 017.072 0l-4.072 4.072zM19 8a1 1 0 011 1v6a1 1 0 01-2 0V9a1 1 0 011-1zm-4-2a1 1 0 011 1v10a1 1 0 01-2 0V7a1 1 0 011-1z"/>
</svg>
<span class="text-sm text-zinc-300 font-medium">Audio Narration</span>
</div>
{#if status === 'idle'}
<div class="flex gap-2">
<button
onclick={loadAudio}
class="px-3 py-1.5 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors"
>
Check for audio
</button>
<button
onclick={generateAudio}
class="px-3 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors"
>
Generate audio
</button>
</div>
{:else if status === 'loading' || status === 'generating'}
<div class="flex items-center gap-2 text-zinc-400 text-sm">
<svg class="w-4 h-4 animate-spin" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
</svg>
{status === 'generating' ? 'Generating audio (this may take a few minutes)…' : 'Loading…'}
</div>
{:else if status === 'error'}
<div class="text-red-400 text-sm">
<p>{errorMsg || 'Failed to load audio.'}</p>
<button
onclick={() => { status = 'idle'; }}
class="mt-1 text-xs underline text-zinc-400 hover:text-zinc-200"
>
Dismiss
</button>
</div>
{:else if status === 'ready' && audioUrl}
<!-- Native audio element (hidden, controlled by custom UI) -->
<audio
bind:this={audioEl}
src={audioUrl}
bind:currentTime
bind:duration
onplay={() => (isPlaying = true)}
onpause={() => (isPlaying = false)}
onended={() => (isPlaying = false)}
preload="metadata"
></audio>
<!-- Custom controls -->
<div class="flex flex-col gap-2">
<!-- Play/pause + time -->
<div class="flex items-center gap-3">
<button
onclick={togglePlay}
class="w-9 h-9 rounded-full bg-amber-400 text-zinc-900 flex items-center justify-center hover:bg-amber-300 transition-colors flex-shrink-0"
aria-label={isPlaying ? 'Pause' : 'Play'}
>
{#if isPlaying}
<svg class="w-4 h-4" fill="currentColor" viewBox="0 0 24 24">
<path d="M6 4h4v16H6V4zm8 0h4v16h-4V4z"/>
</svg>
{:else}
<svg class="w-4 h-4 ml-0.5" fill="currentColor" viewBox="0 0 24 24">
<path d="M8 5v14l11-7z"/>
</svg>
{/if}
</button>
<span class="text-xs text-zinc-400 w-20 flex-shrink-0">
{formatTime(currentTime)} / {formatTime(duration || 0)}
</span>
<!-- Seek bar -->
<input
type="range"
min="0"
max={duration || 0}
value={currentTime}
oninput={seek}
class="flex-1 accent-amber-400 h-1.5 rounded"
/>
</div>
</div>
{/if}
</div>

1
ui/src/lib/index.ts Normal file
View File

@@ -0,0 +1 @@
// place files you want to import through the `$lib` alias in this folder.

View File

@@ -0,0 +1,67 @@
/**
* Server-side MinIO presign helper.
* Calls the scraper API to get presigned URLs, then optionally rewrites
* the MinIO host to the public-facing URL for browser use.
*
* Never import this from client-side code.
*/
import { env } from '$env/dynamic/private';
import { env as pubEnv } from '$env/dynamic/public';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
// Public MinIO URL — used to rewrite presigned URLs so the browser can reach MinIO directly.
// In docker-compose this would differ from the internal endpoint.
const MINIO_PUBLIC_URL = pubEnv.PUBLIC_MINIO_PUBLIC_URL ?? 'http://localhost:9000';
/**
* Rewrites the MinIO host in a presigned URL to the public-facing URL.
* The presigned URL is signed against the internal endpoint (e.g. minio:9000),
* but the browser needs the public URL (e.g. localhost:9000 in dev, or a CDN in prod).
* Rewriting the host preserves all query params (signature, expiry, etc).
*/
function rewriteHost(presignedUrl: string): string {
try {
const u = new URL(presignedUrl);
const pub = new URL(MINIO_PUBLIC_URL);
u.protocol = pub.protocol;
u.hostname = pub.hostname;
u.port = pub.port;
return u.toString();
} catch {
return presignedUrl;
}
}
/**
* Returns a presigned URL for a chapter markdown file.
* URL is valid for ~15 minutes (set by the scraper).
* The returned URL points to the public MinIO endpoint and can be used
* server-side (in a +page.server.ts load function) to fetch the markdown content.
*/
export async function presignChapter(slug: string, n: number): Promise<string> {
const res = await fetch(`${SCRAPER_URL}/api/presign/chapter/${slug}/${n}`);
if (!res.ok) throw new Error(`presign chapter ${slug}/${n}: ${res.status}`);
const data = (await res.json()) as { url: string };
return rewriteHost(data.url);
}
/**
* Returns a presigned URL for an audio file.
* URL is valid for ~1 hour. The URL is returned to the browser for direct streaming.
*/
export async function presignAudio(
slug: string,
n: number,
voice?: string,
speed?: number
): Promise<string> {
const params = new URLSearchParams();
if (voice) params.set('voice', voice);
if (speed) params.set('speed', String(speed));
const qs = params.toString() ? `?${params.toString()}` : '';
const res = await fetch(`${SCRAPER_URL}/api/presign/audio/${slug}/${n}${qs}`);
if (!res.ok) throw new Error(`presign audio ${slug}/${n}: ${res.status}`);
const data = (await res.json()) as { url: string };
return rewriteHost(data.url);
}

View File

@@ -0,0 +1,169 @@
/**
* Server-side PocketBase client.
* Uses admin credentials — never import this from client-side code.
* All methods talk directly to PocketBase REST API.
*/
import { env } from '$env/dynamic/private';
const PB_URL = env.POCKETBASE_URL ?? 'http://localhost:8090';
const PB_EMAIL = env.POCKETBASE_ADMIN_EMAIL ?? 'admin@libnovel.local';
const PB_PASSWORD = env.POCKETBASE_ADMIN_PASSWORD ?? 'changeme123';
// ─── Types ────────────────────────────────────────────────────────────────────
export interface Book {
id: string;
slug: string;
title: string;
author: string;
cover: string;
status: string;
genres: string[] | string;
summary: string;
total_chapters: number;
source_url: string;
ranking: number;
meta_updated: string;
}
export interface ChapterIdx {
id: string;
slug: string;
number: number;
title: string;
date_label: string;
}
export interface Progress {
session_id: string;
slug: string;
chapter: number;
updated: string;
}
// ─── Auth token cache ─────────────────────────────────────────────────────────
let _token = '';
let _tokenExp = 0;
async function getToken(): Promise<string> {
if (_token && Date.now() < _tokenExp) return _token;
const res = await fetch(`${PB_URL}/api/admins/auth-with-password`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ identity: PB_EMAIL, password: PB_PASSWORD })
});
if (!res.ok) {
throw new Error(`PocketBase auth failed: ${res.status}`);
}
const data = await res.json();
_token = data.token as string;
_tokenExp = Date.now() + 12 * 60 * 60 * 1000; // 12 hours
return _token;
}
// ─── Generic helpers ──────────────────────────────────────────────────────────
async function pbGet<T>(path: string): Promise<T> {
const token = await getToken();
const res = await fetch(`${PB_URL}${path}`, {
headers: { Authorization: token }
});
if (!res.ok) throw new Error(`PocketBase GET ${path} failed: ${res.status}`);
return res.json() as Promise<T>;
}
async function pbPost(path: string, body: unknown): Promise<Response> {
const token = await getToken();
return fetch(`${PB_URL}${path}`, {
method: 'POST',
headers: { Authorization: token, 'Content-Type': 'application/json' },
body: JSON.stringify(body)
});
}
async function pbPatch(path: string, body: unknown): Promise<Response> {
const token = await getToken();
return fetch(`${PB_URL}${path}`, {
method: 'PATCH',
headers: { Authorization: token, 'Content-Type': 'application/json' },
body: JSON.stringify(body)
});
}
function encodeFilter(filter: string) {
return encodeURIComponent(filter);
}
interface PBList<T> {
items: T[];
totalItems: number;
}
async function listAll<T>(collection: string, filter = '', sort = ''): Promise<T[]> {
const params = new URLSearchParams({ perPage: '500' });
if (filter) params.set('filter', filter);
if (sort) params.set('sort', sort);
const data = await pbGet<PBList<T>>(
`/api/collections/${collection}/records?${params.toString()}`
);
return data.items;
}
async function listOne<T>(collection: string, filter: string): Promise<T | null> {
const params = new URLSearchParams({ perPage: '1', filter });
const data = await pbGet<PBList<T>>(
`/api/collections/${collection}/records?${params.toString()}`
);
return data.items[0] ?? null;
}
// ─── Books ────────────────────────────────────────────────────────────────────
export async function listBooks(): Promise<Book[]> {
return listAll<Book>('books', '', '+title');
}
export async function getBook(slug: string): Promise<Book | null> {
return listOne<Book>('books', `slug="${slug}"`);
}
// ─── Chapter index ────────────────────────────────────────────────────────────
export async function listChapterIdx(slug: string): Promise<ChapterIdx[]> {
return listAll<ChapterIdx>('chapters_idx', `slug="${slug}"`, '+number');
}
// ─── Reading progress ─────────────────────────────────────────────────────────
export async function getProgress(sessionId: string, slug: string): Promise<Progress | null> {
return listOne<Progress>('progress', `session_id="${sessionId}"&&slug="${slug}"`);
}
export async function allProgress(sessionId: string): Promise<Progress[]> {
return listAll<Progress>('progress', `session_id="${sessionId}"`, '-updated');
}
export async function setProgress(sessionId: string, slug: string, chapter: number): Promise<void> {
const existing = await listOne<Progress & { id: string }>(
'progress',
`session_id="${sessionId}"&&slug="${slug}"`
);
const payload = {
session_id: sessionId,
slug,
chapter,
updated: new Date().toISOString()
};
if (existing) {
await pbPatch(`/api/collections/progress/records/${existing.id}`, payload);
} else {
await pbPost('/api/collections/progress/records', payload);
}
}

View File

@@ -0,0 +1,32 @@
<script lang="ts">
import '../app.css';
let { children } = $props();
</script>
<svelte:head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>libnovel</title>
</svelte:head>
<div class="min-h-screen flex flex-col">
<header class="border-b border-zinc-700 bg-zinc-900 sticky top-0 z-50">
<nav class="max-w-6xl mx-auto px-4 h-14 flex items-center gap-6">
<a href="/" class="text-amber-400 font-bold text-lg tracking-tight hover:text-amber-300">
libnovel
</a>
<a href="/books" class="text-zinc-400 hover:text-zinc-100 text-sm transition-colors">
Library
</a>
</nav>
</header>
<main class="flex-1 max-w-6xl mx-auto w-full px-4 py-8">
{@render children()}
</main>
<footer class="border-t border-zinc-800 text-zinc-600 text-xs text-center py-4">
libnovel
</footer>
</div>

View File

@@ -0,0 +1,3 @@
<a href="/books" class="inline-block mt-4 px-6 py-3 bg-amber-400 text-zinc-900 font-semibold rounded hover:bg-amber-300 transition-colors">
Browse Library
</a>

View File

@@ -0,0 +1,92 @@
import { error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { env } from '$env/dynamic/private';
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
/**
* POST /api/audio/[slug]/[n]
* Proxies the audio generation request to the scraper's /api/audio endpoint.
* Keeps the scraper URL server-side — the browser never needs to know it.
*
* Body: { voice?: string, speed?: number }
* Response: { url: string, filename: string }
* where `url` is a relative path to GET /api/audio/[slug]/[n]?voice=...&speed=...
*/
export const POST: RequestHandler = async ({ params, request }) => {
const { slug, n } = params;
const chapter = parseInt(n, 10);
if (!slug || !chapter || chapter < 1) {
error(400, 'Invalid slug or chapter number');
}
let body: { voice?: string; speed?: number } = {};
try {
body = await request.json();
} catch {
// empty body is fine — scraper will use defaults
}
const scraperRes = await fetch(`${SCRAPER_URL}/api/audio/${slug}/${chapter}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
});
if (!scraperRes.ok) {
const text = await scraperRes.text().catch(() => '');
error(scraperRes.status as Parameters<typeof error>[0], text || 'Audio generation failed');
}
const data = (await scraperRes.json()) as { url: string; filename: string };
// The scraper returns a proxy URL pointing to /api/audio-proxy/... — we rewrite
// it to our own /api/audio/[slug]/[n]?... so the browser never calls the scraper directly.
const voice = body.voice ?? '';
const speed = body.speed ?? 1.0;
const qs = new URLSearchParams();
if (voice) qs.set('voice', voice);
qs.set('speed', String(speed));
return new Response(
JSON.stringify({
url: `/api/audio/${slug}/${chapter}?${qs.toString()}`,
filename: data.filename
}),
{ headers: { 'Content-Type': 'application/json' } }
);
};
/**
* GET /api/audio/[slug]/[n]?voice=...&speed=...
* Proxies the audio stream from the scraper's /api/audio-proxy endpoint.
* This is the URL the browser's <audio> element uses as its src.
*/
export const GET: RequestHandler = async ({ params, url }) => {
const { slug, n } = params;
const chapter = parseInt(n, 10);
if (!slug || !chapter || chapter < 1) {
error(400, 'Invalid slug or chapter number');
}
const voice = url.searchParams.get('voice') ?? '';
const speed = url.searchParams.get('speed') ?? '1';
const qs = new URLSearchParams();
if (voice) qs.set('voice', voice);
qs.set('speed', speed);
const scraperRes = await fetch(`${SCRAPER_URL}/api/audio-proxy/${slug}/${chapter}?${qs.toString()}`);
if (!scraperRes.ok) {
error(scraperRes.status as Parameters<typeof error>[0], 'Audio not found');
}
// Stream the audio body through — preserve Content-Type and Content-Length.
const headers = new Headers();
headers.set('Content-Type', scraperRes.headers.get('Content-Type') ?? 'audio/mpeg');
headers.set('Cache-Control', 'public, max-age=3600');
const cl = scraperRes.headers.get('Content-Length');
if (cl) headers.set('Content-Length', cl);
return new Response(scraperRes.body, { headers });
};

View File

@@ -0,0 +1,26 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { presignAudio } from '$lib/server/minio';
/**
* GET /api/presign/audio?slug=...&n=...&voice=...&speed=...
* Returns a presigned MinIO URL for the audio file so the browser
* can stream it directly without going through the server.
*/
export const GET: RequestHandler = async ({ url }) => {
const slug = url.searchParams.get('slug');
const n = parseInt(url.searchParams.get('n') ?? '', 10);
const voice = url.searchParams.get('voice') ?? undefined;
const speed = parseFloat(url.searchParams.get('speed') ?? '1') || 1;
if (!slug || !n || n < 1) {
error(400, 'Missing slug or n');
}
try {
const presignedUrl = await presignAudio(slug, n, voice, speed);
return json({ url: presignedUrl });
} catch (e) {
error(500, `Could not get presigned URL: ${e}`);
}
};

View File

@@ -0,0 +1,19 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { setProgress } from '$lib/server/pocketbase';
/**
* POST /api/progress
* Body: { slug: string, chapter: number }
* Records the user's reading position for the current session.
*/
export const POST: RequestHandler = async ({ request, locals }) => {
const body = await request.json().catch(() => null);
if (!body || typeof body.slug !== 'string' || typeof body.chapter !== 'number') {
error(400, 'Invalid body — expected { slug, chapter }');
}
await setProgress(locals.sessionId, body.slug, body.chapter);
return json({ ok: true });
};

View File

@@ -0,0 +1,17 @@
import type { PageServerLoad } from './$types';
import { listBooks, allProgress } from '$lib/server/pocketbase';
export const load: PageServerLoad = async ({ locals }) => {
const [books, progressList] = await Promise.all([
listBooks(),
allProgress(locals.sessionId)
]);
// Build a quick lookup: slug → last chapter read
const progressMap: Record<string, number> = {};
for (const p of progressList) {
progressMap[p.slug] = p.chapter;
}
return { books, progressMap };
};

View File

@@ -0,0 +1,91 @@
<script lang="ts">
import type { PageData } from './$types';
let { data }: { data: PageData } = $props();
function parseGenres(genres: string[] | string): string[] {
if (Array.isArray(genres)) return genres;
try {
return JSON.parse(genres);
} catch {
return [];
}
}
</script>
<svelte:head>
<title>Library — libnovel</title>
</svelte:head>
<div class="mb-6">
<h1 class="text-2xl font-bold text-zinc-100">Library</h1>
<p class="text-zinc-400 text-sm mt-1">{data.books.length} books</p>
</div>
{#if data.books.length === 0}
<div class="text-center py-20 text-zinc-500">
<p class="text-lg">No books scraped yet.</p>
<p class="text-sm mt-2">Use the scraper API to add books.</p>
</div>
{:else}
<div class="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 xl:grid-cols-6 gap-4">
{#each data.books as book}
{@const lastChapter = data.progressMap[book.slug]}
{@const genres = parseGenres(book.genres)}
<a
href="/books/{book.slug}"
class="group flex flex-col rounded-lg overflow-hidden bg-zinc-800 hover:bg-zinc-700 transition-colors border border-zinc-700 hover:border-zinc-500"
>
<!-- Cover image -->
<div class="aspect-[2/3] bg-zinc-900 overflow-hidden">
{#if book.cover}
<img
src={book.cover}
alt={book.title}
class="w-full h-full object-cover group-hover:scale-105 transition-transform duration-300"
loading="lazy"
/>
{:else}
<div class="w-full h-full flex items-center justify-center text-zinc-600">
<svg class="w-12 h-12" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"
d="M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.332.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253" />
</svg>
</div>
{/if}
</div>
<!-- Info -->
<div class="p-2 flex flex-col gap-1 flex-1">
<h2 class="text-xs font-semibold text-zinc-100 line-clamp-2 leading-snug">
{book.title}
</h2>
{#if book.author}
<p class="text-xs text-zinc-400 truncate">{book.author}</p>
{/if}
<div class="mt-auto pt-1 flex items-center justify-between gap-1">
{#if book.status}
<span class="text-xs px-1.5 py-0.5 rounded bg-zinc-700 text-zinc-300 truncate max-w-[60%]">
{book.status}
</span>
{/if}
{#if lastChapter}
<span class="text-xs text-amber-400 font-medium ml-auto whitespace-nowrap">
ch.{lastChapter}
</span>
{/if}
</div>
{#if genres.length > 0}
<div class="flex flex-wrap gap-1 mt-1">
{#each genres.slice(0, 2) as genre}
<span class="text-xs px-1 py-0.5 rounded bg-zinc-900 text-zinc-500">{genre}</span>
{/each}
</div>
{/if}
</div>
</a>
{/each}
</div>
{/if}

View File

@@ -0,0 +1,23 @@
import { error } from '@sveltejs/kit';
import type { PageServerLoad } from './$types';
import { getBook, listChapterIdx, getProgress } from '$lib/server/pocketbase';
export const load: PageServerLoad = async ({ params, locals }) => {
const { slug } = params;
const [book, chapters, progress] = await Promise.all([
getBook(slug),
listChapterIdx(slug),
getProgress(locals.sessionId, slug)
]);
if (!book) {
error(404, `Book "${slug}" not found`);
}
return {
book,
chapters,
lastChapter: progress?.chapter ?? null
};
};

View File

@@ -0,0 +1,142 @@
<script lang="ts">
import type { PageData } from './$types';
let { data }: { data: PageData } = $props();
function parseGenres(genres: string[] | string): string[] {
if (Array.isArray(genres)) return genres;
try {
return JSON.parse(genres);
} catch {
return [];
}
}
const genres = $derived(parseGenres(data.book.genres));
// Paginate chapter list — show 100 at a time
let page = $state(0);
const PAGE_SIZE = 100;
const totalPages = $derived(Math.ceil(data.chapters.length / PAGE_SIZE));
const visibleChapters = $derived(
data.chapters.slice(page * PAGE_SIZE, (page + 1) * PAGE_SIZE)
);
</script>
<svelte:head>
<title>{data.book.title} — libnovel</title>
</svelte:head>
<!-- Book header -->
<div class="flex gap-6 mb-8">
{#if data.book.cover}
<img
src={data.book.cover}
alt={data.book.title}
class="w-32 sm:w-40 rounded-lg object-cover flex-shrink-0 border border-zinc-700"
/>
{/if}
<div class="flex flex-col gap-2 min-w-0">
<h1 class="text-2xl font-bold text-zinc-100 leading-tight">{data.book.title}</h1>
{#if data.book.author}
<p class="text-zinc-400 text-sm">{data.book.author}</p>
{/if}
<div class="flex flex-wrap gap-2 mt-1">
{#if data.book.status}
<span class="text-xs px-2 py-1 rounded bg-zinc-700 text-zinc-300">{data.book.status}</span>
{/if}
{#each genres as genre}
<span class="text-xs px-2 py-1 rounded bg-zinc-800 text-zinc-400">{genre}</span>
{/each}
</div>
{#if data.book.summary}
<p class="text-zinc-400 text-sm leading-relaxed line-clamp-4 mt-1">{data.book.summary}</p>
{/if}
<div class="flex gap-3 mt-2">
{#if data.lastChapter}
<a
href="/books/{data.book.slug}/chapters/{data.lastChapter}"
class="px-4 py-2 bg-amber-400 text-zinc-900 font-semibold rounded text-sm hover:bg-amber-300 transition-colors"
>
Continue ch.{data.lastChapter}
</a>
{/if}
{#if data.chapters.length > 0}
<a
href="/books/{data.book.slug}/chapters/1"
class="px-4 py-2 bg-zinc-700 text-zinc-100 font-semibold rounded text-sm hover:bg-zinc-600 transition-colors"
>
Start from ch.1
</a>
{/if}
</div>
</div>
</div>
<!-- Chapter list -->
<div class="mt-4">
<div class="flex items-center justify-between mb-3">
<h2 class="text-lg font-semibold text-zinc-100">
Chapters
<span class="text-zinc-500 font-normal text-sm ml-1">({data.chapters.length})</span>
</h2>
{#if totalPages > 1}
<div class="flex gap-2 items-center text-sm">
<button
onclick={() => (page = Math.max(0, page - 1))}
disabled={page === 0}
class="px-2 py-1 rounded bg-zinc-700 text-zinc-300 disabled:opacity-40 hover:bg-zinc-600 transition-colors"
>
&larr;
</button>
<span class="text-zinc-400">{page + 1} / {totalPages}</span>
<button
onclick={() => (page = Math.min(totalPages - 1, page + 1))}
disabled={page === totalPages - 1}
class="px-2 py-1 rounded bg-zinc-700 text-zinc-300 disabled:opacity-40 hover:bg-zinc-600 transition-colors"
>
&rarr;
</button>
</div>
{/if}
</div>
{#if data.chapters.length === 0}
<p class="text-zinc-500 text-sm">No chapters available yet.</p>
{:else}
<div class="grid grid-cols-1 sm:grid-cols-2 gap-1">
{#each visibleChapters as chapter}
{@const isCurrent = data.lastChapter === chapter.number}
<a
href="/books/{data.book.slug}/chapters/{chapter.number}"
class="flex items-center gap-3 px-3 py-2 rounded hover:bg-zinc-800 transition-colors group {isCurrent
? 'bg-zinc-800'
: ''}"
>
<span
class="text-xs font-mono w-10 text-right flex-shrink-0 {isCurrent
? 'text-amber-400'
: 'text-zinc-500'}"
>
{chapter.number}
</span>
<span class="text-sm text-zinc-300 group-hover:text-zinc-100 truncate flex-1">
{chapter.title || `Chapter ${chapter.number}`}
</span>
{#if isCurrent}
<span class="text-xs text-amber-400 flex-shrink-0">reading</span>
{/if}
{#if chapter.date_label}
<span class="text-xs text-zinc-600 flex-shrink-0 hidden sm:block">{chapter.date_label}</span>
{/if}
</a>
{/each}
</div>
{/if}
</div>

View File

@@ -0,0 +1,45 @@
import { error } from '@sveltejs/kit';
import { marked } from 'marked';
import type { PageServerLoad } from './$types';
import { getBook, listChapterIdx } from '$lib/server/pocketbase';
import { presignChapter } from '$lib/server/minio';
export const load: PageServerLoad = async ({ params, locals }) => {
const { slug } = params;
const n = parseInt(params.n, 10);
if (!n || n < 1) error(400, 'Invalid chapter number');
// Fetch book metadata and chapter index in parallel
const [book, chapters] = await Promise.all([getBook(slug), listChapterIdx(slug)]);
if (!book) error(404, `Book "${slug}" not found`);
const chapterIdx = chapters.find((c) => c.number === n);
if (!chapterIdx) error(404, `Chapter ${n} not found`);
// Get presigned URL and fetch chapter markdown server-side
let html = '';
try {
const url = await presignChapter(slug, n);
const res = await fetch(url);
if (!res.ok) throw new Error(`MinIO returned ${res.status}`);
const markdown = await res.text();
html = await marked(markdown, { async: true });
} catch (e) {
// Don't hard-fail — show empty content with error message
console.error('Failed to fetch chapter content:', e);
}
const prevChapter = chapters.find((c) => c.number === n - 1) ?? null;
const nextChapter = chapters.find((c) => c.number === n + 1) ?? null;
return {
book: { slug: book.slug, title: book.title },
chapter: chapterIdx,
html,
prev: prevChapter ? prevChapter.number : null,
next: nextChapter ? nextChapter.number : null,
sessionId: locals.sessionId
};
};

View File

@@ -0,0 +1,103 @@
<script lang="ts">
import { onMount } from 'svelte';
import AudioPlayer from '$lib/components/AudioPlayer.svelte';
import type { PageData } from './$types';
let { data }: { data: PageData } = $props();
// Record reading progress when the chapter is opened
onMount(async () => {
try {
await fetch('/api/progress', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ slug: data.book.slug, chapter: data.chapter.number })
});
} catch {
// Non-critical — silently ignore
}
});
</script>
<svelte:head>
<title>{data.chapter.title || `Chapter ${data.chapter.number}`} — {data.book.title} — libnovel</title>
</svelte:head>
<!-- Top nav -->
<div class="flex items-center justify-between mb-6 gap-4">
<a
href="/books/{data.book.slug}"
class="text-zinc-400 hover:text-zinc-100 text-sm flex items-center gap-1 transition-colors"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 19l-7-7 7-7" />
</svg>
{data.book.title}
</a>
<div class="flex gap-2">
{#if data.prev}
<a
href="/books/{data.book.slug}/chapters/{data.prev}"
class="px-3 py-1.5 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors"
>
&larr; Ch.{data.prev}
</a>
{/if}
{#if data.next}
<a
href="/books/{data.book.slug}/chapters/{data.next}"
class="px-3 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors"
>
Ch.{data.next} &rarr;
</a>
{/if}
</div>
</div>
<!-- Chapter heading -->
<div class="mb-6">
<p class="text-zinc-500 text-sm mb-1">Chapter {data.chapter.number}</p>
<h1 class="text-xl font-bold text-zinc-100">
{data.chapter.title || `Chapter ${data.chapter.number}`}
</h1>
{#if data.chapter.date_label}
<p class="text-zinc-600 text-xs mt-1">{data.chapter.date_label}</p>
{/if}
</div>
<!-- Audio player -->
<AudioPlayer slug={data.book.slug} chapter={data.chapter.number} />
<!-- Chapter content -->
{#if !data.html}
<div class="text-zinc-500 text-center py-16">
<p>Chapter content not available.</p>
</div>
{:else}
<div class="prose-chapter mt-8">
{@html data.html}
</div>
{/if}
<!-- Bottom nav -->
<div class="flex justify-between mt-12 pt-6 border-t border-zinc-800 gap-4">
{#if data.prev}
<a
href="/books/{data.book.slug}/chapters/{data.prev}"
class="px-4 py-2 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors"
>
&larr; Previous chapter
</a>
{:else}
<div></div>
{/if}
{#if data.next}
<a
href="/books/{data.book.slug}/chapters/{data.next}"
class="px-4 py-2 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors"
>
Next chapter &rarr;
</a>
{/if}
</div>

3
ui/static/robots.txt Normal file
View File

@@ -0,0 +1,3 @@
# allow crawling everything by default
User-agent: *
Disallow:

10
ui/svelte.config.js Normal file
View File

@@ -0,0 +1,10 @@
import adapter from '@sveltejs/adapter-node';
/** @type {import('@sveltejs/kit').Config} */
const config = {
kit: {
adapter: adapter()
}
};
export default config;

20
ui/tsconfig.json Normal file
View File

@@ -0,0 +1,20 @@
{
"extends": "./.svelte-kit/tsconfig.json",
"compilerOptions": {
"rewriteRelativeImportExtensions": true,
"allowJs": true,
"checkJs": true,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"skipLibCheck": true,
"sourceMap": true,
"strict": true,
"moduleResolution": "bundler"
}
// Path aliases are handled by https://svelte.dev/docs/kit/configuration#alias
// except $lib which is handled by https://svelte.dev/docs/kit/configuration#files
//
// To make changes to top-level options such as include and exclude, we recommend extending
// the generated config; see https://svelte.dev/docs/kit/configuration#typescript
}

7
ui/vite.config.ts Normal file
View File

@@ -0,0 +1,7 @@
import { sveltekit } from '@sveltejs/kit/vite';
import tailwindcss from '@tailwindcss/vite';
import { defineConfig } from 'vite';
export default defineConfig({
plugins: [tailwindcss(), sveltekit()]
});