Compare commits
21 Commits
main
...
38e400a4c7
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
38e400a4c7 | ||
|
|
cb90771248 | ||
|
|
59b1cfab1d | ||
|
|
f95ad3ed29 | ||
|
|
e4c4f8de66 | ||
|
|
4f84bd29c9 | ||
|
|
6bf79ab392 | ||
|
|
4ae6f0ab42 | ||
|
|
33e2a4dc01 | ||
|
|
cb4be0848f | ||
|
|
2f948f2a50 | ||
|
|
baab66823d | ||
|
|
11d2eaa0e5 | ||
|
|
9c115f00c4 | ||
|
|
5ac89da513 | ||
|
|
af86c6f96f | ||
|
|
da4a182f85 | ||
|
|
18e76c9668 | ||
|
|
9add9033b9 | ||
|
|
66d8481637 | ||
|
|
7f92a58fd7 |
46
.env.example
46
.env.example
@@ -1,6 +1,26 @@
|
||||
# libnovel scraper — environment overrides
|
||||
# Copy to .env and adjust values; do NOT commit this file with real secrets.
|
||||
|
||||
# ── Service ports (host-side) ─────────────────────────────────────────────────
|
||||
# Port the scraper HTTP API listens on (default 8080)
|
||||
SCRAPER_PORT=8080
|
||||
|
||||
# Port PocketBase listens on (default 8090)
|
||||
POCKETBASE_PORT=8090
|
||||
|
||||
# Port MinIO S3 API listens on (default 9000)
|
||||
MINIO_PORT=9000
|
||||
|
||||
# Port MinIO web console listens on (default 9001)
|
||||
MINIO_CONSOLE_PORT=9001
|
||||
|
||||
# Port Browserless Chrome listens on (default 3030)
|
||||
BROWSERLESS_PORT=3030
|
||||
|
||||
# Port the SvelteKit UI listens on (default 3000)
|
||||
UI_PORT=3000
|
||||
|
||||
# ── Browserless ───────────────────────────────────────────────────────────────
|
||||
# Browserless API token (leave empty to disable auth)
|
||||
BROWSERLESS_TOKEN=
|
||||
|
||||
@@ -23,6 +43,7 @@ BROWSERLESS_STRATEGY=direct
|
||||
# Set to direct to use plain HTTP, or content/scrape/cdp for browserless.
|
||||
BROWSERLESS_URL_STRATEGY=content
|
||||
|
||||
# ── Scraper ───────────────────────────────────────────────────────────────────
|
||||
# Chapter worker goroutines (0 = NumCPU inside the container)
|
||||
SCRAPER_WORKERS=0
|
||||
|
||||
@@ -39,3 +60,28 @@ KOKORO_URL=http://kokoro:8880
|
||||
# Single voices: af_bella, af_sky, af_heart, am_adam, …
|
||||
# Mixed voices: af_bella+af_sky or af_bella(2)+af_sky(1) (weighted blend)
|
||||
KOKORO_VOICE=af_bella
|
||||
|
||||
# ── MinIO / S3 object storage ─────────────────────────────────────────────────
|
||||
MINIO_ROOT_USER=admin
|
||||
MINIO_ROOT_PASSWORD=changeme123
|
||||
MINIO_BUCKET_CHAPTERS=libnovel-chapters
|
||||
MINIO_BUCKET_AUDIO=libnovel-audio
|
||||
|
||||
# ── PocketBase ────────────────────────────────────────────────────────────────
|
||||
# Admin credentials (used by scraper + UI server-side)
|
||||
POCKETBASE_ADMIN_EMAIL=admin@libnovel.local
|
||||
POCKETBASE_ADMIN_PASSWORD=changeme123
|
||||
|
||||
# ── SvelteKit UI ─────────────────────────────────────────────────────────────
|
||||
# Internal URL the SvelteKit server uses to reach the scraper API.
|
||||
# In docker-compose this is http://scraper:8080 (wired automatically).
|
||||
# Override here only if running the UI outside of docker-compose.
|
||||
SCRAPER_API_URL=http://localhost:8080
|
||||
|
||||
# Internal URL the SvelteKit server uses to reach PocketBase.
|
||||
# In docker-compose this is http://pocketbase:8090 (wired automatically).
|
||||
POCKETBASE_URL=http://localhost:8090
|
||||
|
||||
# Public MinIO URL reachable from the browser (for audio/presigned URLs).
|
||||
# In production, point this at your MinIO reverse-proxy or CDN domain.
|
||||
PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000
|
||||
|
||||
@@ -1,10 +1,64 @@
|
||||
version: "3.9"
|
||||
|
||||
services:
|
||||
# ─── MinIO (object storage for chapter .md files + audio cache) ─────────────
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
#container_name: libnovel-minio
|
||||
restart: unless-stopped
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "${MINIO_PORT:-9000}:9000" # S3 API
|
||||
- "${MINIO_CONSOLE_PORT:-9001}:9001" # Web console
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── MinIO bucket initialisation ─────────────────────────────────────────────
|
||||
# Runs once to create the default buckets and then exits.
|
||||
minio-init:
|
||||
image: minio/mc:latest
|
||||
#container_name: libnovel-minio-init
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
|
||||
mc mb --ignore-existing local/libnovel-chapters;
|
||||
mc mb --ignore-existing local/libnovel-audio;
|
||||
echo 'buckets ready';
|
||||
"
|
||||
environment:
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
|
||||
# ─── PocketBase (auth + structured data: books, chapters index, ranking, progress) ──
|
||||
pocketbase:
|
||||
image: ghcr.io/muchobien/pocketbase:latest
|
||||
#container_name: libnovel-pocketbase
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${POCKETBASE_PORT:-8090}:8090"
|
||||
volumes:
|
||||
- pb_data:/pb/pb_data
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── Browserless ────────────────────────────────────────────────────────────
|
||||
browserless:
|
||||
image: ghcr.io/browserless/chromium:latest
|
||||
container_name: libnovel-browserless
|
||||
#container_name: libnovel-browserless
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# Set a token to lock down the endpoint; the scraper reads it via
|
||||
@@ -19,7 +73,7 @@ services:
|
||||
# Optional webhook URL for Browserless error alerts.
|
||||
ERROR_ALERT_URL: "${ERROR_ALERT_URL:-}"
|
||||
ports:
|
||||
- "3030:3000"
|
||||
- "${BROWSERLESS_PORT:-3030}:3000"
|
||||
# Shared memory is required for Chrome.
|
||||
shm_size: "2gb"
|
||||
healthcheck:
|
||||
@@ -28,30 +82,17 @@ services:
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── Kokoro-FastAPI (TTS) ────────────────────────────────────────────────────
|
||||
# CPU image; swap for ghcr.io/remsky/kokoro-fastapi-gpu:latest on NVIDIA hosts.
|
||||
# Models are baked in — no volume mount required for the default voice set.
|
||||
kokoro:
|
||||
image: ghcr.io/remsky/kokoro-fastapi-cpu:latest
|
||||
container_name: libnovel-kokoro
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8880:8880"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8880/health"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── Scraper ─────────────────────────────────────────────────────────────────
|
||||
scraper:
|
||||
build:
|
||||
context: ./scraper
|
||||
dockerfile: Dockerfile
|
||||
container_name: libnovel-scraper
|
||||
#container_name: libnovel-scraper
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
kokoro:
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
BROWSERLESS_URL: "http://browserless:3000"
|
||||
@@ -66,10 +107,21 @@ services:
|
||||
SCRAPER_HTTP_ADDR: ":8080"
|
||||
LOG_LEVEL: "debug"
|
||||
# Kokoro-FastAPI TTS endpoint.
|
||||
KOKORO_URL: "${KOKORO_URL:-http://localhost:8880}"
|
||||
KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc}"
|
||||
KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}"
|
||||
# MinIO / S3 object storage
|
||||
MINIO_ENDPOINT: "minio:9000"
|
||||
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
MINIO_USE_SSL: "false"
|
||||
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
|
||||
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
|
||||
# PocketBase
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- "${SCRAPER_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- static_books:/app/static/books
|
||||
healthcheck:
|
||||
@@ -78,5 +130,33 @@ services:
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# ─── SvelteKit UI ────────────────────────────────────────────────────────────
|
||||
ui:
|
||||
build:
|
||||
context: ./ui
|
||||
dockerfile: Dockerfile
|
||||
container_name: libnovel-ui
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
scraper:
|
||||
condition: service_healthy
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
SCRAPER_API_URL: "http://scraper:8080"
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
PUBLIC_MINIO_PUBLIC_URL: "${PUBLIC_MINIO_PUBLIC_URL:-http://localhost:9000}"
|
||||
ports:
|
||||
- "${UI_PORT:-3000}:3000"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:3000/"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
volumes:
|
||||
static_books:
|
||||
minio_data:
|
||||
pb_data:
|
||||
|
||||
@@ -10,16 +10,24 @@
|
||||
//
|
||||
// Environment variables:
|
||||
//
|
||||
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||
// BROWSERLESS_TOKEN Browserless API token (default: "")
|
||||
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
|
||||
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||
// BROWSERLESS_TOKEN Browserless API token (default: "")
|
||||
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
|
||||
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
|
||||
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
|
||||
// SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
|
||||
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
|
||||
// KOKORO_VOICE Default TTS voice (default: af_bella)
|
||||
// LOG_LEVEL debug | info | warn | error (default: info)
|
||||
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
|
||||
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
|
||||
// KOKORO_VOICE Default TTS voice (default: af_bella)
|
||||
// POCKETBASE_URL PocketBase API base URL (default: http://localhost:8090)
|
||||
// POCKETBASE_EMAIL PocketBase admin email (default: admin@libnovel.local)
|
||||
// POCKETBASE_PASSWORD PocketBase admin password (default: adminpassword)
|
||||
// MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000)
|
||||
// MINIO_ACCESS_KEY MinIO access key (default: minioadmin)
|
||||
// MINIO_SECRET_KEY MinIO secret key (default: minioadmin)
|
||||
// MINIO_USE_SSL Use TLS for MinIO (default: false)
|
||||
// MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters)
|
||||
// MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio)
|
||||
// LOG_LEVEL debug | info | warn | error (default: info)
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -38,7 +46,7 @@ import (
|
||||
"github.com/libnovel/scraper/internal/novelfire"
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/server"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -88,9 +96,30 @@ func run(log *slog.Logger) error {
|
||||
bc := newBrowserClient(strategy, browserCfg)
|
||||
urlClient := newBrowserClient(urlStrategy, browserCfg)
|
||||
|
||||
staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books")
|
||||
w := writer.New(staticRoot)
|
||||
nf := novelfire.New(bc, log, urlClient, w)
|
||||
// ── Storage backends ────────────────────────────────────────────────────
|
||||
minioCfg := storage.MinioConfig{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "minioadmin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "minioadmin"),
|
||||
UseSSL: strings.ToLower(os.Getenv("MINIO_USE_SSL")) == "true",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
pbCfg := storage.PocketBaseConfig{
|
||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||
AdminEmail: envOr("POCKETBASE_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_PASSWORD", "adminpassword"),
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
store, err := storage.NewHybridStore(ctx, pbCfg, minioCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage init failed: %w", err)
|
||||
}
|
||||
|
||||
nf := novelfire.New(bc, log, urlClient, &rankingCacheAdapter{store: store})
|
||||
|
||||
workers := 0
|
||||
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
|
||||
@@ -104,13 +133,9 @@ func run(log *slog.Logger) error {
|
||||
}
|
||||
|
||||
oCfg := orchestrator.Config{
|
||||
Workers: workers,
|
||||
StaticRoot: staticRoot,
|
||||
Workers: workers,
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
switch cmd {
|
||||
case "run":
|
||||
// Optional --url flag.
|
||||
@@ -121,10 +146,9 @@ func run(log *slog.Logger) error {
|
||||
"strategy", strategy,
|
||||
"workers", workers,
|
||||
"max_concurrent", browserCfg.MaxConcurrent,
|
||||
"static_root", oCfg.StaticRoot,
|
||||
"single_book", oCfg.SingleBookURL,
|
||||
)
|
||||
o := orchestrator.New(oCfg, nf, log)
|
||||
o := orchestrator.New(oCfg, nf, log, store)
|
||||
return o.Run(ctx)
|
||||
|
||||
case "refresh":
|
||||
@@ -133,13 +157,12 @@ func run(log *slog.Logger) error {
|
||||
return fmt.Errorf("refresh command requires a book slug argument")
|
||||
}
|
||||
slug := args[1]
|
||||
w := writer.New(oCfg.StaticRoot)
|
||||
meta, ok, err := w.ReadMetadata(slug)
|
||||
meta, ok, err := store.ReadMetadata(ctx, slug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read metadata for %s: %w", slug, err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot)
|
||||
return fmt.Errorf("book %q not found in store", slug)
|
||||
}
|
||||
if meta.SourceURL == "" {
|
||||
return fmt.Errorf("book %q has no source_url in metadata", slug)
|
||||
@@ -149,12 +172,12 @@ func run(log *slog.Logger) error {
|
||||
"slug", slug,
|
||||
"source_url", meta.SourceURL,
|
||||
)
|
||||
o := orchestrator.New(oCfg, nf, log)
|
||||
o := orchestrator.New(oCfg, nf, log, store)
|
||||
return o.Run(ctx)
|
||||
|
||||
case "serve":
|
||||
addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
|
||||
kokoroURL := envOr("KOKORO_URL", "")
|
||||
kokoroURL := envOr("KOKORO_URL", "https://kokoro.kalekber.cc")
|
||||
kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
|
||||
log.Info("starting HTTP server",
|
||||
"addr", addr,
|
||||
@@ -164,7 +187,7 @@ func run(log *slog.Logger) error {
|
||||
"kokoro_url", kokoroURL,
|
||||
"kokoro_voice", kokoroVoice,
|
||||
)
|
||||
srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice)
|
||||
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
|
||||
return srv.ListenAndServe(ctx)
|
||||
|
||||
default:
|
||||
@@ -192,6 +215,20 @@ func envOr(key, fallback string) string {
|
||||
return fallback
|
||||
}
|
||||
|
||||
// rankingCacheAdapter bridges storage.HybridStore (context-aware) to the
|
||||
// context-free scraper.RankingPageCacher interface expected by novelfire.New.
|
||||
type rankingCacheAdapter struct {
|
||||
store *storage.HybridStore
|
||||
}
|
||||
|
||||
func (a *rankingCacheAdapter) WriteRankingPageCache(page int, html string) error {
|
||||
return a.store.WriteRankingPageCache(context.Background(), page, html)
|
||||
}
|
||||
|
||||
func (a *rankingCacheAdapter) ReadRankingPageCache(page int) (string, error) {
|
||||
return a.store.ReadRankingPageCache(context.Background(), page)
|
||||
}
|
||||
|
||||
func printUsage() {
|
||||
fmt.Fprintf(os.Stderr, `libnovel scraper
|
||||
|
||||
|
||||
@@ -3,8 +3,28 @@ module github.com/libnovel/scraper
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/yuin/goldmark v1.7.16 // indirect
|
||||
golang.org/x/net v0.51.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/minio/minio-go/v7 v7.0.98
|
||||
golang.org/x/net v0.51.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,9 +1,47 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE=
|
||||
github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
|
||||
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
|
||||
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// Config holds tunable parameters for the orchestrator.
|
||||
@@ -28,7 +28,8 @@ type Config struct {
|
||||
// Defaults to runtime.NumCPU() when 0.
|
||||
Workers int
|
||||
|
||||
// StaticRoot is the path to the static/books output directory.
|
||||
// StaticRoot is kept for backwards-compatibility but is no longer used
|
||||
// when a Store is provided.
|
||||
StaticRoot string
|
||||
|
||||
// SingleBookURL when non-empty causes the orchestrator to scrape only
|
||||
@@ -40,13 +41,13 @@ type Config struct {
|
||||
type Orchestrator struct {
|
||||
cfg Config
|
||||
novel scraper.NovelScraper
|
||||
writer *writer.Writer
|
||||
store storage.Store
|
||||
log *slog.Logger
|
||||
workers int
|
||||
}
|
||||
|
||||
// New returns a new Orchestrator.
|
||||
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator {
|
||||
// New returns a new Orchestrator backed by the provided Store.
|
||||
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store) *Orchestrator {
|
||||
workers := cfg.Workers
|
||||
if workers <= 0 {
|
||||
workers = runtime.NumCPU()
|
||||
@@ -54,7 +55,7 @@ func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator
|
||||
return &Orchestrator{
|
||||
cfg: cfg,
|
||||
novel: novel,
|
||||
writer: writer.New(cfg.StaticRoot),
|
||||
store: store,
|
||||
log: log,
|
||||
workers: workers,
|
||||
}
|
||||
@@ -66,7 +67,6 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
o.log.Info("orchestrator starting",
|
||||
"source", o.novel.SourceName(),
|
||||
"workers", o.workers,
|
||||
"static_root", o.cfg.StaticRoot,
|
||||
)
|
||||
|
||||
// chapterWork is the shared queue consumed by chapter worker goroutines.
|
||||
@@ -89,8 +89,8 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
default:
|
||||
}
|
||||
|
||||
// Skip if already on disk.
|
||||
if o.writer.ChapterExists(job.slug, job.ref) {
|
||||
// Skip if already stored.
|
||||
if o.store.ChapterExists(ctx, job.slug, job.ref) {
|
||||
o.log.Debug("chapter already exists, skipping",
|
||||
"book", job.slug, "chapter", job.ref.Number)
|
||||
continue
|
||||
@@ -107,7 +107,7 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := o.writer.WriteChapter(job.slug, chapter); err != nil {
|
||||
if err := o.store.WriteChapter(ctx, job.slug, chapter); err != nil {
|
||||
o.log.Error("chapter write failed",
|
||||
"book", job.slug,
|
||||
"chapter", job.ref.Number,
|
||||
@@ -135,8 +135,8 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
return
|
||||
}
|
||||
|
||||
// Persist / update metadata.yaml.
|
||||
if err := o.writer.WriteMetadata(meta); err != nil {
|
||||
// Persist / update metadata.
|
||||
if err := o.store.WriteMetadata(ctx, meta); err != nil {
|
||||
o.log.Error("metadata write failed", "slug", meta.Slug, "err", err)
|
||||
// Continue — chapters can still be scraped.
|
||||
}
|
||||
|
||||
62
scraper/internal/server/helpers.go
Normal file
62
scraper/internal/server/helpers.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// kokoroVoices is the built-in fallback list of voices shipped with Kokoro-FastAPI.
|
||||
// Used when the live GET /v1/audio/voices request to Kokoro fails.
|
||||
// Grouped by language prefix:
|
||||
//
|
||||
// af_ / am_ American English female / male
|
||||
// bf_ / bm_ British English female / male
|
||||
// ef_ / em_ Spanish female / male
|
||||
// ff_ French female
|
||||
// hf_ / hm_ Hindi female / male
|
||||
// if_ / im_ Italian female / male
|
||||
// jf_ / jm_ Japanese female / male
|
||||
// pf_ / pm_ Portuguese female / male
|
||||
// zf_ / zm_ Chinese female / male
|
||||
var kokoroVoices = []string{
|
||||
// American English
|
||||
"af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia",
|
||||
"af_jessica", "af_kore", "af_nicole", "af_nova", "af_river",
|
||||
"af_sarah", "af_sky",
|
||||
"am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam",
|
||||
"am_michael", "am_onyx", "am_puck",
|
||||
// British English
|
||||
"bf_alice", "bf_emma", "bf_lily",
|
||||
"bm_daniel", "bm_fable", "bm_george", "bm_lewis",
|
||||
// Spanish
|
||||
"ef_dora", "em_alex",
|
||||
// French
|
||||
"ff_siwis",
|
||||
// Hindi
|
||||
"hf_alpha", "hf_beta", "hm_omega", "hm_psi",
|
||||
// Italian
|
||||
"if_sara", "im_nicola",
|
||||
// Japanese
|
||||
"jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo",
|
||||
// Portuguese
|
||||
"pf_dora", "pm_alex",
|
||||
// Chinese
|
||||
"zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi",
|
||||
"zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang",
|
||||
}
|
||||
|
||||
// stripMarkdown removes common markdown syntax from src, returning plain text
|
||||
// suitable for TTS or display. Not a full markdown parser — handles the most
|
||||
// common constructs (headings, bold/italic, code blocks, links, blockquotes).
|
||||
func stripMarkdown(src string) string {
|
||||
src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1")
|
||||
src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n")
|
||||
return strings.TrimSpace(src)
|
||||
}
|
||||
@@ -1,15 +1,25 @@
|
||||
// Package server exposes the scraper as an HTTP service.
|
||||
// Package server exposes the scraper as an HTTP API service.
|
||||
//
|
||||
// Endpoints:
|
||||
//
|
||||
// POST /scrape — enqueue a full catalogue scrape
|
||||
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
|
||||
// GET /health — liveness probe
|
||||
// POST /scrape — enqueue a full catalogue scrape
|
||||
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
|
||||
// GET /health — liveness probe
|
||||
// GET /api/progress — get reading progress map (session-scoped)
|
||||
// POST /api/progress/{slug} — set reading progress
|
||||
// DELETE /api/progress/{slug} — delete reading progress
|
||||
// GET /api/presign/chapter/{slug}/{n} — presigned MinIO URL for chapter markdown
|
||||
// GET /api/presign/audio/{slug}/{n} — presigned MinIO URL for chapter audio
|
||||
// GET /api/chapter-text/{slug}/{n} — plain text of chapter (markdown stripped)
|
||||
// POST /api/audio/{slug}/{n} — trigger Kokoro audio generation
|
||||
// GET /api/audio-proxy/{slug}/{n} — proxy generated audio from Kokoro
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -22,7 +32,7 @@ import (
|
||||
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// Server wraps an HTTP mux with the scraping endpoints.
|
||||
@@ -31,7 +41,7 @@ type Server struct {
|
||||
oCfg orchestrator.Config
|
||||
novel scraper.NovelScraper
|
||||
log *slog.Logger
|
||||
writer *writer.Writer
|
||||
store storage.Store
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
rankingRunning bool
|
||||
@@ -42,26 +52,23 @@ type Server struct {
|
||||
voiceMu sync.RWMutex
|
||||
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
|
||||
|
||||
// audioMu guards audioCache and audioInFlight.
|
||||
// audioCache maps a cache key to the Kokoro download filename returned by
|
||||
// POST /v1/audio/speech with return_download_link=true.
|
||||
// audioMu guards audioInFlight only.
|
||||
// Completed audio filenames are persisted to the Store (PocketBase).
|
||||
// audioInFlight deduplicates concurrent generation requests for the same key.
|
||||
audioMu sync.Mutex
|
||||
audioCache map[string]string // cacheKey → kokoro download filename
|
||||
audioInFlight map[string]chan struct{} // cacheKey → closed when done
|
||||
}
|
||||
|
||||
// New creates a new Server.
|
||||
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server {
|
||||
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
|
||||
return &Server{
|
||||
addr: addr,
|
||||
oCfg: oCfg,
|
||||
novel: novel,
|
||||
log: log,
|
||||
writer: writer.New(oCfg.StaticRoot),
|
||||
store: store,
|
||||
kokoroURL: kokoroURL,
|
||||
kokoroVoice: kokoroVoice,
|
||||
audioCache: make(map[string]string),
|
||||
audioInFlight: make(map[string]chan struct{}),
|
||||
}
|
||||
}
|
||||
@@ -112,20 +119,15 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /health", s.handleHealth)
|
||||
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
||||
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
||||
// UI routes
|
||||
mux.HandleFunc("GET /", s.handleHome)
|
||||
mux.HandleFunc("GET /scrape", s.handleScrape)
|
||||
mux.HandleFunc("GET /ranking", s.handleRanking)
|
||||
mux.HandleFunc("POST /ranking/refresh", s.handleRankingRefresh)
|
||||
mux.HandleFunc("GET /ranking/view", s.handleRankingView)
|
||||
mux.HandleFunc("GET /books/{slug}", s.handleBook)
|
||||
mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter)
|
||||
mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage)
|
||||
mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
|
||||
mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
|
||||
mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus)
|
||||
// Plain-text chapter content for browser-side TTS
|
||||
mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)
|
||||
// Progress API
|
||||
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
||||
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
||||
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
|
||||
// Presigned URL API (for SvelteKit UI)
|
||||
mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter)
|
||||
mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio)
|
||||
// Plain-text chapter content (used server-side for audio generation)
|
||||
mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText)
|
||||
// Server-side audio generation via Kokoro /v1/audio/speech.
|
||||
// Generation can take several minutes, so wrap in its own timeout handler.
|
||||
audioGenHandler := http.TimeoutHandler(
|
||||
@@ -133,9 +135,9 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
10*time.Minute,
|
||||
`{"error":"audio generation timed out"}`,
|
||||
)
|
||||
mux.Handle("POST /ui/audio/{slug}/{n}", audioGenHandler)
|
||||
mux.Handle("POST /api/audio/{slug}/{n}", audioGenHandler)
|
||||
// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
|
||||
mux.HandleFunc("GET /ui/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
||||
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: s.addr,
|
||||
@@ -165,8 +167,125 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// ─── Session cookie helpers ───────────────────────────────────────────────────
|
||||
|
||||
const sessionCookieName = "libnovel_session"
|
||||
|
||||
// sessionID returns the session ID from the request cookie, or "" if absent.
|
||||
func sessionID(r *http.Request) string {
|
||||
c, err := r.Cookie(sessionCookieName)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return c.Value
|
||||
}
|
||||
|
||||
// newSessionID generates a random 16-byte hex session ID.
|
||||
func newSessionID() (string, error) {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// ensureSession issues a new session cookie if the request does not already
|
||||
// carry one, and returns the session ID (either existing or newly issued).
|
||||
func ensureSession(w http.ResponseWriter, r *http.Request) string {
|
||||
if id := sessionID(r); id != "" {
|
||||
return id
|
||||
}
|
||||
id, err := newSessionID()
|
||||
if err != nil {
|
||||
// Very unlikely, but fall back to a timestamp-based ID.
|
||||
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: sessionCookieName,
|
||||
Value: id,
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
MaxAge: 365 * 24 * 60 * 60, // 1 year
|
||||
})
|
||||
return id
|
||||
}
|
||||
|
||||
// ─── Reading progress API ─────────────────────────────────────────────────────
|
||||
|
||||
// handleGetProgress handles GET /api/progress.
|
||||
// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...}
|
||||
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
entries, err := s.store.AllProgress(r.Context(), sid)
|
||||
if err != nil {
|
||||
s.log.Error("AllProgress failed", "err", err)
|
||||
entries = nil
|
||||
}
|
||||
|
||||
progress := make(map[string]interface{}, len(entries)*2)
|
||||
for _, p := range entries {
|
||||
progress[p.Slug] = p.Chapter
|
||||
progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(progress)
|
||||
}
|
||||
|
||||
// handleSetProgress handles POST /api/progress/{slug}.
|
||||
// Body: {"chapter": N}
|
||||
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Chapter int `json:"chapter"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
|
||||
http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
p := storage.ReadingProgress{
|
||||
Slug: slug,
|
||||
Chapter: body.Chapter,
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
if err := s.store.SetProgress(r.Context(), sid, p); err != nil {
|
||||
s.log.Error("SetProgress failed", "slug", slug, "err", err)
|
||||
http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{})
|
||||
}
|
||||
|
||||
// handleDeleteProgress handles DELETE /api/progress/{slug}.
|
||||
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil {
|
||||
s.log.Error("DeleteProgress failed", "slug", slug, "err", err)
|
||||
// Non-fatal — treat as success.
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{})
|
||||
}
|
||||
|
||||
// handleChapterText returns the plain text of a chapter (markdown stripped)
|
||||
// for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI.
|
||||
// for server-side audio generation. Called by handleAudioGenerate internally.
|
||||
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
@@ -174,7 +293,7 @@ func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
raw, err := s.writer.ReadChapter(slug, n)
|
||||
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
@@ -186,7 +305,7 @@ func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// ─── Audio generation via Kokoro /v1/audio/speech ────────────────────────────
|
||||
//
|
||||
// handleAudioGenerate handles POST /ui/audio/{slug}/{n}.
|
||||
// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
|
||||
//
|
||||
// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
|
||||
// Kokoro generates the audio, saves it to its own temp storage, and returns
|
||||
@@ -223,15 +342,14 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
||||
|
||||
// Fast path: already generated this session.
|
||||
s.audioMu.Lock()
|
||||
if filename, ok := s.audioCache[cacheKey]; ok {
|
||||
s.audioMu.Unlock()
|
||||
// Fast path: already generated (check persistent store first).
|
||||
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
||||
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
||||
return
|
||||
}
|
||||
|
||||
// Deduplicate concurrent generation for the same key.
|
||||
s.audioMu.Lock()
|
||||
if ch, ok := s.audioInFlight[cacheKey]; ok {
|
||||
s.audioMu.Unlock()
|
||||
select {
|
||||
@@ -240,10 +358,8 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
s.audioMu.Lock()
|
||||
filename, ok := s.audioCache[cacheKey]
|
||||
s.audioMu.Unlock()
|
||||
if ok {
|
||||
// Check store again after waiting.
|
||||
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
||||
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
||||
} else {
|
||||
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
|
||||
@@ -262,7 +378,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
}()
|
||||
|
||||
// Load and validate chapter text.
|
||||
raw, err := s.writer.ReadChapter(slug, n)
|
||||
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
|
||||
return
|
||||
@@ -287,9 +403,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
s.audioMu.Lock()
|
||||
s.audioCache[cacheKey] = filename
|
||||
s.audioMu.Unlock()
|
||||
_ = s.store.SetAudioCache(r.Context(), cacheKey, filename)
|
||||
|
||||
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
|
||||
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
||||
@@ -345,9 +459,9 @@ func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed f
|
||||
}
|
||||
|
||||
// writeAudioResponse writes the JSON response for a generated audio chapter.
|
||||
// The URL points to our proxy handler which fetches from Kokoro on demand.
|
||||
// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
|
||||
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) {
|
||||
proxyURL := fmt.Sprintf("/ui/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed)
|
||||
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"url": proxyURL,
|
||||
@@ -355,7 +469,7 @@ func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, v
|
||||
})
|
||||
}
|
||||
|
||||
// handleAudioProxy handles GET /ui/audio-proxy/{slug}/{n}.
|
||||
// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}.
|
||||
// It looks up the Kokoro download filename for this chapter (voice/speed) and
|
||||
// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
|
||||
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -378,10 +492,7 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
||||
s.audioMu.Lock()
|
||||
filename, ok := s.audioCache[cacheKey]
|
||||
s.audioMu.Unlock()
|
||||
|
||||
filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
|
||||
if !ok {
|
||||
http.Error(w, "audio not generated yet", http.StatusNotFound)
|
||||
return
|
||||
@@ -414,6 +525,64 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
// ─── Presigned URL handlers ───────────────────────────────────────────────────
|
||||
|
||||
// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}.
|
||||
// Returns a short-lived presigned MinIO URL for the chapter markdown object.
|
||||
// The SvelteKit server uses this to fetch chapter content server-side.
|
||||
func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
url, err := s.store.PresignChapter(r.Context(), slug, n, 15*time.Minute)
|
||||
if err != nil {
|
||||
s.log.Error("presign chapter failed", "slug", slug, "n", n, "err", err)
|
||||
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
||||
}
|
||||
|
||||
// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}.
|
||||
// Returns a presigned MinIO URL for the audio object (if it has been generated).
|
||||
// Query params: voice, speed (optional, defaults to server defaults).
|
||||
func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
voice := r.URL.Query().Get("voice")
|
||||
if voice == "" {
|
||||
voice = s.kokoroVoice
|
||||
}
|
||||
speed := 1.0
|
||||
if sv := r.URL.Query().Get("speed"); sv != "" {
|
||||
if v, err := strconv.ParseFloat(sv, 64); err == nil && v > 0 {
|
||||
speed = v
|
||||
}
|
||||
}
|
||||
|
||||
key := s.store.AudioObjectKey(slug, n, voice, speed)
|
||||
url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
|
||||
if err != nil {
|
||||
s.log.Error("presign audio failed", "slug", slug, "n", n, "err", err)
|
||||
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
||||
}
|
||||
|
||||
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = "" // full catalogue
|
||||
@@ -462,7 +631,7 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
o := orchestrator.New(cfg, s.novel, s.log)
|
||||
o := orchestrator.New(cfg, s.novel, s.log, s.store)
|
||||
if err := o.Run(ctx); err != nil {
|
||||
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err))
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
310
scraper/internal/storage/hybrid.go
Normal file
310
scraper/internal/storage/hybrid.go
Normal file
@@ -0,0 +1,310 @@
|
||||
// hybrid.go implements the Store interface using PocketBase for structured data
|
||||
// and MinIO for binary chapter/audio blobs.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// HybridStore satisfies Store by routing structured data to PocketBase and
|
||||
// binary objects (chapters, audio) to MinIO.
|
||||
type HybridStore struct {
|
||||
pb *PocketBaseStore
|
||||
minio *MinioClient
|
||||
}
|
||||
|
||||
// NewHybridStore constructs a HybridStore. It connects to both backends and
|
||||
// calls EnsureCollections to bootstrap any missing PocketBase collections.
|
||||
func NewHybridStore(ctx context.Context, pbCfg PocketBaseConfig, minioCfg MinioConfig) (*HybridStore, error) {
|
||||
mc, err := NewMinioClient(ctx, minioCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: minio: %w", err)
|
||||
}
|
||||
pb := NewPocketBaseStore(pbCfg)
|
||||
if err := pb.EnsureCollections(ctx); err != nil {
|
||||
// Log but don't fail — collection creation errors are often "already exists"
|
||||
_ = err
|
||||
}
|
||||
return &HybridStore{pb: pb, minio: mc}, nil
|
||||
}
|
||||
|
||||
// ─── Book metadata ────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteMetadata(ctx context.Context, meta scraper.BookMeta) error {
|
||||
return h.pb.UpsertBook(ctx,
|
||||
meta.Slug, meta.Title, meta.Author, meta.Cover,
|
||||
meta.Status, meta.Summary, meta.SourceURL,
|
||||
meta.Genres, meta.TotalChapters, meta.Ranking,
|
||||
)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) {
|
||||
rec, found, err := h.pb.GetBook(ctx, slug)
|
||||
if err != nil || !found {
|
||||
return scraper.BookMeta{}, found, err
|
||||
}
|
||||
return recToBookMeta(rec), true, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListBooks(ctx context.Context) ([]scraper.BookMeta, error) {
|
||||
rows, err := h.pb.ListBooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
books := make([]scraper.BookMeta, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
books = append(books, recToBookMeta(r))
|
||||
}
|
||||
return books, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) LocalSlugs(ctx context.Context) (map[string]bool, error) {
|
||||
books, err := h.ListBooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slugs := make(map[string]bool, len(books))
|
||||
for _, b := range books {
|
||||
slugs[b.Slug] = true
|
||||
}
|
||||
return slugs, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) MetadataMtime(ctx context.Context, slug string) int64 {
|
||||
t, err := h.pb.BookMetaUpdated(ctx, slug)
|
||||
if err != nil || t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
// ─── Chapters ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool {
|
||||
return h.minio.ChapterExists(ctx, slug, ref.Volume, ref.Number)
|
||||
}
|
||||
|
||||
func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error {
|
||||
content := "# " + chapter.Ref.Title + "\n\n" + chapter.Text + "\n"
|
||||
if err := h.minio.PutChapter(ctx, slug, chapter.Ref.Volume, chapter.Ref.Number, content); err != nil {
|
||||
return err
|
||||
}
|
||||
// Update chapter index in PocketBase.
|
||||
title, dateLabel := splitChapterTitle(chapter.Ref.Title)
|
||||
_ = h.pb.UpsertChapterIdx(ctx, slug, chapter.Ref.Number, title, dateLabel)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
|
||||
return h.minio.GetChapter(ctx, slug, 0, n)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) {
|
||||
rows, err := h.pb.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
infos := make([]ChapterInfo, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
n := int(floatVal(r, "number"))
|
||||
title, _ := r["title"].(string)
|
||||
date, _ := r["date_label"].(string)
|
||||
infos = append(infos, ChapterInfo{Number: n, Title: title, Date: date})
|
||||
}
|
||||
sort.Slice(infos, func(i, j int) bool { return infos[i].Number < infos[j].Number })
|
||||
return infos, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) CountChapters(ctx context.Context, slug string) int {
|
||||
return h.pb.CountChapterIdx(ctx, slug)
|
||||
}
|
||||
|
||||
// ─── Ranking ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteRanking(ctx context.Context, items []RankingItem) error {
|
||||
data, err := json.Marshal(items)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: marshal ranking: %w", err)
|
||||
}
|
||||
return h.pb.SetRanking(ctx, string(data))
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadRankingItems(ctx context.Context) ([]RankingItem, error) {
|
||||
dataStr, _, err := h.pb.GetRanking(ctx)
|
||||
if err != nil || dataStr == "" {
|
||||
return nil, err
|
||||
}
|
||||
var items []RankingItem
|
||||
if err := json.Unmarshal([]byte(dataStr), &items); err != nil {
|
||||
return nil, fmt.Errorf("storage: unmarshal ranking: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) RankingFileInfo(ctx context.Context) (os.FileInfo, error) {
|
||||
return h.pb.RankingModTime(ctx)
|
||||
}
|
||||
|
||||
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteRankingPageCache(ctx context.Context, page int, html string) error {
|
||||
return h.pb.SetRankingPageHTML(ctx, page, html)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadRankingPageCache(ctx context.Context, page int) (string, error) {
|
||||
html, _, err := h.pb.GetRankingPageHTML(ctx, page)
|
||||
return html, err
|
||||
}
|
||||
|
||||
func (h *HybridStore) RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error) {
|
||||
return h.pb.RankingPageCacheModTime(ctx, page)
|
||||
}
|
||||
|
||||
// ─── Audio cache ──────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool) {
|
||||
filename, ok, _ := h.pb.GetAudioCache(ctx, cacheKey)
|
||||
return filename, ok
|
||||
}
|
||||
|
||||
func (h *HybridStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
|
||||
return h.pb.SetAudioCache(ctx, cacheKey, filename)
|
||||
}
|
||||
|
||||
// ─── Reading progress ─────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) {
|
||||
ch, updated, ok, err := h.pb.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil || !ok {
|
||||
return ReadingProgress{}, false
|
||||
}
|
||||
return ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}, true
|
||||
}
|
||||
|
||||
func (h *HybridStore) SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error {
|
||||
return h.pb.SetProgress(ctx, sessionID, p.Slug, p.Chapter)
|
||||
}
|
||||
|
||||
func (h *HybridStore) AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) {
|
||||
rows, err := h.pb.AllProgress(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]ReadingProgress, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
slug, _ := r["slug"].(string)
|
||||
ch := int(floatVal(r, "chapter"))
|
||||
var updated time.Time
|
||||
if ts, ok := r["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
out = append(out, ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
|
||||
return h.pb.DeleteProgress(ctx, sessionID, slug)
|
||||
}
|
||||
|
||||
// ─── AudioObjectKey ───────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) AudioObjectKey(slug string, n int, voice string, speed float64) string {
|
||||
return AudioObjectKey(slug, n, voice, speed)
|
||||
}
|
||||
|
||||
// ─── Presigned URLs ───────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) {
|
||||
return h.minio.PresignChapter(ctx, slug, 0, n, expires)
|
||||
}
|
||||
|
||||
func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
|
||||
return h.minio.PresignAudio(ctx, key, expires)
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func recToBookMeta(rec map[string]interface{}) scraper.BookMeta {
|
||||
m := scraper.BookMeta{
|
||||
Slug: strVal(rec, "slug"),
|
||||
Title: strVal(rec, "title"),
|
||||
Author: strVal(rec, "author"),
|
||||
Cover: strVal(rec, "cover"),
|
||||
Status: strVal(rec, "status"),
|
||||
Summary: strVal(rec, "summary"),
|
||||
SourceURL: strVal(rec, "source_url"),
|
||||
}
|
||||
if tc := floatVal(rec, "total_chapters"); tc > 0 {
|
||||
m.TotalChapters = int(tc)
|
||||
}
|
||||
if rk := floatVal(rec, "ranking"); rk > 0 {
|
||||
m.Ranking = int(rk)
|
||||
}
|
||||
// Genres stored as JSON string or array.
|
||||
switch v := rec["genres"].(type) {
|
||||
case string:
|
||||
_ = json.Unmarshal([]byte(v), &m.Genres)
|
||||
case []interface{}:
|
||||
for _, g := range v {
|
||||
if s, ok := g.(string); ok {
|
||||
m.Genres = append(m.Genres, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func strVal(m map[string]interface{}, key string) string {
|
||||
if v, ok := m[key].(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// splitChapterTitle mirrors writer.SplitChapterTitle logic (simplified).
|
||||
func splitChapterTitle(raw string) (title, date string) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
// Strip leading numeric index.
|
||||
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
|
||||
prefix := raw[:idx]
|
||||
allDigit := true
|
||||
for _, c := range prefix {
|
||||
if c < '0' || c > '9' {
|
||||
allDigit = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allDigit {
|
||||
raw = strings.TrimSpace(raw[idx:])
|
||||
}
|
||||
}
|
||||
// Detect trailing relative date.
|
||||
units := []string{"second", "minute", "hour", "day", "week", "month", "year"}
|
||||
lower := strings.ToLower(raw)
|
||||
for _, u := range units {
|
||||
for _, suffix := range []string{u + "s ago", u + " ago"} {
|
||||
if idx := strings.LastIndex(lower, suffix); idx > 0 {
|
||||
// Find start of date token (digit before the unit).
|
||||
start := strings.LastIndex(raw[:idx], " ")
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
numPart := strings.TrimSpace(raw[start:idx])
|
||||
if _, err := strconv.Atoi(strings.Fields(numPart)[0]); err == nil {
|
||||
return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return raw, ""
|
||||
}
|
||||
203
scraper/internal/storage/minio.go
Normal file
203
scraper/internal/storage/minio.go
Normal file
@@ -0,0 +1,203 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// MinioConfig holds connection parameters for MinIO.
|
||||
type MinioConfig struct {
|
||||
Endpoint string // e.g. "minio:9000"
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
UseSSL bool
|
||||
BucketChapters string // e.g. "libnovel-chapters"
|
||||
BucketAudio string // e.g. "libnovel-audio"
|
||||
}
|
||||
|
||||
// MinioClient wraps a minio.Client and exposes object operations for
|
||||
// chapters and audio files.
|
||||
type MinioClient struct {
|
||||
c *minio.Client
|
||||
cfg MinioConfig
|
||||
}
|
||||
|
||||
// NewMinioClient creates a connected MinIO client and ensures the required
|
||||
// buckets exist.
|
||||
func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error) {
|
||||
c, err := minio.New(cfg.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio: new client: %w", err)
|
||||
}
|
||||
|
||||
mc := &MinioClient{c: c, cfg: cfg}
|
||||
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio} {
|
||||
if err := mc.ensureBucket(ctx, bucket); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return mc, nil
|
||||
}
|
||||
|
||||
// ensureBucket creates a bucket if it does not exist.
|
||||
func (m *MinioClient) ensureBucket(ctx context.Context, bucket string) error {
|
||||
exists, err := m.c.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: bucket exists %q: %w", bucket, err)
|
||||
}
|
||||
if !exists {
|
||||
if err := m.c.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
return fmt.Errorf("minio: make bucket %q: %w", bucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── Chapter objects ──────────────────────────────────────────────────────────
|
||||
|
||||
// chapterKey returns the MinIO object key for a chapter.
|
||||
// Layout: {slug}/vol-{vol}/{lo}-{hi}/chapter-{n}.md
|
||||
func chapterKey(slug string, vol, n int) string {
|
||||
const chaptersPerFolder = 50
|
||||
lo := ((n-1)/chaptersPerFolder)*chaptersPerFolder + 1
|
||||
hi := lo + chaptersPerFolder - 1
|
||||
return fmt.Sprintf("%s/vol-%d/%d-%d/chapter-%d.md", slug, vol, lo, hi, n)
|
||||
}
|
||||
|
||||
// PutChapter stores chapter markdown in MinIO.
|
||||
func (m *MinioClient) PutChapter(ctx context.Context, slug string, vol, n int, content string) error {
|
||||
key := chapterKey(slug, vol, n)
|
||||
data := []byte(content)
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketChapters, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: "text/markdown; charset=utf-8"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put chapter %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetChapter retrieves chapter markdown from MinIO.
|
||||
func (m *MinioClient) GetChapter(ctx context.Context, slug string, vol, n int) (string, error) {
|
||||
key := chapterKey(slug, vol, n)
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketChapters, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio: get chapter %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio: read chapter %s: %w", key, err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
// ChapterExists returns true if the object for this chapter is present.
|
||||
func (m *MinioClient) ChapterExists(ctx context.Context, slug string, vol, n int) bool {
|
||||
key := chapterKey(slug, vol, n)
|
||||
_, err := m.c.StatObject(ctx, m.cfg.BucketChapters, key, minio.StatObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ListChapterKeys returns all object keys under slug/ in the chapters bucket,
|
||||
// sorted lexicographically (MinIO returns them in order).
|
||||
func (m *MinioClient) ListChapterKeys(ctx context.Context, slug string) ([]string, error) {
|
||||
prefix := slug + "/"
|
||||
var keys []string
|
||||
for obj := range m.c.ListObjects(ctx, m.cfg.BucketChapters,
|
||||
minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
|
||||
if obj.Err != nil {
|
||||
return nil, fmt.Errorf("minio: list chapters %s: %w", slug, obj.Err)
|
||||
}
|
||||
keys = append(keys, obj.Key)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// CountChapters returns the number of chapter objects for a slug.
|
||||
func (m *MinioClient) CountChapters(ctx context.Context, slug string) int {
|
||||
keys, _ := m.ListChapterKeys(ctx, slug)
|
||||
return len(keys)
|
||||
}
|
||||
|
||||
// ─── Audio objects ────────────────────────────────────────────────────────────
|
||||
|
||||
// AudioObjectKey returns the MinIO key for a cached audio file.
|
||||
// Key: {slug}/ch{n}-{voice}-{speed:.1f}.mp3
|
||||
func AudioObjectKey(slug string, n int, voice string, speed float64) string {
|
||||
safe := sanitiseVoice(voice)
|
||||
return fmt.Sprintf("%s/ch%d-%s-%.1f.mp3", slug, n, safe, speed)
|
||||
}
|
||||
|
||||
// PutAudio stores an audio file in the audio bucket.
|
||||
func (m *MinioClient) PutAudio(ctx context.Context, key string, data []byte) error {
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketAudio, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: "audio/mpeg"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put audio %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAudio retrieves audio bytes from the audio bucket.
|
||||
func (m *MinioClient) GetAudio(ctx context.Context, key string) ([]byte, error) {
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketAudio, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio: get audio %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
return io.ReadAll(obj)
|
||||
}
|
||||
|
||||
// AudioExists returns true if the audio object is present in the bucket.
|
||||
func (m *MinioClient) AudioExists(ctx context.Context, key string) bool {
|
||||
_, err := m.c.StatObject(ctx, m.cfg.BucketAudio, key, minio.StatObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ─── Presigned URLs ───────────────────────────────────────────────────────────
|
||||
|
||||
// PresignChapter returns a presigned GET URL for a chapter object, valid for
|
||||
// the given duration. The URL is signed with the MinIO credentials and can be
|
||||
// fetched directly by the browser without authentication.
|
||||
func (m *MinioClient) PresignChapter(ctx context.Context, slug string, vol, n int, expires time.Duration) (string, error) {
|
||||
key := chapterKey(slug, vol, n)
|
||||
u, err := m.c.PresignedGetObject(ctx, m.cfg.BucketChapters, key, expires, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio: presign chapter %s: %w", key, err)
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
// PresignAudio returns a presigned GET URL for an audio object.
|
||||
func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
|
||||
u, err := m.c.PresignedGetObject(ctx, m.cfg.BucketAudio, key, expires, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio: presign audio %s: %w", key, err)
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// sanitiseVoice converts a voice name to a filename-safe string.
|
||||
func sanitiseVoice(voice string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
|
||||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '_'
|
||||
}, voice)
|
||||
}
|
||||
560
scraper/internal/storage/pocketbase.go
Normal file
560
scraper/internal/storage/pocketbase.go
Normal file
@@ -0,0 +1,560 @@
|
||||
// Package storage — PocketBase REST client.
|
||||
//
|
||||
// Collections expected in PocketBase:
|
||||
//
|
||||
// books — slug(text,unique), title, author, cover, status, genres(json),
|
||||
// summary, total_chapters(number), source_url, ranking(number), updated(date)
|
||||
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
|
||||
// ranking — data(json), updated(date) [single row, upserted by slug="_ranking_"]
|
||||
// ranking_html — page(number,unique), html(text), updated(date)
|
||||
// progress — session_id(text), slug(text), chapter(number), updated(date)
|
||||
// audio_cache — cache_key(text,unique), filename(text), updated(date)
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PocketBaseConfig holds PocketBase connection settings.
|
||||
type PocketBaseConfig struct {
|
||||
BaseURL string // e.g. "http://pocketbase:8090"
|
||||
AdminEmail string
|
||||
AdminPassword string
|
||||
}
|
||||
|
||||
// pbClient is a minimal PocketBase admin REST client.
|
||||
type pbClient struct {
|
||||
cfg PocketBaseConfig
|
||||
httpClient *http.Client
|
||||
|
||||
tokenMu sync.RWMutex
|
||||
token string
|
||||
tokenExp time.Time
|
||||
}
|
||||
|
||||
// newPBClient creates a new PocketBase client. It does not authenticate yet;
|
||||
// authentication happens lazily on the first API call.
|
||||
func newPBClient(cfg PocketBaseConfig) *pbClient {
|
||||
return &pbClient{
|
||||
cfg: cfg,
|
||||
httpClient: &http.Client{Timeout: 15 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Auth ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
func (p *pbClient) authenticate(ctx context.Context) error {
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
"identity": p.cfg.AdminEmail,
|
||||
"password": p.cfg.AdminPassword,
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
p.cfg.BaseURL+"/api/admins/auth-with-password", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pocketbase: auth: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pocketbase: auth status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var result struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return fmt.Errorf("pocketbase: decode auth: %w", err)
|
||||
}
|
||||
p.tokenMu.Lock()
|
||||
p.token = result.Token
|
||||
p.tokenExp = time.Now().Add(12 * time.Hour)
|
||||
p.tokenMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *pbClient) authToken(ctx context.Context) (string, error) {
|
||||
p.tokenMu.RLock()
|
||||
tok, exp := p.token, p.tokenExp
|
||||
p.tokenMu.RUnlock()
|
||||
if tok != "" && time.Now().Before(exp) {
|
||||
return tok, nil
|
||||
}
|
||||
if err := p.authenticate(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
p.tokenMu.RLock()
|
||||
defer p.tokenMu.RUnlock()
|
||||
return p.token, nil
|
||||
}
|
||||
|
||||
// ─── Generic CRUD helpers ──────────────────────────────────────────────────────
|
||||
|
||||
func (p *pbClient) do(ctx context.Context, method, path string, body interface{}) (*http.Response, error) {
|
||||
tok, err := p.authToken(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var bodyReader io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
bodyReader = bytes.NewReader(b)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, p.cfg.BaseURL+path, bodyReader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", tok)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
return p.httpClient.Do(req)
|
||||
}
|
||||
|
||||
// listOne fetches the first matching record from a collection.
|
||||
func (p *pbClient) listOne(ctx context.Context, collection, filter string) (map[string]interface{}, error) {
|
||||
q := url.Values{}
|
||||
q.Set("filter", filter)
|
||||
q.Set("perPage", "1")
|
||||
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||
resp, err := p.do(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
var result struct {
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(result.Items) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return result.Items[0], nil
|
||||
}
|
||||
|
||||
// listAll returns all records (up to 500) from a collection matching filter.
|
||||
func (p *pbClient) listAll(ctx context.Context, collection, filter, sort string) ([]map[string]interface{}, error) {
|
||||
q := url.Values{}
|
||||
if filter != "" {
|
||||
q.Set("filter", filter)
|
||||
}
|
||||
if sort != "" {
|
||||
q.Set("sort", sort)
|
||||
}
|
||||
q.Set("perPage", "500")
|
||||
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||
resp, err := p.do(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var result struct {
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result.Items, nil
|
||||
}
|
||||
|
||||
// upsert creates a record; if one matching filter already exists it updates it.
|
||||
func (p *pbClient) upsert(ctx context.Context, collection, filter string, data map[string]interface{}) error {
|
||||
existing, err := p.listOne(ctx, collection, filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != nil {
|
||||
id := existing["id"].(string)
|
||||
resp, err := p.do(ctx, http.MethodPatch,
|
||||
fmt.Sprintf("/api/collections/%s/records/%s", collection, id), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
resp, err := p.do(ctx, http.MethodPost,
|
||||
fmt.Sprintf("/api/collections/%s/records", collection), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteWhere deletes all records matching filter in collection.
|
||||
func (p *pbClient) deleteWhere(ctx context.Context, collection, filter string) error {
|
||||
items, err := p.listAll(ctx, collection, filter, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
id, _ := item["id"].(string)
|
||||
resp, err := p.do(ctx, http.MethodDelete,
|
||||
fmt.Sprintf("/api/collections/%s/records/%s", collection, id), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── PocketBaseStore ──────────────────────────────────────────────────────────
|
||||
|
||||
// PocketBaseStore implements the structured-data portion of the Store interface
|
||||
// backed by PocketBase REST API.
|
||||
type PocketBaseStore struct {
|
||||
pb *pbClient
|
||||
}
|
||||
|
||||
// NewPocketBaseStore returns a connected PocketBaseStore.
|
||||
func NewPocketBaseStore(cfg PocketBaseConfig) *PocketBaseStore {
|
||||
return &PocketBaseStore{pb: newPBClient(cfg)}
|
||||
}
|
||||
|
||||
// Ping verifies connectivity by authenticating.
|
||||
func (s *PocketBaseStore) Ping(ctx context.Context) error {
|
||||
_, err := s.pb.authToken(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// ─── Collections schema bootstrap ────────────────────────────────────────────
|
||||
// CollectionDef maps a collection name to its fields for auto-creation.
|
||||
|
||||
// EnsureCollections creates missing collections via the PocketBase API.
|
||||
// Safe to call on every startup — existing collections are skipped.
|
||||
func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
|
||||
// We just attempt to create each collection; 400/422 errors for "already
|
||||
// exists" are silently ignored.
|
||||
collections := []map[string]interface{}{
|
||||
{
|
||||
"name": "books",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "slug", "type": "text", "required": true, "options": map[string]interface{}{"min": 1}},
|
||||
{"name": "title", "type": "text", "required": true},
|
||||
{"name": "author", "type": "text"},
|
||||
{"name": "cover", "type": "url"},
|
||||
{"name": "status", "type": "text"},
|
||||
{"name": "genres", "type": "json"},
|
||||
{"name": "summary", "type": "text"},
|
||||
{"name": "total_chapters", "type": "number"},
|
||||
{"name": "source_url", "type": "url"},
|
||||
{"name": "ranking", "type": "number"},
|
||||
{"name": "meta_updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "chapters_idx",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "number", "type": "number", "required": true},
|
||||
{"name": "title", "type": "text"},
|
||||
{"name": "date_label", "type": "text"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "ranking",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "key", "type": "text", "required": true},
|
||||
{"name": "data", "type": "json"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "ranking_html",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "page", "type": "number", "required": true},
|
||||
{"name": "html", "type": "text"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "progress",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "session_id", "type": "text", "required": true},
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "chapter", "type": "number"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "audio_cache",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "cache_key", "type": "text", "required": true},
|
||||
{"name": "filename", "type": "text"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, col := range collections {
|
||||
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections", col)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pocketbase: ensure collection %v: %w", col["name"], err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
// 400/422 = already exists or schema mismatch — ignore
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── Book metadata ────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) UpsertBook(ctx context.Context, slug, title, author, cover, status, summary, sourceURL string, genres []string, totalChapters, ranking int) error {
|
||||
genresJSON, _ := json.Marshal(genres)
|
||||
return s.pb.upsert(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)), map[string]interface{}{
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"author": author,
|
||||
"cover": cover,
|
||||
"status": status,
|
||||
"genres": string(genresJSON),
|
||||
"summary": summary,
|
||||
"total_chapters": totalChapters,
|
||||
"source_url": sourceURL,
|
||||
"ranking": ranking,
|
||||
"meta_updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetBook(ctx context.Context, slug string) (map[string]interface{}, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if rec == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
return rec, true, nil
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) ListBooks(ctx context.Context) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "books", "", "+title")
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) BookMetaUpdated(ctx context.Context, slug string) (time.Time, error) {
|
||||
rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)))
|
||||
if err != nil || rec == nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if ts, ok := rec["meta_updated"].(string); ok {
|
||||
t, err := time.Parse(time.RFC3339, ts)
|
||||
if err == nil {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, nil
|
||||
}
|
||||
|
||||
// ─── Chapter index ────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) UpsertChapterIdx(ctx context.Context, slug string, number int, title, dateLabel string) error {
|
||||
return s.pb.upsert(ctx, "chapters_idx",
|
||||
fmt.Sprintf(`slug="%s"&&number=%d`, pbEsc(slug), number),
|
||||
map[string]interface{}{
|
||||
"slug": slug,
|
||||
"number": number,
|
||||
"title": title,
|
||||
"date_label": dateLabel,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) ListChapterIdx(ctx context.Context, slug string) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "chapters_idx",
|
||||
fmt.Sprintf(`slug="%s"`, pbEsc(slug)), "+number")
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) CountChapterIdx(ctx context.Context, slug string) int {
|
||||
rows, _ := s.ListChapterIdx(ctx, slug)
|
||||
return len(rows)
|
||||
}
|
||||
|
||||
// ─── Ranking ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) SetRanking(ctx context.Context, dataJSON string) error {
|
||||
return s.pb.upsert(ctx, "ranking", `key="_ranking_"`, map[string]interface{}{
|
||||
"key": "_ranking_",
|
||||
"data": dataJSON,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetRanking(ctx context.Context) (string, time.Time, error) {
|
||||
rec, err := s.pb.listOne(ctx, "ranking", `key="_ranking_"`)
|
||||
if err != nil || rec == nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
data, _ := rec["data"].(string)
|
||||
var updated time.Time
|
||||
if ts, ok := rec["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
return data, updated, nil
|
||||
}
|
||||
|
||||
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) SetRankingPageHTML(ctx context.Context, page int, html string) error {
|
||||
return s.pb.upsert(ctx, "ranking_html",
|
||||
fmt.Sprintf(`page=%d`, page),
|
||||
map[string]interface{}{
|
||||
"page": page,
|
||||
"html": html,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetRankingPageHTML(ctx context.Context, page int) (string, time.Time, error) {
|
||||
rec, err := s.pb.listOne(ctx, "ranking_html", fmt.Sprintf(`page=%d`, page))
|
||||
if err != nil || rec == nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
html, _ := rec["html"].(string)
|
||||
var updated time.Time
|
||||
if ts, ok := rec["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
return html, updated, nil
|
||||
}
|
||||
|
||||
// ─── Reading progress ─────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) SetProgress(ctx context.Context, sessionID, slug string, chapter int) error {
|
||||
return s.pb.upsert(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)),
|
||||
map[string]interface{}{
|
||||
"session_id": sessionID,
|
||||
"slug": slug,
|
||||
"chapter": chapter,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetProgress(ctx context.Context, sessionID, slug string) (int, time.Time, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)))
|
||||
if err != nil {
|
||||
return 0, time.Time{}, false, err
|
||||
}
|
||||
if rec == nil {
|
||||
return 0, time.Time{}, false, nil
|
||||
}
|
||||
ch := int(floatVal(rec, "chapter"))
|
||||
var updated time.Time
|
||||
if ts, ok := rec["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
return ch, updated, true, nil
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) AllProgress(ctx context.Context, sessionID string) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"`, pbEsc(sessionID)), "-updated")
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
|
||||
return s.pb.deleteWhere(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)))
|
||||
}
|
||||
|
||||
// ─── Audio cache ──────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
|
||||
return s.pb.upsert(ctx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)),
|
||||
map[string]interface{}{
|
||||
"cache_key": cacheKey,
|
||||
"filename": filename,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)))
|
||||
if err != nil || rec == nil {
|
||||
return "", false, err
|
||||
}
|
||||
filename, _ := rec["filename"].(string)
|
||||
return filename, filename != "", nil
|
||||
}
|
||||
|
||||
// ─── rankingFileInfo is a minimal os.FileInfo implementation ─────────────────
|
||||
|
||||
type rankingFileInfo struct {
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
func (r rankingFileInfo) Name() string { return "ranking" }
|
||||
func (r rankingFileInfo) Size() int64 { return 0 }
|
||||
func (r rankingFileInfo) Mode() os.FileMode { return 0o444 }
|
||||
func (r rankingFileInfo) ModTime() time.Time { return r.modTime }
|
||||
func (r rankingFileInfo) IsDir() bool { return false }
|
||||
func (r rankingFileInfo) Sys() interface{} { return nil }
|
||||
|
||||
var _ os.FileInfo = rankingFileInfo{}
|
||||
|
||||
// RankingModTime returns file-info-compatible data for the ranking record.
|
||||
func (s *PocketBaseStore) RankingModTime(ctx context.Context) (os.FileInfo, error) {
|
||||
_, updated, err := s.GetRanking(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if updated.IsZero() {
|
||||
return nil, nil
|
||||
}
|
||||
return rankingFileInfo{modTime: updated}, nil
|
||||
}
|
||||
|
||||
// RankingPageCacheModTime returns file-info for a cached ranking page.
|
||||
func (s *PocketBaseStore) RankingPageCacheModTime(ctx context.Context, page int) (os.FileInfo, error) {
|
||||
_, updated, err := s.GetRankingPageHTML(ctx, page)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if updated.IsZero() {
|
||||
return nil, nil
|
||||
}
|
||||
return rankingFileInfo{modTime: updated}, nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// pbEsc escapes a string for use in a PocketBase filter expression.
|
||||
// Only escapes double-quotes to prevent injection.
|
||||
func pbEsc(s string) string {
|
||||
return strings.ReplaceAll(s, `"`, `\"`)
|
||||
}
|
||||
|
||||
func floatVal(m map[string]interface{}, key string) float64 {
|
||||
if v, ok := m[key].(float64); ok {
|
||||
return v
|
||||
}
|
||||
return 0
|
||||
}
|
||||
132
scraper/internal/storage/store.go
Normal file
132
scraper/internal/storage/store.go
Normal file
@@ -0,0 +1,132 @@
|
||||
// Package storage defines the unified Store interface and helper types used by
|
||||
// the server and orchestrator. Concrete implementations back the interface
|
||||
// with PocketBase (structured data) and MinIO (binary objects).
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// ─── Shared types ─────────────────────────────────────────────────────────────
|
||||
|
||||
// ChapterInfo is a lightweight chapter descriptor (mirrors writer.ChapterInfo).
|
||||
type ChapterInfo struct {
|
||||
Number int
|
||||
Title string
|
||||
Date string
|
||||
}
|
||||
|
||||
// RankingItem represents a single entry in the novel ranking list.
|
||||
type RankingItem struct {
|
||||
Rank int `json:"rank"`
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Cover string `json:"cover,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Genres []string `json:"genres,omitempty"`
|
||||
SourceURL string `json:"source_url,omitempty"`
|
||||
}
|
||||
|
||||
// ReadingProgress holds a single user's reading position for one book.
|
||||
type ReadingProgress struct {
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro
|
||||
// download filename so audio is not re-generated after a server restart.
|
||||
type AudioCacheEntry struct {
|
||||
CacheKey string `json:"cache_key"`
|
||||
Filename string `json:"filename"`
|
||||
}
|
||||
|
||||
// ─── Store interface ──────────────────────────────────────────────────────────
|
||||
|
||||
// Store is the single persistence abstraction consumed by the server and the
|
||||
// orchestrator. Implementations may route calls to different backends
|
||||
// (PocketBase for structured records, MinIO for binary blobs).
|
||||
type Store interface {
|
||||
// ── Book metadata ──────────────────────────────────────────────────────
|
||||
|
||||
// WriteMetadata upserts book metadata.
|
||||
WriteMetadata(ctx context.Context, meta scraper.BookMeta) error
|
||||
// ReadMetadata returns the metadata for slug. Returns (zero, false, nil)
|
||||
// when the book is not found.
|
||||
ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error)
|
||||
// ListBooks returns all books, sorted alphabetically by title.
|
||||
ListBooks(ctx context.Context) ([]scraper.BookMeta, error)
|
||||
// LocalSlugs returns the set of slugs that have metadata stored.
|
||||
LocalSlugs(ctx context.Context) (map[string]bool, error)
|
||||
// MetadataMtime returns the Unix-second mtime of the metadata record, or 0.
|
||||
MetadataMtime(ctx context.Context, slug string) int64
|
||||
|
||||
// ── Chapters (binary blobs in MinIO) ───────────────────────────────────
|
||||
|
||||
// ChapterExists returns true if the markdown file for the given ref exists.
|
||||
ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool
|
||||
// WriteChapter stores the chapter markdown.
|
||||
WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error
|
||||
// ReadChapter returns the raw markdown for chapter number n.
|
||||
ReadChapter(ctx context.Context, slug string, n int) (string, error)
|
||||
// ListChapters returns all stored chapters for slug, sorted by number.
|
||||
ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error)
|
||||
// CountChapters returns the number of stored chapters for slug.
|
||||
CountChapters(ctx context.Context, slug string) int
|
||||
|
||||
// ── Ranking ────────────────────────────────────────────────────────────
|
||||
|
||||
// WriteRanking persists the ranking list.
|
||||
WriteRanking(ctx context.Context, items []RankingItem) error
|
||||
// ReadRankingItems returns the stored ranking items.
|
||||
ReadRankingItems(ctx context.Context) ([]RankingItem, error)
|
||||
// RankingFileInfo returns os.FileInfo-like data for the ranking record.
|
||||
// Returns (nil, nil) when no ranking has been stored yet.
|
||||
RankingFileInfo(ctx context.Context) (os.FileInfo, error)
|
||||
|
||||
// ── Ranking page HTML cache ────────────────────────────────────────────
|
||||
|
||||
// WriteRankingPageCache stores raw HTML for a ranking page.
|
||||
WriteRankingPageCache(ctx context.Context, page int, html string) error
|
||||
// ReadRankingPageCache returns cached HTML for a ranking page, or "" on miss.
|
||||
ReadRankingPageCache(ctx context.Context, page int) (string, error)
|
||||
// RankingPageCacheInfo returns file-like info for a cached ranking page.
|
||||
RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error)
|
||||
|
||||
// ── Audio cache ────────────────────────────────────────────────────────
|
||||
|
||||
// GetAudioCache returns the Kokoro filename for cacheKey, or ("", false).
|
||||
GetAudioCache(ctx context.Context, cacheKey string) (string, bool)
|
||||
// SetAudioCache persists a Kokoro filename for cacheKey.
|
||||
SetAudioCache(ctx context.Context, cacheKey, filename string) error
|
||||
|
||||
// ── Reading progress ───────────────────────────────────────────────────
|
||||
|
||||
// GetProgress returns the reading progress for the given session ID and slug.
|
||||
// Returns (zero, false) if no progress is recorded.
|
||||
GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool)
|
||||
// SetProgress saves or updates reading progress.
|
||||
SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error
|
||||
// AllProgress returns all progress entries for a session.
|
||||
AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error)
|
||||
// DeleteProgress removes progress for a specific slug.
|
||||
DeleteProgress(ctx context.Context, sessionID, slug string) error
|
||||
|
||||
// ── Audio object paths (MinIO) ─────────────────────────────────────────
|
||||
|
||||
// AudioObjectKey returns the MinIO object key for a cached audio file.
|
||||
AudioObjectKey(slug string, n int, voice string, speed float64) string
|
||||
|
||||
// ── Presigned URLs ─────────────────────────────────────────────────────
|
||||
|
||||
// PresignChapter returns a presigned GET URL for a chapter markdown object.
|
||||
PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error)
|
||||
|
||||
// PresignAudio returns a presigned GET URL for an audio object.
|
||||
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
|
||||
}
|
||||
13
ui/.env.example
Normal file
13
ui/.env.example
Normal file
@@ -0,0 +1,13 @@
|
||||
# libnovel UI — environment variables
|
||||
# Copy to .env and adjust; do NOT commit with real secrets.
|
||||
|
||||
# Public URL of the scraper API (used by SvelteKit server-side load functions)
|
||||
# In docker-compose this is the internal service name
|
||||
SCRAPER_API_URL=http://localhost:8080
|
||||
|
||||
# Public URL of PocketBase (used by SvelteKit server-side load functions)
|
||||
POCKETBASE_URL=http://localhost:8090
|
||||
|
||||
# Public-facing MinIO URL (used to rewrite presigned URLs for the browser)
|
||||
# In dev this is localhost; in prod set to your MinIO public domain
|
||||
PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000
|
||||
23
ui/.gitignore
vendored
Normal file
23
ui/.gitignore
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
node_modules
|
||||
|
||||
# Output
|
||||
.output
|
||||
.vercel
|
||||
.netlify
|
||||
.wrangler
|
||||
/.svelte-kit
|
||||
/build
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Env
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
!.env.test
|
||||
|
||||
# Vite
|
||||
vite.config.js.timestamp-*
|
||||
vite.config.ts.timestamp-*
|
||||
23
ui/Dockerfile
Normal file
23
ui/Dockerfile
Normal file
@@ -0,0 +1,23 @@
|
||||
FROM node:22-alpine AS builder
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
# ── Runtime image ──────────────────────────────────────────────────────────────
|
||||
FROM node:22-alpine
|
||||
WORKDIR /app
|
||||
|
||||
# adapter-node produces a standalone build/
|
||||
COPY --from=builder /app/build ./build
|
||||
COPY --from=builder /app/package.json ./
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PORT=3000
|
||||
ENV HOST=0.0.0.0
|
||||
|
||||
EXPOSE 3000
|
||||
CMD ["node", "build"]
|
||||
42
ui/README.md
Normal file
42
ui/README.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# sv
|
||||
|
||||
Everything you need to build a Svelte project, powered by [`sv`](https://github.com/sveltejs/cli).
|
||||
|
||||
## Creating a project
|
||||
|
||||
If you're seeing this, you've probably already done this step. Congrats!
|
||||
|
||||
```sh
|
||||
# create a new project
|
||||
npx sv create my-app
|
||||
```
|
||||
|
||||
To recreate this project with the same configuration:
|
||||
|
||||
```sh
|
||||
# recreate this project
|
||||
npx sv@0.12.4 create --template minimal --types ts --install npm ui
|
||||
```
|
||||
|
||||
## Developing
|
||||
|
||||
Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server:
|
||||
|
||||
```sh
|
||||
npm run dev
|
||||
|
||||
# or start the server and open the app in a new browser tab
|
||||
npm run dev -- --open
|
||||
```
|
||||
|
||||
## Building
|
||||
|
||||
To create a production version of your app:
|
||||
|
||||
```sh
|
||||
npm run build
|
||||
```
|
||||
|
||||
You can preview the production build with `npm run preview`.
|
||||
|
||||
> To deploy your app, you may need to install an [adapter](https://svelte.dev/docs/kit/adapters) for your target environment.
|
||||
2462
ui/package-lock.json
generated
Normal file
2462
ui/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
30
ui/package.json
Normal file
30
ui/package.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "ui",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite dev",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
"prepare": "svelte-kit sync || echo ''",
|
||||
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
|
||||
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@sveltejs/adapter-auto": "^7.0.0",
|
||||
"@sveltejs/adapter-node": "^5.5.4",
|
||||
"@sveltejs/kit": "^2.50.2",
|
||||
"@sveltejs/vite-plugin-svelte": "^6.2.4",
|
||||
"@tailwindcss/vite": "^4.2.1",
|
||||
"svelte": "^5.51.0",
|
||||
"svelte-check": "^4.4.2",
|
||||
"tailwindcss": "^4.2.1",
|
||||
"typescript": "^5.9.3",
|
||||
"vite": "^7.3.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"marked": "^17.0.3",
|
||||
"pocketbase": "^0.26.8"
|
||||
}
|
||||
}
|
||||
55
ui/src/app.css
Normal file
55
ui/src/app.css
Normal file
@@ -0,0 +1,55 @@
|
||||
@import "tailwindcss";
|
||||
|
||||
@theme {
|
||||
--color-brand: #f59e0b; /* amber-400 */
|
||||
--color-brand-dim: #d97706; /* amber-600 */
|
||||
--color-surface: #18181b; /* zinc-900 */
|
||||
--color-surface-2: #27272a; /* zinc-800 */
|
||||
--color-surface-3: #3f3f46; /* zinc-700 */
|
||||
--color-muted: #a1a1aa; /* zinc-400 */
|
||||
--color-text: #f4f4f5; /* zinc-100 */
|
||||
}
|
||||
|
||||
html {
|
||||
background-color: var(--color-surface);
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
/* ── Chapter prose ─────────────────────────────────────────────────── */
|
||||
.prose-chapter {
|
||||
max-width: 72ch;
|
||||
line-height: 1.85;
|
||||
font-size: 1.05rem;
|
||||
color: #d4d4d8; /* zinc-300 */
|
||||
}
|
||||
|
||||
.prose-chapter h1,
|
||||
.prose-chapter h2,
|
||||
.prose-chapter h3 {
|
||||
color: #f4f4f5;
|
||||
font-weight: 700;
|
||||
margin-top: 1.5em;
|
||||
margin-bottom: 0.5em;
|
||||
}
|
||||
|
||||
.prose-chapter h1 { font-size: 1.4rem; }
|
||||
.prose-chapter h2 { font-size: 1.2rem; }
|
||||
.prose-chapter h3 { font-size: 1.05rem; }
|
||||
|
||||
.prose-chapter p {
|
||||
margin-bottom: 1.2em;
|
||||
}
|
||||
|
||||
.prose-chapter em {
|
||||
color: #a1a1aa;
|
||||
}
|
||||
|
||||
.prose-chapter strong {
|
||||
color: #f4f4f5;
|
||||
}
|
||||
|
||||
.prose-chapter hr {
|
||||
border-color: #3f3f46;
|
||||
margin: 2em 0;
|
||||
}
|
||||
|
||||
15
ui/src/app.d.ts
vendored
Normal file
15
ui/src/app.d.ts
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
// See https://svelte.dev/docs/kit/types#app.d.ts
|
||||
// for information about these interfaces
|
||||
declare global {
|
||||
namespace App {
|
||||
// interface Error {}
|
||||
interface Locals {
|
||||
sessionId: string;
|
||||
}
|
||||
// interface PageData {}
|
||||
// interface PageState {}
|
||||
// interface Platform {}
|
||||
}
|
||||
}
|
||||
|
||||
export {};
|
||||
11
ui/src/app.html
Normal file
11
ui/src/app.html
Normal file
@@ -0,0 +1,11 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
%sveltekit.head%
|
||||
</head>
|
||||
<body data-sveltekit-preload-data="hover">
|
||||
<div style="display: contents">%sveltekit.body%</div>
|
||||
</body>
|
||||
</html>
|
||||
23
ui/src/hooks.server.ts
Normal file
23
ui/src/hooks.server.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import type { Handle } from '@sveltejs/kit';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
|
||||
const SESSION_COOKIE = 'libnovel_session';
|
||||
const ONE_YEAR = 60 * 60 * 24 * 365;
|
||||
|
||||
export const handle: Handle = async ({ event, resolve }) => {
|
||||
let sessionId = event.cookies.get(SESSION_COOKIE);
|
||||
|
||||
if (!sessionId) {
|
||||
sessionId = randomBytes(16).toString('hex');
|
||||
event.cookies.set(SESSION_COOKIE, sessionId, {
|
||||
path: '/',
|
||||
httpOnly: true,
|
||||
sameSite: 'lax',
|
||||
maxAge: ONE_YEAR
|
||||
});
|
||||
}
|
||||
|
||||
event.locals.sessionId = sessionId;
|
||||
|
||||
return resolve(event);
|
||||
};
|
||||
1
ui/src/lib/assets/favicon.svg
Normal file
1
ui/src/lib/assets/favicon.svg
Normal file
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="107" height="128" viewBox="0 0 107 128"><title>svelte-logo</title><path d="M94.157 22.819c-10.4-14.885-30.94-19.297-45.792-9.835L22.282 29.608A29.92 29.92 0 0 0 8.764 49.65a31.5 31.5 0 0 0 3.108 20.231 30 30 0 0 0-4.477 11.183 31.9 31.9 0 0 0 5.448 24.116c10.402 14.887 30.942 19.297 45.791 9.835l26.083-16.624A29.92 29.92 0 0 0 98.235 78.35a31.53 31.53 0 0 0-3.105-20.232 30 30 0 0 0 4.474-11.182 31.88 31.88 0 0 0-5.447-24.116" style="fill:#ff3e00"/><path d="M45.817 106.582a20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.503 18 18 0 0 1 .624-2.435l.49-1.498 1.337.981a33.6 33.6 0 0 0 10.203 5.098l.97.294-.09.968a5.85 5.85 0 0 0 1.052 3.878 6.24 6.24 0 0 0 6.695 2.485 5.8 5.8 0 0 0 1.603-.704L69.27 76.28a5.43 5.43 0 0 0 2.45-3.631 5.8 5.8 0 0 0-.987-4.371 6.24 6.24 0 0 0-6.698-2.487 5.7 5.7 0 0 0-1.6.704l-9.953 6.345a19 19 0 0 1-5.296 2.326 20.72 20.72 0 0 1-22.237-8.243 19.17 19.17 0 0 1-3.277-14.502 17.99 17.99 0 0 1 8.13-12.052l26.081-16.623a19 19 0 0 1 5.3-2.329 20.72 20.72 0 0 1 22.237 8.243 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-.624 2.435l-.49 1.498-1.337-.98a33.6 33.6 0 0 0-10.203-5.1l-.97-.294.09-.968a5.86 5.86 0 0 0-1.052-3.878 6.24 6.24 0 0 0-6.696-2.485 5.8 5.8 0 0 0-1.602.704L37.73 51.72a5.42 5.42 0 0 0-2.449 3.63 5.79 5.79 0 0 0 .986 4.372 6.24 6.24 0 0 0 6.698 2.486 5.8 5.8 0 0 0 1.602-.704l9.952-6.342a19 19 0 0 1 5.295-2.328 20.72 20.72 0 0 1 22.237 8.242 19.17 19.17 0 0 1 3.277 14.503 18 18 0 0 1-8.13 12.053l-26.081 16.622a19 19 0 0 1-5.3 2.328" style="fill:#fff"/></svg>
|
||||
|
After Width: | Height: | Size: 1.5 KiB |
187
ui/src/lib/components/AudioPlayer.svelte
Normal file
187
ui/src/lib/components/AudioPlayer.svelte
Normal file
@@ -0,0 +1,187 @@
|
||||
<script lang="ts">
|
||||
/**
|
||||
* AudioPlayer — fetches a presigned MinIO URL for the chapter audio
|
||||
* and renders a native <audio> element with custom controls.
|
||||
*
|
||||
* The audio is generated server-side by Kokoro and cached.
|
||||
* If no audio exists yet, the user can trigger generation via the scraper API.
|
||||
*/
|
||||
|
||||
interface Props {
|
||||
slug: string;
|
||||
chapter: number;
|
||||
voice?: string;
|
||||
speed?: number;
|
||||
}
|
||||
|
||||
let { slug, chapter, voice = 'af_bella', speed = 1.0 }: Props = $props();
|
||||
|
||||
type Status = 'idle' | 'loading' | 'ready' | 'error' | 'generating';
|
||||
|
||||
let status = $state<Status>('idle');
|
||||
let audioUrl = $state('');
|
||||
let errorMsg = $state('');
|
||||
let audioEl = $state<HTMLAudioElement | null>(null);
|
||||
let currentTime = $state(0);
|
||||
let duration = $state(0);
|
||||
let isPlaying = $state(false);
|
||||
|
||||
async function loadAudio() {
|
||||
status = 'loading';
|
||||
errorMsg = '';
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
slug,
|
||||
n: String(chapter),
|
||||
voice,
|
||||
speed: String(speed)
|
||||
});
|
||||
const res = await fetch(`/api/presign/audio?${params.toString()}`);
|
||||
if (res.status === 404) {
|
||||
status = 'idle';
|
||||
return;
|
||||
}
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const data = (await res.json()) as { url: string };
|
||||
audioUrl = data.url;
|
||||
status = 'ready';
|
||||
} catch (e) {
|
||||
status = 'error';
|
||||
errorMsg = String(e);
|
||||
}
|
||||
}
|
||||
|
||||
async function generateAudio() {
|
||||
status = 'generating';
|
||||
errorMsg = '';
|
||||
try {
|
||||
const res = await fetch(`/api/audio/${slug}/${chapter}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ voice, speed })
|
||||
});
|
||||
if (!res.ok) throw new Error(`Generation failed: ${res.status}`);
|
||||
// After generation, fetch the presigned URL
|
||||
await loadAudio();
|
||||
} catch (e) {
|
||||
status = 'error';
|
||||
errorMsg = `Generation failed: ${e}`;
|
||||
}
|
||||
}
|
||||
|
||||
function formatTime(s: number): string {
|
||||
const m = Math.floor(s / 60);
|
||||
const sec = Math.floor(s % 60);
|
||||
return `${m}:${sec.toString().padStart(2, '0')}`;
|
||||
}
|
||||
|
||||
function togglePlay() {
|
||||
if (!audioEl) return;
|
||||
if (isPlaying) {
|
||||
audioEl.pause();
|
||||
} else {
|
||||
audioEl.play();
|
||||
}
|
||||
}
|
||||
|
||||
function seek(e: Event) {
|
||||
if (!audioEl) return;
|
||||
const input = e.target as HTMLInputElement;
|
||||
audioEl.currentTime = parseFloat(input.value);
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="mt-6 p-4 rounded-lg bg-zinc-800 border border-zinc-700">
|
||||
<div class="flex items-center gap-2 mb-3">
|
||||
<svg class="w-4 h-4 text-amber-400" fill="currentColor" viewBox="0 0 24 24">
|
||||
<path d="M11.536 3.464a5 5 0 010 7.072L8 14H5v3H2V7h3l3.536-3.536a5 5 0 017.072 0l-4.072 4.072zM19 8a1 1 0 011 1v6a1 1 0 01-2 0V9a1 1 0 011-1zm-4-2a1 1 0 011 1v10a1 1 0 01-2 0V7a1 1 0 011-1z"/>
|
||||
</svg>
|
||||
<span class="text-sm text-zinc-300 font-medium">Audio Narration</span>
|
||||
</div>
|
||||
|
||||
{#if status === 'idle'}
|
||||
<div class="flex gap-2">
|
||||
<button
|
||||
onclick={loadAudio}
|
||||
class="px-3 py-1.5 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors"
|
||||
>
|
||||
Check for audio
|
||||
</button>
|
||||
<button
|
||||
onclick={generateAudio}
|
||||
class="px-3 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors"
|
||||
>
|
||||
Generate audio
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{:else if status === 'loading' || status === 'generating'}
|
||||
<div class="flex items-center gap-2 text-zinc-400 text-sm">
|
||||
<svg class="w-4 h-4 animate-spin" fill="none" viewBox="0 0 24 24">
|
||||
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
|
||||
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
|
||||
</svg>
|
||||
{status === 'generating' ? 'Generating audio (this may take a few minutes)…' : 'Loading…'}
|
||||
</div>
|
||||
|
||||
{:else if status === 'error'}
|
||||
<div class="text-red-400 text-sm">
|
||||
<p>{errorMsg || 'Failed to load audio.'}</p>
|
||||
<button
|
||||
onclick={() => { status = 'idle'; }}
|
||||
class="mt-1 text-xs underline text-zinc-400 hover:text-zinc-200"
|
||||
>
|
||||
Dismiss
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{:else if status === 'ready' && audioUrl}
|
||||
<!-- Native audio element (hidden, controlled by custom UI) -->
|
||||
<audio
|
||||
bind:this={audioEl}
|
||||
src={audioUrl}
|
||||
bind:currentTime
|
||||
bind:duration
|
||||
onplay={() => (isPlaying = true)}
|
||||
onpause={() => (isPlaying = false)}
|
||||
onended={() => (isPlaying = false)}
|
||||
preload="metadata"
|
||||
></audio>
|
||||
|
||||
<!-- Custom controls -->
|
||||
<div class="flex flex-col gap-2">
|
||||
<!-- Play/pause + time -->
|
||||
<div class="flex items-center gap-3">
|
||||
<button
|
||||
onclick={togglePlay}
|
||||
class="w-9 h-9 rounded-full bg-amber-400 text-zinc-900 flex items-center justify-center hover:bg-amber-300 transition-colors flex-shrink-0"
|
||||
aria-label={isPlaying ? 'Pause' : 'Play'}
|
||||
>
|
||||
{#if isPlaying}
|
||||
<svg class="w-4 h-4" fill="currentColor" viewBox="0 0 24 24">
|
||||
<path d="M6 4h4v16H6V4zm8 0h4v16h-4V4z"/>
|
||||
</svg>
|
||||
{:else}
|
||||
<svg class="w-4 h-4 ml-0.5" fill="currentColor" viewBox="0 0 24 24">
|
||||
<path d="M8 5v14l11-7z"/>
|
||||
</svg>
|
||||
{/if}
|
||||
</button>
|
||||
|
||||
<span class="text-xs text-zinc-400 w-20 flex-shrink-0">
|
||||
{formatTime(currentTime)} / {formatTime(duration || 0)}
|
||||
</span>
|
||||
|
||||
<!-- Seek bar -->
|
||||
<input
|
||||
type="range"
|
||||
min="0"
|
||||
max={duration || 0}
|
||||
value={currentTime}
|
||||
oninput={seek}
|
||||
class="flex-1 accent-amber-400 h-1.5 rounded"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
1
ui/src/lib/index.ts
Normal file
1
ui/src/lib/index.ts
Normal file
@@ -0,0 +1 @@
|
||||
// place files you want to import through the `$lib` alias in this folder.
|
||||
67
ui/src/lib/server/minio.ts
Normal file
67
ui/src/lib/server/minio.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* Server-side MinIO presign helper.
|
||||
* Calls the scraper API to get presigned URLs, then optionally rewrites
|
||||
* the MinIO host to the public-facing URL for browser use.
|
||||
*
|
||||
* Never import this from client-side code.
|
||||
*/
|
||||
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { env as pubEnv } from '$env/dynamic/public';
|
||||
|
||||
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||
// Public MinIO URL — used to rewrite presigned URLs so the browser can reach MinIO directly.
|
||||
// In docker-compose this would differ from the internal endpoint.
|
||||
const MINIO_PUBLIC_URL = pubEnv.PUBLIC_MINIO_PUBLIC_URL ?? 'http://localhost:9000';
|
||||
|
||||
/**
|
||||
* Rewrites the MinIO host in a presigned URL to the public-facing URL.
|
||||
* The presigned URL is signed against the internal endpoint (e.g. minio:9000),
|
||||
* but the browser needs the public URL (e.g. localhost:9000 in dev, or a CDN in prod).
|
||||
* Rewriting the host preserves all query params (signature, expiry, etc).
|
||||
*/
|
||||
function rewriteHost(presignedUrl: string): string {
|
||||
try {
|
||||
const u = new URL(presignedUrl);
|
||||
const pub = new URL(MINIO_PUBLIC_URL);
|
||||
u.protocol = pub.protocol;
|
||||
u.hostname = pub.hostname;
|
||||
u.port = pub.port;
|
||||
return u.toString();
|
||||
} catch {
|
||||
return presignedUrl;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a presigned URL for a chapter markdown file.
|
||||
* URL is valid for ~15 minutes (set by the scraper).
|
||||
* The returned URL points to the public MinIO endpoint and can be used
|
||||
* server-side (in a +page.server.ts load function) to fetch the markdown content.
|
||||
*/
|
||||
export async function presignChapter(slug: string, n: number): Promise<string> {
|
||||
const res = await fetch(`${SCRAPER_URL}/api/presign/chapter/${slug}/${n}`);
|
||||
if (!res.ok) throw new Error(`presign chapter ${slug}/${n}: ${res.status}`);
|
||||
const data = (await res.json()) as { url: string };
|
||||
return rewriteHost(data.url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a presigned URL for an audio file.
|
||||
* URL is valid for ~1 hour. The URL is returned to the browser for direct streaming.
|
||||
*/
|
||||
export async function presignAudio(
|
||||
slug: string,
|
||||
n: number,
|
||||
voice?: string,
|
||||
speed?: number
|
||||
): Promise<string> {
|
||||
const params = new URLSearchParams();
|
||||
if (voice) params.set('voice', voice);
|
||||
if (speed) params.set('speed', String(speed));
|
||||
const qs = params.toString() ? `?${params.toString()}` : '';
|
||||
const res = await fetch(`${SCRAPER_URL}/api/presign/audio/${slug}/${n}${qs}`);
|
||||
if (!res.ok) throw new Error(`presign audio ${slug}/${n}: ${res.status}`);
|
||||
const data = (await res.json()) as { url: string };
|
||||
return rewriteHost(data.url);
|
||||
}
|
||||
169
ui/src/lib/server/pocketbase.ts
Normal file
169
ui/src/lib/server/pocketbase.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* Server-side PocketBase client.
|
||||
* Uses admin credentials — never import this from client-side code.
|
||||
* All methods talk directly to PocketBase REST API.
|
||||
*/
|
||||
|
||||
import { env } from '$env/dynamic/private';
|
||||
|
||||
const PB_URL = env.POCKETBASE_URL ?? 'http://localhost:8090';
|
||||
const PB_EMAIL = env.POCKETBASE_ADMIN_EMAIL ?? 'admin@libnovel.local';
|
||||
const PB_PASSWORD = env.POCKETBASE_ADMIN_PASSWORD ?? 'changeme123';
|
||||
|
||||
// ─── Types ────────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface Book {
|
||||
id: string;
|
||||
slug: string;
|
||||
title: string;
|
||||
author: string;
|
||||
cover: string;
|
||||
status: string;
|
||||
genres: string[] | string;
|
||||
summary: string;
|
||||
total_chapters: number;
|
||||
source_url: string;
|
||||
ranking: number;
|
||||
meta_updated: string;
|
||||
}
|
||||
|
||||
export interface ChapterIdx {
|
||||
id: string;
|
||||
slug: string;
|
||||
number: number;
|
||||
title: string;
|
||||
date_label: string;
|
||||
}
|
||||
|
||||
export interface Progress {
|
||||
session_id: string;
|
||||
slug: string;
|
||||
chapter: number;
|
||||
updated: string;
|
||||
}
|
||||
|
||||
// ─── Auth token cache ─────────────────────────────────────────────────────────
|
||||
|
||||
let _token = '';
|
||||
let _tokenExp = 0;
|
||||
|
||||
async function getToken(): Promise<string> {
|
||||
if (_token && Date.now() < _tokenExp) return _token;
|
||||
|
||||
const res = await fetch(`${PB_URL}/api/admins/auth-with-password`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ identity: PB_EMAIL, password: PB_PASSWORD })
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`PocketBase auth failed: ${res.status}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
_token = data.token as string;
|
||||
_tokenExp = Date.now() + 12 * 60 * 60 * 1000; // 12 hours
|
||||
return _token;
|
||||
}
|
||||
|
||||
// ─── Generic helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
async function pbGet<T>(path: string): Promise<T> {
|
||||
const token = await getToken();
|
||||
const res = await fetch(`${PB_URL}${path}`, {
|
||||
headers: { Authorization: token }
|
||||
});
|
||||
if (!res.ok) throw new Error(`PocketBase GET ${path} failed: ${res.status}`);
|
||||
return res.json() as Promise<T>;
|
||||
}
|
||||
|
||||
async function pbPost(path: string, body: unknown): Promise<Response> {
|
||||
const token = await getToken();
|
||||
return fetch(`${PB_URL}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { Authorization: token, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
}
|
||||
|
||||
async function pbPatch(path: string, body: unknown): Promise<Response> {
|
||||
const token = await getToken();
|
||||
return fetch(`${PB_URL}${path}`, {
|
||||
method: 'PATCH',
|
||||
headers: { Authorization: token, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
}
|
||||
|
||||
function encodeFilter(filter: string) {
|
||||
return encodeURIComponent(filter);
|
||||
}
|
||||
|
||||
interface PBList<T> {
|
||||
items: T[];
|
||||
totalItems: number;
|
||||
}
|
||||
|
||||
async function listAll<T>(collection: string, filter = '', sort = ''): Promise<T[]> {
|
||||
const params = new URLSearchParams({ perPage: '500' });
|
||||
if (filter) params.set('filter', filter);
|
||||
if (sort) params.set('sort', sort);
|
||||
const data = await pbGet<PBList<T>>(
|
||||
`/api/collections/${collection}/records?${params.toString()}`
|
||||
);
|
||||
return data.items;
|
||||
}
|
||||
|
||||
async function listOne<T>(collection: string, filter: string): Promise<T | null> {
|
||||
const params = new URLSearchParams({ perPage: '1', filter });
|
||||
const data = await pbGet<PBList<T>>(
|
||||
`/api/collections/${collection}/records?${params.toString()}`
|
||||
);
|
||||
return data.items[0] ?? null;
|
||||
}
|
||||
|
||||
// ─── Books ────────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function listBooks(): Promise<Book[]> {
|
||||
return listAll<Book>('books', '', '+title');
|
||||
}
|
||||
|
||||
export async function getBook(slug: string): Promise<Book | null> {
|
||||
return listOne<Book>('books', `slug="${slug}"`);
|
||||
}
|
||||
|
||||
// ─── Chapter index ────────────────────────────────────────────────────────────
|
||||
|
||||
export async function listChapterIdx(slug: string): Promise<ChapterIdx[]> {
|
||||
return listAll<ChapterIdx>('chapters_idx', `slug="${slug}"`, '+number');
|
||||
}
|
||||
|
||||
// ─── Reading progress ─────────────────────────────────────────────────────────
|
||||
|
||||
export async function getProgress(sessionId: string, slug: string): Promise<Progress | null> {
|
||||
return listOne<Progress>('progress', `session_id="${sessionId}"&&slug="${slug}"`);
|
||||
}
|
||||
|
||||
export async function allProgress(sessionId: string): Promise<Progress[]> {
|
||||
return listAll<Progress>('progress', `session_id="${sessionId}"`, '-updated');
|
||||
}
|
||||
|
||||
export async function setProgress(sessionId: string, slug: string, chapter: number): Promise<void> {
|
||||
const existing = await listOne<Progress & { id: string }>(
|
||||
'progress',
|
||||
`session_id="${sessionId}"&&slug="${slug}"`
|
||||
);
|
||||
|
||||
const payload = {
|
||||
session_id: sessionId,
|
||||
slug,
|
||||
chapter,
|
||||
updated: new Date().toISOString()
|
||||
};
|
||||
|
||||
if (existing) {
|
||||
await pbPatch(`/api/collections/progress/records/${existing.id}`, payload);
|
||||
} else {
|
||||
await pbPost('/api/collections/progress/records', payload);
|
||||
}
|
||||
}
|
||||
32
ui/src/routes/+layout.svelte
Normal file
32
ui/src/routes/+layout.svelte
Normal file
@@ -0,0 +1,32 @@
|
||||
<script lang="ts">
|
||||
import '../app.css';
|
||||
|
||||
let { children } = $props();
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>libnovel</title>
|
||||
</svelte:head>
|
||||
|
||||
<div class="min-h-screen flex flex-col">
|
||||
<header class="border-b border-zinc-700 bg-zinc-900 sticky top-0 z-50">
|
||||
<nav class="max-w-6xl mx-auto px-4 h-14 flex items-center gap-6">
|
||||
<a href="/" class="text-amber-400 font-bold text-lg tracking-tight hover:text-amber-300">
|
||||
libnovel
|
||||
</a>
|
||||
<a href="/books" class="text-zinc-400 hover:text-zinc-100 text-sm transition-colors">
|
||||
Library
|
||||
</a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<main class="flex-1 max-w-6xl mx-auto w-full px-4 py-8">
|
||||
{@render children()}
|
||||
</main>
|
||||
|
||||
<footer class="border-t border-zinc-800 text-zinc-600 text-xs text-center py-4">
|
||||
libnovel
|
||||
</footer>
|
||||
</div>
|
||||
3
ui/src/routes/+page.svelte
Normal file
3
ui/src/routes/+page.svelte
Normal file
@@ -0,0 +1,3 @@
|
||||
<a href="/books" class="inline-block mt-4 px-6 py-3 bg-amber-400 text-zinc-900 font-semibold rounded hover:bg-amber-300 transition-colors">
|
||||
Browse Library
|
||||
</a>
|
||||
92
ui/src/routes/api/audio/[slug]/[n]/+server.ts
Normal file
92
ui/src/routes/api/audio/[slug]/[n]/+server.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
import { error } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { env } from '$env/dynamic/private';
|
||||
|
||||
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||
|
||||
/**
|
||||
* POST /api/audio/[slug]/[n]
|
||||
* Proxies the audio generation request to the scraper's /api/audio endpoint.
|
||||
* Keeps the scraper URL server-side — the browser never needs to know it.
|
||||
*
|
||||
* Body: { voice?: string, speed?: number }
|
||||
* Response: { url: string, filename: string }
|
||||
* where `url` is a relative path to GET /api/audio/[slug]/[n]?voice=...&speed=...
|
||||
*/
|
||||
export const POST: RequestHandler = async ({ params, request }) => {
|
||||
const { slug, n } = params;
|
||||
const chapter = parseInt(n, 10);
|
||||
if (!slug || !chapter || chapter < 1) {
|
||||
error(400, 'Invalid slug or chapter number');
|
||||
}
|
||||
|
||||
let body: { voice?: string; speed?: number } = {};
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
// empty body is fine — scraper will use defaults
|
||||
}
|
||||
|
||||
const scraperRes = await fetch(`${SCRAPER_URL}/api/audio/${slug}/${chapter}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
|
||||
if (!scraperRes.ok) {
|
||||
const text = await scraperRes.text().catch(() => '');
|
||||
error(scraperRes.status as Parameters<typeof error>[0], text || 'Audio generation failed');
|
||||
}
|
||||
|
||||
const data = (await scraperRes.json()) as { url: string; filename: string };
|
||||
|
||||
// The scraper returns a proxy URL pointing to /api/audio-proxy/... — we rewrite
|
||||
// it to our own /api/audio/[slug]/[n]?... so the browser never calls the scraper directly.
|
||||
const voice = body.voice ?? '';
|
||||
const speed = body.speed ?? 1.0;
|
||||
const qs = new URLSearchParams();
|
||||
if (voice) qs.set('voice', voice);
|
||||
qs.set('speed', String(speed));
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
url: `/api/audio/${slug}/${chapter}?${qs.toString()}`,
|
||||
filename: data.filename
|
||||
}),
|
||||
{ headers: { 'Content-Type': 'application/json' } }
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* GET /api/audio/[slug]/[n]?voice=...&speed=...
|
||||
* Proxies the audio stream from the scraper's /api/audio-proxy endpoint.
|
||||
* This is the URL the browser's <audio> element uses as its src.
|
||||
*/
|
||||
export const GET: RequestHandler = async ({ params, url }) => {
|
||||
const { slug, n } = params;
|
||||
const chapter = parseInt(n, 10);
|
||||
if (!slug || !chapter || chapter < 1) {
|
||||
error(400, 'Invalid slug or chapter number');
|
||||
}
|
||||
|
||||
const voice = url.searchParams.get('voice') ?? '';
|
||||
const speed = url.searchParams.get('speed') ?? '1';
|
||||
const qs = new URLSearchParams();
|
||||
if (voice) qs.set('voice', voice);
|
||||
qs.set('speed', speed);
|
||||
|
||||
const scraperRes = await fetch(`${SCRAPER_URL}/api/audio-proxy/${slug}/${chapter}?${qs.toString()}`);
|
||||
|
||||
if (!scraperRes.ok) {
|
||||
error(scraperRes.status as Parameters<typeof error>[0], 'Audio not found');
|
||||
}
|
||||
|
||||
// Stream the audio body through — preserve Content-Type and Content-Length.
|
||||
const headers = new Headers();
|
||||
headers.set('Content-Type', scraperRes.headers.get('Content-Type') ?? 'audio/mpeg');
|
||||
headers.set('Cache-Control', 'public, max-age=3600');
|
||||
const cl = scraperRes.headers.get('Content-Length');
|
||||
if (cl) headers.set('Content-Length', cl);
|
||||
|
||||
return new Response(scraperRes.body, { headers });
|
||||
};
|
||||
26
ui/src/routes/api/presign/audio/+server.ts
Normal file
26
ui/src/routes/api/presign/audio/+server.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import { json, error } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { presignAudio } from '$lib/server/minio';
|
||||
|
||||
/**
|
||||
* GET /api/presign/audio?slug=...&n=...&voice=...&speed=...
|
||||
* Returns a presigned MinIO URL for the audio file so the browser
|
||||
* can stream it directly without going through the server.
|
||||
*/
|
||||
export const GET: RequestHandler = async ({ url }) => {
|
||||
const slug = url.searchParams.get('slug');
|
||||
const n = parseInt(url.searchParams.get('n') ?? '', 10);
|
||||
const voice = url.searchParams.get('voice') ?? undefined;
|
||||
const speed = parseFloat(url.searchParams.get('speed') ?? '1') || 1;
|
||||
|
||||
if (!slug || !n || n < 1) {
|
||||
error(400, 'Missing slug or n');
|
||||
}
|
||||
|
||||
try {
|
||||
const presignedUrl = await presignAudio(slug, n, voice, speed);
|
||||
return json({ url: presignedUrl });
|
||||
} catch (e) {
|
||||
error(500, `Could not get presigned URL: ${e}`);
|
||||
}
|
||||
};
|
||||
19
ui/src/routes/api/progress/+server.ts
Normal file
19
ui/src/routes/api/progress/+server.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { json, error } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { setProgress } from '$lib/server/pocketbase';
|
||||
|
||||
/**
|
||||
* POST /api/progress
|
||||
* Body: { slug: string, chapter: number }
|
||||
* Records the user's reading position for the current session.
|
||||
*/
|
||||
export const POST: RequestHandler = async ({ request, locals }) => {
|
||||
const body = await request.json().catch(() => null);
|
||||
|
||||
if (!body || typeof body.slug !== 'string' || typeof body.chapter !== 'number') {
|
||||
error(400, 'Invalid body — expected { slug, chapter }');
|
||||
}
|
||||
|
||||
await setProgress(locals.sessionId, body.slug, body.chapter);
|
||||
return json({ ok: true });
|
||||
};
|
||||
17
ui/src/routes/books/+page.server.ts
Normal file
17
ui/src/routes/books/+page.server.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import type { PageServerLoad } from './$types';
|
||||
import { listBooks, allProgress } from '$lib/server/pocketbase';
|
||||
|
||||
export const load: PageServerLoad = async ({ locals }) => {
|
||||
const [books, progressList] = await Promise.all([
|
||||
listBooks(),
|
||||
allProgress(locals.sessionId)
|
||||
]);
|
||||
|
||||
// Build a quick lookup: slug → last chapter read
|
||||
const progressMap: Record<string, number> = {};
|
||||
for (const p of progressList) {
|
||||
progressMap[p.slug] = p.chapter;
|
||||
}
|
||||
|
||||
return { books, progressMap };
|
||||
};
|
||||
91
ui/src/routes/books/+page.svelte
Normal file
91
ui/src/routes/books/+page.svelte
Normal file
@@ -0,0 +1,91 @@
|
||||
<script lang="ts">
|
||||
import type { PageData } from './$types';
|
||||
|
||||
let { data }: { data: PageData } = $props();
|
||||
|
||||
function parseGenres(genres: string[] | string): string[] {
|
||||
if (Array.isArray(genres)) return genres;
|
||||
try {
|
||||
return JSON.parse(genres);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
<title>Library — libnovel</title>
|
||||
</svelte:head>
|
||||
|
||||
<div class="mb-6">
|
||||
<h1 class="text-2xl font-bold text-zinc-100">Library</h1>
|
||||
<p class="text-zinc-400 text-sm mt-1">{data.books.length} books</p>
|
||||
</div>
|
||||
|
||||
{#if data.books.length === 0}
|
||||
<div class="text-center py-20 text-zinc-500">
|
||||
<p class="text-lg">No books scraped yet.</p>
|
||||
<p class="text-sm mt-2">Use the scraper API to add books.</p>
|
||||
</div>
|
||||
{:else}
|
||||
<div class="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 xl:grid-cols-6 gap-4">
|
||||
{#each data.books as book}
|
||||
{@const lastChapter = data.progressMap[book.slug]}
|
||||
{@const genres = parseGenres(book.genres)}
|
||||
<a
|
||||
href="/books/{book.slug}"
|
||||
class="group flex flex-col rounded-lg overflow-hidden bg-zinc-800 hover:bg-zinc-700 transition-colors border border-zinc-700 hover:border-zinc-500"
|
||||
>
|
||||
<!-- Cover image -->
|
||||
<div class="aspect-[2/3] bg-zinc-900 overflow-hidden">
|
||||
{#if book.cover}
|
||||
<img
|
||||
src={book.cover}
|
||||
alt={book.title}
|
||||
class="w-full h-full object-cover group-hover:scale-105 transition-transform duration-300"
|
||||
loading="lazy"
|
||||
/>
|
||||
{:else}
|
||||
<div class="w-full h-full flex items-center justify-center text-zinc-600">
|
||||
<svg class="w-12 h-12" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5"
|
||||
d="M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.332.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253" />
|
||||
</svg>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- Info -->
|
||||
<div class="p-2 flex flex-col gap-1 flex-1">
|
||||
<h2 class="text-xs font-semibold text-zinc-100 line-clamp-2 leading-snug">
|
||||
{book.title}
|
||||
</h2>
|
||||
{#if book.author}
|
||||
<p class="text-xs text-zinc-400 truncate">{book.author}</p>
|
||||
{/if}
|
||||
|
||||
<div class="mt-auto pt-1 flex items-center justify-between gap-1">
|
||||
{#if book.status}
|
||||
<span class="text-xs px-1.5 py-0.5 rounded bg-zinc-700 text-zinc-300 truncate max-w-[60%]">
|
||||
{book.status}
|
||||
</span>
|
||||
{/if}
|
||||
{#if lastChapter}
|
||||
<span class="text-xs text-amber-400 font-medium ml-auto whitespace-nowrap">
|
||||
ch.{lastChapter}
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if genres.length > 0}
|
||||
<div class="flex flex-wrap gap-1 mt-1">
|
||||
{#each genres.slice(0, 2) as genre}
|
||||
<span class="text-xs px-1 py-0.5 rounded bg-zinc-900 text-zinc-500">{genre}</span>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</a>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
23
ui/src/routes/books/[slug]/+page.server.ts
Normal file
23
ui/src/routes/books/[slug]/+page.server.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import { error } from '@sveltejs/kit';
|
||||
import type { PageServerLoad } from './$types';
|
||||
import { getBook, listChapterIdx, getProgress } from '$lib/server/pocketbase';
|
||||
|
||||
export const load: PageServerLoad = async ({ params, locals }) => {
|
||||
const { slug } = params;
|
||||
|
||||
const [book, chapters, progress] = await Promise.all([
|
||||
getBook(slug),
|
||||
listChapterIdx(slug),
|
||||
getProgress(locals.sessionId, slug)
|
||||
]);
|
||||
|
||||
if (!book) {
|
||||
error(404, `Book "${slug}" not found`);
|
||||
}
|
||||
|
||||
return {
|
||||
book,
|
||||
chapters,
|
||||
lastChapter: progress?.chapter ?? null
|
||||
};
|
||||
};
|
||||
142
ui/src/routes/books/[slug]/+page.svelte
Normal file
142
ui/src/routes/books/[slug]/+page.svelte
Normal file
@@ -0,0 +1,142 @@
|
||||
<script lang="ts">
|
||||
import type { PageData } from './$types';
|
||||
|
||||
let { data }: { data: PageData } = $props();
|
||||
|
||||
function parseGenres(genres: string[] | string): string[] {
|
||||
if (Array.isArray(genres)) return genres;
|
||||
try {
|
||||
return JSON.parse(genres);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const genres = $derived(parseGenres(data.book.genres));
|
||||
|
||||
// Paginate chapter list — show 100 at a time
|
||||
let page = $state(0);
|
||||
const PAGE_SIZE = 100;
|
||||
const totalPages = $derived(Math.ceil(data.chapters.length / PAGE_SIZE));
|
||||
const visibleChapters = $derived(
|
||||
data.chapters.slice(page * PAGE_SIZE, (page + 1) * PAGE_SIZE)
|
||||
);
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
<title>{data.book.title} — libnovel</title>
|
||||
</svelte:head>
|
||||
|
||||
<!-- Book header -->
|
||||
<div class="flex gap-6 mb-8">
|
||||
{#if data.book.cover}
|
||||
<img
|
||||
src={data.book.cover}
|
||||
alt={data.book.title}
|
||||
class="w-32 sm:w-40 rounded-lg object-cover flex-shrink-0 border border-zinc-700"
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<div class="flex flex-col gap-2 min-w-0">
|
||||
<h1 class="text-2xl font-bold text-zinc-100 leading-tight">{data.book.title}</h1>
|
||||
|
||||
{#if data.book.author}
|
||||
<p class="text-zinc-400 text-sm">{data.book.author}</p>
|
||||
{/if}
|
||||
|
||||
<div class="flex flex-wrap gap-2 mt-1">
|
||||
{#if data.book.status}
|
||||
<span class="text-xs px-2 py-1 rounded bg-zinc-700 text-zinc-300">{data.book.status}</span>
|
||||
{/if}
|
||||
{#each genres as genre}
|
||||
<span class="text-xs px-2 py-1 rounded bg-zinc-800 text-zinc-400">{genre}</span>
|
||||
{/each}
|
||||
</div>
|
||||
|
||||
{#if data.book.summary}
|
||||
<p class="text-zinc-400 text-sm leading-relaxed line-clamp-4 mt-1">{data.book.summary}</p>
|
||||
{/if}
|
||||
|
||||
<div class="flex gap-3 mt-2">
|
||||
{#if data.lastChapter}
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/{data.lastChapter}"
|
||||
class="px-4 py-2 bg-amber-400 text-zinc-900 font-semibold rounded text-sm hover:bg-amber-300 transition-colors"
|
||||
>
|
||||
Continue ch.{data.lastChapter}
|
||||
</a>
|
||||
{/if}
|
||||
{#if data.chapters.length > 0}
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/1"
|
||||
class="px-4 py-2 bg-zinc-700 text-zinc-100 font-semibold rounded text-sm hover:bg-zinc-600 transition-colors"
|
||||
>
|
||||
Start from ch.1
|
||||
</a>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Chapter list -->
|
||||
<div class="mt-4">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<h2 class="text-lg font-semibold text-zinc-100">
|
||||
Chapters
|
||||
<span class="text-zinc-500 font-normal text-sm ml-1">({data.chapters.length})</span>
|
||||
</h2>
|
||||
|
||||
{#if totalPages > 1}
|
||||
<div class="flex gap-2 items-center text-sm">
|
||||
<button
|
||||
onclick={() => (page = Math.max(0, page - 1))}
|
||||
disabled={page === 0}
|
||||
class="px-2 py-1 rounded bg-zinc-700 text-zinc-300 disabled:opacity-40 hover:bg-zinc-600 transition-colors"
|
||||
>
|
||||
←
|
||||
</button>
|
||||
<span class="text-zinc-400">{page + 1} / {totalPages}</span>
|
||||
<button
|
||||
onclick={() => (page = Math.min(totalPages - 1, page + 1))}
|
||||
disabled={page === totalPages - 1}
|
||||
class="px-2 py-1 rounded bg-zinc-700 text-zinc-300 disabled:opacity-40 hover:bg-zinc-600 transition-colors"
|
||||
>
|
||||
→
|
||||
</button>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if data.chapters.length === 0}
|
||||
<p class="text-zinc-500 text-sm">No chapters available yet.</p>
|
||||
{:else}
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 gap-1">
|
||||
{#each visibleChapters as chapter}
|
||||
{@const isCurrent = data.lastChapter === chapter.number}
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/{chapter.number}"
|
||||
class="flex items-center gap-3 px-3 py-2 rounded hover:bg-zinc-800 transition-colors group {isCurrent
|
||||
? 'bg-zinc-800'
|
||||
: ''}"
|
||||
>
|
||||
<span
|
||||
class="text-xs font-mono w-10 text-right flex-shrink-0 {isCurrent
|
||||
? 'text-amber-400'
|
||||
: 'text-zinc-500'}"
|
||||
>
|
||||
{chapter.number}
|
||||
</span>
|
||||
<span class="text-sm text-zinc-300 group-hover:text-zinc-100 truncate flex-1">
|
||||
{chapter.title || `Chapter ${chapter.number}`}
|
||||
</span>
|
||||
{#if isCurrent}
|
||||
<span class="text-xs text-amber-400 flex-shrink-0">reading</span>
|
||||
{/if}
|
||||
{#if chapter.date_label}
|
||||
<span class="text-xs text-zinc-600 flex-shrink-0 hidden sm:block">{chapter.date_label}</span>
|
||||
{/if}
|
||||
</a>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
45
ui/src/routes/books/[slug]/chapters/[n]/+page.server.ts
Normal file
45
ui/src/routes/books/[slug]/chapters/[n]/+page.server.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import { error } from '@sveltejs/kit';
|
||||
import { marked } from 'marked';
|
||||
import type { PageServerLoad } from './$types';
|
||||
import { getBook, listChapterIdx } from '$lib/server/pocketbase';
|
||||
import { presignChapter } from '$lib/server/minio';
|
||||
|
||||
export const load: PageServerLoad = async ({ params, locals }) => {
|
||||
const { slug } = params;
|
||||
const n = parseInt(params.n, 10);
|
||||
|
||||
if (!n || n < 1) error(400, 'Invalid chapter number');
|
||||
|
||||
// Fetch book metadata and chapter index in parallel
|
||||
const [book, chapters] = await Promise.all([getBook(slug), listChapterIdx(slug)]);
|
||||
|
||||
if (!book) error(404, `Book "${slug}" not found`);
|
||||
|
||||
const chapterIdx = chapters.find((c) => c.number === n);
|
||||
if (!chapterIdx) error(404, `Chapter ${n} not found`);
|
||||
|
||||
// Get presigned URL and fetch chapter markdown server-side
|
||||
let html = '';
|
||||
try {
|
||||
const url = await presignChapter(slug, n);
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`MinIO returned ${res.status}`);
|
||||
const markdown = await res.text();
|
||||
html = await marked(markdown, { async: true });
|
||||
} catch (e) {
|
||||
// Don't hard-fail — show empty content with error message
|
||||
console.error('Failed to fetch chapter content:', e);
|
||||
}
|
||||
|
||||
const prevChapter = chapters.find((c) => c.number === n - 1) ?? null;
|
||||
const nextChapter = chapters.find((c) => c.number === n + 1) ?? null;
|
||||
|
||||
return {
|
||||
book: { slug: book.slug, title: book.title },
|
||||
chapter: chapterIdx,
|
||||
html,
|
||||
prev: prevChapter ? prevChapter.number : null,
|
||||
next: nextChapter ? nextChapter.number : null,
|
||||
sessionId: locals.sessionId
|
||||
};
|
||||
};
|
||||
103
ui/src/routes/books/[slug]/chapters/[n]/+page.svelte
Normal file
103
ui/src/routes/books/[slug]/chapters/[n]/+page.svelte
Normal file
@@ -0,0 +1,103 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import AudioPlayer from '$lib/components/AudioPlayer.svelte';
|
||||
import type { PageData } from './$types';
|
||||
|
||||
let { data }: { data: PageData } = $props();
|
||||
|
||||
// Record reading progress when the chapter is opened
|
||||
onMount(async () => {
|
||||
try {
|
||||
await fetch('/api/progress', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ slug: data.book.slug, chapter: data.chapter.number })
|
||||
});
|
||||
} catch {
|
||||
// Non-critical — silently ignore
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
<title>{data.chapter.title || `Chapter ${data.chapter.number}`} — {data.book.title} — libnovel</title>
|
||||
</svelte:head>
|
||||
|
||||
<!-- Top nav -->
|
||||
<div class="flex items-center justify-between mb-6 gap-4">
|
||||
<a
|
||||
href="/books/{data.book.slug}"
|
||||
class="text-zinc-400 hover:text-zinc-100 text-sm flex items-center gap-1 transition-colors"
|
||||
>
|
||||
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 19l-7-7 7-7" />
|
||||
</svg>
|
||||
{data.book.title}
|
||||
</a>
|
||||
|
||||
<div class="flex gap-2">
|
||||
{#if data.prev}
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/{data.prev}"
|
||||
class="px-3 py-1.5 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors"
|
||||
>
|
||||
← Ch.{data.prev}
|
||||
</a>
|
||||
{/if}
|
||||
{#if data.next}
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/{data.next}"
|
||||
class="px-3 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors"
|
||||
>
|
||||
Ch.{data.next} →
|
||||
</a>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Chapter heading -->
|
||||
<div class="mb-6">
|
||||
<p class="text-zinc-500 text-sm mb-1">Chapter {data.chapter.number}</p>
|
||||
<h1 class="text-xl font-bold text-zinc-100">
|
||||
{data.chapter.title || `Chapter ${data.chapter.number}`}
|
||||
</h1>
|
||||
{#if data.chapter.date_label}
|
||||
<p class="text-zinc-600 text-xs mt-1">{data.chapter.date_label}</p>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- Audio player -->
|
||||
<AudioPlayer slug={data.book.slug} chapter={data.chapter.number} />
|
||||
|
||||
<!-- Chapter content -->
|
||||
{#if !data.html}
|
||||
<div class="text-zinc-500 text-center py-16">
|
||||
<p>Chapter content not available.</p>
|
||||
</div>
|
||||
{:else}
|
||||
<div class="prose-chapter mt-8">
|
||||
{@html data.html}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Bottom nav -->
|
||||
<div class="flex justify-between mt-12 pt-6 border-t border-zinc-800 gap-4">
|
||||
{#if data.prev}
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/{data.prev}"
|
||||
class="px-4 py-2 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors"
|
||||
>
|
||||
← Previous chapter
|
||||
</a>
|
||||
{:else}
|
||||
<div></div>
|
||||
{/if}
|
||||
{#if data.next}
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/{data.next}"
|
||||
class="px-4 py-2 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors"
|
||||
>
|
||||
Next chapter →
|
||||
</a>
|
||||
{/if}
|
||||
</div>
|
||||
3
ui/static/robots.txt
Normal file
3
ui/static/robots.txt
Normal file
@@ -0,0 +1,3 @@
|
||||
# allow crawling everything by default
|
||||
User-agent: *
|
||||
Disallow:
|
||||
10
ui/svelte.config.js
Normal file
10
ui/svelte.config.js
Normal file
@@ -0,0 +1,10 @@
|
||||
import adapter from '@sveltejs/adapter-node';
|
||||
|
||||
/** @type {import('@sveltejs/kit').Config} */
|
||||
const config = {
|
||||
kit: {
|
||||
adapter: adapter()
|
||||
}
|
||||
};
|
||||
|
||||
export default config;
|
||||
20
ui/tsconfig.json
Normal file
20
ui/tsconfig.json
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"extends": "./.svelte-kit/tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"rewriteRelativeImportExtensions": true,
|
||||
"allowJs": true,
|
||||
"checkJs": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"skipLibCheck": true,
|
||||
"sourceMap": true,
|
||||
"strict": true,
|
||||
"moduleResolution": "bundler"
|
||||
}
|
||||
// Path aliases are handled by https://svelte.dev/docs/kit/configuration#alias
|
||||
// except $lib which is handled by https://svelte.dev/docs/kit/configuration#files
|
||||
//
|
||||
// To make changes to top-level options such as include and exclude, we recommend extending
|
||||
// the generated config; see https://svelte.dev/docs/kit/configuration#typescript
|
||||
}
|
||||
7
ui/vite.config.ts
Normal file
7
ui/vite.config.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
import { sveltekit } from '@sveltejs/kit/vite';
|
||||
import tailwindcss from '@tailwindcss/vite';
|
||||
import { defineConfig } from 'vite';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [tailwindcss(), sveltekit()]
|
||||
});
|
||||
Reference in New Issue
Block a user