Compare commits

..

10 Commits

Author SHA1 Message Date
Admin
baab66823d fix: remove container names for name collisions 2026-03-02 20:08:46 +05:00
Admin
11d2eaa0e5 chore: trigger deploy 2026-03-02 19:56:09 +05:00
Admin
9c115f00c4 chore: remove kokoro from compose, make host ports env-configurable
- Drop kokoro service (deployed separately); update default KOKORO_URL to kokoro.kalekber.cc
- Expose host ports via env vars (MINIO_PORT, MINIO_CONSOLE_PORT, POCKETBASE_PORT, BROWSERLESS_PORT, SCRAPER_PORT) for preview deployments
2026-03-02 15:13:26 +05:00
Admin
5ac89da513 Steps 11-12: migrate chapter + book page reading-progress JS from localStorage to /api/progress 2026-03-02 14:52:01 +05:00
Admin
af86c6f96f Step 10: migrate home page reading-progress JS from localStorage to /api/progress 2026-03-02 14:51:07 +05:00
Admin
da4a182f85 Step 9: add session cookie + /api/progress GET/POST/DELETE endpoints 2026-03-02 14:47:58 +05:00
Admin
18e76c9668 steps 6-8: wire HybridStore into orchestrator, server, and main
- Add storage/hybrid.go: HybridStore composing PocketBase + MinIO backends
- Rewrite orchestrator to accept storage.Store instead of *writer.Writer
- Replace *writer.Writer with storage.Store in server.go and ui.go
- Wire audio cache, reading progress, chapter reads/writes through store
- Add rankingCacheAdapter in main.go to bridge context-free RankingPageCacher
  interface to HybridStore's context-aware methods
2026-03-02 14:44:02 +05:00
Admin
9add9033b9 feat(v2): add internal/storage package with MinIO + PocketBase clients and Store interface 2026-03-02 14:34:25 +05:00
Admin
66d8481637 chore(v2): add minio-go/v7 dependency 2026-03-02 14:31:36 +05:00
Admin
7f92a58fd7 chore(v2): add PocketBase and MinIO services to docker-compose 2026-03-02 14:30:47 +05:00
12 changed files with 1758 additions and 397 deletions

View File

@@ -33,7 +33,9 @@ jobs:
run: go vet ./...
- name: staticcheck
run: go tool staticcheck ./...
run: |
go install honnef.co/go/tools/cmd/staticcheck@latest
staticcheck ./...
# ── tests ────────────────────────────────────────────────────────────────────
test:

View File

@@ -1,10 +1,64 @@
version: "3.9"
services:
# ─── MinIO (object storage for chapter .md files + audio cache) ─────────────
minio:
image: minio/minio:latest
#container_name: libnovel-minio
restart: unless-stopped
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
ports:
- "${MINIO_PORT:-9000}:9000" # S3 API
- "${MINIO_CONSOLE_PORT:-9001}:9001" # Web console
volumes:
- minio_data:/data
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 10s
timeout: 5s
retries: 5
# ─── MinIO bucket initialisation ─────────────────────────────────────────────
# Runs once to create the default buckets and then exits.
minio-init:
image: minio/mc:latest
#container_name: libnovel-minio-init
depends_on:
minio:
condition: service_healthy
entrypoint: >
/bin/sh -c "
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
mc mb --ignore-existing local/libnovel-chapters;
mc mb --ignore-existing local/libnovel-audio;
echo 'buckets ready';
"
environment:
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
# ─── PocketBase (auth + structured data: books, chapters index, ranking, progress) ──
pocketbase:
image: ghcr.io/muchobien/pocketbase:latest
#container_name: libnovel-pocketbase
restart: unless-stopped
ports:
- "${POCKETBASE_PORT:-8090}:8090"
volumes:
- pb_data:/pb/pb_data
healthcheck:
test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"]
interval: 10s
timeout: 5s
retries: 5
# ─── Browserless ────────────────────────────────────────────────────────────
browserless:
image: ghcr.io/browserless/chromium:latest
container_name: libnovel-browserless
#container_name: libnovel-browserless
restart: unless-stopped
environment:
# Set a token to lock down the endpoint; the scraper reads it via
@@ -19,7 +73,7 @@ services:
# Optional webhook URL for Browserless error alerts.
ERROR_ALERT_URL: "${ERROR_ALERT_URL:-}"
ports:
- "3030:3000"
- "${BROWSERLESS_PORT:-3030}:3000"
# Shared memory is required for Chrome.
shm_size: "2gb"
healthcheck:
@@ -28,30 +82,17 @@ services:
timeout: 5s
retries: 5
# ─── Kokoro-FastAPI (TTS) ────────────────────────────────────────────────────
# CPU image; swap for ghcr.io/remsky/kokoro-fastapi-gpu:latest on NVIDIA hosts.
# Models are baked in — no volume mount required for the default voice set.
kokoro:
image: ghcr.io/remsky/kokoro-fastapi-cpu:latest
container_name: libnovel-kokoro
restart: unless-stopped
ports:
- "8880:8880"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8880/health"]
interval: 15s
timeout: 5s
retries: 5
# ─── Scraper ─────────────────────────────────────────────────────────────────
scraper:
build:
context: ./scraper
dockerfile: Dockerfile
container_name: libnovel-scraper
#container_name: libnovel-scraper
restart: unless-stopped
depends_on:
kokoro:
pocketbase:
condition: service_healthy
minio:
condition: service_healthy
environment:
BROWSERLESS_URL: "http://browserless:3000"
@@ -66,10 +107,21 @@ services:
SCRAPER_HTTP_ADDR: ":8080"
LOG_LEVEL: "debug"
# Kokoro-FastAPI TTS endpoint.
KOKORO_URL: "${KOKORO_URL:-http://localhost:8880}"
KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc}"
KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}"
# MinIO / S3 object storage
MINIO_ENDPOINT: "minio:9000"
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
MINIO_USE_SSL: "false"
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
# PocketBase
POCKETBASE_URL: "http://pocketbase:8090"
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
ports:
- "8080:8080"
- "${SCRAPER_PORT:-8080}:8080"
volumes:
- static_books:/app/static/books
healthcheck:
@@ -80,3 +132,5 @@ services:
volumes:
static_books:
minio_data:
pb_data:

View File

@@ -10,16 +10,24 @@
//
// Environment variables:
//
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
// KOKORO_VOICE Default TTS voice (default: af_bella)
// LOG_LEVEL debug | info | warn | error (default: info)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
// KOKORO_VOICE Default TTS voice (default: af_bella)
// POCKETBASE_URL PocketBase API base URL (default: http://localhost:8090)
// POCKETBASE_EMAIL PocketBase admin email (default: admin@libnovel.local)
// POCKETBASE_PASSWORD PocketBase admin password (default: adminpassword)
// MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000)
// MINIO_ACCESS_KEY MinIO access key (default: minioadmin)
// MINIO_SECRET_KEY MinIO secret key (default: minioadmin)
// MINIO_USE_SSL Use TLS for MinIO (default: false)
// MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters)
// MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio)
// LOG_LEVEL debug | info | warn | error (default: info)
package main
import (
@@ -38,7 +46,7 @@ import (
"github.com/libnovel/scraper/internal/novelfire"
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/server"
"github.com/libnovel/scraper/internal/writer"
"github.com/libnovel/scraper/internal/storage"
)
func main() {
@@ -88,9 +96,30 @@ func run(log *slog.Logger) error {
bc := newBrowserClient(strategy, browserCfg)
urlClient := newBrowserClient(urlStrategy, browserCfg)
staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books")
w := writer.New(staticRoot)
nf := novelfire.New(bc, log, urlClient, w)
// ── Storage backends ────────────────────────────────────────────────────
minioCfg := storage.MinioConfig{
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
AccessKey: envOr("MINIO_ACCESS_KEY", "minioadmin"),
SecretKey: envOr("MINIO_SECRET_KEY", "minioadmin"),
UseSSL: strings.ToLower(os.Getenv("MINIO_USE_SSL")) == "true",
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
}
pbCfg := storage.PocketBaseConfig{
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
AdminEmail: envOr("POCKETBASE_EMAIL", "admin@libnovel.local"),
AdminPassword: envOr("POCKETBASE_PASSWORD", "adminpassword"),
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
store, err := storage.NewHybridStore(ctx, pbCfg, minioCfg)
if err != nil {
return fmt.Errorf("storage init failed: %w", err)
}
nf := novelfire.New(bc, log, urlClient, &rankingCacheAdapter{store: store})
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -104,13 +133,9 @@ func run(log *slog.Logger) error {
}
oCfg := orchestrator.Config{
Workers: workers,
StaticRoot: staticRoot,
Workers: workers,
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
switch cmd {
case "run":
// Optional --url flag.
@@ -121,10 +146,9 @@ func run(log *slog.Logger) error {
"strategy", strategy,
"workers", workers,
"max_concurrent", browserCfg.MaxConcurrent,
"static_root", oCfg.StaticRoot,
"single_book", oCfg.SingleBookURL,
)
o := orchestrator.New(oCfg, nf, log)
o := orchestrator.New(oCfg, nf, log, store)
return o.Run(ctx)
case "refresh":
@@ -133,13 +157,12 @@ func run(log *slog.Logger) error {
return fmt.Errorf("refresh command requires a book slug argument")
}
slug := args[1]
w := writer.New(oCfg.StaticRoot)
meta, ok, err := w.ReadMetadata(slug)
meta, ok, err := store.ReadMetadata(ctx, slug)
if err != nil {
return fmt.Errorf("failed to read metadata for %s: %w", slug, err)
}
if !ok {
return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot)
return fmt.Errorf("book %q not found in store", slug)
}
if meta.SourceURL == "" {
return fmt.Errorf("book %q has no source_url in metadata", slug)
@@ -149,12 +172,12 @@ func run(log *slog.Logger) error {
"slug", slug,
"source_url", meta.SourceURL,
)
o := orchestrator.New(oCfg, nf, log)
o := orchestrator.New(oCfg, nf, log, store)
return o.Run(ctx)
case "serve":
addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
kokoroURL := envOr("KOKORO_URL", "")
kokoroURL := envOr("KOKORO_URL", "https://kokoro.kalekber.cc")
kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
log.Info("starting HTTP server",
"addr", addr,
@@ -164,7 +187,7 @@ func run(log *slog.Logger) error {
"kokoro_url", kokoroURL,
"kokoro_voice", kokoroVoice,
)
srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice)
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
return srv.ListenAndServe(ctx)
default:
@@ -192,6 +215,20 @@ func envOr(key, fallback string) string {
return fallback
}
// rankingCacheAdapter bridges storage.HybridStore (context-aware) to the
// context-free scraper.RankingPageCacher interface expected by novelfire.New.
type rankingCacheAdapter struct {
store *storage.HybridStore
}
func (a *rankingCacheAdapter) WriteRankingPageCache(page int, html string) error {
return a.store.WriteRankingPageCache(context.Background(), page, html)
}
func (a *rankingCacheAdapter) ReadRankingPageCache(page int) (string, error) {
return a.store.ReadRankingPageCache(context.Background(), page)
}
func printUsage() {
fmt.Fprintf(os.Stderr, `libnovel scraper

View File

@@ -3,16 +3,29 @@ module github.com/libnovel/scraper
go 1.25.0
require (
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/yuin/goldmark v1.7.16 // indirect
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect
golang.org/x/mod v0.31.0 // indirect
golang.org/x/net v0.51.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/tools v0.40.1-0.20260108161641-ca281cf95054 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
honnef.co/go/tools v0.7.0 // indirect
github.com/gorilla/websocket v1.5.3
github.com/minio/minio-go/v7 v7.0.98
github.com/yuin/goldmark v1.7.16
golang.org/x/net v0.51.0
gopkg.in/yaml.v3 v3.0.1
)
tool honnef.co/go/tools/cmd/staticcheck
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
)

View File

@@ -1,22 +1,49 @@
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE=
github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 h1:1P7xPZEwZMoBoz0Yze5Nx2/4pxj6nw9ZqHWXqP0iRgQ=
golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/tools v0.40.1-0.20260108161641-ca281cf95054 h1:CHVDrNHx9ZoOrNN9kKWYIbT5Rj+WF2rlwPkhbQQ5V4U=
golang.org/x/tools v0.40.1-0.20260108161641-ca281cf95054/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU=
honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc=

View File

@@ -19,7 +19,7 @@ import (
"sync"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/writer"
"github.com/libnovel/scraper/internal/storage"
)
// Config holds tunable parameters for the orchestrator.
@@ -28,7 +28,8 @@ type Config struct {
// Defaults to runtime.NumCPU() when 0.
Workers int
// StaticRoot is the path to the static/books output directory.
// StaticRoot is kept for backwards-compatibility but is no longer used
// when a Store is provided.
StaticRoot string
// SingleBookURL when non-empty causes the orchestrator to scrape only
@@ -40,13 +41,13 @@ type Config struct {
type Orchestrator struct {
cfg Config
novel scraper.NovelScraper
writer *writer.Writer
store storage.Store
log *slog.Logger
workers int
}
// New returns a new Orchestrator.
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator {
// New returns a new Orchestrator backed by the provided Store.
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store) *Orchestrator {
workers := cfg.Workers
if workers <= 0 {
workers = runtime.NumCPU()
@@ -54,7 +55,7 @@ func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator
return &Orchestrator{
cfg: cfg,
novel: novel,
writer: writer.New(cfg.StaticRoot),
store: store,
log: log,
workers: workers,
}
@@ -66,7 +67,6 @@ func (o *Orchestrator) Run(ctx context.Context) error {
o.log.Info("orchestrator starting",
"source", o.novel.SourceName(),
"workers", o.workers,
"static_root", o.cfg.StaticRoot,
)
// chapterWork is the shared queue consumed by chapter worker goroutines.
@@ -89,8 +89,8 @@ func (o *Orchestrator) Run(ctx context.Context) error {
default:
}
// Skip if already on disk.
if o.writer.ChapterExists(job.slug, job.ref) {
// Skip if already stored.
if o.store.ChapterExists(ctx, job.slug, job.ref) {
o.log.Debug("chapter already exists, skipping",
"book", job.slug, "chapter", job.ref.Number)
continue
@@ -107,7 +107,7 @@ func (o *Orchestrator) Run(ctx context.Context) error {
continue
}
if err := o.writer.WriteChapter(job.slug, chapter); err != nil {
if err := o.store.WriteChapter(ctx, job.slug, chapter); err != nil {
o.log.Error("chapter write failed",
"book", job.slug,
"chapter", job.ref.Number,
@@ -135,8 +135,8 @@ func (o *Orchestrator) Run(ctx context.Context) error {
return
}
// Persist / update metadata.yaml.
if err := o.writer.WriteMetadata(meta); err != nil {
// Persist / update metadata.
if err := o.store.WriteMetadata(ctx, meta); err != nil {
o.log.Error("metadata write failed", "slug", meta.Slug, "err", err)
// Continue — chapters can still be scraped.
}

View File

@@ -10,6 +10,8 @@ package server
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"io"
@@ -22,7 +24,7 @@ import (
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/writer"
"github.com/libnovel/scraper/internal/storage"
)
// Server wraps an HTTP mux with the scraping endpoints.
@@ -31,7 +33,7 @@ type Server struct {
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
writer *writer.Writer
store storage.Store
mu sync.Mutex
running bool
rankingRunning bool
@@ -42,26 +44,23 @@ type Server struct {
voiceMu sync.RWMutex
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
// audioMu guards audioCache and audioInFlight.
// audioCache maps a cache key to the Kokoro download filename returned by
// POST /v1/audio/speech with return_download_link=true.
// audioMu guards audioInFlight only.
// Completed audio filenames are persisted to the Store (PocketBase).
// audioInFlight deduplicates concurrent generation requests for the same key.
audioMu sync.Mutex
audioCache map[string]string // cacheKey → kokoro download filename
audioInFlight map[string]chan struct{} // cacheKey → closed when done
}
// New creates a new Server.
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server {
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
return &Server{
addr: addr,
oCfg: oCfg,
novel: novel,
log: log,
writer: writer.New(oCfg.StaticRoot),
store: store,
kokoroURL: kokoroURL,
kokoroVoice: kokoroVoice,
audioCache: make(map[string]string),
audioInFlight: make(map[string]chan struct{}),
}
}
@@ -112,6 +111,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /health", s.handleHealth)
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
// Progress API
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
// UI routes
mux.HandleFunc("GET /", s.handleHome)
mux.HandleFunc("GET /scrape", s.handleScrape)
@@ -134,8 +137,6 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
`{"error":"audio generation timed out"}`,
)
mux.Handle("POST /ui/audio/{slug}/{n}", audioGenHandler)
// Status route: returns the proxy URL if audio was already generated, 404 otherwise.
mux.HandleFunc("GET /ui/audio/{slug}/{n}", s.handleAudioStatus)
// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
mux.HandleFunc("GET /ui/audio-proxy/{slug}/{n}", s.handleAudioProxy)
@@ -167,6 +168,123 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}
// ─── Session cookie helpers ───────────────────────────────────────────────────
const sessionCookieName = "libnovel_session"
// sessionID returns the session ID from the request cookie, or "" if absent.
func sessionID(r *http.Request) string {
c, err := r.Cookie(sessionCookieName)
if err != nil {
return ""
}
return c.Value
}
// newSessionID generates a random 16-byte hex session ID.
func newSessionID() (string, error) {
b := make([]byte, 16)
if _, err := rand.Read(b); err != nil {
return "", err
}
return hex.EncodeToString(b), nil
}
// ensureSession issues a new session cookie if the request does not already
// carry one, and returns the session ID (either existing or newly issued).
func ensureSession(w http.ResponseWriter, r *http.Request) string {
if id := sessionID(r); id != "" {
return id
}
id, err := newSessionID()
if err != nil {
// Very unlikely, but fall back to a timestamp-based ID.
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
}
http.SetCookie(w, &http.Cookie{
Name: sessionCookieName,
Value: id,
Path: "/",
HttpOnly: true,
SameSite: http.SameSiteLaxMode,
MaxAge: 365 * 24 * 60 * 60, // 1 year
})
return id
}
// ─── Reading progress API ─────────────────────────────────────────────────────
// handleGetProgress handles GET /api/progress.
// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...}
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
sid := ensureSession(w, r)
entries, err := s.store.AllProgress(r.Context(), sid)
if err != nil {
s.log.Error("AllProgress failed", "err", err)
entries = nil
}
progress := make(map[string]interface{}, len(entries)*2)
for _, p := range entries {
progress[p.Slug] = p.Chapter
progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(progress)
}
// handleSetProgress handles POST /api/progress/{slug}.
// Body: {"chapter": N}
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
sid := ensureSession(w, r)
slug := r.PathValue("slug")
if slug == "" {
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
return
}
var body struct {
Chapter int `json:"chapter"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest)
return
}
p := storage.ReadingProgress{
Slug: slug,
Chapter: body.Chapter,
UpdatedAt: time.Now(),
}
if err := s.store.SetProgress(r.Context(), sid, p); err != nil {
s.log.Error("SetProgress failed", "slug", slug, "err", err)
http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{})
}
// handleDeleteProgress handles DELETE /api/progress/{slug}.
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
sid := ensureSession(w, r)
slug := r.PathValue("slug")
if slug == "" {
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
return
}
if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil {
s.log.Error("DeleteProgress failed", "slug", slug, "err", err)
// Non-fatal — treat as success.
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{})
}
// handleChapterText returns the plain text of a chapter (markdown stripped)
// for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI.
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
@@ -176,7 +294,7 @@ func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r)
return
}
raw, err := s.writer.ReadChapter(slug, n)
raw, err := s.store.ReadChapter(r.Context(), slug, n)
if err != nil {
http.NotFound(w, r)
return
@@ -225,15 +343,14 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
// Fast path: already generated this session.
s.audioMu.Lock()
if filename, ok := s.audioCache[cacheKey]; ok {
s.audioMu.Unlock()
// Fast path: already generated (check persistent store first).
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
s.writeAudioResponse(w, slug, n, voice, speed, filename)
return
}
// Deduplicate concurrent generation for the same key.
s.audioMu.Lock()
if ch, ok := s.audioInFlight[cacheKey]; ok {
s.audioMu.Unlock()
select {
@@ -242,10 +359,8 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
return
}
s.audioMu.Lock()
filename, ok := s.audioCache[cacheKey]
s.audioMu.Unlock()
if ok {
// Check store again after waiting.
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
s.writeAudioResponse(w, slug, n, voice, speed, filename)
} else {
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
@@ -264,7 +379,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
}()
// Load and validate chapter text.
raw, err := s.writer.ReadChapter(slug, n)
raw, err := s.store.ReadChapter(r.Context(), slug, n)
if err != nil {
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
return
@@ -289,9 +404,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
return
}
s.audioMu.Lock()
s.audioCache[cacheKey] = filename
s.audioMu.Unlock()
_ = s.store.SetAudioCache(r.Context(), cacheKey, filename)
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
s.writeAudioResponse(w, slug, n, voice, speed, filename)
@@ -357,39 +470,6 @@ func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, v
})
}
// handleAudioStatus handles GET /ui/audio/{slug}/{n}.
// Returns the proxy URL if audio was already generated this session, 404 otherwise.
func (s *Server) handleAudioStatus(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 {
http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
return
}
voice := r.URL.Query().Get("voice")
if voice == "" {
voice = s.kokoroVoice
}
speedStr := r.URL.Query().Get("speed")
speed := 1.0
if speedStr != "" {
if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 {
speed = v
}
}
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
s.audioMu.Lock()
filename, ok := s.audioCache[cacheKey]
s.audioMu.Unlock()
if !ok {
http.Error(w, `{"error":"not generated"}`, http.StatusNotFound)
return
}
s.writeAudioResponse(w, slug, n, voice, speed, filename)
}
// handleAudioProxy handles GET /ui/audio-proxy/{slug}/{n}.
// It looks up the Kokoro download filename for this chapter (voice/speed) and
// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
@@ -413,10 +493,7 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
}
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
s.audioMu.Lock()
filename, ok := s.audioCache[cacheKey]
s.audioMu.Unlock()
filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
if !ok {
http.Error(w, "audio not generated yet", http.StatusNotFound)
return
@@ -497,7 +574,7 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
defer cancel()
o := orchestrator.New(cfg, s.novel, s.log)
o := orchestrator.New(cfg, s.novel, s.log, s.store)
if err := o.Run(ctx); err != nil {
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err))
}

View File

@@ -15,7 +15,7 @@ import (
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/writer"
"github.com/libnovel/scraper/internal/storage"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/extension"
goldhtml "github.com/yuin/goldmark/renderer/html"
@@ -300,14 +300,9 @@ const homeTmpl = `
<script>
(function () {
/* ── state ─────────────────────────────────────────────────────────────── */
var progress = {};
try { progress = JSON.parse(localStorage.getItem('reading_progress') || '{}'); } catch(_) {}
// progress = { slug: chapterNum, ... }
var progressTs = {};
try { progressTs = JSON.parse(localStorage.getItem('reading_progress_ts') || '{}'); } catch(_) {}
// progressTs = { slug: timestampMs, ... }
/* ── state (populated asynchronously from /api/progress) ───────────────── */
var progress = {}; // { slug: chapterNum }
var progressTs = {}; // { slug: timestampMs }
var booksGrid = document.getElementById('books-grid');
var continueSection = document.getElementById('continue-reading-section');
@@ -321,149 +316,7 @@ const homeTmpl = `
var allCards = booksGrid ? Array.from(booksGrid.querySelectorAll('[data-slug]')) : [];
/* ── reading_progress_ts: record timestamps for "recently read" sort ───── */
// On first load after upgrade, seed timestamps for existing progress entries.
var tsUpdated = false;
Object.keys(progress).forEach(function(slug) {
if (!progressTs[slug]) { progressTs[slug] = Date.now(); tsUpdated = true; }
});
if (tsUpdated) {
try { localStorage.setItem('reading_progress_ts', JSON.stringify(progressTs)); } catch(_) {}
}
/* ── progress bars + unread badges ─────────────────────────────────────── */
allCards.forEach(function(card) {
var slug = card.dataset.slug;
var total = parseInt(card.dataset.total, 10) || 0;
var downloaded = parseInt(card.dataset.downloaded, 10) || 0;
var chapterNum = progress[slug] ? parseInt(progress[slug], 10) : 0;
var base = downloaded || total;
if (chapterNum > 0 && base > 0) {
var pct = Math.min(100, Math.round(chapterNum / base * 100));
var wrap = card.querySelector('.book-progress-wrap');
var bar = card.querySelector('.book-progress-bar');
var lbl = card.querySelector('.book-progress-label');
if (wrap && bar && lbl) {
bar.style.width = pct + '%';
lbl.textContent = 'Ch.\u00a0' + chapterNum + (base ? '\u00a0/\u00a0' + base : '') + '\u00a0\u2014\u00a0' + pct + '%';
wrap.classList.remove('hidden');
}
}
// Unread badge: downloaded > last-read chapter
if (downloaded > 0 && chapterNum > 0 && downloaded > chapterNum) {
var unread = downloaded - chapterNum;
var badge = card.querySelector('.book-unread-badge');
if (badge) {
badge.textContent = '+' + unread + ' new';
badge.classList.remove('hidden');
}
}
});
/* ── stats strip ─────────────────────────────────────────────────────────── */
var totalChaptersRead = 0;
var booksCompleted = 0;
Object.keys(progress).forEach(function(slug) {
var card = booksGrid ? booksGrid.querySelector('[data-slug="' + slug + '"]') : null;
var ch = parseInt(progress[slug], 10) || 0;
totalChaptersRead += ch;
if (card) {
var total = parseInt(card.dataset.total, 10) || 0;
var dl = parseInt(card.dataset.downloaded, 10) || 0;
var base = dl || total;
if (base > 0 && ch >= base) booksCompleted++;
}
});
if (totalChaptersRead > 0) {
var estMins = Math.round(totalChaptersRead * 12); // ~12 min/chapter avg
var timeLabel = estMins >= 60
? Math.floor(estMins / 60) + 'h ' + (estMins % 60) + 'm'
: estMins + 'm';
document.getElementById('stat-books').textContent = allCards.length + ' book' + (allCards.length !== 1 ? 's' : '');
document.getElementById('stat-chapters').textContent = totalChaptersRead + ' ch read';
document.getElementById('stat-time').textContent = '~' + timeLabel + ' read';
if (booksCompleted > 0) {
document.getElementById('stat-completed').textContent = booksCompleted + ' completed';
document.getElementById('stat-completed-wrap').classList.remove('hidden');
}
statsStrip.classList.remove('hidden');
statsStrip.style.display = 'flex';
if (statsFallback) statsFallback.classList.add('hidden');
}
/* ── hero card: most recently read ─────────────────────────────────────── */
var heroSlug = null, heroTs = 0;
Object.keys(progressTs).forEach(function(slug) {
if (progressTs[slug] > heroTs && progress[slug]) {
heroTs = progressTs[slug]; heroSlug = slug;
}
});
if (heroSlug && heroSection) {
var heroCard = booksGrid ? booksGrid.querySelector('[data-slug="' + heroSlug + '"]') : null;
if (heroCard) {
var heroChapter = parseInt(progress[heroSlug], 10) || 0;
var heroTotal = parseInt(heroCard.dataset.total, 10) || parseInt(heroCard.dataset.downloaded, 10) || 0;
var heroPct = heroTotal > 0 ? Math.min(100, Math.round(heroChapter / heroTotal * 100)) : 0;
var heroCover = heroCard.dataset.cover || '';
var heroTitle = heroCard.dataset.title || '';
var heroHref = heroCard.getAttribute('href');
var heroGet = heroCard.getAttribute('hx-get');
heroSection.innerHTML =
'<a href="' + heroHref + '"' +
' hx-get="' + heroGet + '" hx-target="#main-content" hx-push-url="true" hx-swap="innerHTML"' +
' class="group relative block rounded-2xl overflow-hidden border border-zinc-700 hover:border-amber-500 transition-colors cursor-pointer"' +
' style="' + (heroCover ? 'background-image:linear-gradient(to right,#09090b 40%,rgba(9,9,11,0.4)),url(' + heroCover + ');background-size:cover;background-position:right center;' : 'background:#18181b;') + '">' +
'<div class="relative z-10 p-5 sm:p-6">' +
'<p class="text-[0.65rem] font-semibold uppercase tracking-widest text-amber-500 mb-1">Continue reading</p>' +
'<h2 class="text-xl sm:text-2xl font-bold text-zinc-100 group-hover:text-amber-400 transition-colors leading-snug mb-2 max-w-sm">' + escHtml(heroTitle) + '</h2>' +
'<p class="text-sm text-zinc-400 mb-4">Chapter\u00a0' + heroChapter + (heroTotal ? '\u00a0/\u00a0' + heroTotal : '') + (heroPct ? '\u00a0\u2014\u00a0' + heroPct + '%' : '') + '</p>' +
'<div class="w-48 h-1 rounded-full bg-zinc-700 overflow-hidden mb-4">' +
'<div class="h-full bg-amber-500 rounded-full" style="width:' + heroPct + '%"></div>' +
'</div>' +
'<span class="inline-block text-sm font-semibold px-4 py-2 rounded-xl bg-amber-500 text-zinc-900 group-hover:bg-amber-400 transition-colors">Resume</span>' +
'</div>' +
'</a>';
heroSection.classList.remove('hidden');
htmx.process(heroSection);
}
}
/* ── continue reading (excluding hero) ──────────────────────────────────── */
if (continueGrid) continueGrid.innerHTML = '';
var inProgress = Object.keys(progress).sort(function(a,b) {
return (progressTs[b]||0) - (progressTs[a]||0);
});
var moved = 0;
inProgress.forEach(function(slug) {
if (slug === heroSlug) return; // hero already shown
var card = booksGrid ? booksGrid.querySelector('[data-slug="' + slug + '"]') : null;
if (!card) return;
var a = buildContinueCard(card, progress[slug]);
continueGrid.appendChild(a);
moved++;
});
if (moved > 0) continueSection.classList.remove('hidden');
/* ── recently added (top 4, excluding books already in continue reading) ── */
if (recentlyAddedGrid) recentlyAddedGrid.innerHTML = '';
var progressSlugs = new Set(Object.keys(progress));
var byAdded = allCards.slice().sort(function(a, b) {
return (parseInt(b.dataset.added,10)||0) - (parseInt(a.dataset.added,10)||0);
});
var addedShown = 0;
byAdded.forEach(function(card) {
if (addedShown >= 4) return;
if (progressSlugs.has(card.dataset.slug)) return; // already in continue reading
recentlyAddedGrid.appendChild(buildSimpleCard(card));
addedShown++;
});
if (addedShown > 0) recentlyAddedSec.classList.remove('hidden');
/* ── genre chips ──────────────────────────────────────────────────────── */
/* ── genre chips (no progress dependency) ────────────────────────────── */
var allGenres = {};
allCards.forEach(function(card) {
(card.dataset.genres || '').split('|').forEach(function(g) {
@@ -508,7 +361,6 @@ const homeTmpl = `
function applySortAndFilter() {
var q = input ? input.value.trim().toLowerCase() : '';
// Sort cards
var sorted = allCards.slice().sort(function(a, b) {
if (currentSort === 'alpha') return (a.dataset.title||'').localeCompare(b.dataset.title||'');
if (currentSort === 'downloaded') return (parseInt(b.dataset.downloaded,10)||0) - (parseInt(a.dataset.downloaded,10)||0);
@@ -521,7 +373,6 @@ const homeTmpl = `
}
return 0;
});
// Re-append in sorted order, apply visibility
var shown = 0;
sorted.forEach(function(card) {
var matchText = !q || card.dataset.filter.toLowerCase().indexOf(q) !== -1;
@@ -532,7 +383,7 @@ const homeTmpl = `
var match = matchText && matchGenre;
card.style.display = match ? '' : 'none';
if (match) shown++;
booksGrid.appendChild(card); // re-order in DOM
booksGrid.appendChild(card);
});
if ((q || activeGenres.size > 0) && countEl) {
countEl.textContent = shown + ' result' + (shown !== 1 ? 's' : '');
@@ -579,7 +430,7 @@ const homeTmpl = `
}
}
}
// dismiss button
// dismiss button — calls DELETE /api/progress/{slug}
var dismissBtn = document.createElement('button');
dismissBtn.type = 'button';
dismissBtn.title = 'Remove from continue reading';
@@ -590,12 +441,9 @@ const homeTmpl = `
var cardEl = this.closest('[data-slug]');
var cardSlug = cardEl ? cardEl.dataset.slug : null;
if (cardSlug) {
var p = {}; try { p = JSON.parse(localStorage.getItem('reading_progress') || '{}'); } catch(_) {}
delete p[cardSlug];
localStorage.setItem('reading_progress', JSON.stringify(p));
var ts = {}; try { ts = JSON.parse(localStorage.getItem('reading_progress_ts') || '{}'); } catch(_) {}
delete ts[cardSlug];
localStorage.setItem('reading_progress_ts', JSON.stringify(ts));
delete progress[cardSlug];
delete progressTs[cardSlug];
fetch('/api/progress/' + encodeURIComponent(cardSlug), { method: 'DELETE' });
}
var target = cardEl || this.parentElement;
target.style.transition = 'transform 0.25s ease, opacity 0.25s ease';
@@ -612,6 +460,158 @@ const homeTmpl = `
return a;
}
/* ── render: called once progress is loaded ──────────────────────────── */
function renderWithProgress() {
/* ── progress bars + unread badges ─────────────────────────────────── */
allCards.forEach(function(card) {
var slug = card.dataset.slug;
var total = parseInt(card.dataset.total, 10) || 0;
var downloaded = parseInt(card.dataset.downloaded, 10) || 0;
var chapterNum = progress[slug] ? parseInt(progress[slug], 10) : 0;
var base = downloaded || total;
if (chapterNum > 0 && base > 0) {
var pct = Math.min(100, Math.round(chapterNum / base * 100));
var wrap = card.querySelector('.book-progress-wrap');
var bar = card.querySelector('.book-progress-bar');
var lbl = card.querySelector('.book-progress-label');
if (wrap && bar && lbl) {
bar.style.width = pct + '%';
lbl.textContent = 'Ch.\u00a0' + chapterNum + (base ? '\u00a0/\u00a0' + base : '') + '\u00a0\u2014\u00a0' + pct + '%';
wrap.classList.remove('hidden');
}
}
if (downloaded > 0 && chapterNum > 0 && downloaded > chapterNum) {
var unread = downloaded - chapterNum;
var badge = card.querySelector('.book-unread-badge');
if (badge) {
badge.textContent = '+' + unread + ' new';
badge.classList.remove('hidden');
}
}
});
/* ── stats strip ────────────────────────────────────────────────────── */
var totalChaptersRead = 0;
var booksCompleted = 0;
Object.keys(progress).forEach(function(slug) {
var card = booksGrid ? booksGrid.querySelector('[data-slug="' + slug + '"]') : null;
var ch = parseInt(progress[slug], 10) || 0;
totalChaptersRead += ch;
if (card) {
var total = parseInt(card.dataset.total, 10) || 0;
var dl = parseInt(card.dataset.downloaded, 10) || 0;
var base = dl || total;
if (base > 0 && ch >= base) booksCompleted++;
}
});
if (totalChaptersRead > 0) {
var estMins = Math.round(totalChaptersRead * 12);
var timeLabel = estMins >= 60
? Math.floor(estMins / 60) + 'h ' + (estMins % 60) + 'm'
: estMins + 'm';
document.getElementById('stat-books').textContent = allCards.length + ' book' + (allCards.length !== 1 ? 's' : '');
document.getElementById('stat-chapters').textContent = totalChaptersRead + ' ch read';
document.getElementById('stat-time').textContent = '~' + timeLabel + ' read';
if (booksCompleted > 0) {
document.getElementById('stat-completed').textContent = booksCompleted + ' completed';
document.getElementById('stat-completed-wrap').classList.remove('hidden');
}
statsStrip.classList.remove('hidden');
statsStrip.style.display = 'flex';
if (statsFallback) statsFallback.classList.add('hidden');
}
/* ── hero card: most recently read ─────────────────────────────────── */
var heroSlug = null, heroTs = 0;
Object.keys(progressTs).forEach(function(slug) {
if (progressTs[slug] > heroTs && progress[slug]) {
heroTs = progressTs[slug]; heroSlug = slug;
}
});
if (heroSlug && heroSection) {
var heroCard = booksGrid ? booksGrid.querySelector('[data-slug="' + heroSlug + '"]') : null;
if (heroCard) {
var heroChapter = parseInt(progress[heroSlug], 10) || 0;
var heroTotal = parseInt(heroCard.dataset.total, 10) || parseInt(heroCard.dataset.downloaded, 10) || 0;
var heroPct = heroTotal > 0 ? Math.min(100, Math.round(heroChapter / heroTotal * 100)) : 0;
var heroCover = heroCard.dataset.cover || '';
var heroTitle = heroCard.dataset.title || '';
var heroHref = heroCard.getAttribute('href');
var heroGet = heroCard.getAttribute('hx-get');
heroSection.innerHTML =
'<a href="' + heroHref + '"' +
' hx-get="' + heroGet + '" hx-target="#main-content" hx-push-url="true" hx-swap="innerHTML"' +
' class="group relative block rounded-2xl overflow-hidden border border-zinc-700 hover:border-amber-500 transition-colors cursor-pointer"' +
' style="' + (heroCover ? 'background-image:linear-gradient(to right,#09090b 40%,rgba(9,9,11,0.4)),url(' + heroCover + ');background-size:cover;background-position:right center;' : 'background:#18181b;') + '">' +
'<div class="relative z-10 p-5 sm:p-6">' +
'<p class="text-[0.65rem] font-semibold uppercase tracking-widest text-amber-500 mb-1">Continue reading</p>' +
'<h2 class="text-xl sm:text-2xl font-bold text-zinc-100 group-hover:text-amber-400 transition-colors leading-snug mb-2 max-w-sm">' + escHtml(heroTitle) + '</h2>' +
'<p class="text-sm text-zinc-400 mb-4">Chapter\u00a0' + heroChapter + (heroTotal ? '\u00a0/\u00a0' + heroTotal : '') + (heroPct ? '\u00a0\u2014\u00a0' + heroPct + '%' : '') + '</p>' +
'<div class="w-48 h-1 rounded-full bg-zinc-700 overflow-hidden mb-4">' +
'<div class="h-full bg-amber-500 rounded-full" style="width:' + heroPct + '%"></div>' +
'</div>' +
'<span class="inline-block text-sm font-semibold px-4 py-2 rounded-xl bg-amber-500 text-zinc-900 group-hover:bg-amber-400 transition-colors">Resume</span>' +
'</div>' +
'</a>';
heroSection.classList.remove('hidden');
htmx.process(heroSection);
}
}
/* ── continue reading (excluding hero) ──────────────────────────────── */
if (continueGrid) continueGrid.innerHTML = '';
var inProgress = Object.keys(progress).sort(function(a,b) {
return (progressTs[b]||0) - (progressTs[a]||0);
});
var moved = 0;
inProgress.forEach(function(slug) {
if (slug === heroSlug) return;
var card = booksGrid ? booksGrid.querySelector('[data-slug="' + slug + '"]') : null;
if (!card) return;
var a = buildContinueCard(card, progress[slug]);
continueGrid.appendChild(a);
moved++;
});
if (moved > 0) continueSection.classList.remove('hidden');
/* ── recently added (top 4, excluding books in continue reading) ────── */
if (recentlyAddedGrid) recentlyAddedGrid.innerHTML = '';
var progressSlugs = new Set(Object.keys(progress));
var byAdded = allCards.slice().sort(function(a, b) {
return (parseInt(b.dataset.added,10)||0) - (parseInt(a.dataset.added,10)||0);
});
var addedShown = 0;
byAdded.forEach(function(card) {
if (addedShown >= 4) return;
if (progressSlugs.has(card.dataset.slug)) return;
recentlyAddedGrid.appendChild(buildSimpleCard(card));
addedShown++;
});
if (addedShown > 0) recentlyAddedSec.classList.remove('hidden');
/* ── re-apply sort in case "recently read" is active ─────────────────── */
applySortAndFilter();
}
/* ── load progress from server, then render ──────────────────────────── */
fetch('/api/progress')
.then(function(r) { return r.ok ? r.json() : {}; })
.catch(function() { return {}; })
.then(function(data) {
Object.keys(data).forEach(function(key) {
if (key.slice(-3) === '_ts') {
progressTs[key.slice(0, -3)] = data[key];
} else {
progress[key] = data[key];
}
});
renderWithProgress();
});
}());
</script>`
@@ -629,7 +629,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
return
}
books, err := s.writer.ListBooks()
books, err := s.store.ListBooks(r.Context())
if err != nil {
http.Error(w, "failed to list books: "+err.Error(), http.StatusInternalServerError)
return
@@ -639,8 +639,8 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
for i, b := range books {
items[i] = homeBookItem{
BookMeta: b,
Downloaded: s.writer.CountChapters(b.Slug),
AddedAt: s.writer.MetadataMtime(b.Slug),
Downloaded: s.store.CountChapters(r.Context(), b.Slug),
AddedAt: s.store.MetadataMtime(r.Context(), b.Slug),
}
}
@@ -877,7 +877,7 @@ const scrapeTmpl = `
</script>`
func (s *Server) handleScrape(w http.ResponseWriter, r *http.Request) {
rankingItems, _ := s.writer.ReadRankingItems()
rankingItems, _ := s.store.ReadRankingItems(r.Context())
rankingJSON, _ := json.Marshal(rankingItems)
t := template.Must(template.New("scrape").Parse(scrapeTmpl))
@@ -1324,12 +1324,12 @@ const rankingTmpl = `
// rankingViewItem enriches a RankingItem with whether it is present in the
// local book library, so the template can highlight it differently.
type rankingViewItem struct {
writer.RankingItem
storage.RankingItem
Local bool
}
// toRankingViewItems annotates items with Local=true for slugs found in localSlugs.
func toRankingViewItems(items []writer.RankingItem, localSlugs map[string]bool) []rankingViewItem {
func toRankingViewItems(items []storage.RankingItem, localSlugs map[string]bool) []rankingViewItem {
out := make([]rankingViewItem, len(items))
for i, it := range items {
out[i] = rankingViewItem{
@@ -1403,13 +1403,13 @@ const rankingPageSize = 20
// It does NOT trigger a live scrape; use POST /ranking/refresh for that.
// Supports ?page=N for browsing through cached items (20 per page).
func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
rankingItems, err := s.writer.ReadRankingItems()
rankingItems, err := s.store.ReadRankingItems(r.Context())
if err != nil {
s.log.Error("failed to read cached ranking", "err", err)
}
cachedAt := ""
if info, statErr := s.writer.RankingFileInfo(); statErr == nil {
if info, statErr := s.store.RankingFileInfo(r.Context()); statErr == nil && info != nil {
cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04")
}
@@ -1482,7 +1482,7 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
}
// Encode full dataset + local slugs for client-side cross-page filtering.
localSlugs := s.writer.LocalSlugs()
localSlugs, _ := s.store.LocalSlugs(r.Context())
type rankingJSONItem struct {
Rank int `json:"rank"`
Slug string `json:"slug"`
@@ -1577,14 +1577,14 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages)
var rankingItems []writer.RankingItem
var rankingItems []storage.RankingItem
for rankingCh != nil || errCh != nil {
select {
case meta, ok := <-rankingCh:
if !ok {
rankingCh = nil
} else {
rankingItems = append(rankingItems, writer.RankingItem{
rankingItems = append(rankingItems, storage.RankingItem{
Rank: meta.Ranking,
Slug: meta.Slug,
Title: meta.Title,
@@ -1605,7 +1605,7 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
}
if len(rankingItems) > 0 {
if err := s.writer.WriteRanking(rankingItems); err != nil {
if err := s.store.WriteRanking(ctx, rankingItems); err != nil {
s.log.Error("failed to save ranking", "err", err)
}
}
@@ -1669,7 +1669,7 @@ const rankingViewTmpl = `
</div>`
func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) {
items, err := s.writer.ReadRankingItems()
items, err := s.store.ReadRankingItems(r.Context())
if err != nil {
http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError)
return
@@ -1769,7 +1769,7 @@ const bookTmpl = `
</div>
</div>
<!-- Resume reading button — populated by JS from localStorage -->
<!-- Resume reading button — populated by JS from /api/progress -->
<div id="resume-bar" class="mb-4 hidden">
<a id="resume-link"
hx-target="#main-content"
@@ -1874,16 +1874,8 @@ const bookTmpl = `
(function () {
var SLUG = '{{.Slug}}';
var LS_KEY = 'reading_progress';
function getProgress() {
try { return JSON.parse(localStorage.getItem(LS_KEY) || '{}'); } catch(_) { return {}; }
}
window.applyProgress = function(slug) {
var progress = getProgress();
var saved = progress[slug];
function applyProgressData(slug, saved) {
// Show/hide latest bar (only when no progress recorded for this book).
var latestBar = document.getElementById('latest-bar');
if (latestBar) {
@@ -1918,6 +1910,13 @@ const bookTmpl = `
htmx.process(link);
bar.classList.remove('hidden');
}
}
window.applyProgress = function(slug) {
fetch('/api/progress')
.then(function(r) { return r.ok ? r.json() : {}; })
.then(function(data) { applyProgressData(slug, data[slug] || null); })
.catch(function() { applyProgressData(slug, null); });
};
applyProgress(SLUG);
@@ -1927,7 +1926,7 @@ const bookTmpl = `
func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
meta, ok, err := s.writer.ReadMetadata(slug)
meta, ok, err := s.store.ReadMetadata(r.Context(), slug)
if err != nil {
http.Error(w, "failed to read metadata: "+err.Error(), http.StatusInternalServerError)
return
@@ -1937,7 +1936,7 @@ func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) {
return
}
chapters, err := s.writer.ListChapters(slug)
chapters, err := s.store.ListChapters(r.Context(), slug)
if err != nil {
http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError)
return
@@ -2054,7 +2053,7 @@ func (s *Server) handleBookChaptersPage(w http.ResponseWriter, r *http.Request)
}
}
chapters, err := s.writer.ListChapters(slug)
chapters, err := s.store.ListChapters(r.Context(), slug)
if err != nil {
http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError)
return
@@ -2471,14 +2470,11 @@ const chapterTmpl = `
// ── reading progress ──────────────────────────────────────────────────────
(function saveProgress() {
try {
var p = JSON.parse(localStorage.getItem('reading_progress') || '{}');
p[SLUG] = CHAPTER_N;
localStorage.setItem('reading_progress', JSON.stringify(p));
var ts = JSON.parse(localStorage.getItem('reading_progress_ts') || '{}');
ts[SLUG] = Date.now();
localStorage.setItem('reading_progress_ts', JSON.stringify(ts));
} catch(_) {}
fetch('/api/progress/' + encodeURIComponent(SLUG), {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ chapter: CHAPTER_N })
}).catch(function() {});
})();
// ── generation counter (stale-closure guard) ──────────────────────────────
@@ -2704,7 +2700,6 @@ const chapterTmpl = `
b.classList.toggle('text-zinc-300', !active);
});
try { localStorage.setItem(LS_SPEED, btn.dataset.speed); } catch(_) {}
if (audio.src) audio.playbackRate = parseFloat(btn.dataset.speed);
}
window.selectSpeed = selectSpeed;
(function loadSettings() {
@@ -2842,7 +2837,7 @@ const chapterTmpl = `
fetch('/ui/audio/' + SLUG + '/' + chapterN, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ voice: voiceSel.value, speed: 1 }),
body: JSON.stringify({ voice: voiceSel.value, speed: getSpeed() }),
signal: ctrl.signal
})
.then(function (res) {
@@ -2872,7 +2867,7 @@ const chapterTmpl = `
if (!activePara || activePara !== paras[idx]) highlightPara(idx);
}
function onCanPlay() { if (stale()) { audio.removeEventListener('canplay', onCanPlay); return; } audio.autoplay = false; if (audio.paused) audio.play().then(setPlaying).catch(function(e){ if (e.name === 'NotAllowedError') setPaused(); else setError(e.message); }); }
function onCanPlay() { if (stale()) { audio.removeEventListener('canplay', onCanPlay); return; } if (audio.paused) audio.play().then(setPlaying).catch(function(e){ setError(e.message); }); }
function onWaiting() { if (!stale()) setStatus('Buffering\u2026'); }
function onPlaying() { if (!stale()) setPlaying(); }
function onPause() { if (!stale() && !audio.ended) setPaused(); }
@@ -2922,43 +2917,14 @@ const chapterTmpl = `
navigator.mediaSession.setActionHandler('seekto', function(d) { if (isFinite(d.seekTime)) { audio.currentTime = d.seekTime; updateSeek(); } });
}
function loadAndPlay(url) {
// Always set autoplay=true before load(): the browser needs this hint set
// synchronously before load() regardless of whether we got here via a user
// gesture or a programmatic trigger (e.g. chapter transition). The canplay
// handler will call play() as well, but autoplay on the element is the only
// reliable path after an async fetch where the gesture stack is gone.
audio.autoplay = true;
audio.src = url; audio.load();
audio.playbackRate = getSpeed();
registerMediaSession();
}
function startAudio(autoplay) {
function startAudio() {
setGenerating();
highlightPara(0);
// Check if audio was already generated for this chapter this session.
// If so, play it directly without re-generating.
fetch('/ui/audio/' + SLUG + '/' + CHAPTER_N + '?voice=' + encodeURIComponent(voiceSel.value) + '&speed=1')
.then(function(res) {
if (stale()) return;
if (res.ok) {
return res.json().then(function(data) {
if (stale()) return;
if (data && data.url) { loadAndPlay(data.url); return; }
throw new Error('no url in response');
});
}
// Not cached yet — fall through to generation.
generateAudio(CHAPTER_N, function(url) {
if (stale()) return;
loadAndPlay(url);
});
})
.catch(function(e) {
if (stale()) return;
setError(e.message);
});
generateAudio(CHAPTER_N, function (url) {
if (stale()) return;
audio.src = url; audio.load();
registerMediaSession();
});
}
window.ttsToggle = function () {
@@ -2970,15 +2936,7 @@ const chapterTmpl = `
startAudio();
};
window.__ttsBeforeSwap = function () {
// If TTS is active (generating or has audio loaded), persist the autostart
// flag so the incoming chapter's init code will restart TTS automatically.
var wasActive = currentAudioCtrl !== null || (audio.src && !audio.ended);
if (wasActive) {
try { localStorage.setItem('tts_autostart', '1'); } catch(_) {}
}
window.__ttsGen++; stop();
};
window.__ttsBeforeSwap = function () { window.__ttsGen++; stop(); };
document.body.addEventListener('htmx:beforeSwap', window.__ttsBeforeSwap);
var _autostart = false;
@@ -2986,10 +2944,10 @@ const chapterTmpl = `
if (_autostart) {
try { localStorage.removeItem('tts_autostart'); } catch(_) {}
autoplayChk.checked = true;
setTimeout(function() { startAudio(true); }, 100);
setTimeout(startAudio, 100);
} else if (new URLSearchParams(window.location.search).get('autoplay') === '1') {
autoplayChk.checked = true;
setTimeout(function() { startAudio(true); }, 100);
setTimeout(startAudio, 100);
}
// ── double-tap left/right to navigate ────────────────────────────────────
@@ -3073,7 +3031,7 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) {
return
}
raw, err := s.writer.ReadChapter(slug, n)
raw, err := s.store.ReadChapter(r.Context(), slug, n)
if err != nil {
http.NotFound(w, r)
return
@@ -3089,15 +3047,15 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) {
return
}
chapters, _ := s.writer.ListChapters(slug)
chapters, _ := s.store.ListChapters(r.Context(), slug)
prevN, nextN := adjacentChapters(chapters, n)
title := firstHeading(raw, fmt.Sprintf("Chapter %d", n))
chapterTitle, chapterDate := writer.SplitChapterTitle(title)
chapterTitle, chapterDate := splitChapterTitle(title)
// Load cover URL for Media Session artwork (best-effort; ignore errors).
var coverURL string
if meta, ok, err := s.writer.ReadMetadata(slug); err == nil && ok {
if meta, ok, err := s.store.ReadMetadata(r.Context(), slug); err == nil && ok {
coverURL = meta.Cover
}
@@ -3134,6 +3092,36 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) {
// ─── helpers ──────────────────────────────────────────────────────────────────
// splitChapterTitle splits a raw chapter heading into a human-readable title
// and a trailing relative-date string (e.g. "1 year ago"). It mirrors the
// same logic in internal/writer and internal/storage/hybrid.
func splitChapterTitle(raw string) (title, date string) {
raw = strings.TrimSpace(raw)
// Strip leading numeric index.
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
prefix := raw[:idx]
allDigit := true
for _, c := range prefix {
if c < '0' || c > '9' {
allDigit = false
break
}
}
if allDigit {
raw = strings.TrimSpace(raw[idx:])
}
}
// Strip "Chapter N - N: " prefix.
chNumRe := regexp.MustCompile(`(?i)^chapter\s+\d+(?:\s*-\s*\d+)?\s*:\s*`)
raw = strings.TrimSpace(chNumRe.ReplaceAllString(raw, ""))
// Detect trailing relative date.
dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`)
if m := dateRe.FindStringSubmatchIndex(raw); m != nil {
return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]])
}
return raw, ""
}
// sortedKeys returns the keys of a string-bool map in sorted order.
func sortedKeys(m map[string]bool) []string {
out := make([]string, 0, len(m))
@@ -3165,7 +3153,7 @@ func stripMarkdown(src string) string {
// adjacentChapters returns the chapter numbers immediately before and after n
// in the sorted chapters list. 0 means "does not exist".
func adjacentChapters(chapters []writer.ChapterInfo, n int) (prev, next int) {
func adjacentChapters(chapters []storage.ChapterInfo, n int) (prev, next int) {
for i, ch := range chapters {
if ch.Number == n {
if i > 0 {
@@ -3249,7 +3237,7 @@ func (s *Server) handleUIScrapeBook(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
defer cancel()
o := orchestrator.New(cfg, s.novel, s.log)
o := orchestrator.New(cfg, s.novel, s.log, s.store)
if err := o.Run(ctx); err != nil {
s.log.Error("UI scrape job failed", "url", bookURL, "err", err)
}

View File

@@ -0,0 +1,300 @@
// hybrid.go implements the Store interface using PocketBase for structured data
// and MinIO for binary chapter/audio blobs.
package storage
import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"strconv"
"strings"
"time"
"github.com/libnovel/scraper/internal/scraper"
)
// HybridStore satisfies Store by routing structured data to PocketBase and
// binary objects (chapters, audio) to MinIO.
type HybridStore struct {
pb *PocketBaseStore
minio *MinioClient
}
// NewHybridStore constructs a HybridStore. It connects to both backends and
// calls EnsureCollections to bootstrap any missing PocketBase collections.
func NewHybridStore(ctx context.Context, pbCfg PocketBaseConfig, minioCfg MinioConfig) (*HybridStore, error) {
mc, err := NewMinioClient(ctx, minioCfg)
if err != nil {
return nil, fmt.Errorf("storage: minio: %w", err)
}
pb := NewPocketBaseStore(pbCfg)
if err := pb.EnsureCollections(ctx); err != nil {
// Log but don't fail — collection creation errors are often "already exists"
_ = err
}
return &HybridStore{pb: pb, minio: mc}, nil
}
// ─── Book metadata ────────────────────────────────────────────────────────────
func (h *HybridStore) WriteMetadata(ctx context.Context, meta scraper.BookMeta) error {
return h.pb.UpsertBook(ctx,
meta.Slug, meta.Title, meta.Author, meta.Cover,
meta.Status, meta.Summary, meta.SourceURL,
meta.Genres, meta.TotalChapters, meta.Ranking,
)
}
func (h *HybridStore) ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) {
rec, found, err := h.pb.GetBook(ctx, slug)
if err != nil || !found {
return scraper.BookMeta{}, found, err
}
return recToBookMeta(rec), true, nil
}
func (h *HybridStore) ListBooks(ctx context.Context) ([]scraper.BookMeta, error) {
rows, err := h.pb.ListBooks(ctx)
if err != nil {
return nil, err
}
books := make([]scraper.BookMeta, 0, len(rows))
for _, r := range rows {
books = append(books, recToBookMeta(r))
}
return books, nil
}
func (h *HybridStore) LocalSlugs(ctx context.Context) (map[string]bool, error) {
books, err := h.ListBooks(ctx)
if err != nil {
return nil, err
}
slugs := make(map[string]bool, len(books))
for _, b := range books {
slugs[b.Slug] = true
}
return slugs, nil
}
func (h *HybridStore) MetadataMtime(ctx context.Context, slug string) int64 {
t, err := h.pb.BookMetaUpdated(ctx, slug)
if err != nil || t.IsZero() {
return 0
}
return t.Unix()
}
// ─── Chapters ─────────────────────────────────────────────────────────────────
func (h *HybridStore) ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool {
return h.minio.ChapterExists(ctx, slug, ref.Volume, ref.Number)
}
func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error {
content := "# " + chapter.Ref.Title + "\n\n" + chapter.Text + "\n"
if err := h.minio.PutChapter(ctx, slug, chapter.Ref.Volume, chapter.Ref.Number, content); err != nil {
return err
}
// Update chapter index in PocketBase.
title, dateLabel := splitChapterTitle(chapter.Ref.Title)
_ = h.pb.UpsertChapterIdx(ctx, slug, chapter.Ref.Number, title, dateLabel)
return nil
}
func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
return h.minio.GetChapter(ctx, slug, 0, n)
}
func (h *HybridStore) ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) {
rows, err := h.pb.ListChapterIdx(ctx, slug)
if err != nil {
return nil, err
}
infos := make([]ChapterInfo, 0, len(rows))
for _, r := range rows {
n := int(floatVal(r, "number"))
title, _ := r["title"].(string)
date, _ := r["date_label"].(string)
infos = append(infos, ChapterInfo{Number: n, Title: title, Date: date})
}
sort.Slice(infos, func(i, j int) bool { return infos[i].Number < infos[j].Number })
return infos, nil
}
func (h *HybridStore) CountChapters(ctx context.Context, slug string) int {
return h.pb.CountChapterIdx(ctx, slug)
}
// ─── Ranking ─────────────────────────────────────────────────────────────────
func (h *HybridStore) WriteRanking(ctx context.Context, items []RankingItem) error {
data, err := json.Marshal(items)
if err != nil {
return fmt.Errorf("storage: marshal ranking: %w", err)
}
return h.pb.SetRanking(ctx, string(data))
}
func (h *HybridStore) ReadRankingItems(ctx context.Context) ([]RankingItem, error) {
dataStr, _, err := h.pb.GetRanking(ctx)
if err != nil || dataStr == "" {
return nil, err
}
var items []RankingItem
if err := json.Unmarshal([]byte(dataStr), &items); err != nil {
return nil, fmt.Errorf("storage: unmarshal ranking: %w", err)
}
return items, nil
}
func (h *HybridStore) RankingFileInfo(ctx context.Context) (os.FileInfo, error) {
return h.pb.RankingModTime(ctx)
}
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
func (h *HybridStore) WriteRankingPageCache(ctx context.Context, page int, html string) error {
return h.pb.SetRankingPageHTML(ctx, page, html)
}
func (h *HybridStore) ReadRankingPageCache(ctx context.Context, page int) (string, error) {
html, _, err := h.pb.GetRankingPageHTML(ctx, page)
return html, err
}
func (h *HybridStore) RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error) {
return h.pb.RankingPageCacheModTime(ctx, page)
}
// ─── Audio cache ──────────────────────────────────────────────────────────────
func (h *HybridStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool) {
filename, ok, _ := h.pb.GetAudioCache(ctx, cacheKey)
return filename, ok
}
func (h *HybridStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
return h.pb.SetAudioCache(ctx, cacheKey, filename)
}
// ─── Reading progress ─────────────────────────────────────────────────────────
func (h *HybridStore) GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) {
ch, updated, ok, err := h.pb.GetProgress(ctx, sessionID, slug)
if err != nil || !ok {
return ReadingProgress{}, false
}
return ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}, true
}
func (h *HybridStore) SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error {
return h.pb.SetProgress(ctx, sessionID, p.Slug, p.Chapter)
}
func (h *HybridStore) AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) {
rows, err := h.pb.AllProgress(ctx, sessionID)
if err != nil {
return nil, err
}
out := make([]ReadingProgress, 0, len(rows))
for _, r := range rows {
slug, _ := r["slug"].(string)
ch := int(floatVal(r, "chapter"))
var updated time.Time
if ts, ok := r["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
}
out = append(out, ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated})
}
return out, nil
}
func (h *HybridStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
return h.pb.DeleteProgress(ctx, sessionID, slug)
}
// ─── AudioObjectKey ───────────────────────────────────────────────────────────
func (h *HybridStore) AudioObjectKey(slug string, n int, voice string, speed float64) string {
return AudioObjectKey(slug, n, voice, speed)
}
// ─── helpers ──────────────────────────────────────────────────────────────────
func recToBookMeta(rec map[string]interface{}) scraper.BookMeta {
m := scraper.BookMeta{
Slug: strVal(rec, "slug"),
Title: strVal(rec, "title"),
Author: strVal(rec, "author"),
Cover: strVal(rec, "cover"),
Status: strVal(rec, "status"),
Summary: strVal(rec, "summary"),
SourceURL: strVal(rec, "source_url"),
}
if tc := floatVal(rec, "total_chapters"); tc > 0 {
m.TotalChapters = int(tc)
}
if rk := floatVal(rec, "ranking"); rk > 0 {
m.Ranking = int(rk)
}
// Genres stored as JSON string or array.
switch v := rec["genres"].(type) {
case string:
_ = json.Unmarshal([]byte(v), &m.Genres)
case []interface{}:
for _, g := range v {
if s, ok := g.(string); ok {
m.Genres = append(m.Genres, s)
}
}
}
return m
}
func strVal(m map[string]interface{}, key string) string {
if v, ok := m[key].(string); ok {
return v
}
return ""
}
// splitChapterTitle mirrors writer.SplitChapterTitle logic (simplified).
func splitChapterTitle(raw string) (title, date string) {
raw = strings.TrimSpace(raw)
// Strip leading numeric index.
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
prefix := raw[:idx]
allDigit := true
for _, c := range prefix {
if c < '0' || c > '9' {
allDigit = false
break
}
}
if allDigit {
raw = strings.TrimSpace(raw[idx:])
}
}
// Detect trailing relative date.
units := []string{"second", "minute", "hour", "day", "week", "month", "year"}
lower := strings.ToLower(raw)
for _, u := range units {
for _, suffix := range []string{u + "s ago", u + " ago"} {
if idx := strings.LastIndex(lower, suffix); idx > 0 {
// Find start of date token (digit before the unit).
start := strings.LastIndex(raw[:idx], " ")
if start < 0 {
start = 0
}
numPart := strings.TrimSpace(raw[start:idx])
if _, err := strconv.Atoi(strings.Fields(numPart)[0]); err == nil {
return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)])
}
}
}
}
return raw, ""
}

View File

@@ -0,0 +1,179 @@
package storage
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
// MinioConfig holds connection parameters for MinIO.
type MinioConfig struct {
Endpoint string // e.g. "minio:9000"
AccessKey string
SecretKey string
UseSSL bool
BucketChapters string // e.g. "libnovel-chapters"
BucketAudio string // e.g. "libnovel-audio"
}
// MinioClient wraps a minio.Client and exposes object operations for
// chapters and audio files.
type MinioClient struct {
c *minio.Client
cfg MinioConfig
}
// NewMinioClient creates a connected MinIO client and ensures the required
// buckets exist.
func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error) {
c, err := minio.New(cfg.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: cfg.UseSSL,
})
if err != nil {
return nil, fmt.Errorf("minio: new client: %w", err)
}
mc := &MinioClient{c: c, cfg: cfg}
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio} {
if err := mc.ensureBucket(ctx, bucket); err != nil {
return nil, err
}
}
return mc, nil
}
// ensureBucket creates a bucket if it does not exist.
func (m *MinioClient) ensureBucket(ctx context.Context, bucket string) error {
exists, err := m.c.BucketExists(ctx, bucket)
if err != nil {
return fmt.Errorf("minio: bucket exists %q: %w", bucket, err)
}
if !exists {
if err := m.c.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
return fmt.Errorf("minio: make bucket %q: %w", bucket, err)
}
}
return nil
}
// ─── Chapter objects ──────────────────────────────────────────────────────────
// chapterKey returns the MinIO object key for a chapter.
// Layout: {slug}/vol-{vol}/{lo}-{hi}/chapter-{n}.md
func chapterKey(slug string, vol, n int) string {
const chaptersPerFolder = 50
lo := ((n-1)/chaptersPerFolder)*chaptersPerFolder + 1
hi := lo + chaptersPerFolder - 1
return fmt.Sprintf("%s/vol-%d/%d-%d/chapter-%d.md", slug, vol, lo, hi, n)
}
// PutChapter stores chapter markdown in MinIO.
func (m *MinioClient) PutChapter(ctx context.Context, slug string, vol, n int, content string) error {
key := chapterKey(slug, vol, n)
data := []byte(content)
_, err := m.c.PutObject(ctx, m.cfg.BucketChapters, key,
bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: "text/markdown; charset=utf-8"})
if err != nil {
return fmt.Errorf("minio: put chapter %s: %w", key, err)
}
return nil
}
// GetChapter retrieves chapter markdown from MinIO.
func (m *MinioClient) GetChapter(ctx context.Context, slug string, vol, n int) (string, error) {
key := chapterKey(slug, vol, n)
obj, err := m.c.GetObject(ctx, m.cfg.BucketChapters, key, minio.GetObjectOptions{})
if err != nil {
return "", fmt.Errorf("minio: get chapter %s: %w", key, err)
}
defer obj.Close()
data, err := io.ReadAll(obj)
if err != nil {
return "", fmt.Errorf("minio: read chapter %s: %w", key, err)
}
return string(data), nil
}
// ChapterExists returns true if the object for this chapter is present.
func (m *MinioClient) ChapterExists(ctx context.Context, slug string, vol, n int) bool {
key := chapterKey(slug, vol, n)
_, err := m.c.StatObject(ctx, m.cfg.BucketChapters, key, minio.StatObjectOptions{})
return err == nil
}
// ListChapterKeys returns all object keys under slug/ in the chapters bucket,
// sorted lexicographically (MinIO returns them in order).
func (m *MinioClient) ListChapterKeys(ctx context.Context, slug string) ([]string, error) {
prefix := slug + "/"
var keys []string
for obj := range m.c.ListObjects(ctx, m.cfg.BucketChapters,
minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
if obj.Err != nil {
return nil, fmt.Errorf("minio: list chapters %s: %w", slug, obj.Err)
}
keys = append(keys, obj.Key)
}
return keys, nil
}
// CountChapters returns the number of chapter objects for a slug.
func (m *MinioClient) CountChapters(ctx context.Context, slug string) int {
keys, _ := m.ListChapterKeys(ctx, slug)
return len(keys)
}
// ─── Audio objects ────────────────────────────────────────────────────────────
// AudioObjectKey returns the MinIO key for a cached audio file.
// Key: {slug}/ch{n}-{voice}-{speed:.1f}.mp3
func AudioObjectKey(slug string, n int, voice string, speed float64) string {
safe := sanitiseVoice(voice)
return fmt.Sprintf("%s/ch%d-%s-%.1f.mp3", slug, n, safe, speed)
}
// PutAudio stores an audio file in the audio bucket.
func (m *MinioClient) PutAudio(ctx context.Context, key string, data []byte) error {
_, err := m.c.PutObject(ctx, m.cfg.BucketAudio, key,
bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: "audio/mpeg"})
if err != nil {
return fmt.Errorf("minio: put audio %s: %w", key, err)
}
return nil
}
// GetAudio retrieves audio bytes from the audio bucket.
func (m *MinioClient) GetAudio(ctx context.Context, key string) ([]byte, error) {
obj, err := m.c.GetObject(ctx, m.cfg.BucketAudio, key, minio.GetObjectOptions{})
if err != nil {
return nil, fmt.Errorf("minio: get audio %s: %w", key, err)
}
defer obj.Close()
return io.ReadAll(obj)
}
// AudioExists returns true if the audio object is present in the bucket.
func (m *MinioClient) AudioExists(ctx context.Context, key string) bool {
_, err := m.c.StatObject(ctx, m.cfg.BucketAudio, key, minio.StatObjectOptions{})
return err == nil
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// sanitiseVoice converts a voice name to a filename-safe string.
func sanitiseVoice(voice string) string {
return strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
return r
}
return '_'
}, voice)
}

View File

@@ -0,0 +1,560 @@
// Package storage — PocketBase REST client.
//
// Collections expected in PocketBase:
//
// books — slug(text,unique), title, author, cover, status, genres(json),
// summary, total_chapters(number), source_url, ranking(number), updated(date)
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
// ranking — data(json), updated(date) [single row, upserted by slug="_ranking_"]
// ranking_html — page(number,unique), html(text), updated(date)
// progress — session_id(text), slug(text), chapter(number), updated(date)
// audio_cache — cache_key(text,unique), filename(text), updated(date)
package storage
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
)
// PocketBaseConfig holds PocketBase connection settings.
type PocketBaseConfig struct {
BaseURL string // e.g. "http://pocketbase:8090"
AdminEmail string
AdminPassword string
}
// pbClient is a minimal PocketBase admin REST client.
type pbClient struct {
cfg PocketBaseConfig
httpClient *http.Client
tokenMu sync.RWMutex
token string
tokenExp time.Time
}
// newPBClient creates a new PocketBase client. It does not authenticate yet;
// authentication happens lazily on the first API call.
func newPBClient(cfg PocketBaseConfig) *pbClient {
return &pbClient{
cfg: cfg,
httpClient: &http.Client{Timeout: 15 * time.Second},
}
}
// ─── Auth ─────────────────────────────────────────────────────────────────────
func (p *pbClient) authenticate(ctx context.Context) error {
body, _ := json.Marshal(map[string]string{
"identity": p.cfg.AdminEmail,
"password": p.cfg.AdminPassword,
})
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
p.cfg.BaseURL+"/api/admins/auth-with-password", bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
resp, err := p.httpClient.Do(req)
if err != nil {
return fmt.Errorf("pocketbase: auth: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pocketbase: auth status %d: %s", resp.StatusCode, b)
}
var result struct {
Token string `json:"token"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return fmt.Errorf("pocketbase: decode auth: %w", err)
}
p.tokenMu.Lock()
p.token = result.Token
p.tokenExp = time.Now().Add(12 * time.Hour)
p.tokenMu.Unlock()
return nil
}
func (p *pbClient) authToken(ctx context.Context) (string, error) {
p.tokenMu.RLock()
tok, exp := p.token, p.tokenExp
p.tokenMu.RUnlock()
if tok != "" && time.Now().Before(exp) {
return tok, nil
}
if err := p.authenticate(ctx); err != nil {
return "", err
}
p.tokenMu.RLock()
defer p.tokenMu.RUnlock()
return p.token, nil
}
// ─── Generic CRUD helpers ──────────────────────────────────────────────────────
func (p *pbClient) do(ctx context.Context, method, path string, body interface{}) (*http.Response, error) {
tok, err := p.authToken(ctx)
if err != nil {
return nil, err
}
var bodyReader io.Reader
if body != nil {
b, _ := json.Marshal(body)
bodyReader = bytes.NewReader(b)
}
req, err := http.NewRequestWithContext(ctx, method, p.cfg.BaseURL+path, bodyReader)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", tok)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
return p.httpClient.Do(req)
}
// listOne fetches the first matching record from a collection.
func (p *pbClient) listOne(ctx context.Context, collection, filter string) (map[string]interface{}, error) {
q := url.Values{}
q.Set("filter", filter)
q.Set("perPage", "1")
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
resp, err := p.do(ctx, http.MethodGet, path, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return nil, nil
}
var result struct {
Items []map[string]interface{} `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
if len(result.Items) == 0 {
return nil, nil
}
return result.Items[0], nil
}
// listAll returns all records (up to 500) from a collection matching filter.
func (p *pbClient) listAll(ctx context.Context, collection, filter, sort string) ([]map[string]interface{}, error) {
q := url.Values{}
if filter != "" {
q.Set("filter", filter)
}
if sort != "" {
q.Set("sort", sort)
}
q.Set("perPage", "500")
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
resp, err := p.do(ctx, http.MethodGet, path, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var result struct {
Items []map[string]interface{} `json:"items"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return nil, err
}
return result.Items, nil
}
// upsert creates a record; if one matching filter already exists it updates it.
func (p *pbClient) upsert(ctx context.Context, collection, filter string, data map[string]interface{}) error {
existing, err := p.listOne(ctx, collection, filter)
if err != nil {
return err
}
if existing != nil {
id := existing["id"].(string)
resp, err := p.do(ctx, http.MethodPatch,
fmt.Sprintf("/api/collections/%s/records/%s", collection, id), data)
if err != nil {
return err
}
resp.Body.Close()
return nil
}
resp, err := p.do(ctx, http.MethodPost,
fmt.Sprintf("/api/collections/%s/records", collection), data)
if err != nil {
return err
}
resp.Body.Close()
return nil
}
// deleteWhere deletes all records matching filter in collection.
func (p *pbClient) deleteWhere(ctx context.Context, collection, filter string) error {
items, err := p.listAll(ctx, collection, filter, "")
if err != nil {
return err
}
for _, item := range items {
id, _ := item["id"].(string)
resp, err := p.do(ctx, http.MethodDelete,
fmt.Sprintf("/api/collections/%s/records/%s", collection, id), nil)
if err != nil {
return err
}
resp.Body.Close()
}
return nil
}
// ─── PocketBaseStore ──────────────────────────────────────────────────────────
// PocketBaseStore implements the structured-data portion of the Store interface
// backed by PocketBase REST API.
type PocketBaseStore struct {
pb *pbClient
}
// NewPocketBaseStore returns a connected PocketBaseStore.
func NewPocketBaseStore(cfg PocketBaseConfig) *PocketBaseStore {
return &PocketBaseStore{pb: newPBClient(cfg)}
}
// Ping verifies connectivity by authenticating.
func (s *PocketBaseStore) Ping(ctx context.Context) error {
_, err := s.pb.authToken(ctx)
return err
}
// ─── Collections schema bootstrap ────────────────────────────────────────────
// CollectionDef maps a collection name to its fields for auto-creation.
// EnsureCollections creates missing collections via the PocketBase API.
// Safe to call on every startup — existing collections are skipped.
func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
// We just attempt to create each collection; 400/422 errors for "already
// exists" are silently ignored.
collections := []map[string]interface{}{
{
"name": "books",
"type": "base",
"schema": []map[string]interface{}{
{"name": "slug", "type": "text", "required": true, "options": map[string]interface{}{"min": 1}},
{"name": "title", "type": "text", "required": true},
{"name": "author", "type": "text"},
{"name": "cover", "type": "url"},
{"name": "status", "type": "text"},
{"name": "genres", "type": "json"},
{"name": "summary", "type": "text"},
{"name": "total_chapters", "type": "number"},
{"name": "source_url", "type": "url"},
{"name": "ranking", "type": "number"},
{"name": "meta_updated", "type": "date"},
},
},
{
"name": "chapters_idx",
"type": "base",
"schema": []map[string]interface{}{
{"name": "slug", "type": "text", "required": true},
{"name": "number", "type": "number", "required": true},
{"name": "title", "type": "text"},
{"name": "date_label", "type": "text"},
},
},
{
"name": "ranking",
"type": "base",
"schema": []map[string]interface{}{
{"name": "key", "type": "text", "required": true},
{"name": "data", "type": "json"},
{"name": "updated", "type": "date"},
},
},
{
"name": "ranking_html",
"type": "base",
"schema": []map[string]interface{}{
{"name": "page", "type": "number", "required": true},
{"name": "html", "type": "text"},
{"name": "updated", "type": "date"},
},
},
{
"name": "progress",
"type": "base",
"schema": []map[string]interface{}{
{"name": "session_id", "type": "text", "required": true},
{"name": "slug", "type": "text", "required": true},
{"name": "chapter", "type": "number"},
{"name": "updated", "type": "date"},
},
},
{
"name": "audio_cache",
"type": "base",
"schema": []map[string]interface{}{
{"name": "cache_key", "type": "text", "required": true},
{"name": "filename", "type": "text"},
{"name": "updated", "type": "date"},
},
},
}
for _, col := range collections {
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections", col)
if err != nil {
return fmt.Errorf("pocketbase: ensure collection %v: %w", col["name"], err)
}
resp.Body.Close()
// 400/422 = already exists or schema mismatch — ignore
}
return nil
}
// ─── Book metadata ────────────────────────────────────────────────────────────
func (s *PocketBaseStore) UpsertBook(ctx context.Context, slug, title, author, cover, status, summary, sourceURL string, genres []string, totalChapters, ranking int) error {
genresJSON, _ := json.Marshal(genres)
return s.pb.upsert(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)), map[string]interface{}{
"slug": slug,
"title": title,
"author": author,
"cover": cover,
"status": status,
"genres": string(genresJSON),
"summary": summary,
"total_chapters": totalChapters,
"source_url": sourceURL,
"ranking": ranking,
"meta_updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetBook(ctx context.Context, slug string) (map[string]interface{}, bool, error) {
rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)))
if err != nil {
return nil, false, err
}
if rec == nil {
return nil, false, nil
}
return rec, true, nil
}
func (s *PocketBaseStore) ListBooks(ctx context.Context) ([]map[string]interface{}, error) {
return s.pb.listAll(ctx, "books", "", "+title")
}
func (s *PocketBaseStore) BookMetaUpdated(ctx context.Context, slug string) (time.Time, error) {
rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)))
if err != nil || rec == nil {
return time.Time{}, err
}
if ts, ok := rec["meta_updated"].(string); ok {
t, err := time.Parse(time.RFC3339, ts)
if err == nil {
return t, nil
}
}
return time.Time{}, nil
}
// ─── Chapter index ────────────────────────────────────────────────────────────
func (s *PocketBaseStore) UpsertChapterIdx(ctx context.Context, slug string, number int, title, dateLabel string) error {
return s.pb.upsert(ctx, "chapters_idx",
fmt.Sprintf(`slug="%s"&&number=%d`, pbEsc(slug), number),
map[string]interface{}{
"slug": slug,
"number": number,
"title": title,
"date_label": dateLabel,
})
}
func (s *PocketBaseStore) ListChapterIdx(ctx context.Context, slug string) ([]map[string]interface{}, error) {
return s.pb.listAll(ctx, "chapters_idx",
fmt.Sprintf(`slug="%s"`, pbEsc(slug)), "+number")
}
func (s *PocketBaseStore) CountChapterIdx(ctx context.Context, slug string) int {
rows, _ := s.ListChapterIdx(ctx, slug)
return len(rows)
}
// ─── Ranking ──────────────────────────────────────────────────────────────────
func (s *PocketBaseStore) SetRanking(ctx context.Context, dataJSON string) error {
return s.pb.upsert(ctx, "ranking", `key="_ranking_"`, map[string]interface{}{
"key": "_ranking_",
"data": dataJSON,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetRanking(ctx context.Context) (string, time.Time, error) {
rec, err := s.pb.listOne(ctx, "ranking", `key="_ranking_"`)
if err != nil || rec == nil {
return "", time.Time{}, err
}
data, _ := rec["data"].(string)
var updated time.Time
if ts, ok := rec["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
}
return data, updated, nil
}
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
func (s *PocketBaseStore) SetRankingPageHTML(ctx context.Context, page int, html string) error {
return s.pb.upsert(ctx, "ranking_html",
fmt.Sprintf(`page=%d`, page),
map[string]interface{}{
"page": page,
"html": html,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetRankingPageHTML(ctx context.Context, page int) (string, time.Time, error) {
rec, err := s.pb.listOne(ctx, "ranking_html", fmt.Sprintf(`page=%d`, page))
if err != nil || rec == nil {
return "", time.Time{}, err
}
html, _ := rec["html"].(string)
var updated time.Time
if ts, ok := rec["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
}
return html, updated, nil
}
// ─── Reading progress ─────────────────────────────────────────────────────────
func (s *PocketBaseStore) SetProgress(ctx context.Context, sessionID, slug string, chapter int) error {
return s.pb.upsert(ctx, "progress",
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)),
map[string]interface{}{
"session_id": sessionID,
"slug": slug,
"chapter": chapter,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetProgress(ctx context.Context, sessionID, slug string) (int, time.Time, bool, error) {
rec, err := s.pb.listOne(ctx, "progress",
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)))
if err != nil {
return 0, time.Time{}, false, err
}
if rec == nil {
return 0, time.Time{}, false, nil
}
ch := int(floatVal(rec, "chapter"))
var updated time.Time
if ts, ok := rec["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
}
return ch, updated, true, nil
}
func (s *PocketBaseStore) AllProgress(ctx context.Context, sessionID string) ([]map[string]interface{}, error) {
return s.pb.listAll(ctx, "progress",
fmt.Sprintf(`session_id="%s"`, pbEsc(sessionID)), "-updated")
}
func (s *PocketBaseStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
return s.pb.deleteWhere(ctx, "progress",
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)))
}
// ─── Audio cache ──────────────────────────────────────────────────────────────
func (s *PocketBaseStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
return s.pb.upsert(ctx, "audio_cache",
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)),
map[string]interface{}{
"cache_key": cacheKey,
"filename": filename,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool, error) {
rec, err := s.pb.listOne(ctx, "audio_cache",
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)))
if err != nil || rec == nil {
return "", false, err
}
filename, _ := rec["filename"].(string)
return filename, filename != "", nil
}
// ─── rankingFileInfo is a minimal os.FileInfo implementation ─────────────────
type rankingFileInfo struct {
modTime time.Time
}
func (r rankingFileInfo) Name() string { return "ranking" }
func (r rankingFileInfo) Size() int64 { return 0 }
func (r rankingFileInfo) Mode() os.FileMode { return 0o444 }
func (r rankingFileInfo) ModTime() time.Time { return r.modTime }
func (r rankingFileInfo) IsDir() bool { return false }
func (r rankingFileInfo) Sys() interface{} { return nil }
var _ os.FileInfo = rankingFileInfo{}
// RankingModTime returns file-info-compatible data for the ranking record.
func (s *PocketBaseStore) RankingModTime(ctx context.Context) (os.FileInfo, error) {
_, updated, err := s.GetRanking(ctx)
if err != nil {
return nil, err
}
if updated.IsZero() {
return nil, nil
}
return rankingFileInfo{modTime: updated}, nil
}
// RankingPageCacheModTime returns file-info for a cached ranking page.
func (s *PocketBaseStore) RankingPageCacheModTime(ctx context.Context, page int) (os.FileInfo, error) {
_, updated, err := s.GetRankingPageHTML(ctx, page)
if err != nil {
return nil, err
}
if updated.IsZero() {
return nil, nil
}
return rankingFileInfo{modTime: updated}, nil
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// pbEsc escapes a string for use in a PocketBase filter expression.
// Only escapes double-quotes to prevent injection.
func pbEsc(s string) string {
return strings.ReplaceAll(s, `"`, `\"`)
}
func floatVal(m map[string]interface{}, key string) float64 {
if v, ok := m[key].(float64); ok {
return v
}
return 0
}

View File

@@ -0,0 +1,124 @@
// Package storage defines the unified Store interface and helper types used by
// the server and orchestrator. Concrete implementations back the interface
// with PocketBase (structured data) and MinIO (binary objects).
package storage
import (
"context"
"os"
"time"
"github.com/libnovel/scraper/internal/scraper"
)
// ─── Shared types ─────────────────────────────────────────────────────────────
// ChapterInfo is a lightweight chapter descriptor (mirrors writer.ChapterInfo).
type ChapterInfo struct {
Number int
Title string
Date string
}
// RankingItem represents a single entry in the novel ranking list.
type RankingItem struct {
Rank int `json:"rank"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author,omitempty"`
Cover string `json:"cover,omitempty"`
Status string `json:"status,omitempty"`
Genres []string `json:"genres,omitempty"`
SourceURL string `json:"source_url,omitempty"`
}
// ReadingProgress holds a single user's reading position for one book.
type ReadingProgress struct {
Slug string `json:"slug"`
Chapter int `json:"chapter"`
UpdatedAt time.Time `json:"updated_at"`
}
// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro
// download filename so audio is not re-generated after a server restart.
type AudioCacheEntry struct {
CacheKey string `json:"cache_key"`
Filename string `json:"filename"`
}
// ─── Store interface ──────────────────────────────────────────────────────────
// Store is the single persistence abstraction consumed by the server and the
// orchestrator. Implementations may route calls to different backends
// (PocketBase for structured records, MinIO for binary blobs).
type Store interface {
// ── Book metadata ──────────────────────────────────────────────────────
// WriteMetadata upserts book metadata.
WriteMetadata(ctx context.Context, meta scraper.BookMeta) error
// ReadMetadata returns the metadata for slug. Returns (zero, false, nil)
// when the book is not found.
ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error)
// ListBooks returns all books, sorted alphabetically by title.
ListBooks(ctx context.Context) ([]scraper.BookMeta, error)
// LocalSlugs returns the set of slugs that have metadata stored.
LocalSlugs(ctx context.Context) (map[string]bool, error)
// MetadataMtime returns the Unix-second mtime of the metadata record, or 0.
MetadataMtime(ctx context.Context, slug string) int64
// ── Chapters (binary blobs in MinIO) ───────────────────────────────────
// ChapterExists returns true if the markdown file for the given ref exists.
ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool
// WriteChapter stores the chapter markdown.
WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error
// ReadChapter returns the raw markdown for chapter number n.
ReadChapter(ctx context.Context, slug string, n int) (string, error)
// ListChapters returns all stored chapters for slug, sorted by number.
ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error)
// CountChapters returns the number of stored chapters for slug.
CountChapters(ctx context.Context, slug string) int
// ── Ranking ────────────────────────────────────────────────────────────
// WriteRanking persists the ranking list.
WriteRanking(ctx context.Context, items []RankingItem) error
// ReadRankingItems returns the stored ranking items.
ReadRankingItems(ctx context.Context) ([]RankingItem, error)
// RankingFileInfo returns os.FileInfo-like data for the ranking record.
// Returns (nil, nil) when no ranking has been stored yet.
RankingFileInfo(ctx context.Context) (os.FileInfo, error)
// ── Ranking page HTML cache ────────────────────────────────────────────
// WriteRankingPageCache stores raw HTML for a ranking page.
WriteRankingPageCache(ctx context.Context, page int, html string) error
// ReadRankingPageCache returns cached HTML for a ranking page, or "" on miss.
ReadRankingPageCache(ctx context.Context, page int) (string, error)
// RankingPageCacheInfo returns file-like info for a cached ranking page.
RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error)
// ── Audio cache ────────────────────────────────────────────────────────
// GetAudioCache returns the Kokoro filename for cacheKey, or ("", false).
GetAudioCache(ctx context.Context, cacheKey string) (string, bool)
// SetAudioCache persists a Kokoro filename for cacheKey.
SetAudioCache(ctx context.Context, cacheKey, filename string) error
// ── Reading progress ───────────────────────────────────────────────────
// GetProgress returns the reading progress for the given session ID and slug.
// Returns (zero, false) if no progress is recorded.
GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool)
// SetProgress saves or updates reading progress.
SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error
// AllProgress returns all progress entries for a session.
AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error)
// DeleteProgress removes progress for a specific slug.
DeleteProgress(ctx context.Context, sessionID, slug string) error
// ── Audio object paths (MinIO) ─────────────────────────────────────────
// AudioObjectKey returns the MinIO object key for a cached audio file.
AudioObjectKey(slug string, n int, voice string, speed float64) string
}