Compare commits
10 Commits
main
...
baab66823d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
baab66823d | ||
|
|
11d2eaa0e5 | ||
|
|
9c115f00c4 | ||
|
|
5ac89da513 | ||
|
|
af86c6f96f | ||
|
|
da4a182f85 | ||
|
|
18e76c9668 | ||
|
|
9add9033b9 | ||
|
|
66d8481637 | ||
|
|
7f92a58fd7 |
@@ -1,10 +1,64 @@
|
||||
version: "3.9"
|
||||
|
||||
services:
|
||||
# ─── MinIO (object storage for chapter .md files + audio cache) ─────────────
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
#container_name: libnovel-minio
|
||||
restart: unless-stopped
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "${MINIO_PORT:-9000}:9000" # S3 API
|
||||
- "${MINIO_CONSOLE_PORT:-9001}:9001" # Web console
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── MinIO bucket initialisation ─────────────────────────────────────────────
|
||||
# Runs once to create the default buckets and then exits.
|
||||
minio-init:
|
||||
image: minio/mc:latest
|
||||
#container_name: libnovel-minio-init
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
entrypoint: >
|
||||
/bin/sh -c "
|
||||
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
|
||||
mc mb --ignore-existing local/libnovel-chapters;
|
||||
mc mb --ignore-existing local/libnovel-audio;
|
||||
echo 'buckets ready';
|
||||
"
|
||||
environment:
|
||||
MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
|
||||
# ─── PocketBase (auth + structured data: books, chapters index, ranking, progress) ──
|
||||
pocketbase:
|
||||
image: ghcr.io/muchobien/pocketbase:latest
|
||||
#container_name: libnovel-pocketbase
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${POCKETBASE_PORT:-8090}:8090"
|
||||
volumes:
|
||||
- pb_data:/pb/pb_data
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── Browserless ────────────────────────────────────────────────────────────
|
||||
browserless:
|
||||
image: ghcr.io/browserless/chromium:latest
|
||||
container_name: libnovel-browserless
|
||||
#container_name: libnovel-browserless
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# Set a token to lock down the endpoint; the scraper reads it via
|
||||
@@ -19,7 +73,7 @@ services:
|
||||
# Optional webhook URL for Browserless error alerts.
|
||||
ERROR_ALERT_URL: "${ERROR_ALERT_URL:-}"
|
||||
ports:
|
||||
- "3030:3000"
|
||||
- "${BROWSERLESS_PORT:-3030}:3000"
|
||||
# Shared memory is required for Chrome.
|
||||
shm_size: "2gb"
|
||||
healthcheck:
|
||||
@@ -28,30 +82,17 @@ services:
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── Kokoro-FastAPI (TTS) ────────────────────────────────────────────────────
|
||||
# CPU image; swap for ghcr.io/remsky/kokoro-fastapi-gpu:latest on NVIDIA hosts.
|
||||
# Models are baked in — no volume mount required for the default voice set.
|
||||
kokoro:
|
||||
image: ghcr.io/remsky/kokoro-fastapi-cpu:latest
|
||||
container_name: libnovel-kokoro
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8880:8880"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8880/health"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ─── Scraper ─────────────────────────────────────────────────────────────────
|
||||
scraper:
|
||||
build:
|
||||
context: ./scraper
|
||||
dockerfile: Dockerfile
|
||||
container_name: libnovel-scraper
|
||||
#container_name: libnovel-scraper
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
kokoro:
|
||||
pocketbase:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
BROWSERLESS_URL: "http://browserless:3000"
|
||||
@@ -66,10 +107,21 @@ services:
|
||||
SCRAPER_HTTP_ADDR: ":8080"
|
||||
LOG_LEVEL: "debug"
|
||||
# Kokoro-FastAPI TTS endpoint.
|
||||
KOKORO_URL: "${KOKORO_URL:-http://localhost:8880}"
|
||||
KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc}"
|
||||
KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}"
|
||||
# MinIO / S3 object storage
|
||||
MINIO_ENDPOINT: "minio:9000"
|
||||
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
|
||||
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
|
||||
MINIO_USE_SSL: "false"
|
||||
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
|
||||
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
|
||||
# PocketBase
|
||||
POCKETBASE_URL: "http://pocketbase:8090"
|
||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- "${SCRAPER_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- static_books:/app/static/books
|
||||
healthcheck:
|
||||
@@ -80,3 +132,5 @@ services:
|
||||
|
||||
volumes:
|
||||
static_books:
|
||||
minio_data:
|
||||
pb_data:
|
||||
|
||||
@@ -10,16 +10,24 @@
|
||||
//
|
||||
// Environment variables:
|
||||
//
|
||||
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||
// BROWSERLESS_TOKEN Browserless API token (default: "")
|
||||
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
|
||||
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||
// BROWSERLESS_TOKEN Browserless API token (default: "")
|
||||
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
|
||||
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
|
||||
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
|
||||
// SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
|
||||
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
|
||||
// KOKORO_VOICE Default TTS voice (default: af_bella)
|
||||
// LOG_LEVEL debug | info | warn | error (default: info)
|
||||
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
|
||||
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
|
||||
// KOKORO_VOICE Default TTS voice (default: af_bella)
|
||||
// POCKETBASE_URL PocketBase API base URL (default: http://localhost:8090)
|
||||
// POCKETBASE_EMAIL PocketBase admin email (default: admin@libnovel.local)
|
||||
// POCKETBASE_PASSWORD PocketBase admin password (default: adminpassword)
|
||||
// MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000)
|
||||
// MINIO_ACCESS_KEY MinIO access key (default: minioadmin)
|
||||
// MINIO_SECRET_KEY MinIO secret key (default: minioadmin)
|
||||
// MINIO_USE_SSL Use TLS for MinIO (default: false)
|
||||
// MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters)
|
||||
// MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio)
|
||||
// LOG_LEVEL debug | info | warn | error (default: info)
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -38,7 +46,7 @@ import (
|
||||
"github.com/libnovel/scraper/internal/novelfire"
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/server"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -88,9 +96,30 @@ func run(log *slog.Logger) error {
|
||||
bc := newBrowserClient(strategy, browserCfg)
|
||||
urlClient := newBrowserClient(urlStrategy, browserCfg)
|
||||
|
||||
staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books")
|
||||
w := writer.New(staticRoot)
|
||||
nf := novelfire.New(bc, log, urlClient, w)
|
||||
// ── Storage backends ────────────────────────────────────────────────────
|
||||
minioCfg := storage.MinioConfig{
|
||||
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
|
||||
AccessKey: envOr("MINIO_ACCESS_KEY", "minioadmin"),
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "minioadmin"),
|
||||
UseSSL: strings.ToLower(os.Getenv("MINIO_USE_SSL")) == "true",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
}
|
||||
pbCfg := storage.PocketBaseConfig{
|
||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||
AdminEmail: envOr("POCKETBASE_EMAIL", "admin@libnovel.local"),
|
||||
AdminPassword: envOr("POCKETBASE_PASSWORD", "adminpassword"),
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
store, err := storage.NewHybridStore(ctx, pbCfg, minioCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage init failed: %w", err)
|
||||
}
|
||||
|
||||
nf := novelfire.New(bc, log, urlClient, &rankingCacheAdapter{store: store})
|
||||
|
||||
workers := 0
|
||||
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
|
||||
@@ -104,13 +133,9 @@ func run(log *slog.Logger) error {
|
||||
}
|
||||
|
||||
oCfg := orchestrator.Config{
|
||||
Workers: workers,
|
||||
StaticRoot: staticRoot,
|
||||
Workers: workers,
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
switch cmd {
|
||||
case "run":
|
||||
// Optional --url flag.
|
||||
@@ -121,10 +146,9 @@ func run(log *slog.Logger) error {
|
||||
"strategy", strategy,
|
||||
"workers", workers,
|
||||
"max_concurrent", browserCfg.MaxConcurrent,
|
||||
"static_root", oCfg.StaticRoot,
|
||||
"single_book", oCfg.SingleBookURL,
|
||||
)
|
||||
o := orchestrator.New(oCfg, nf, log)
|
||||
o := orchestrator.New(oCfg, nf, log, store)
|
||||
return o.Run(ctx)
|
||||
|
||||
case "refresh":
|
||||
@@ -133,13 +157,12 @@ func run(log *slog.Logger) error {
|
||||
return fmt.Errorf("refresh command requires a book slug argument")
|
||||
}
|
||||
slug := args[1]
|
||||
w := writer.New(oCfg.StaticRoot)
|
||||
meta, ok, err := w.ReadMetadata(slug)
|
||||
meta, ok, err := store.ReadMetadata(ctx, slug)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read metadata for %s: %w", slug, err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot)
|
||||
return fmt.Errorf("book %q not found in store", slug)
|
||||
}
|
||||
if meta.SourceURL == "" {
|
||||
return fmt.Errorf("book %q has no source_url in metadata", slug)
|
||||
@@ -149,12 +172,12 @@ func run(log *slog.Logger) error {
|
||||
"slug", slug,
|
||||
"source_url", meta.SourceURL,
|
||||
)
|
||||
o := orchestrator.New(oCfg, nf, log)
|
||||
o := orchestrator.New(oCfg, nf, log, store)
|
||||
return o.Run(ctx)
|
||||
|
||||
case "serve":
|
||||
addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
|
||||
kokoroURL := envOr("KOKORO_URL", "")
|
||||
kokoroURL := envOr("KOKORO_URL", "https://kokoro.kalekber.cc")
|
||||
kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
|
||||
log.Info("starting HTTP server",
|
||||
"addr", addr,
|
||||
@@ -164,7 +187,7 @@ func run(log *slog.Logger) error {
|
||||
"kokoro_url", kokoroURL,
|
||||
"kokoro_voice", kokoroVoice,
|
||||
)
|
||||
srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice)
|
||||
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
|
||||
return srv.ListenAndServe(ctx)
|
||||
|
||||
default:
|
||||
@@ -192,6 +215,20 @@ func envOr(key, fallback string) string {
|
||||
return fallback
|
||||
}
|
||||
|
||||
// rankingCacheAdapter bridges storage.HybridStore (context-aware) to the
|
||||
// context-free scraper.RankingPageCacher interface expected by novelfire.New.
|
||||
type rankingCacheAdapter struct {
|
||||
store *storage.HybridStore
|
||||
}
|
||||
|
||||
func (a *rankingCacheAdapter) WriteRankingPageCache(page int, html string) error {
|
||||
return a.store.WriteRankingPageCache(context.Background(), page, html)
|
||||
}
|
||||
|
||||
func (a *rankingCacheAdapter) ReadRankingPageCache(page int) (string, error) {
|
||||
return a.store.ReadRankingPageCache(context.Background(), page)
|
||||
}
|
||||
|
||||
func printUsage() {
|
||||
fmt.Fprintf(os.Stderr, `libnovel scraper
|
||||
|
||||
|
||||
@@ -3,8 +3,29 @@ module github.com/libnovel/scraper
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/yuin/goldmark v1.7.16 // indirect
|
||||
golang.org/x/net v0.51.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/minio/minio-go/v7 v7.0.98
|
||||
github.com/yuin/goldmark v1.7.16
|
||||
golang.org/x/net v0.51.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.48.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,9 +1,49 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
|
||||
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE=
|
||||
github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
|
||||
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
|
||||
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
|
||||
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// Config holds tunable parameters for the orchestrator.
|
||||
@@ -28,7 +28,8 @@ type Config struct {
|
||||
// Defaults to runtime.NumCPU() when 0.
|
||||
Workers int
|
||||
|
||||
// StaticRoot is the path to the static/books output directory.
|
||||
// StaticRoot is kept for backwards-compatibility but is no longer used
|
||||
// when a Store is provided.
|
||||
StaticRoot string
|
||||
|
||||
// SingleBookURL when non-empty causes the orchestrator to scrape only
|
||||
@@ -40,13 +41,13 @@ type Config struct {
|
||||
type Orchestrator struct {
|
||||
cfg Config
|
||||
novel scraper.NovelScraper
|
||||
writer *writer.Writer
|
||||
store storage.Store
|
||||
log *slog.Logger
|
||||
workers int
|
||||
}
|
||||
|
||||
// New returns a new Orchestrator.
|
||||
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator {
|
||||
// New returns a new Orchestrator backed by the provided Store.
|
||||
func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store) *Orchestrator {
|
||||
workers := cfg.Workers
|
||||
if workers <= 0 {
|
||||
workers = runtime.NumCPU()
|
||||
@@ -54,7 +55,7 @@ func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator
|
||||
return &Orchestrator{
|
||||
cfg: cfg,
|
||||
novel: novel,
|
||||
writer: writer.New(cfg.StaticRoot),
|
||||
store: store,
|
||||
log: log,
|
||||
workers: workers,
|
||||
}
|
||||
@@ -66,7 +67,6 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
o.log.Info("orchestrator starting",
|
||||
"source", o.novel.SourceName(),
|
||||
"workers", o.workers,
|
||||
"static_root", o.cfg.StaticRoot,
|
||||
)
|
||||
|
||||
// chapterWork is the shared queue consumed by chapter worker goroutines.
|
||||
@@ -89,8 +89,8 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
default:
|
||||
}
|
||||
|
||||
// Skip if already on disk.
|
||||
if o.writer.ChapterExists(job.slug, job.ref) {
|
||||
// Skip if already stored.
|
||||
if o.store.ChapterExists(ctx, job.slug, job.ref) {
|
||||
o.log.Debug("chapter already exists, skipping",
|
||||
"book", job.slug, "chapter", job.ref.Number)
|
||||
continue
|
||||
@@ -107,7 +107,7 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := o.writer.WriteChapter(job.slug, chapter); err != nil {
|
||||
if err := o.store.WriteChapter(ctx, job.slug, chapter); err != nil {
|
||||
o.log.Error("chapter write failed",
|
||||
"book", job.slug,
|
||||
"chapter", job.ref.Number,
|
||||
@@ -135,8 +135,8 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
return
|
||||
}
|
||||
|
||||
// Persist / update metadata.yaml.
|
||||
if err := o.writer.WriteMetadata(meta); err != nil {
|
||||
// Persist / update metadata.
|
||||
if err := o.store.WriteMetadata(ctx, meta); err != nil {
|
||||
o.log.Error("metadata write failed", "slug", meta.Slug, "err", err)
|
||||
// Continue — chapters can still be scraped.
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@ package server
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -22,7 +24,7 @@ import (
|
||||
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
)
|
||||
|
||||
// Server wraps an HTTP mux with the scraping endpoints.
|
||||
@@ -31,7 +33,7 @@ type Server struct {
|
||||
oCfg orchestrator.Config
|
||||
novel scraper.NovelScraper
|
||||
log *slog.Logger
|
||||
writer *writer.Writer
|
||||
store storage.Store
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
rankingRunning bool
|
||||
@@ -42,26 +44,23 @@ type Server struct {
|
||||
voiceMu sync.RWMutex
|
||||
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
|
||||
|
||||
// audioMu guards audioCache and audioInFlight.
|
||||
// audioCache maps a cache key to the Kokoro download filename returned by
|
||||
// POST /v1/audio/speech with return_download_link=true.
|
||||
// audioMu guards audioInFlight only.
|
||||
// Completed audio filenames are persisted to the Store (PocketBase).
|
||||
// audioInFlight deduplicates concurrent generation requests for the same key.
|
||||
audioMu sync.Mutex
|
||||
audioCache map[string]string // cacheKey → kokoro download filename
|
||||
audioInFlight map[string]chan struct{} // cacheKey → closed when done
|
||||
}
|
||||
|
||||
// New creates a new Server.
|
||||
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server {
|
||||
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
|
||||
return &Server{
|
||||
addr: addr,
|
||||
oCfg: oCfg,
|
||||
novel: novel,
|
||||
log: log,
|
||||
writer: writer.New(oCfg.StaticRoot),
|
||||
store: store,
|
||||
kokoroURL: kokoroURL,
|
||||
kokoroVoice: kokoroVoice,
|
||||
audioCache: make(map[string]string),
|
||||
audioInFlight: make(map[string]chan struct{}),
|
||||
}
|
||||
}
|
||||
@@ -112,6 +111,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /health", s.handleHealth)
|
||||
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
||||
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
||||
// Progress API
|
||||
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
||||
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
||||
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
|
||||
// UI routes
|
||||
mux.HandleFunc("GET /", s.handleHome)
|
||||
mux.HandleFunc("GET /scrape", s.handleScrape)
|
||||
@@ -165,6 +168,123 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// ─── Session cookie helpers ───────────────────────────────────────────────────
|
||||
|
||||
const sessionCookieName = "libnovel_session"
|
||||
|
||||
// sessionID returns the session ID from the request cookie, or "" if absent.
|
||||
func sessionID(r *http.Request) string {
|
||||
c, err := r.Cookie(sessionCookieName)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return c.Value
|
||||
}
|
||||
|
||||
// newSessionID generates a random 16-byte hex session ID.
|
||||
func newSessionID() (string, error) {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// ensureSession issues a new session cookie if the request does not already
|
||||
// carry one, and returns the session ID (either existing or newly issued).
|
||||
func ensureSession(w http.ResponseWriter, r *http.Request) string {
|
||||
if id := sessionID(r); id != "" {
|
||||
return id
|
||||
}
|
||||
id, err := newSessionID()
|
||||
if err != nil {
|
||||
// Very unlikely, but fall back to a timestamp-based ID.
|
||||
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
|
||||
}
|
||||
http.SetCookie(w, &http.Cookie{
|
||||
Name: sessionCookieName,
|
||||
Value: id,
|
||||
Path: "/",
|
||||
HttpOnly: true,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
MaxAge: 365 * 24 * 60 * 60, // 1 year
|
||||
})
|
||||
return id
|
||||
}
|
||||
|
||||
// ─── Reading progress API ─────────────────────────────────────────────────────
|
||||
|
||||
// handleGetProgress handles GET /api/progress.
|
||||
// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...}
|
||||
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
entries, err := s.store.AllProgress(r.Context(), sid)
|
||||
if err != nil {
|
||||
s.log.Error("AllProgress failed", "err", err)
|
||||
entries = nil
|
||||
}
|
||||
|
||||
progress := make(map[string]interface{}, len(entries)*2)
|
||||
for _, p := range entries {
|
||||
progress[p.Slug] = p.Chapter
|
||||
progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(progress)
|
||||
}
|
||||
|
||||
// handleSetProgress handles POST /api/progress/{slug}.
|
||||
// Body: {"chapter": N}
|
||||
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Chapter int `json:"chapter"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
|
||||
http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
p := storage.ReadingProgress{
|
||||
Slug: slug,
|
||||
Chapter: body.Chapter,
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
if err := s.store.SetProgress(r.Context(), sid, p); err != nil {
|
||||
s.log.Error("SetProgress failed", "slug", slug, "err", err)
|
||||
http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{})
|
||||
}
|
||||
|
||||
// handleDeleteProgress handles DELETE /api/progress/{slug}.
|
||||
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
|
||||
sid := ensureSession(w, r)
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil {
|
||||
s.log.Error("DeleteProgress failed", "slug", slug, "err", err)
|
||||
// Non-fatal — treat as success.
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{})
|
||||
}
|
||||
|
||||
// handleChapterText returns the plain text of a chapter (markdown stripped)
|
||||
// for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI.
|
||||
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -174,7 +294,7 @@ func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
raw, err := s.writer.ReadChapter(slug, n)
|
||||
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
@@ -223,15 +343,14 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
||||
|
||||
// Fast path: already generated this session.
|
||||
s.audioMu.Lock()
|
||||
if filename, ok := s.audioCache[cacheKey]; ok {
|
||||
s.audioMu.Unlock()
|
||||
// Fast path: already generated (check persistent store first).
|
||||
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
||||
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
||||
return
|
||||
}
|
||||
|
||||
// Deduplicate concurrent generation for the same key.
|
||||
s.audioMu.Lock()
|
||||
if ch, ok := s.audioInFlight[cacheKey]; ok {
|
||||
s.audioMu.Unlock()
|
||||
select {
|
||||
@@ -240,10 +359,8 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
s.audioMu.Lock()
|
||||
filename, ok := s.audioCache[cacheKey]
|
||||
s.audioMu.Unlock()
|
||||
if ok {
|
||||
// Check store again after waiting.
|
||||
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
||||
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
||||
} else {
|
||||
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
|
||||
@@ -262,7 +379,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
}()
|
||||
|
||||
// Load and validate chapter text.
|
||||
raw, err := s.writer.ReadChapter(slug, n)
|
||||
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
|
||||
return
|
||||
@@ -287,9 +404,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
s.audioMu.Lock()
|
||||
s.audioCache[cacheKey] = filename
|
||||
s.audioMu.Unlock()
|
||||
_ = s.store.SetAudioCache(r.Context(), cacheKey, filename)
|
||||
|
||||
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
|
||||
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
||||
@@ -378,10 +493,7 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
||||
s.audioMu.Lock()
|
||||
filename, ok := s.audioCache[cacheKey]
|
||||
s.audioMu.Unlock()
|
||||
|
||||
filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
|
||||
if !ok {
|
||||
http.Error(w, "audio not generated yet", http.StatusNotFound)
|
||||
return
|
||||
@@ -462,7 +574,7 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
o := orchestrator.New(cfg, s.novel, s.log)
|
||||
o := orchestrator.New(cfg, s.novel, s.log, s.store)
|
||||
if err := o.Run(ctx); err != nil {
|
||||
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err))
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/writer"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
"github.com/yuin/goldmark"
|
||||
"github.com/yuin/goldmark/extension"
|
||||
goldhtml "github.com/yuin/goldmark/renderer/html"
|
||||
@@ -300,14 +300,9 @@ const homeTmpl = `
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
/* ── state ─────────────────────────────────────────────────────────────── */
|
||||
var progress = {};
|
||||
try { progress = JSON.parse(localStorage.getItem('reading_progress') || '{}'); } catch(_) {}
|
||||
// progress = { slug: chapterNum, ... }
|
||||
|
||||
var progressTs = {};
|
||||
try { progressTs = JSON.parse(localStorage.getItem('reading_progress_ts') || '{}'); } catch(_) {}
|
||||
// progressTs = { slug: timestampMs, ... }
|
||||
/* ── state (populated asynchronously from /api/progress) ───────────────── */
|
||||
var progress = {}; // { slug: chapterNum }
|
||||
var progressTs = {}; // { slug: timestampMs }
|
||||
|
||||
var booksGrid = document.getElementById('books-grid');
|
||||
var continueSection = document.getElementById('continue-reading-section');
|
||||
@@ -321,149 +316,7 @@ const homeTmpl = `
|
||||
|
||||
var allCards = booksGrid ? Array.from(booksGrid.querySelectorAll('[data-slug]')) : [];
|
||||
|
||||
/* ── reading_progress_ts: record timestamps for "recently read" sort ───── */
|
||||
// On first load after upgrade, seed timestamps for existing progress entries.
|
||||
var tsUpdated = false;
|
||||
Object.keys(progress).forEach(function(slug) {
|
||||
if (!progressTs[slug]) { progressTs[slug] = Date.now(); tsUpdated = true; }
|
||||
});
|
||||
if (tsUpdated) {
|
||||
try { localStorage.setItem('reading_progress_ts', JSON.stringify(progressTs)); } catch(_) {}
|
||||
}
|
||||
|
||||
/* ── progress bars + unread badges ─────────────────────────────────────── */
|
||||
allCards.forEach(function(card) {
|
||||
var slug = card.dataset.slug;
|
||||
var total = parseInt(card.dataset.total, 10) || 0;
|
||||
var downloaded = parseInt(card.dataset.downloaded, 10) || 0;
|
||||
var chapterNum = progress[slug] ? parseInt(progress[slug], 10) : 0;
|
||||
var base = downloaded || total;
|
||||
|
||||
if (chapterNum > 0 && base > 0) {
|
||||
var pct = Math.min(100, Math.round(chapterNum / base * 100));
|
||||
var wrap = card.querySelector('.book-progress-wrap');
|
||||
var bar = card.querySelector('.book-progress-bar');
|
||||
var lbl = card.querySelector('.book-progress-label');
|
||||
if (wrap && bar && lbl) {
|
||||
bar.style.width = pct + '%';
|
||||
lbl.textContent = 'Ch.\u00a0' + chapterNum + (base ? '\u00a0/\u00a0' + base : '') + '\u00a0\u2014\u00a0' + pct + '%';
|
||||
wrap.classList.remove('hidden');
|
||||
}
|
||||
}
|
||||
|
||||
// Unread badge: downloaded > last-read chapter
|
||||
if (downloaded > 0 && chapterNum > 0 && downloaded > chapterNum) {
|
||||
var unread = downloaded - chapterNum;
|
||||
var badge = card.querySelector('.book-unread-badge');
|
||||
if (badge) {
|
||||
badge.textContent = '+' + unread + ' new';
|
||||
badge.classList.remove('hidden');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/* ── stats strip ─────────────────────────────────────────────────────────── */
|
||||
var totalChaptersRead = 0;
|
||||
var booksCompleted = 0;
|
||||
Object.keys(progress).forEach(function(slug) {
|
||||
var card = booksGrid ? booksGrid.querySelector('[data-slug="' + slug + '"]') : null;
|
||||
var ch = parseInt(progress[slug], 10) || 0;
|
||||
totalChaptersRead += ch;
|
||||
if (card) {
|
||||
var total = parseInt(card.dataset.total, 10) || 0;
|
||||
var dl = parseInt(card.dataset.downloaded, 10) || 0;
|
||||
var base = dl || total;
|
||||
if (base > 0 && ch >= base) booksCompleted++;
|
||||
}
|
||||
});
|
||||
|
||||
if (totalChaptersRead > 0) {
|
||||
var estMins = Math.round(totalChaptersRead * 12); // ~12 min/chapter avg
|
||||
var timeLabel = estMins >= 60
|
||||
? Math.floor(estMins / 60) + 'h ' + (estMins % 60) + 'm'
|
||||
: estMins + 'm';
|
||||
document.getElementById('stat-books').textContent = allCards.length + ' book' + (allCards.length !== 1 ? 's' : '');
|
||||
document.getElementById('stat-chapters').textContent = totalChaptersRead + ' ch read';
|
||||
document.getElementById('stat-time').textContent = '~' + timeLabel + ' read';
|
||||
if (booksCompleted > 0) {
|
||||
document.getElementById('stat-completed').textContent = booksCompleted + ' completed';
|
||||
document.getElementById('stat-completed-wrap').classList.remove('hidden');
|
||||
}
|
||||
statsStrip.classList.remove('hidden');
|
||||
statsStrip.style.display = 'flex';
|
||||
if (statsFallback) statsFallback.classList.add('hidden');
|
||||
}
|
||||
|
||||
/* ── hero card: most recently read ─────────────────────────────────────── */
|
||||
var heroSlug = null, heroTs = 0;
|
||||
Object.keys(progressTs).forEach(function(slug) {
|
||||
if (progressTs[slug] > heroTs && progress[slug]) {
|
||||
heroTs = progressTs[slug]; heroSlug = slug;
|
||||
}
|
||||
});
|
||||
if (heroSlug && heroSection) {
|
||||
var heroCard = booksGrid ? booksGrid.querySelector('[data-slug="' + heroSlug + '"]') : null;
|
||||
if (heroCard) {
|
||||
var heroChapter = parseInt(progress[heroSlug], 10) || 0;
|
||||
var heroTotal = parseInt(heroCard.dataset.total, 10) || parseInt(heroCard.dataset.downloaded, 10) || 0;
|
||||
var heroPct = heroTotal > 0 ? Math.min(100, Math.round(heroChapter / heroTotal * 100)) : 0;
|
||||
var heroCover = heroCard.dataset.cover || '';
|
||||
var heroTitle = heroCard.dataset.title || '';
|
||||
var heroHref = heroCard.getAttribute('href');
|
||||
var heroGet = heroCard.getAttribute('hx-get');
|
||||
|
||||
heroSection.innerHTML =
|
||||
'<a href="' + heroHref + '"' +
|
||||
' hx-get="' + heroGet + '" hx-target="#main-content" hx-push-url="true" hx-swap="innerHTML"' +
|
||||
' class="group relative block rounded-2xl overflow-hidden border border-zinc-700 hover:border-amber-500 transition-colors cursor-pointer"' +
|
||||
' style="' + (heroCover ? 'background-image:linear-gradient(to right,#09090b 40%,rgba(9,9,11,0.4)),url(' + heroCover + ');background-size:cover;background-position:right center;' : 'background:#18181b;') + '">' +
|
||||
'<div class="relative z-10 p-5 sm:p-6">' +
|
||||
'<p class="text-[0.65rem] font-semibold uppercase tracking-widest text-amber-500 mb-1">Continue reading</p>' +
|
||||
'<h2 class="text-xl sm:text-2xl font-bold text-zinc-100 group-hover:text-amber-400 transition-colors leading-snug mb-2 max-w-sm">' + escHtml(heroTitle) + '</h2>' +
|
||||
'<p class="text-sm text-zinc-400 mb-4">Chapter\u00a0' + heroChapter + (heroTotal ? '\u00a0/\u00a0' + heroTotal : '') + (heroPct ? '\u00a0\u2014\u00a0' + heroPct + '%' : '') + '</p>' +
|
||||
'<div class="w-48 h-1 rounded-full bg-zinc-700 overflow-hidden mb-4">' +
|
||||
'<div class="h-full bg-amber-500 rounded-full" style="width:' + heroPct + '%"></div>' +
|
||||
'</div>' +
|
||||
'<span class="inline-block text-sm font-semibold px-4 py-2 rounded-xl bg-amber-500 text-zinc-900 group-hover:bg-amber-400 transition-colors">Resume</span>' +
|
||||
'</div>' +
|
||||
'</a>';
|
||||
heroSection.classList.remove('hidden');
|
||||
htmx.process(heroSection);
|
||||
}
|
||||
}
|
||||
|
||||
/* ── continue reading (excluding hero) ──────────────────────────────────── */
|
||||
if (continueGrid) continueGrid.innerHTML = '';
|
||||
var inProgress = Object.keys(progress).sort(function(a,b) {
|
||||
return (progressTs[b]||0) - (progressTs[a]||0);
|
||||
});
|
||||
var moved = 0;
|
||||
inProgress.forEach(function(slug) {
|
||||
if (slug === heroSlug) return; // hero already shown
|
||||
var card = booksGrid ? booksGrid.querySelector('[data-slug="' + slug + '"]') : null;
|
||||
if (!card) return;
|
||||
var a = buildContinueCard(card, progress[slug]);
|
||||
continueGrid.appendChild(a);
|
||||
moved++;
|
||||
});
|
||||
if (moved > 0) continueSection.classList.remove('hidden');
|
||||
|
||||
/* ── recently added (top 4, excluding books already in continue reading) ── */
|
||||
if (recentlyAddedGrid) recentlyAddedGrid.innerHTML = '';
|
||||
var progressSlugs = new Set(Object.keys(progress));
|
||||
var byAdded = allCards.slice().sort(function(a, b) {
|
||||
return (parseInt(b.dataset.added,10)||0) - (parseInt(a.dataset.added,10)||0);
|
||||
});
|
||||
var addedShown = 0;
|
||||
byAdded.forEach(function(card) {
|
||||
if (addedShown >= 4) return;
|
||||
if (progressSlugs.has(card.dataset.slug)) return; // already in continue reading
|
||||
recentlyAddedGrid.appendChild(buildSimpleCard(card));
|
||||
addedShown++;
|
||||
});
|
||||
if (addedShown > 0) recentlyAddedSec.classList.remove('hidden');
|
||||
|
||||
/* ── genre chips ──────────────────────────────────────────────────────── */
|
||||
/* ── genre chips (no progress dependency) ────────────────────────────── */
|
||||
var allGenres = {};
|
||||
allCards.forEach(function(card) {
|
||||
(card.dataset.genres || '').split('|').forEach(function(g) {
|
||||
@@ -508,7 +361,6 @@ const homeTmpl = `
|
||||
|
||||
function applySortAndFilter() {
|
||||
var q = input ? input.value.trim().toLowerCase() : '';
|
||||
// Sort cards
|
||||
var sorted = allCards.slice().sort(function(a, b) {
|
||||
if (currentSort === 'alpha') return (a.dataset.title||'').localeCompare(b.dataset.title||'');
|
||||
if (currentSort === 'downloaded') return (parseInt(b.dataset.downloaded,10)||0) - (parseInt(a.dataset.downloaded,10)||0);
|
||||
@@ -521,7 +373,6 @@ const homeTmpl = `
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
// Re-append in sorted order, apply visibility
|
||||
var shown = 0;
|
||||
sorted.forEach(function(card) {
|
||||
var matchText = !q || card.dataset.filter.toLowerCase().indexOf(q) !== -1;
|
||||
@@ -532,7 +383,7 @@ const homeTmpl = `
|
||||
var match = matchText && matchGenre;
|
||||
card.style.display = match ? '' : 'none';
|
||||
if (match) shown++;
|
||||
booksGrid.appendChild(card); // re-order in DOM
|
||||
booksGrid.appendChild(card);
|
||||
});
|
||||
if ((q || activeGenres.size > 0) && countEl) {
|
||||
countEl.textContent = shown + ' result' + (shown !== 1 ? 's' : '');
|
||||
@@ -579,7 +430,7 @@ const homeTmpl = `
|
||||
}
|
||||
}
|
||||
}
|
||||
// dismiss button
|
||||
// dismiss button — calls DELETE /api/progress/{slug}
|
||||
var dismissBtn = document.createElement('button');
|
||||
dismissBtn.type = 'button';
|
||||
dismissBtn.title = 'Remove from continue reading';
|
||||
@@ -590,12 +441,9 @@ const homeTmpl = `
|
||||
var cardEl = this.closest('[data-slug]');
|
||||
var cardSlug = cardEl ? cardEl.dataset.slug : null;
|
||||
if (cardSlug) {
|
||||
var p = {}; try { p = JSON.parse(localStorage.getItem('reading_progress') || '{}'); } catch(_) {}
|
||||
delete p[cardSlug];
|
||||
localStorage.setItem('reading_progress', JSON.stringify(p));
|
||||
var ts = {}; try { ts = JSON.parse(localStorage.getItem('reading_progress_ts') || '{}'); } catch(_) {}
|
||||
delete ts[cardSlug];
|
||||
localStorage.setItem('reading_progress_ts', JSON.stringify(ts));
|
||||
delete progress[cardSlug];
|
||||
delete progressTs[cardSlug];
|
||||
fetch('/api/progress/' + encodeURIComponent(cardSlug), { method: 'DELETE' });
|
||||
}
|
||||
var target = cardEl || this.parentElement;
|
||||
target.style.transition = 'transform 0.25s ease, opacity 0.25s ease';
|
||||
@@ -612,6 +460,158 @@ const homeTmpl = `
|
||||
return a;
|
||||
}
|
||||
|
||||
/* ── render: called once progress is loaded ──────────────────────────── */
|
||||
function renderWithProgress() {
|
||||
/* ── progress bars + unread badges ─────────────────────────────────── */
|
||||
allCards.forEach(function(card) {
|
||||
var slug = card.dataset.slug;
|
||||
var total = parseInt(card.dataset.total, 10) || 0;
|
||||
var downloaded = parseInt(card.dataset.downloaded, 10) || 0;
|
||||
var chapterNum = progress[slug] ? parseInt(progress[slug], 10) : 0;
|
||||
var base = downloaded || total;
|
||||
|
||||
if (chapterNum > 0 && base > 0) {
|
||||
var pct = Math.min(100, Math.round(chapterNum / base * 100));
|
||||
var wrap = card.querySelector('.book-progress-wrap');
|
||||
var bar = card.querySelector('.book-progress-bar');
|
||||
var lbl = card.querySelector('.book-progress-label');
|
||||
if (wrap && bar && lbl) {
|
||||
bar.style.width = pct + '%';
|
||||
lbl.textContent = 'Ch.\u00a0' + chapterNum + (base ? '\u00a0/\u00a0' + base : '') + '\u00a0\u2014\u00a0' + pct + '%';
|
||||
wrap.classList.remove('hidden');
|
||||
}
|
||||
}
|
||||
|
||||
if (downloaded > 0 && chapterNum > 0 && downloaded > chapterNum) {
|
||||
var unread = downloaded - chapterNum;
|
||||
var badge = card.querySelector('.book-unread-badge');
|
||||
if (badge) {
|
||||
badge.textContent = '+' + unread + ' new';
|
||||
badge.classList.remove('hidden');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/* ── stats strip ────────────────────────────────────────────────────── */
|
||||
var totalChaptersRead = 0;
|
||||
var booksCompleted = 0;
|
||||
Object.keys(progress).forEach(function(slug) {
|
||||
var card = booksGrid ? booksGrid.querySelector('[data-slug="' + slug + '"]') : null;
|
||||
var ch = parseInt(progress[slug], 10) || 0;
|
||||
totalChaptersRead += ch;
|
||||
if (card) {
|
||||
var total = parseInt(card.dataset.total, 10) || 0;
|
||||
var dl = parseInt(card.dataset.downloaded, 10) || 0;
|
||||
var base = dl || total;
|
||||
if (base > 0 && ch >= base) booksCompleted++;
|
||||
}
|
||||
});
|
||||
|
||||
if (totalChaptersRead > 0) {
|
||||
var estMins = Math.round(totalChaptersRead * 12);
|
||||
var timeLabel = estMins >= 60
|
||||
? Math.floor(estMins / 60) + 'h ' + (estMins % 60) + 'm'
|
||||
: estMins + 'm';
|
||||
document.getElementById('stat-books').textContent = allCards.length + ' book' + (allCards.length !== 1 ? 's' : '');
|
||||
document.getElementById('stat-chapters').textContent = totalChaptersRead + ' ch read';
|
||||
document.getElementById('stat-time').textContent = '~' + timeLabel + ' read';
|
||||
if (booksCompleted > 0) {
|
||||
document.getElementById('stat-completed').textContent = booksCompleted + ' completed';
|
||||
document.getElementById('stat-completed-wrap').classList.remove('hidden');
|
||||
}
|
||||
statsStrip.classList.remove('hidden');
|
||||
statsStrip.style.display = 'flex';
|
||||
if (statsFallback) statsFallback.classList.add('hidden');
|
||||
}
|
||||
|
||||
/* ── hero card: most recently read ─────────────────────────────────── */
|
||||
var heroSlug = null, heroTs = 0;
|
||||
Object.keys(progressTs).forEach(function(slug) {
|
||||
if (progressTs[slug] > heroTs && progress[slug]) {
|
||||
heroTs = progressTs[slug]; heroSlug = slug;
|
||||
}
|
||||
});
|
||||
if (heroSlug && heroSection) {
|
||||
var heroCard = booksGrid ? booksGrid.querySelector('[data-slug="' + heroSlug + '"]') : null;
|
||||
if (heroCard) {
|
||||
var heroChapter = parseInt(progress[heroSlug], 10) || 0;
|
||||
var heroTotal = parseInt(heroCard.dataset.total, 10) || parseInt(heroCard.dataset.downloaded, 10) || 0;
|
||||
var heroPct = heroTotal > 0 ? Math.min(100, Math.round(heroChapter / heroTotal * 100)) : 0;
|
||||
var heroCover = heroCard.dataset.cover || '';
|
||||
var heroTitle = heroCard.dataset.title || '';
|
||||
var heroHref = heroCard.getAttribute('href');
|
||||
var heroGet = heroCard.getAttribute('hx-get');
|
||||
|
||||
heroSection.innerHTML =
|
||||
'<a href="' + heroHref + '"' +
|
||||
' hx-get="' + heroGet + '" hx-target="#main-content" hx-push-url="true" hx-swap="innerHTML"' +
|
||||
' class="group relative block rounded-2xl overflow-hidden border border-zinc-700 hover:border-amber-500 transition-colors cursor-pointer"' +
|
||||
' style="' + (heroCover ? 'background-image:linear-gradient(to right,#09090b 40%,rgba(9,9,11,0.4)),url(' + heroCover + ');background-size:cover;background-position:right center;' : 'background:#18181b;') + '">' +
|
||||
'<div class="relative z-10 p-5 sm:p-6">' +
|
||||
'<p class="text-[0.65rem] font-semibold uppercase tracking-widest text-amber-500 mb-1">Continue reading</p>' +
|
||||
'<h2 class="text-xl sm:text-2xl font-bold text-zinc-100 group-hover:text-amber-400 transition-colors leading-snug mb-2 max-w-sm">' + escHtml(heroTitle) + '</h2>' +
|
||||
'<p class="text-sm text-zinc-400 mb-4">Chapter\u00a0' + heroChapter + (heroTotal ? '\u00a0/\u00a0' + heroTotal : '') + (heroPct ? '\u00a0\u2014\u00a0' + heroPct + '%' : '') + '</p>' +
|
||||
'<div class="w-48 h-1 rounded-full bg-zinc-700 overflow-hidden mb-4">' +
|
||||
'<div class="h-full bg-amber-500 rounded-full" style="width:' + heroPct + '%"></div>' +
|
||||
'</div>' +
|
||||
'<span class="inline-block text-sm font-semibold px-4 py-2 rounded-xl bg-amber-500 text-zinc-900 group-hover:bg-amber-400 transition-colors">Resume</span>' +
|
||||
'</div>' +
|
||||
'</a>';
|
||||
heroSection.classList.remove('hidden');
|
||||
htmx.process(heroSection);
|
||||
}
|
||||
}
|
||||
|
||||
/* ── continue reading (excluding hero) ──────────────────────────────── */
|
||||
if (continueGrid) continueGrid.innerHTML = '';
|
||||
var inProgress = Object.keys(progress).sort(function(a,b) {
|
||||
return (progressTs[b]||0) - (progressTs[a]||0);
|
||||
});
|
||||
var moved = 0;
|
||||
inProgress.forEach(function(slug) {
|
||||
if (slug === heroSlug) return;
|
||||
var card = booksGrid ? booksGrid.querySelector('[data-slug="' + slug + '"]') : null;
|
||||
if (!card) return;
|
||||
var a = buildContinueCard(card, progress[slug]);
|
||||
continueGrid.appendChild(a);
|
||||
moved++;
|
||||
});
|
||||
if (moved > 0) continueSection.classList.remove('hidden');
|
||||
|
||||
/* ── recently added (top 4, excluding books in continue reading) ────── */
|
||||
if (recentlyAddedGrid) recentlyAddedGrid.innerHTML = '';
|
||||
var progressSlugs = new Set(Object.keys(progress));
|
||||
var byAdded = allCards.slice().sort(function(a, b) {
|
||||
return (parseInt(b.dataset.added,10)||0) - (parseInt(a.dataset.added,10)||0);
|
||||
});
|
||||
var addedShown = 0;
|
||||
byAdded.forEach(function(card) {
|
||||
if (addedShown >= 4) return;
|
||||
if (progressSlugs.has(card.dataset.slug)) return;
|
||||
recentlyAddedGrid.appendChild(buildSimpleCard(card));
|
||||
addedShown++;
|
||||
});
|
||||
if (addedShown > 0) recentlyAddedSec.classList.remove('hidden');
|
||||
|
||||
/* ── re-apply sort in case "recently read" is active ─────────────────── */
|
||||
applySortAndFilter();
|
||||
}
|
||||
|
||||
/* ── load progress from server, then render ──────────────────────────── */
|
||||
fetch('/api/progress')
|
||||
.then(function(r) { return r.ok ? r.json() : {}; })
|
||||
.catch(function() { return {}; })
|
||||
.then(function(data) {
|
||||
Object.keys(data).forEach(function(key) {
|
||||
if (key.slice(-3) === '_ts') {
|
||||
progressTs[key.slice(0, -3)] = data[key];
|
||||
} else {
|
||||
progress[key] = data[key];
|
||||
}
|
||||
});
|
||||
renderWithProgress();
|
||||
});
|
||||
|
||||
}());
|
||||
</script>`
|
||||
|
||||
@@ -629,7 +629,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
books, err := s.writer.ListBooks()
|
||||
books, err := s.store.ListBooks(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, "failed to list books: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -639,8 +639,8 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
|
||||
for i, b := range books {
|
||||
items[i] = homeBookItem{
|
||||
BookMeta: b,
|
||||
Downloaded: s.writer.CountChapters(b.Slug),
|
||||
AddedAt: s.writer.MetadataMtime(b.Slug),
|
||||
Downloaded: s.store.CountChapters(r.Context(), b.Slug),
|
||||
AddedAt: s.store.MetadataMtime(r.Context(), b.Slug),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -877,7 +877,7 @@ const scrapeTmpl = `
|
||||
</script>`
|
||||
|
||||
func (s *Server) handleScrape(w http.ResponseWriter, r *http.Request) {
|
||||
rankingItems, _ := s.writer.ReadRankingItems()
|
||||
rankingItems, _ := s.store.ReadRankingItems(r.Context())
|
||||
rankingJSON, _ := json.Marshal(rankingItems)
|
||||
|
||||
t := template.Must(template.New("scrape").Parse(scrapeTmpl))
|
||||
@@ -1324,12 +1324,12 @@ const rankingTmpl = `
|
||||
// rankingViewItem enriches a RankingItem with whether it is present in the
|
||||
// local book library, so the template can highlight it differently.
|
||||
type rankingViewItem struct {
|
||||
writer.RankingItem
|
||||
storage.RankingItem
|
||||
Local bool
|
||||
}
|
||||
|
||||
// toRankingViewItems annotates items with Local=true for slugs found in localSlugs.
|
||||
func toRankingViewItems(items []writer.RankingItem, localSlugs map[string]bool) []rankingViewItem {
|
||||
func toRankingViewItems(items []storage.RankingItem, localSlugs map[string]bool) []rankingViewItem {
|
||||
out := make([]rankingViewItem, len(items))
|
||||
for i, it := range items {
|
||||
out[i] = rankingViewItem{
|
||||
@@ -1403,13 +1403,13 @@ const rankingPageSize = 20
|
||||
// It does NOT trigger a live scrape; use POST /ranking/refresh for that.
|
||||
// Supports ?page=N for browsing through cached items (20 per page).
|
||||
func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
|
||||
rankingItems, err := s.writer.ReadRankingItems()
|
||||
rankingItems, err := s.store.ReadRankingItems(r.Context())
|
||||
if err != nil {
|
||||
s.log.Error("failed to read cached ranking", "err", err)
|
||||
}
|
||||
|
||||
cachedAt := ""
|
||||
if info, statErr := s.writer.RankingFileInfo(); statErr == nil {
|
||||
if info, statErr := s.store.RankingFileInfo(r.Context()); statErr == nil && info != nil {
|
||||
cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04")
|
||||
}
|
||||
|
||||
@@ -1482,7 +1482,7 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Encode full dataset + local slugs for client-side cross-page filtering.
|
||||
localSlugs := s.writer.LocalSlugs()
|
||||
localSlugs, _ := s.store.LocalSlugs(r.Context())
|
||||
type rankingJSONItem struct {
|
||||
Rank int `json:"rank"`
|
||||
Slug string `json:"slug"`
|
||||
@@ -1577,14 +1577,14 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages)
|
||||
|
||||
var rankingItems []writer.RankingItem
|
||||
var rankingItems []storage.RankingItem
|
||||
for rankingCh != nil || errCh != nil {
|
||||
select {
|
||||
case meta, ok := <-rankingCh:
|
||||
if !ok {
|
||||
rankingCh = nil
|
||||
} else {
|
||||
rankingItems = append(rankingItems, writer.RankingItem{
|
||||
rankingItems = append(rankingItems, storage.RankingItem{
|
||||
Rank: meta.Ranking,
|
||||
Slug: meta.Slug,
|
||||
Title: meta.Title,
|
||||
@@ -1605,7 +1605,7 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
if len(rankingItems) > 0 {
|
||||
if err := s.writer.WriteRanking(rankingItems); err != nil {
|
||||
if err := s.store.WriteRanking(ctx, rankingItems); err != nil {
|
||||
s.log.Error("failed to save ranking", "err", err)
|
||||
}
|
||||
}
|
||||
@@ -1669,7 +1669,7 @@ const rankingViewTmpl = `
|
||||
</div>`
|
||||
|
||||
func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) {
|
||||
items, err := s.writer.ReadRankingItems()
|
||||
items, err := s.store.ReadRankingItems(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -1769,7 +1769,7 @@ const bookTmpl = `
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Resume reading button — populated by JS from localStorage -->
|
||||
<!-- Resume reading button — populated by JS from /api/progress -->
|
||||
<div id="resume-bar" class="mb-4 hidden">
|
||||
<a id="resume-link"
|
||||
hx-target="#main-content"
|
||||
@@ -1874,16 +1874,8 @@ const bookTmpl = `
|
||||
|
||||
(function () {
|
||||
var SLUG = '{{.Slug}}';
|
||||
var LS_KEY = 'reading_progress';
|
||||
|
||||
function getProgress() {
|
||||
try { return JSON.parse(localStorage.getItem(LS_KEY) || '{}'); } catch(_) { return {}; }
|
||||
}
|
||||
|
||||
window.applyProgress = function(slug) {
|
||||
var progress = getProgress();
|
||||
var saved = progress[slug];
|
||||
|
||||
function applyProgressData(slug, saved) {
|
||||
// Show/hide latest bar (only when no progress recorded for this book).
|
||||
var latestBar = document.getElementById('latest-bar');
|
||||
if (latestBar) {
|
||||
@@ -1918,6 +1910,13 @@ const bookTmpl = `
|
||||
htmx.process(link);
|
||||
bar.classList.remove('hidden');
|
||||
}
|
||||
}
|
||||
|
||||
window.applyProgress = function(slug) {
|
||||
fetch('/api/progress')
|
||||
.then(function(r) { return r.ok ? r.json() : {}; })
|
||||
.then(function(data) { applyProgressData(slug, data[slug] || null); })
|
||||
.catch(function() { applyProgressData(slug, null); });
|
||||
};
|
||||
|
||||
applyProgress(SLUG);
|
||||
@@ -1927,7 +1926,7 @@ const bookTmpl = `
|
||||
func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
|
||||
meta, ok, err := s.writer.ReadMetadata(slug)
|
||||
meta, ok, err := s.store.ReadMetadata(r.Context(), slug)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to read metadata: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -1937,7 +1936,7 @@ func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
chapters, err := s.writer.ListChapters(slug)
|
||||
chapters, err := s.store.ListChapters(r.Context(), slug)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -2054,7 +2053,7 @@ func (s *Server) handleBookChaptersPage(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
chapters, err := s.writer.ListChapters(slug)
|
||||
chapters, err := s.store.ListChapters(r.Context(), slug)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -2471,14 +2470,11 @@ const chapterTmpl = `
|
||||
|
||||
// ── reading progress ──────────────────────────────────────────────────────
|
||||
(function saveProgress() {
|
||||
try {
|
||||
var p = JSON.parse(localStorage.getItem('reading_progress') || '{}');
|
||||
p[SLUG] = CHAPTER_N;
|
||||
localStorage.setItem('reading_progress', JSON.stringify(p));
|
||||
var ts = JSON.parse(localStorage.getItem('reading_progress_ts') || '{}');
|
||||
ts[SLUG] = Date.now();
|
||||
localStorage.setItem('reading_progress_ts', JSON.stringify(ts));
|
||||
} catch(_) {}
|
||||
fetch('/api/progress/' + encodeURIComponent(SLUG), {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ chapter: CHAPTER_N })
|
||||
}).catch(function() {});
|
||||
})();
|
||||
|
||||
// ── generation counter (stale-closure guard) ──────────────────────────────
|
||||
@@ -3035,7 +3031,7 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
raw, err := s.writer.ReadChapter(slug, n)
|
||||
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
@@ -3051,15 +3047,15 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
chapters, _ := s.writer.ListChapters(slug)
|
||||
chapters, _ := s.store.ListChapters(r.Context(), slug)
|
||||
prevN, nextN := adjacentChapters(chapters, n)
|
||||
|
||||
title := firstHeading(raw, fmt.Sprintf("Chapter %d", n))
|
||||
chapterTitle, chapterDate := writer.SplitChapterTitle(title)
|
||||
chapterTitle, chapterDate := splitChapterTitle(title)
|
||||
|
||||
// Load cover URL for Media Session artwork (best-effort; ignore errors).
|
||||
var coverURL string
|
||||
if meta, ok, err := s.writer.ReadMetadata(slug); err == nil && ok {
|
||||
if meta, ok, err := s.store.ReadMetadata(r.Context(), slug); err == nil && ok {
|
||||
coverURL = meta.Cover
|
||||
}
|
||||
|
||||
@@ -3096,6 +3092,36 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// splitChapterTitle splits a raw chapter heading into a human-readable title
|
||||
// and a trailing relative-date string (e.g. "1 year ago"). It mirrors the
|
||||
// same logic in internal/writer and internal/storage/hybrid.
|
||||
func splitChapterTitle(raw string) (title, date string) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
// Strip leading numeric index.
|
||||
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
|
||||
prefix := raw[:idx]
|
||||
allDigit := true
|
||||
for _, c := range prefix {
|
||||
if c < '0' || c > '9' {
|
||||
allDigit = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allDigit {
|
||||
raw = strings.TrimSpace(raw[idx:])
|
||||
}
|
||||
}
|
||||
// Strip "Chapter N - N: " prefix.
|
||||
chNumRe := regexp.MustCompile(`(?i)^chapter\s+\d+(?:\s*-\s*\d+)?\s*:\s*`)
|
||||
raw = strings.TrimSpace(chNumRe.ReplaceAllString(raw, ""))
|
||||
// Detect trailing relative date.
|
||||
dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`)
|
||||
if m := dateRe.FindStringSubmatchIndex(raw); m != nil {
|
||||
return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]])
|
||||
}
|
||||
return raw, ""
|
||||
}
|
||||
|
||||
// sortedKeys returns the keys of a string-bool map in sorted order.
|
||||
func sortedKeys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
@@ -3127,7 +3153,7 @@ func stripMarkdown(src string) string {
|
||||
|
||||
// adjacentChapters returns the chapter numbers immediately before and after n
|
||||
// in the sorted chapters list. 0 means "does not exist".
|
||||
func adjacentChapters(chapters []writer.ChapterInfo, n int) (prev, next int) {
|
||||
func adjacentChapters(chapters []storage.ChapterInfo, n int) (prev, next int) {
|
||||
for i, ch := range chapters {
|
||||
if ch.Number == n {
|
||||
if i > 0 {
|
||||
@@ -3211,7 +3237,7 @@ func (s *Server) handleUIScrapeBook(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
o := orchestrator.New(cfg, s.novel, s.log)
|
||||
o := orchestrator.New(cfg, s.novel, s.log, s.store)
|
||||
if err := o.Run(ctx); err != nil {
|
||||
s.log.Error("UI scrape job failed", "url", bookURL, "err", err)
|
||||
}
|
||||
|
||||
300
scraper/internal/storage/hybrid.go
Normal file
300
scraper/internal/storage/hybrid.go
Normal file
@@ -0,0 +1,300 @@
|
||||
// hybrid.go implements the Store interface using PocketBase for structured data
|
||||
// and MinIO for binary chapter/audio blobs.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// HybridStore satisfies Store by routing structured data to PocketBase and
|
||||
// binary objects (chapters, audio) to MinIO.
|
||||
type HybridStore struct {
|
||||
pb *PocketBaseStore
|
||||
minio *MinioClient
|
||||
}
|
||||
|
||||
// NewHybridStore constructs a HybridStore. It connects to both backends and
|
||||
// calls EnsureCollections to bootstrap any missing PocketBase collections.
|
||||
func NewHybridStore(ctx context.Context, pbCfg PocketBaseConfig, minioCfg MinioConfig) (*HybridStore, error) {
|
||||
mc, err := NewMinioClient(ctx, minioCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: minio: %w", err)
|
||||
}
|
||||
pb := NewPocketBaseStore(pbCfg)
|
||||
if err := pb.EnsureCollections(ctx); err != nil {
|
||||
// Log but don't fail — collection creation errors are often "already exists"
|
||||
_ = err
|
||||
}
|
||||
return &HybridStore{pb: pb, minio: mc}, nil
|
||||
}
|
||||
|
||||
// ─── Book metadata ────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteMetadata(ctx context.Context, meta scraper.BookMeta) error {
|
||||
return h.pb.UpsertBook(ctx,
|
||||
meta.Slug, meta.Title, meta.Author, meta.Cover,
|
||||
meta.Status, meta.Summary, meta.SourceURL,
|
||||
meta.Genres, meta.TotalChapters, meta.Ranking,
|
||||
)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) {
|
||||
rec, found, err := h.pb.GetBook(ctx, slug)
|
||||
if err != nil || !found {
|
||||
return scraper.BookMeta{}, found, err
|
||||
}
|
||||
return recToBookMeta(rec), true, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListBooks(ctx context.Context) ([]scraper.BookMeta, error) {
|
||||
rows, err := h.pb.ListBooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
books := make([]scraper.BookMeta, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
books = append(books, recToBookMeta(r))
|
||||
}
|
||||
return books, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) LocalSlugs(ctx context.Context) (map[string]bool, error) {
|
||||
books, err := h.ListBooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slugs := make(map[string]bool, len(books))
|
||||
for _, b := range books {
|
||||
slugs[b.Slug] = true
|
||||
}
|
||||
return slugs, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) MetadataMtime(ctx context.Context, slug string) int64 {
|
||||
t, err := h.pb.BookMetaUpdated(ctx, slug)
|
||||
if err != nil || t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
// ─── Chapters ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool {
|
||||
return h.minio.ChapterExists(ctx, slug, ref.Volume, ref.Number)
|
||||
}
|
||||
|
||||
func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error {
|
||||
content := "# " + chapter.Ref.Title + "\n\n" + chapter.Text + "\n"
|
||||
if err := h.minio.PutChapter(ctx, slug, chapter.Ref.Volume, chapter.Ref.Number, content); err != nil {
|
||||
return err
|
||||
}
|
||||
// Update chapter index in PocketBase.
|
||||
title, dateLabel := splitChapterTitle(chapter.Ref.Title)
|
||||
_ = h.pb.UpsertChapterIdx(ctx, slug, chapter.Ref.Number, title, dateLabel)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
|
||||
return h.minio.GetChapter(ctx, slug, 0, n)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) {
|
||||
rows, err := h.pb.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
infos := make([]ChapterInfo, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
n := int(floatVal(r, "number"))
|
||||
title, _ := r["title"].(string)
|
||||
date, _ := r["date_label"].(string)
|
||||
infos = append(infos, ChapterInfo{Number: n, Title: title, Date: date})
|
||||
}
|
||||
sort.Slice(infos, func(i, j int) bool { return infos[i].Number < infos[j].Number })
|
||||
return infos, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) CountChapters(ctx context.Context, slug string) int {
|
||||
return h.pb.CountChapterIdx(ctx, slug)
|
||||
}
|
||||
|
||||
// ─── Ranking ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteRanking(ctx context.Context, items []RankingItem) error {
|
||||
data, err := json.Marshal(items)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: marshal ranking: %w", err)
|
||||
}
|
||||
return h.pb.SetRanking(ctx, string(data))
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadRankingItems(ctx context.Context) ([]RankingItem, error) {
|
||||
dataStr, _, err := h.pb.GetRanking(ctx)
|
||||
if err != nil || dataStr == "" {
|
||||
return nil, err
|
||||
}
|
||||
var items []RankingItem
|
||||
if err := json.Unmarshal([]byte(dataStr), &items); err != nil {
|
||||
return nil, fmt.Errorf("storage: unmarshal ranking: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) RankingFileInfo(ctx context.Context) (os.FileInfo, error) {
|
||||
return h.pb.RankingModTime(ctx)
|
||||
}
|
||||
|
||||
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteRankingPageCache(ctx context.Context, page int, html string) error {
|
||||
return h.pb.SetRankingPageHTML(ctx, page, html)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadRankingPageCache(ctx context.Context, page int) (string, error) {
|
||||
html, _, err := h.pb.GetRankingPageHTML(ctx, page)
|
||||
return html, err
|
||||
}
|
||||
|
||||
func (h *HybridStore) RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error) {
|
||||
return h.pb.RankingPageCacheModTime(ctx, page)
|
||||
}
|
||||
|
||||
// ─── Audio cache ──────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool) {
|
||||
filename, ok, _ := h.pb.GetAudioCache(ctx, cacheKey)
|
||||
return filename, ok
|
||||
}
|
||||
|
||||
func (h *HybridStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
|
||||
return h.pb.SetAudioCache(ctx, cacheKey, filename)
|
||||
}
|
||||
|
||||
// ─── Reading progress ─────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) {
|
||||
ch, updated, ok, err := h.pb.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil || !ok {
|
||||
return ReadingProgress{}, false
|
||||
}
|
||||
return ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}, true
|
||||
}
|
||||
|
||||
func (h *HybridStore) SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error {
|
||||
return h.pb.SetProgress(ctx, sessionID, p.Slug, p.Chapter)
|
||||
}
|
||||
|
||||
func (h *HybridStore) AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) {
|
||||
rows, err := h.pb.AllProgress(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]ReadingProgress, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
slug, _ := r["slug"].(string)
|
||||
ch := int(floatVal(r, "chapter"))
|
||||
var updated time.Time
|
||||
if ts, ok := r["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
out = append(out, ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
|
||||
return h.pb.DeleteProgress(ctx, sessionID, slug)
|
||||
}
|
||||
|
||||
// ─── AudioObjectKey ───────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) AudioObjectKey(slug string, n int, voice string, speed float64) string {
|
||||
return AudioObjectKey(slug, n, voice, speed)
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func recToBookMeta(rec map[string]interface{}) scraper.BookMeta {
|
||||
m := scraper.BookMeta{
|
||||
Slug: strVal(rec, "slug"),
|
||||
Title: strVal(rec, "title"),
|
||||
Author: strVal(rec, "author"),
|
||||
Cover: strVal(rec, "cover"),
|
||||
Status: strVal(rec, "status"),
|
||||
Summary: strVal(rec, "summary"),
|
||||
SourceURL: strVal(rec, "source_url"),
|
||||
}
|
||||
if tc := floatVal(rec, "total_chapters"); tc > 0 {
|
||||
m.TotalChapters = int(tc)
|
||||
}
|
||||
if rk := floatVal(rec, "ranking"); rk > 0 {
|
||||
m.Ranking = int(rk)
|
||||
}
|
||||
// Genres stored as JSON string or array.
|
||||
switch v := rec["genres"].(type) {
|
||||
case string:
|
||||
_ = json.Unmarshal([]byte(v), &m.Genres)
|
||||
case []interface{}:
|
||||
for _, g := range v {
|
||||
if s, ok := g.(string); ok {
|
||||
m.Genres = append(m.Genres, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func strVal(m map[string]interface{}, key string) string {
|
||||
if v, ok := m[key].(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// splitChapterTitle mirrors writer.SplitChapterTitle logic (simplified).
|
||||
func splitChapterTitle(raw string) (title, date string) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
// Strip leading numeric index.
|
||||
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
|
||||
prefix := raw[:idx]
|
||||
allDigit := true
|
||||
for _, c := range prefix {
|
||||
if c < '0' || c > '9' {
|
||||
allDigit = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allDigit {
|
||||
raw = strings.TrimSpace(raw[idx:])
|
||||
}
|
||||
}
|
||||
// Detect trailing relative date.
|
||||
units := []string{"second", "minute", "hour", "day", "week", "month", "year"}
|
||||
lower := strings.ToLower(raw)
|
||||
for _, u := range units {
|
||||
for _, suffix := range []string{u + "s ago", u + " ago"} {
|
||||
if idx := strings.LastIndex(lower, suffix); idx > 0 {
|
||||
// Find start of date token (digit before the unit).
|
||||
start := strings.LastIndex(raw[:idx], " ")
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
numPart := strings.TrimSpace(raw[start:idx])
|
||||
if _, err := strconv.Atoi(strings.Fields(numPart)[0]); err == nil {
|
||||
return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return raw, ""
|
||||
}
|
||||
179
scraper/internal/storage/minio.go
Normal file
179
scraper/internal/storage/minio.go
Normal file
@@ -0,0 +1,179 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// MinioConfig holds connection parameters for MinIO.
|
||||
type MinioConfig struct {
|
||||
Endpoint string // e.g. "minio:9000"
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
UseSSL bool
|
||||
BucketChapters string // e.g. "libnovel-chapters"
|
||||
BucketAudio string // e.g. "libnovel-audio"
|
||||
}
|
||||
|
||||
// MinioClient wraps a minio.Client and exposes object operations for
|
||||
// chapters and audio files.
|
||||
type MinioClient struct {
|
||||
c *minio.Client
|
||||
cfg MinioConfig
|
||||
}
|
||||
|
||||
// NewMinioClient creates a connected MinIO client and ensures the required
|
||||
// buckets exist.
|
||||
func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error) {
|
||||
c, err := minio.New(cfg.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio: new client: %w", err)
|
||||
}
|
||||
|
||||
mc := &MinioClient{c: c, cfg: cfg}
|
||||
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio} {
|
||||
if err := mc.ensureBucket(ctx, bucket); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return mc, nil
|
||||
}
|
||||
|
||||
// ensureBucket creates a bucket if it does not exist.
|
||||
func (m *MinioClient) ensureBucket(ctx context.Context, bucket string) error {
|
||||
exists, err := m.c.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: bucket exists %q: %w", bucket, err)
|
||||
}
|
||||
if !exists {
|
||||
if err := m.c.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
return fmt.Errorf("minio: make bucket %q: %w", bucket, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── Chapter objects ──────────────────────────────────────────────────────────
|
||||
|
||||
// chapterKey returns the MinIO object key for a chapter.
|
||||
// Layout: {slug}/vol-{vol}/{lo}-{hi}/chapter-{n}.md
|
||||
func chapterKey(slug string, vol, n int) string {
|
||||
const chaptersPerFolder = 50
|
||||
lo := ((n-1)/chaptersPerFolder)*chaptersPerFolder + 1
|
||||
hi := lo + chaptersPerFolder - 1
|
||||
return fmt.Sprintf("%s/vol-%d/%d-%d/chapter-%d.md", slug, vol, lo, hi, n)
|
||||
}
|
||||
|
||||
// PutChapter stores chapter markdown in MinIO.
|
||||
func (m *MinioClient) PutChapter(ctx context.Context, slug string, vol, n int, content string) error {
|
||||
key := chapterKey(slug, vol, n)
|
||||
data := []byte(content)
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketChapters, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: "text/markdown; charset=utf-8"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put chapter %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetChapter retrieves chapter markdown from MinIO.
|
||||
func (m *MinioClient) GetChapter(ctx context.Context, slug string, vol, n int) (string, error) {
|
||||
key := chapterKey(slug, vol, n)
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketChapters, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio: get chapter %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio: read chapter %s: %w", key, err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
// ChapterExists returns true if the object for this chapter is present.
|
||||
func (m *MinioClient) ChapterExists(ctx context.Context, slug string, vol, n int) bool {
|
||||
key := chapterKey(slug, vol, n)
|
||||
_, err := m.c.StatObject(ctx, m.cfg.BucketChapters, key, minio.StatObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ListChapterKeys returns all object keys under slug/ in the chapters bucket,
|
||||
// sorted lexicographically (MinIO returns them in order).
|
||||
func (m *MinioClient) ListChapterKeys(ctx context.Context, slug string) ([]string, error) {
|
||||
prefix := slug + "/"
|
||||
var keys []string
|
||||
for obj := range m.c.ListObjects(ctx, m.cfg.BucketChapters,
|
||||
minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
|
||||
if obj.Err != nil {
|
||||
return nil, fmt.Errorf("minio: list chapters %s: %w", slug, obj.Err)
|
||||
}
|
||||
keys = append(keys, obj.Key)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// CountChapters returns the number of chapter objects for a slug.
|
||||
func (m *MinioClient) CountChapters(ctx context.Context, slug string) int {
|
||||
keys, _ := m.ListChapterKeys(ctx, slug)
|
||||
return len(keys)
|
||||
}
|
||||
|
||||
// ─── Audio objects ────────────────────────────────────────────────────────────
|
||||
|
||||
// AudioObjectKey returns the MinIO key for a cached audio file.
|
||||
// Key: {slug}/ch{n}-{voice}-{speed:.1f}.mp3
|
||||
func AudioObjectKey(slug string, n int, voice string, speed float64) string {
|
||||
safe := sanitiseVoice(voice)
|
||||
return fmt.Sprintf("%s/ch%d-%s-%.1f.mp3", slug, n, safe, speed)
|
||||
}
|
||||
|
||||
// PutAudio stores an audio file in the audio bucket.
|
||||
func (m *MinioClient) PutAudio(ctx context.Context, key string, data []byte) error {
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketAudio, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: "audio/mpeg"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put audio %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAudio retrieves audio bytes from the audio bucket.
|
||||
func (m *MinioClient) GetAudio(ctx context.Context, key string) ([]byte, error) {
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketAudio, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio: get audio %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
return io.ReadAll(obj)
|
||||
}
|
||||
|
||||
// AudioExists returns true if the audio object is present in the bucket.
|
||||
func (m *MinioClient) AudioExists(ctx context.Context, key string) bool {
|
||||
_, err := m.c.StatObject(ctx, m.cfg.BucketAudio, key, minio.StatObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// sanitiseVoice converts a voice name to a filename-safe string.
|
||||
func sanitiseVoice(voice string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
|
||||
(r >= '0' && r <= '9') || r == '_' || r == '-' {
|
||||
return r
|
||||
}
|
||||
return '_'
|
||||
}, voice)
|
||||
}
|
||||
560
scraper/internal/storage/pocketbase.go
Normal file
560
scraper/internal/storage/pocketbase.go
Normal file
@@ -0,0 +1,560 @@
|
||||
// Package storage — PocketBase REST client.
|
||||
//
|
||||
// Collections expected in PocketBase:
|
||||
//
|
||||
// books — slug(text,unique), title, author, cover, status, genres(json),
|
||||
// summary, total_chapters(number), source_url, ranking(number), updated(date)
|
||||
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
|
||||
// ranking — data(json), updated(date) [single row, upserted by slug="_ranking_"]
|
||||
// ranking_html — page(number,unique), html(text), updated(date)
|
||||
// progress — session_id(text), slug(text), chapter(number), updated(date)
|
||||
// audio_cache — cache_key(text,unique), filename(text), updated(date)
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PocketBaseConfig holds PocketBase connection settings.
|
||||
type PocketBaseConfig struct {
|
||||
BaseURL string // e.g. "http://pocketbase:8090"
|
||||
AdminEmail string
|
||||
AdminPassword string
|
||||
}
|
||||
|
||||
// pbClient is a minimal PocketBase admin REST client.
|
||||
type pbClient struct {
|
||||
cfg PocketBaseConfig
|
||||
httpClient *http.Client
|
||||
|
||||
tokenMu sync.RWMutex
|
||||
token string
|
||||
tokenExp time.Time
|
||||
}
|
||||
|
||||
// newPBClient creates a new PocketBase client. It does not authenticate yet;
|
||||
// authentication happens lazily on the first API call.
|
||||
func newPBClient(cfg PocketBaseConfig) *pbClient {
|
||||
return &pbClient{
|
||||
cfg: cfg,
|
||||
httpClient: &http.Client{Timeout: 15 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Auth ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
func (p *pbClient) authenticate(ctx context.Context) error {
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
"identity": p.cfg.AdminEmail,
|
||||
"password": p.cfg.AdminPassword,
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
p.cfg.BaseURL+"/api/admins/auth-with-password", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := p.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pocketbase: auth: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pocketbase: auth status %d: %s", resp.StatusCode, b)
|
||||
}
|
||||
var result struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return fmt.Errorf("pocketbase: decode auth: %w", err)
|
||||
}
|
||||
p.tokenMu.Lock()
|
||||
p.token = result.Token
|
||||
p.tokenExp = time.Now().Add(12 * time.Hour)
|
||||
p.tokenMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *pbClient) authToken(ctx context.Context) (string, error) {
|
||||
p.tokenMu.RLock()
|
||||
tok, exp := p.token, p.tokenExp
|
||||
p.tokenMu.RUnlock()
|
||||
if tok != "" && time.Now().Before(exp) {
|
||||
return tok, nil
|
||||
}
|
||||
if err := p.authenticate(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
p.tokenMu.RLock()
|
||||
defer p.tokenMu.RUnlock()
|
||||
return p.token, nil
|
||||
}
|
||||
|
||||
// ─── Generic CRUD helpers ──────────────────────────────────────────────────────
|
||||
|
||||
func (p *pbClient) do(ctx context.Context, method, path string, body interface{}) (*http.Response, error) {
|
||||
tok, err := p.authToken(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var bodyReader io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
bodyReader = bytes.NewReader(b)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, p.cfg.BaseURL+path, bodyReader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", tok)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
return p.httpClient.Do(req)
|
||||
}
|
||||
|
||||
// listOne fetches the first matching record from a collection.
|
||||
func (p *pbClient) listOne(ctx context.Context, collection, filter string) (map[string]interface{}, error) {
|
||||
q := url.Values{}
|
||||
q.Set("filter", filter)
|
||||
q.Set("perPage", "1")
|
||||
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||
resp, err := p.do(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
var result struct {
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(result.Items) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return result.Items[0], nil
|
||||
}
|
||||
|
||||
// listAll returns all records (up to 500) from a collection matching filter.
|
||||
func (p *pbClient) listAll(ctx context.Context, collection, filter, sort string) ([]map[string]interface{}, error) {
|
||||
q := url.Values{}
|
||||
if filter != "" {
|
||||
q.Set("filter", filter)
|
||||
}
|
||||
if sort != "" {
|
||||
q.Set("sort", sort)
|
||||
}
|
||||
q.Set("perPage", "500")
|
||||
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||
resp, err := p.do(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var result struct {
|
||||
Items []map[string]interface{} `json:"items"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result.Items, nil
|
||||
}
|
||||
|
||||
// upsert creates a record; if one matching filter already exists it updates it.
|
||||
func (p *pbClient) upsert(ctx context.Context, collection, filter string, data map[string]interface{}) error {
|
||||
existing, err := p.listOne(ctx, collection, filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != nil {
|
||||
id := existing["id"].(string)
|
||||
resp, err := p.do(ctx, http.MethodPatch,
|
||||
fmt.Sprintf("/api/collections/%s/records/%s", collection, id), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
resp, err := p.do(ctx, http.MethodPost,
|
||||
fmt.Sprintf("/api/collections/%s/records", collection), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteWhere deletes all records matching filter in collection.
|
||||
func (p *pbClient) deleteWhere(ctx context.Context, collection, filter string) error {
|
||||
items, err := p.listAll(ctx, collection, filter, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range items {
|
||||
id, _ := item["id"].(string)
|
||||
resp, err := p.do(ctx, http.MethodDelete,
|
||||
fmt.Sprintf("/api/collections/%s/records/%s", collection, id), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── PocketBaseStore ──────────────────────────────────────────────────────────
|
||||
|
||||
// PocketBaseStore implements the structured-data portion of the Store interface
|
||||
// backed by PocketBase REST API.
|
||||
type PocketBaseStore struct {
|
||||
pb *pbClient
|
||||
}
|
||||
|
||||
// NewPocketBaseStore returns a connected PocketBaseStore.
|
||||
func NewPocketBaseStore(cfg PocketBaseConfig) *PocketBaseStore {
|
||||
return &PocketBaseStore{pb: newPBClient(cfg)}
|
||||
}
|
||||
|
||||
// Ping verifies connectivity by authenticating.
|
||||
func (s *PocketBaseStore) Ping(ctx context.Context) error {
|
||||
_, err := s.pb.authToken(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// ─── Collections schema bootstrap ────────────────────────────────────────────
|
||||
// CollectionDef maps a collection name to its fields for auto-creation.
|
||||
|
||||
// EnsureCollections creates missing collections via the PocketBase API.
|
||||
// Safe to call on every startup — existing collections are skipped.
|
||||
func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
|
||||
// We just attempt to create each collection; 400/422 errors for "already
|
||||
// exists" are silently ignored.
|
||||
collections := []map[string]interface{}{
|
||||
{
|
||||
"name": "books",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "slug", "type": "text", "required": true, "options": map[string]interface{}{"min": 1}},
|
||||
{"name": "title", "type": "text", "required": true},
|
||||
{"name": "author", "type": "text"},
|
||||
{"name": "cover", "type": "url"},
|
||||
{"name": "status", "type": "text"},
|
||||
{"name": "genres", "type": "json"},
|
||||
{"name": "summary", "type": "text"},
|
||||
{"name": "total_chapters", "type": "number"},
|
||||
{"name": "source_url", "type": "url"},
|
||||
{"name": "ranking", "type": "number"},
|
||||
{"name": "meta_updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "chapters_idx",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "number", "type": "number", "required": true},
|
||||
{"name": "title", "type": "text"},
|
||||
{"name": "date_label", "type": "text"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "ranking",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "key", "type": "text", "required": true},
|
||||
{"name": "data", "type": "json"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "ranking_html",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "page", "type": "number", "required": true},
|
||||
{"name": "html", "type": "text"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "progress",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "session_id", "type": "text", "required": true},
|
||||
{"name": "slug", "type": "text", "required": true},
|
||||
{"name": "chapter", "type": "number"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "audio_cache",
|
||||
"type": "base",
|
||||
"schema": []map[string]interface{}{
|
||||
{"name": "cache_key", "type": "text", "required": true},
|
||||
{"name": "filename", "type": "text"},
|
||||
{"name": "updated", "type": "date"},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, col := range collections {
|
||||
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections", col)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pocketbase: ensure collection %v: %w", col["name"], err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
// 400/422 = already exists or schema mismatch — ignore
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── Book metadata ────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) UpsertBook(ctx context.Context, slug, title, author, cover, status, summary, sourceURL string, genres []string, totalChapters, ranking int) error {
|
||||
genresJSON, _ := json.Marshal(genres)
|
||||
return s.pb.upsert(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)), map[string]interface{}{
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"author": author,
|
||||
"cover": cover,
|
||||
"status": status,
|
||||
"genres": string(genresJSON),
|
||||
"summary": summary,
|
||||
"total_chapters": totalChapters,
|
||||
"source_url": sourceURL,
|
||||
"ranking": ranking,
|
||||
"meta_updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetBook(ctx context.Context, slug string) (map[string]interface{}, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if rec == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
return rec, true, nil
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) ListBooks(ctx context.Context) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "books", "", "+title")
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) BookMetaUpdated(ctx context.Context, slug string) (time.Time, error) {
|
||||
rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)))
|
||||
if err != nil || rec == nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if ts, ok := rec["meta_updated"].(string); ok {
|
||||
t, err := time.Parse(time.RFC3339, ts)
|
||||
if err == nil {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, nil
|
||||
}
|
||||
|
||||
// ─── Chapter index ────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) UpsertChapterIdx(ctx context.Context, slug string, number int, title, dateLabel string) error {
|
||||
return s.pb.upsert(ctx, "chapters_idx",
|
||||
fmt.Sprintf(`slug="%s"&&number=%d`, pbEsc(slug), number),
|
||||
map[string]interface{}{
|
||||
"slug": slug,
|
||||
"number": number,
|
||||
"title": title,
|
||||
"date_label": dateLabel,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) ListChapterIdx(ctx context.Context, slug string) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "chapters_idx",
|
||||
fmt.Sprintf(`slug="%s"`, pbEsc(slug)), "+number")
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) CountChapterIdx(ctx context.Context, slug string) int {
|
||||
rows, _ := s.ListChapterIdx(ctx, slug)
|
||||
return len(rows)
|
||||
}
|
||||
|
||||
// ─── Ranking ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) SetRanking(ctx context.Context, dataJSON string) error {
|
||||
return s.pb.upsert(ctx, "ranking", `key="_ranking_"`, map[string]interface{}{
|
||||
"key": "_ranking_",
|
||||
"data": dataJSON,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetRanking(ctx context.Context) (string, time.Time, error) {
|
||||
rec, err := s.pb.listOne(ctx, "ranking", `key="_ranking_"`)
|
||||
if err != nil || rec == nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
data, _ := rec["data"].(string)
|
||||
var updated time.Time
|
||||
if ts, ok := rec["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
return data, updated, nil
|
||||
}
|
||||
|
||||
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) SetRankingPageHTML(ctx context.Context, page int, html string) error {
|
||||
return s.pb.upsert(ctx, "ranking_html",
|
||||
fmt.Sprintf(`page=%d`, page),
|
||||
map[string]interface{}{
|
||||
"page": page,
|
||||
"html": html,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetRankingPageHTML(ctx context.Context, page int) (string, time.Time, error) {
|
||||
rec, err := s.pb.listOne(ctx, "ranking_html", fmt.Sprintf(`page=%d`, page))
|
||||
if err != nil || rec == nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
html, _ := rec["html"].(string)
|
||||
var updated time.Time
|
||||
if ts, ok := rec["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
return html, updated, nil
|
||||
}
|
||||
|
||||
// ─── Reading progress ─────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) SetProgress(ctx context.Context, sessionID, slug string, chapter int) error {
|
||||
return s.pb.upsert(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)),
|
||||
map[string]interface{}{
|
||||
"session_id": sessionID,
|
||||
"slug": slug,
|
||||
"chapter": chapter,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetProgress(ctx context.Context, sessionID, slug string) (int, time.Time, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)))
|
||||
if err != nil {
|
||||
return 0, time.Time{}, false, err
|
||||
}
|
||||
if rec == nil {
|
||||
return 0, time.Time{}, false, nil
|
||||
}
|
||||
ch := int(floatVal(rec, "chapter"))
|
||||
var updated time.Time
|
||||
if ts, ok := rec["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
return ch, updated, true, nil
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) AllProgress(ctx context.Context, sessionID string) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"`, pbEsc(sessionID)), "-updated")
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
|
||||
return s.pb.deleteWhere(ctx, "progress",
|
||||
fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)))
|
||||
}
|
||||
|
||||
// ─── Audio cache ──────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *PocketBaseStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
|
||||
return s.pb.upsert(ctx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)),
|
||||
map[string]interface{}{
|
||||
"cache_key": cacheKey,
|
||||
"filename": filename,
|
||||
"updated": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool, error) {
|
||||
rec, err := s.pb.listOne(ctx, "audio_cache",
|
||||
fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)))
|
||||
if err != nil || rec == nil {
|
||||
return "", false, err
|
||||
}
|
||||
filename, _ := rec["filename"].(string)
|
||||
return filename, filename != "", nil
|
||||
}
|
||||
|
||||
// ─── rankingFileInfo is a minimal os.FileInfo implementation ─────────────────
|
||||
|
||||
type rankingFileInfo struct {
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
func (r rankingFileInfo) Name() string { return "ranking" }
|
||||
func (r rankingFileInfo) Size() int64 { return 0 }
|
||||
func (r rankingFileInfo) Mode() os.FileMode { return 0o444 }
|
||||
func (r rankingFileInfo) ModTime() time.Time { return r.modTime }
|
||||
func (r rankingFileInfo) IsDir() bool { return false }
|
||||
func (r rankingFileInfo) Sys() interface{} { return nil }
|
||||
|
||||
var _ os.FileInfo = rankingFileInfo{}
|
||||
|
||||
// RankingModTime returns file-info-compatible data for the ranking record.
|
||||
func (s *PocketBaseStore) RankingModTime(ctx context.Context) (os.FileInfo, error) {
|
||||
_, updated, err := s.GetRanking(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if updated.IsZero() {
|
||||
return nil, nil
|
||||
}
|
||||
return rankingFileInfo{modTime: updated}, nil
|
||||
}
|
||||
|
||||
// RankingPageCacheModTime returns file-info for a cached ranking page.
|
||||
func (s *PocketBaseStore) RankingPageCacheModTime(ctx context.Context, page int) (os.FileInfo, error) {
|
||||
_, updated, err := s.GetRankingPageHTML(ctx, page)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if updated.IsZero() {
|
||||
return nil, nil
|
||||
}
|
||||
return rankingFileInfo{modTime: updated}, nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// pbEsc escapes a string for use in a PocketBase filter expression.
|
||||
// Only escapes double-quotes to prevent injection.
|
||||
func pbEsc(s string) string {
|
||||
return strings.ReplaceAll(s, `"`, `\"`)
|
||||
}
|
||||
|
||||
func floatVal(m map[string]interface{}, key string) float64 {
|
||||
if v, ok := m[key].(float64); ok {
|
||||
return v
|
||||
}
|
||||
return 0
|
||||
}
|
||||
124
scraper/internal/storage/store.go
Normal file
124
scraper/internal/storage/store.go
Normal file
@@ -0,0 +1,124 @@
|
||||
// Package storage defines the unified Store interface and helper types used by
|
||||
// the server and orchestrator. Concrete implementations back the interface
|
||||
// with PocketBase (structured data) and MinIO (binary objects).
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// ─── Shared types ─────────────────────────────────────────────────────────────
|
||||
|
||||
// ChapterInfo is a lightweight chapter descriptor (mirrors writer.ChapterInfo).
|
||||
type ChapterInfo struct {
|
||||
Number int
|
||||
Title string
|
||||
Date string
|
||||
}
|
||||
|
||||
// RankingItem represents a single entry in the novel ranking list.
|
||||
type RankingItem struct {
|
||||
Rank int `json:"rank"`
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author,omitempty"`
|
||||
Cover string `json:"cover,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Genres []string `json:"genres,omitempty"`
|
||||
SourceURL string `json:"source_url,omitempty"`
|
||||
}
|
||||
|
||||
// ReadingProgress holds a single user's reading position for one book.
|
||||
type ReadingProgress struct {
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro
|
||||
// download filename so audio is not re-generated after a server restart.
|
||||
type AudioCacheEntry struct {
|
||||
CacheKey string `json:"cache_key"`
|
||||
Filename string `json:"filename"`
|
||||
}
|
||||
|
||||
// ─── Store interface ──────────────────────────────────────────────────────────
|
||||
|
||||
// Store is the single persistence abstraction consumed by the server and the
|
||||
// orchestrator. Implementations may route calls to different backends
|
||||
// (PocketBase for structured records, MinIO for binary blobs).
|
||||
type Store interface {
|
||||
// ── Book metadata ──────────────────────────────────────────────────────
|
||||
|
||||
// WriteMetadata upserts book metadata.
|
||||
WriteMetadata(ctx context.Context, meta scraper.BookMeta) error
|
||||
// ReadMetadata returns the metadata for slug. Returns (zero, false, nil)
|
||||
// when the book is not found.
|
||||
ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error)
|
||||
// ListBooks returns all books, sorted alphabetically by title.
|
||||
ListBooks(ctx context.Context) ([]scraper.BookMeta, error)
|
||||
// LocalSlugs returns the set of slugs that have metadata stored.
|
||||
LocalSlugs(ctx context.Context) (map[string]bool, error)
|
||||
// MetadataMtime returns the Unix-second mtime of the metadata record, or 0.
|
||||
MetadataMtime(ctx context.Context, slug string) int64
|
||||
|
||||
// ── Chapters (binary blobs in MinIO) ───────────────────────────────────
|
||||
|
||||
// ChapterExists returns true if the markdown file for the given ref exists.
|
||||
ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool
|
||||
// WriteChapter stores the chapter markdown.
|
||||
WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error
|
||||
// ReadChapter returns the raw markdown for chapter number n.
|
||||
ReadChapter(ctx context.Context, slug string, n int) (string, error)
|
||||
// ListChapters returns all stored chapters for slug, sorted by number.
|
||||
ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error)
|
||||
// CountChapters returns the number of stored chapters for slug.
|
||||
CountChapters(ctx context.Context, slug string) int
|
||||
|
||||
// ── Ranking ────────────────────────────────────────────────────────────
|
||||
|
||||
// WriteRanking persists the ranking list.
|
||||
WriteRanking(ctx context.Context, items []RankingItem) error
|
||||
// ReadRankingItems returns the stored ranking items.
|
||||
ReadRankingItems(ctx context.Context) ([]RankingItem, error)
|
||||
// RankingFileInfo returns os.FileInfo-like data for the ranking record.
|
||||
// Returns (nil, nil) when no ranking has been stored yet.
|
||||
RankingFileInfo(ctx context.Context) (os.FileInfo, error)
|
||||
|
||||
// ── Ranking page HTML cache ────────────────────────────────────────────
|
||||
|
||||
// WriteRankingPageCache stores raw HTML for a ranking page.
|
||||
WriteRankingPageCache(ctx context.Context, page int, html string) error
|
||||
// ReadRankingPageCache returns cached HTML for a ranking page, or "" on miss.
|
||||
ReadRankingPageCache(ctx context.Context, page int) (string, error)
|
||||
// RankingPageCacheInfo returns file-like info for a cached ranking page.
|
||||
RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error)
|
||||
|
||||
// ── Audio cache ────────────────────────────────────────────────────────
|
||||
|
||||
// GetAudioCache returns the Kokoro filename for cacheKey, or ("", false).
|
||||
GetAudioCache(ctx context.Context, cacheKey string) (string, bool)
|
||||
// SetAudioCache persists a Kokoro filename for cacheKey.
|
||||
SetAudioCache(ctx context.Context, cacheKey, filename string) error
|
||||
|
||||
// ── Reading progress ───────────────────────────────────────────────────
|
||||
|
||||
// GetProgress returns the reading progress for the given session ID and slug.
|
||||
// Returns (zero, false) if no progress is recorded.
|
||||
GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool)
|
||||
// SetProgress saves or updates reading progress.
|
||||
SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error
|
||||
// AllProgress returns all progress entries for a session.
|
||||
AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error)
|
||||
// DeleteProgress removes progress for a specific slug.
|
||||
DeleteProgress(ctx context.Context, sessionID, slug string) error
|
||||
|
||||
// ── Audio object paths (MinIO) ─────────────────────────────────────────
|
||||
|
||||
// AudioObjectKey returns the MinIO object key for a cached audio file.
|
||||
AudioObjectKey(slug string, n int, voice string, speed float64) string
|
||||
}
|
||||
Reference in New Issue
Block a user