From 7f92a58fd70417692f9bf6d334b044ee59dad2de Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 14:30:47 +0500 Subject: [PATCH 001/231] chore(v2): add PocketBase and MinIO services to docker-compose --- docker-compose.yml | 71 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 750c534..de31555 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,60 @@ version: "3.9" services: + # ─── MinIO (object storage for chapter .md files + audio cache) ───────────── + minio: + image: minio/minio:latest + container_name: libnovel-minio + restart: unless-stopped + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}" + ports: + - "9000:9000" # S3 API + - "9001:9001" # Web console + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 10s + timeout: 5s + retries: 5 + + # ─── MinIO bucket initialisation ───────────────────────────────────────────── + # Runs once to create the default buckets and then exits. + minio-init: + image: minio/mc:latest + container_name: libnovel-minio-init + depends_on: + minio: + condition: service_healthy + entrypoint: > + /bin/sh -c " + mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123}; + mc mb --ignore-existing local/libnovel-chapters; + mc mb --ignore-existing local/libnovel-audio; + echo 'buckets ready'; + " + environment: + MINIO_ROOT_USER: "${MINIO_ROOT_USER:-admin}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-changeme123}" + + # ─── PocketBase (auth + structured data: books, chapters index, ranking, progress) ── + pocketbase: + image: ghcr.io/muchobien/pocketbase:latest + container_name: libnovel-pocketbase + restart: unless-stopped + ports: + - "8090:8090" + volumes: + - pb_data:/pb/pb_data + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:8090/api/health"] + interval: 10s + timeout: 5s + retries: 5 + # ─── Browserless ──────────────────────────────────────────────────────────── browserless: image: ghcr.io/browserless/chromium:latest @@ -53,6 +107,10 @@ services: depends_on: kokoro: condition: service_healthy + pocketbase: + condition: service_healthy + minio: + condition: service_healthy environment: BROWSERLESS_URL: "http://browserless:3000" BROWSERLESS_TOKEN: "${BROWSERLESS_TOKEN:-}" @@ -68,6 +126,17 @@ services: # Kokoro-FastAPI TTS endpoint. KOKORO_URL: "${KOKORO_URL:-http://localhost:8880}" KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}" + # MinIO / S3 object storage + MINIO_ENDPOINT: "minio:9000" + MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}" + MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}" + MINIO_USE_SSL: "false" + MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}" + MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}" + # PocketBase + POCKETBASE_URL: "http://pocketbase:8090" + POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" + POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}" ports: - "8080:8080" volumes: @@ -80,3 +149,5 @@ services: volumes: static_books: + minio_data: + pb_data: -- 2.47.3 From 66d8481637781ce4e4fef4d0cc5208cda565926a Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 14:31:36 +0500 Subject: [PATCH 002/231] chore(v2): add minio-go/v7 dependency --- scraper/go.mod | 8 ++++---- scraper/go.sum | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scraper/go.mod b/scraper/go.mod index 10b6635..ff9ea50 100644 --- a/scraper/go.mod +++ b/scraper/go.mod @@ -3,8 +3,8 @@ module github.com/libnovel/scraper go 1.25.0 require ( - github.com/gorilla/websocket v1.5.3 // indirect - github.com/yuin/goldmark v1.7.16 // indirect - golang.org/x/net v0.51.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/gorilla/websocket v1.5.3 + github.com/yuin/goldmark v1.7.16 + golang.org/x/net v0.51.0 + gopkg.in/yaml.v3 v3.0.1 ) diff --git a/scraper/go.sum b/scraper/go.sum index 5338185..49bcf84 100644 --- a/scraper/go.sum +++ b/scraper/go.sum @@ -4,6 +4,7 @@ github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE= github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -- 2.47.3 From 9add9033b9520f9f293e231c900dac0d181c6cd2 Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 14:34:25 +0500 Subject: [PATCH 003/231] feat(v2): add internal/storage package with MinIO + PocketBase clients and Store interface --- scraper/go.mod | 21 + scraper/go.sum | 39 ++ scraper/internal/storage/minio.go | 179 ++++++++ scraper/internal/storage/pocketbase.go | 560 +++++++++++++++++++++++++ scraper/internal/storage/store.go | 124 ++++++ 5 files changed, 923 insertions(+) create mode 100644 scraper/internal/storage/minio.go create mode 100644 scraper/internal/storage/pocketbase.go create mode 100644 scraper/internal/storage/store.go diff --git a/scraper/go.mod b/scraper/go.mod index ff9ea50..1ad5393 100644 --- a/scraper/go.mod +++ b/scraper/go.mod @@ -4,7 +4,28 @@ go 1.25.0 require ( github.com/gorilla/websocket v1.5.3 + github.com/minio/minio-go/v7 v7.0.98 github.com/yuin/goldmark v1.7.16 golang.org/x/net v0.51.0 gopkg.in/yaml.v3 v3.0.1 ) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect +) diff --git a/scraper/go.sum b/scraper/go.sum index 49bcf84..ebd83af 100644 --- a/scraper/go.sum +++ b/scraper/go.sum @@ -1,9 +1,48 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE= github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/scraper/internal/storage/minio.go b/scraper/internal/storage/minio.go new file mode 100644 index 0000000..932011a --- /dev/null +++ b/scraper/internal/storage/minio.go @@ -0,0 +1,179 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// MinioConfig holds connection parameters for MinIO. +type MinioConfig struct { + Endpoint string // e.g. "minio:9000" + AccessKey string + SecretKey string + UseSSL bool + BucketChapters string // e.g. "libnovel-chapters" + BucketAudio string // e.g. "libnovel-audio" +} + +// MinioClient wraps a minio.Client and exposes object operations for +// chapters and audio files. +type MinioClient struct { + c *minio.Client + cfg MinioConfig +} + +// NewMinioClient creates a connected MinIO client and ensures the required +// buckets exist. +func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error) { + c, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: cfg.UseSSL, + }) + if err != nil { + return nil, fmt.Errorf("minio: new client: %w", err) + } + + mc := &MinioClient{c: c, cfg: cfg} + for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio} { + if err := mc.ensureBucket(ctx, bucket); err != nil { + return nil, err + } + } + return mc, nil +} + +// ensureBucket creates a bucket if it does not exist. +func (m *MinioClient) ensureBucket(ctx context.Context, bucket string) error { + exists, err := m.c.BucketExists(ctx, bucket) + if err != nil { + return fmt.Errorf("minio: bucket exists %q: %w", bucket, err) + } + if !exists { + if err := m.c.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil { + return fmt.Errorf("minio: make bucket %q: %w", bucket, err) + } + } + return nil +} + +// ─── Chapter objects ────────────────────────────────────────────────────────── + +// chapterKey returns the MinIO object key for a chapter. +// Layout: {slug}/vol-{vol}/{lo}-{hi}/chapter-{n}.md +func chapterKey(slug string, vol, n int) string { + const chaptersPerFolder = 50 + lo := ((n-1)/chaptersPerFolder)*chaptersPerFolder + 1 + hi := lo + chaptersPerFolder - 1 + return fmt.Sprintf("%s/vol-%d/%d-%d/chapter-%d.md", slug, vol, lo, hi, n) +} + +// PutChapter stores chapter markdown in MinIO. +func (m *MinioClient) PutChapter(ctx context.Context, slug string, vol, n int, content string) error { + key := chapterKey(slug, vol, n) + data := []byte(content) + _, err := m.c.PutObject(ctx, m.cfg.BucketChapters, key, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "text/markdown; charset=utf-8"}) + if err != nil { + return fmt.Errorf("minio: put chapter %s: %w", key, err) + } + return nil +} + +// GetChapter retrieves chapter markdown from MinIO. +func (m *MinioClient) GetChapter(ctx context.Context, slug string, vol, n int) (string, error) { + key := chapterKey(slug, vol, n) + obj, err := m.c.GetObject(ctx, m.cfg.BucketChapters, key, minio.GetObjectOptions{}) + if err != nil { + return "", fmt.Errorf("minio: get chapter %s: %w", key, err) + } + defer obj.Close() + data, err := io.ReadAll(obj) + if err != nil { + return "", fmt.Errorf("minio: read chapter %s: %w", key, err) + } + return string(data), nil +} + +// ChapterExists returns true if the object for this chapter is present. +func (m *MinioClient) ChapterExists(ctx context.Context, slug string, vol, n int) bool { + key := chapterKey(slug, vol, n) + _, err := m.c.StatObject(ctx, m.cfg.BucketChapters, key, minio.StatObjectOptions{}) + return err == nil +} + +// ListChapterKeys returns all object keys under slug/ in the chapters bucket, +// sorted lexicographically (MinIO returns them in order). +func (m *MinioClient) ListChapterKeys(ctx context.Context, slug string) ([]string, error) { + prefix := slug + "/" + var keys []string + for obj := range m.c.ListObjects(ctx, m.cfg.BucketChapters, + minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + if obj.Err != nil { + return nil, fmt.Errorf("minio: list chapters %s: %w", slug, obj.Err) + } + keys = append(keys, obj.Key) + } + return keys, nil +} + +// CountChapters returns the number of chapter objects for a slug. +func (m *MinioClient) CountChapters(ctx context.Context, slug string) int { + keys, _ := m.ListChapterKeys(ctx, slug) + return len(keys) +} + +// ─── Audio objects ──────────────────────────────────────────────────────────── + +// AudioObjectKey returns the MinIO key for a cached audio file. +// Key: {slug}/ch{n}-{voice}-{speed:.1f}.mp3 +func AudioObjectKey(slug string, n int, voice string, speed float64) string { + safe := sanitiseVoice(voice) + return fmt.Sprintf("%s/ch%d-%s-%.1f.mp3", slug, n, safe, speed) +} + +// PutAudio stores an audio file in the audio bucket. +func (m *MinioClient) PutAudio(ctx context.Context, key string, data []byte) error { + _, err := m.c.PutObject(ctx, m.cfg.BucketAudio, key, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "audio/mpeg"}) + if err != nil { + return fmt.Errorf("minio: put audio %s: %w", key, err) + } + return nil +} + +// GetAudio retrieves audio bytes from the audio bucket. +func (m *MinioClient) GetAudio(ctx context.Context, key string) ([]byte, error) { + obj, err := m.c.GetObject(ctx, m.cfg.BucketAudio, key, minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("minio: get audio %s: %w", key, err) + } + defer obj.Close() + return io.ReadAll(obj) +} + +// AudioExists returns true if the audio object is present in the bucket. +func (m *MinioClient) AudioExists(ctx context.Context, key string) bool { + _, err := m.c.StatObject(ctx, m.cfg.BucketAudio, key, minio.StatObjectOptions{}) + return err == nil +} + +// ─── helpers ────────────────────────────────────────────────────────────────── + +// sanitiseVoice converts a voice name to a filename-safe string. +func sanitiseVoice(voice string) string { + return strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || r == '_' || r == '-' { + return r + } + return '_' + }, voice) +} diff --git a/scraper/internal/storage/pocketbase.go b/scraper/internal/storage/pocketbase.go new file mode 100644 index 0000000..8a34ae3 --- /dev/null +++ b/scraper/internal/storage/pocketbase.go @@ -0,0 +1,560 @@ +// Package storage — PocketBase REST client. +// +// Collections expected in PocketBase: +// +// books — slug(text,unique), title, author, cover, status, genres(json), +// summary, total_chapters(number), source_url, ranking(number), updated(date) +// chapters_idx — slug(text), number(number), title, date_label, updated(date) +// ranking — data(json), updated(date) [single row, upserted by slug="_ranking_"] +// ranking_html — page(number,unique), html(text), updated(date) +// progress — session_id(text), slug(text), chapter(number), updated(date) +// audio_cache — cache_key(text,unique), filename(text), updated(date) +package storage + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" +) + +// PocketBaseConfig holds PocketBase connection settings. +type PocketBaseConfig struct { + BaseURL string // e.g. "http://pocketbase:8090" + AdminEmail string + AdminPassword string +} + +// pbClient is a minimal PocketBase admin REST client. +type pbClient struct { + cfg PocketBaseConfig + httpClient *http.Client + + tokenMu sync.RWMutex + token string + tokenExp time.Time +} + +// newPBClient creates a new PocketBase client. It does not authenticate yet; +// authentication happens lazily on the first API call. +func newPBClient(cfg PocketBaseConfig) *pbClient { + return &pbClient{ + cfg: cfg, + httpClient: &http.Client{Timeout: 15 * time.Second}, + } +} + +// ─── Auth ───────────────────────────────────────────────────────────────────── + +func (p *pbClient) authenticate(ctx context.Context) error { + body, _ := json.Marshal(map[string]string{ + "identity": p.cfg.AdminEmail, + "password": p.cfg.AdminPassword, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + p.cfg.BaseURL+"/api/admins/auth-with-password", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := p.httpClient.Do(req) + if err != nil { + return fmt.Errorf("pocketbase: auth: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pocketbase: auth status %d: %s", resp.StatusCode, b) + } + var result struct { + Token string `json:"token"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return fmt.Errorf("pocketbase: decode auth: %w", err) + } + p.tokenMu.Lock() + p.token = result.Token + p.tokenExp = time.Now().Add(12 * time.Hour) + p.tokenMu.Unlock() + return nil +} + +func (p *pbClient) authToken(ctx context.Context) (string, error) { + p.tokenMu.RLock() + tok, exp := p.token, p.tokenExp + p.tokenMu.RUnlock() + if tok != "" && time.Now().Before(exp) { + return tok, nil + } + if err := p.authenticate(ctx); err != nil { + return "", err + } + p.tokenMu.RLock() + defer p.tokenMu.RUnlock() + return p.token, nil +} + +// ─── Generic CRUD helpers ────────────────────────────────────────────────────── + +func (p *pbClient) do(ctx context.Context, method, path string, body interface{}) (*http.Response, error) { + tok, err := p.authToken(ctx) + if err != nil { + return nil, err + } + + var bodyReader io.Reader + if body != nil { + b, _ := json.Marshal(body) + bodyReader = bytes.NewReader(b) + } + + req, err := http.NewRequestWithContext(ctx, method, p.cfg.BaseURL+path, bodyReader) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", tok) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return p.httpClient.Do(req) +} + +// listOne fetches the first matching record from a collection. +func (p *pbClient) listOne(ctx context.Context, collection, filter string) (map[string]interface{}, error) { + q := url.Values{} + q.Set("filter", filter) + q.Set("perPage", "1") + path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode()) + resp, err := p.do(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + var result struct { + Items []map[string]interface{} `json:"items"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + if len(result.Items) == 0 { + return nil, nil + } + return result.Items[0], nil +} + +// listAll returns all records (up to 500) from a collection matching filter. +func (p *pbClient) listAll(ctx context.Context, collection, filter, sort string) ([]map[string]interface{}, error) { + q := url.Values{} + if filter != "" { + q.Set("filter", filter) + } + if sort != "" { + q.Set("sort", sort) + } + q.Set("perPage", "500") + path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode()) + resp, err := p.do(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + var result struct { + Items []map[string]interface{} `json:"items"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return result.Items, nil +} + +// upsert creates a record; if one matching filter already exists it updates it. +func (p *pbClient) upsert(ctx context.Context, collection, filter string, data map[string]interface{}) error { + existing, err := p.listOne(ctx, collection, filter) + if err != nil { + return err + } + if existing != nil { + id := existing["id"].(string) + resp, err := p.do(ctx, http.MethodPatch, + fmt.Sprintf("/api/collections/%s/records/%s", collection, id), data) + if err != nil { + return err + } + resp.Body.Close() + return nil + } + resp, err := p.do(ctx, http.MethodPost, + fmt.Sprintf("/api/collections/%s/records", collection), data) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +// deleteWhere deletes all records matching filter in collection. +func (p *pbClient) deleteWhere(ctx context.Context, collection, filter string) error { + items, err := p.listAll(ctx, collection, filter, "") + if err != nil { + return err + } + for _, item := range items { + id, _ := item["id"].(string) + resp, err := p.do(ctx, http.MethodDelete, + fmt.Sprintf("/api/collections/%s/records/%s", collection, id), nil) + if err != nil { + return err + } + resp.Body.Close() + } + return nil +} + +// ─── PocketBaseStore ────────────────────────────────────────────────────────── + +// PocketBaseStore implements the structured-data portion of the Store interface +// backed by PocketBase REST API. +type PocketBaseStore struct { + pb *pbClient +} + +// NewPocketBaseStore returns a connected PocketBaseStore. +func NewPocketBaseStore(cfg PocketBaseConfig) *PocketBaseStore { + return &PocketBaseStore{pb: newPBClient(cfg)} +} + +// Ping verifies connectivity by authenticating. +func (s *PocketBaseStore) Ping(ctx context.Context) error { + _, err := s.pb.authToken(ctx) + return err +} + +// ─── Collections schema bootstrap ──────────────────────────────────────────── +// CollectionDef maps a collection name to its fields for auto-creation. + +// EnsureCollections creates missing collections via the PocketBase API. +// Safe to call on every startup — existing collections are skipped. +func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error { + // We just attempt to create each collection; 400/422 errors for "already + // exists" are silently ignored. + collections := []map[string]interface{}{ + { + "name": "books", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "slug", "type": "text", "required": true, "options": map[string]interface{}{"min": 1}}, + {"name": "title", "type": "text", "required": true}, + {"name": "author", "type": "text"}, + {"name": "cover", "type": "url"}, + {"name": "status", "type": "text"}, + {"name": "genres", "type": "json"}, + {"name": "summary", "type": "text"}, + {"name": "total_chapters", "type": "number"}, + {"name": "source_url", "type": "url"}, + {"name": "ranking", "type": "number"}, + {"name": "meta_updated", "type": "date"}, + }, + }, + { + "name": "chapters_idx", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "slug", "type": "text", "required": true}, + {"name": "number", "type": "number", "required": true}, + {"name": "title", "type": "text"}, + {"name": "date_label", "type": "text"}, + }, + }, + { + "name": "ranking", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "key", "type": "text", "required": true}, + {"name": "data", "type": "json"}, + {"name": "updated", "type": "date"}, + }, + }, + { + "name": "ranking_html", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "page", "type": "number", "required": true}, + {"name": "html", "type": "text"}, + {"name": "updated", "type": "date"}, + }, + }, + { + "name": "progress", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "session_id", "type": "text", "required": true}, + {"name": "slug", "type": "text", "required": true}, + {"name": "chapter", "type": "number"}, + {"name": "updated", "type": "date"}, + }, + }, + { + "name": "audio_cache", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "cache_key", "type": "text", "required": true}, + {"name": "filename", "type": "text"}, + {"name": "updated", "type": "date"}, + }, + }, + } + for _, col := range collections { + resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections", col) + if err != nil { + return fmt.Errorf("pocketbase: ensure collection %v: %w", col["name"], err) + } + resp.Body.Close() + // 400/422 = already exists or schema mismatch — ignore + } + return nil +} + +// ─── Book metadata ──────────────────────────────────────────────────────────── + +func (s *PocketBaseStore) UpsertBook(ctx context.Context, slug, title, author, cover, status, summary, sourceURL string, genres []string, totalChapters, ranking int) error { + genresJSON, _ := json.Marshal(genres) + return s.pb.upsert(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)), map[string]interface{}{ + "slug": slug, + "title": title, + "author": author, + "cover": cover, + "status": status, + "genres": string(genresJSON), + "summary": summary, + "total_chapters": totalChapters, + "source_url": sourceURL, + "ranking": ranking, + "meta_updated": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *PocketBaseStore) GetBook(ctx context.Context, slug string) (map[string]interface{}, bool, error) { + rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug))) + if err != nil { + return nil, false, err + } + if rec == nil { + return nil, false, nil + } + return rec, true, nil +} + +func (s *PocketBaseStore) ListBooks(ctx context.Context) ([]map[string]interface{}, error) { + return s.pb.listAll(ctx, "books", "", "+title") +} + +func (s *PocketBaseStore) BookMetaUpdated(ctx context.Context, slug string) (time.Time, error) { + rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug))) + if err != nil || rec == nil { + return time.Time{}, err + } + if ts, ok := rec["meta_updated"].(string); ok { + t, err := time.Parse(time.RFC3339, ts) + if err == nil { + return t, nil + } + } + return time.Time{}, nil +} + +// ─── Chapter index ──────────────────────────────────────────────────────────── + +func (s *PocketBaseStore) UpsertChapterIdx(ctx context.Context, slug string, number int, title, dateLabel string) error { + return s.pb.upsert(ctx, "chapters_idx", + fmt.Sprintf(`slug="%s"&&number=%d`, pbEsc(slug), number), + map[string]interface{}{ + "slug": slug, + "number": number, + "title": title, + "date_label": dateLabel, + }) +} + +func (s *PocketBaseStore) ListChapterIdx(ctx context.Context, slug string) ([]map[string]interface{}, error) { + return s.pb.listAll(ctx, "chapters_idx", + fmt.Sprintf(`slug="%s"`, pbEsc(slug)), "+number") +} + +func (s *PocketBaseStore) CountChapterIdx(ctx context.Context, slug string) int { + rows, _ := s.ListChapterIdx(ctx, slug) + return len(rows) +} + +// ─── Ranking ────────────────────────────────────────────────────────────────── + +func (s *PocketBaseStore) SetRanking(ctx context.Context, dataJSON string) error { + return s.pb.upsert(ctx, "ranking", `key="_ranking_"`, map[string]interface{}{ + "key": "_ranking_", + "data": dataJSON, + "updated": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *PocketBaseStore) GetRanking(ctx context.Context) (string, time.Time, error) { + rec, err := s.pb.listOne(ctx, "ranking", `key="_ranking_"`) + if err != nil || rec == nil { + return "", time.Time{}, err + } + data, _ := rec["data"].(string) + var updated time.Time + if ts, ok := rec["updated"].(string); ok { + updated, _ = time.Parse(time.RFC3339, ts) + } + return data, updated, nil +} + +// ─── Ranking page HTML cache ────────────────────────────────────────────────── + +func (s *PocketBaseStore) SetRankingPageHTML(ctx context.Context, page int, html string) error { + return s.pb.upsert(ctx, "ranking_html", + fmt.Sprintf(`page=%d`, page), + map[string]interface{}{ + "page": page, + "html": html, + "updated": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *PocketBaseStore) GetRankingPageHTML(ctx context.Context, page int) (string, time.Time, error) { + rec, err := s.pb.listOne(ctx, "ranking_html", fmt.Sprintf(`page=%d`, page)) + if err != nil || rec == nil { + return "", time.Time{}, err + } + html, _ := rec["html"].(string) + var updated time.Time + if ts, ok := rec["updated"].(string); ok { + updated, _ = time.Parse(time.RFC3339, ts) + } + return html, updated, nil +} + +// ─── Reading progress ───────────────────────────────────────────────────────── + +func (s *PocketBaseStore) SetProgress(ctx context.Context, sessionID, slug string, chapter int) error { + return s.pb.upsert(ctx, "progress", + fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)), + map[string]interface{}{ + "session_id": sessionID, + "slug": slug, + "chapter": chapter, + "updated": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *PocketBaseStore) GetProgress(ctx context.Context, sessionID, slug string) (int, time.Time, bool, error) { + rec, err := s.pb.listOne(ctx, "progress", + fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug))) + if err != nil { + return 0, time.Time{}, false, err + } + if rec == nil { + return 0, time.Time{}, false, nil + } + ch := int(floatVal(rec, "chapter")) + var updated time.Time + if ts, ok := rec["updated"].(string); ok { + updated, _ = time.Parse(time.RFC3339, ts) + } + return ch, updated, true, nil +} + +func (s *PocketBaseStore) AllProgress(ctx context.Context, sessionID string) ([]map[string]interface{}, error) { + return s.pb.listAll(ctx, "progress", + fmt.Sprintf(`session_id="%s"`, pbEsc(sessionID)), "-updated") +} + +func (s *PocketBaseStore) DeleteProgress(ctx context.Context, sessionID, slug string) error { + return s.pb.deleteWhere(ctx, "progress", + fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug))) +} + +// ─── Audio cache ────────────────────────────────────────────────────────────── + +func (s *PocketBaseStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error { + return s.pb.upsert(ctx, "audio_cache", + fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)), + map[string]interface{}{ + "cache_key": cacheKey, + "filename": filename, + "updated": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool, error) { + rec, err := s.pb.listOne(ctx, "audio_cache", + fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey))) + if err != nil || rec == nil { + return "", false, err + } + filename, _ := rec["filename"].(string) + return filename, filename != "", nil +} + +// ─── rankingFileInfo is a minimal os.FileInfo implementation ───────────────── + +type rankingFileInfo struct { + modTime time.Time +} + +func (r rankingFileInfo) Name() string { return "ranking" } +func (r rankingFileInfo) Size() int64 { return 0 } +func (r rankingFileInfo) Mode() os.FileMode { return 0o444 } +func (r rankingFileInfo) ModTime() time.Time { return r.modTime } +func (r rankingFileInfo) IsDir() bool { return false } +func (r rankingFileInfo) Sys() interface{} { return nil } + +var _ os.FileInfo = rankingFileInfo{} + +// RankingModTime returns file-info-compatible data for the ranking record. +func (s *PocketBaseStore) RankingModTime(ctx context.Context) (os.FileInfo, error) { + _, updated, err := s.GetRanking(ctx) + if err != nil { + return nil, err + } + if updated.IsZero() { + return nil, nil + } + return rankingFileInfo{modTime: updated}, nil +} + +// RankingPageCacheModTime returns file-info for a cached ranking page. +func (s *PocketBaseStore) RankingPageCacheModTime(ctx context.Context, page int) (os.FileInfo, error) { + _, updated, err := s.GetRankingPageHTML(ctx, page) + if err != nil { + return nil, err + } + if updated.IsZero() { + return nil, nil + } + return rankingFileInfo{modTime: updated}, nil +} + +// ─── helpers ────────────────────────────────────────────────────────────────── + +// pbEsc escapes a string for use in a PocketBase filter expression. +// Only escapes double-quotes to prevent injection. +func pbEsc(s string) string { + return strings.ReplaceAll(s, `"`, `\"`) +} + +func floatVal(m map[string]interface{}, key string) float64 { + if v, ok := m[key].(float64); ok { + return v + } + return 0 +} diff --git a/scraper/internal/storage/store.go b/scraper/internal/storage/store.go new file mode 100644 index 0000000..6868594 --- /dev/null +++ b/scraper/internal/storage/store.go @@ -0,0 +1,124 @@ +// Package storage defines the unified Store interface and helper types used by +// the server and orchestrator. Concrete implementations back the interface +// with PocketBase (structured data) and MinIO (binary objects). +package storage + +import ( + "context" + "os" + "time" + + "github.com/libnovel/scraper/internal/scraper" +) + +// ─── Shared types ───────────────────────────────────────────────────────────── + +// ChapterInfo is a lightweight chapter descriptor (mirrors writer.ChapterInfo). +type ChapterInfo struct { + Number int + Title string + Date string +} + +// RankingItem represents a single entry in the novel ranking list. +type RankingItem struct { + Rank int `json:"rank"` + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author,omitempty"` + Cover string `json:"cover,omitempty"` + Status string `json:"status,omitempty"` + Genres []string `json:"genres,omitempty"` + SourceURL string `json:"source_url,omitempty"` +} + +// ReadingProgress holds a single user's reading position for one book. +type ReadingProgress struct { + Slug string `json:"slug"` + Chapter int `json:"chapter"` + UpdatedAt time.Time `json:"updated_at"` +} + +// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro +// download filename so audio is not re-generated after a server restart. +type AudioCacheEntry struct { + CacheKey string `json:"cache_key"` + Filename string `json:"filename"` +} + +// ─── Store interface ────────────────────────────────────────────────────────── + +// Store is the single persistence abstraction consumed by the server and the +// orchestrator. Implementations may route calls to different backends +// (PocketBase for structured records, MinIO for binary blobs). +type Store interface { + // ── Book metadata ────────────────────────────────────────────────────── + + // WriteMetadata upserts book metadata. + WriteMetadata(ctx context.Context, meta scraper.BookMeta) error + // ReadMetadata returns the metadata for slug. Returns (zero, false, nil) + // when the book is not found. + ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) + // ListBooks returns all books, sorted alphabetically by title. + ListBooks(ctx context.Context) ([]scraper.BookMeta, error) + // LocalSlugs returns the set of slugs that have metadata stored. + LocalSlugs(ctx context.Context) (map[string]bool, error) + // MetadataMtime returns the Unix-second mtime of the metadata record, or 0. + MetadataMtime(ctx context.Context, slug string) int64 + + // ── Chapters (binary blobs in MinIO) ─────────────────────────────────── + + // ChapterExists returns true if the markdown file for the given ref exists. + ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool + // WriteChapter stores the chapter markdown. + WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error + // ReadChapter returns the raw markdown for chapter number n. + ReadChapter(ctx context.Context, slug string, n int) (string, error) + // ListChapters returns all stored chapters for slug, sorted by number. + ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) + // CountChapters returns the number of stored chapters for slug. + CountChapters(ctx context.Context, slug string) int + + // ── Ranking ──────────────────────────────────────────────────────────── + + // WriteRanking persists the ranking list. + WriteRanking(ctx context.Context, items []RankingItem) error + // ReadRankingItems returns the stored ranking items. + ReadRankingItems(ctx context.Context) ([]RankingItem, error) + // RankingFileInfo returns os.FileInfo-like data for the ranking record. + // Returns (nil, nil) when no ranking has been stored yet. + RankingFileInfo(ctx context.Context) (os.FileInfo, error) + + // ── Ranking page HTML cache ──────────────────────────────────────────── + + // WriteRankingPageCache stores raw HTML for a ranking page. + WriteRankingPageCache(ctx context.Context, page int, html string) error + // ReadRankingPageCache returns cached HTML for a ranking page, or "" on miss. + ReadRankingPageCache(ctx context.Context, page int) (string, error) + // RankingPageCacheInfo returns file-like info for a cached ranking page. + RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error) + + // ── Audio cache ──────────────────────────────────────────────────────── + + // GetAudioCache returns the Kokoro filename for cacheKey, or ("", false). + GetAudioCache(ctx context.Context, cacheKey string) (string, bool) + // SetAudioCache persists a Kokoro filename for cacheKey. + SetAudioCache(ctx context.Context, cacheKey, filename string) error + + // ── Reading progress ─────────────────────────────────────────────────── + + // GetProgress returns the reading progress for the given session ID and slug. + // Returns (zero, false) if no progress is recorded. + GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) + // SetProgress saves or updates reading progress. + SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error + // AllProgress returns all progress entries for a session. + AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) + // DeleteProgress removes progress for a specific slug. + DeleteProgress(ctx context.Context, sessionID, slug string) error + + // ── Audio object paths (MinIO) ───────────────────────────────────────── + + // AudioObjectKey returns the MinIO object key for a cached audio file. + AudioObjectKey(slug string, n int, voice string, speed float64) string +} -- 2.47.3 From 18e76c9668ebfb6d279f41943e81e7c5c2d194d0 Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 14:44:02 +0500 Subject: [PATCH 004/231] steps 6-8: wire HybridStore into orchestrator, server, and main - Add storage/hybrid.go: HybridStore composing PocketBase + MinIO backends - Rewrite orchestrator to accept storage.Store instead of *writer.Writer - Replace *writer.Writer with storage.Store in server.go and ui.go - Wire audio cache, reading progress, chapter reads/writes through store - Add rankingCacheAdapter in main.go to bridge context-free RankingPageCacher interface to HybridStore's context-aware methods --- scraper/cmd/scraper/main.go | 87 +++-- scraper/internal/orchestrator/orchestrator.go | 24 +- scraper/internal/server/server.go | 43 +-- scraper/internal/server/ui.go | 76 +++-- scraper/internal/storage/hybrid.go | 300 ++++++++++++++++++ 5 files changed, 443 insertions(+), 87 deletions(-) create mode 100644 scraper/internal/storage/hybrid.go diff --git a/scraper/cmd/scraper/main.go b/scraper/cmd/scraper/main.go index f3c6962..326e0e0 100644 --- a/scraper/cmd/scraper/main.go +++ b/scraper/cmd/scraper/main.go @@ -10,16 +10,24 @@ // // Environment variables: // -// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030) -// BROWSERLESS_TOKEN Browserless API token (default: "") -// BROWSERLESS_STRATEGY content | scrape | cdp (default: content) +// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030) +// BROWSERLESS_TOKEN Browserless API token (default: "") +// BROWSERLESS_STRATEGY content | scrape | cdp (default: content) // BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5) -// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU) -// SCRAPER_STATIC_ROOT Output directory (default: ./static/books) -// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) -// KOKORO_URL Kokoro-FastAPI base URL (default: "") -// KOKORO_VOICE Default TTS voice (default: af_bella) -// LOG_LEVEL debug | info | warn | error (default: info) +// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU) +// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) +// KOKORO_URL Kokoro-FastAPI base URL (default: "") +// KOKORO_VOICE Default TTS voice (default: af_bella) +// POCKETBASE_URL PocketBase API base URL (default: http://localhost:8090) +// POCKETBASE_EMAIL PocketBase admin email (default: admin@libnovel.local) +// POCKETBASE_PASSWORD PocketBase admin password (default: adminpassword) +// MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000) +// MINIO_ACCESS_KEY MinIO access key (default: minioadmin) +// MINIO_SECRET_KEY MinIO secret key (default: minioadmin) +// MINIO_USE_SSL Use TLS for MinIO (default: false) +// MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters) +// MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio) +// LOG_LEVEL debug | info | warn | error (default: info) package main import ( @@ -38,7 +46,7 @@ import ( "github.com/libnovel/scraper/internal/novelfire" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/server" - "github.com/libnovel/scraper/internal/writer" + "github.com/libnovel/scraper/internal/storage" ) func main() { @@ -88,9 +96,30 @@ func run(log *slog.Logger) error { bc := newBrowserClient(strategy, browserCfg) urlClient := newBrowserClient(urlStrategy, browserCfg) - staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books") - w := writer.New(staticRoot) - nf := novelfire.New(bc, log, urlClient, w) + // ── Storage backends ──────────────────────────────────────────────────── + minioCfg := storage.MinioConfig{ + Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"), + AccessKey: envOr("MINIO_ACCESS_KEY", "minioadmin"), + SecretKey: envOr("MINIO_SECRET_KEY", "minioadmin"), + UseSSL: strings.ToLower(os.Getenv("MINIO_USE_SSL")) == "true", + BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), + BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), + } + pbCfg := storage.PocketBaseConfig{ + BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"), + AdminEmail: envOr("POCKETBASE_EMAIL", "admin@libnovel.local"), + AdminPassword: envOr("POCKETBASE_PASSWORD", "adminpassword"), + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + store, err := storage.NewHybridStore(ctx, pbCfg, minioCfg) + if err != nil { + return fmt.Errorf("storage init failed: %w", err) + } + + nf := novelfire.New(bc, log, urlClient, &rankingCacheAdapter{store: store}) workers := 0 if s := os.Getenv("SCRAPER_WORKERS"); s != "" { @@ -104,13 +133,9 @@ func run(log *slog.Logger) error { } oCfg := orchestrator.Config{ - Workers: workers, - StaticRoot: staticRoot, + Workers: workers, } - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer stop() - switch cmd { case "run": // Optional --url flag. @@ -121,10 +146,9 @@ func run(log *slog.Logger) error { "strategy", strategy, "workers", workers, "max_concurrent", browserCfg.MaxConcurrent, - "static_root", oCfg.StaticRoot, "single_book", oCfg.SingleBookURL, ) - o := orchestrator.New(oCfg, nf, log) + o := orchestrator.New(oCfg, nf, log, store) return o.Run(ctx) case "refresh": @@ -133,13 +157,12 @@ func run(log *slog.Logger) error { return fmt.Errorf("refresh command requires a book slug argument") } slug := args[1] - w := writer.New(oCfg.StaticRoot) - meta, ok, err := w.ReadMetadata(slug) + meta, ok, err := store.ReadMetadata(ctx, slug) if err != nil { return fmt.Errorf("failed to read metadata for %s: %w", slug, err) } if !ok { - return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot) + return fmt.Errorf("book %q not found in store", slug) } if meta.SourceURL == "" { return fmt.Errorf("book %q has no source_url in metadata", slug) @@ -149,7 +172,7 @@ func run(log *slog.Logger) error { "slug", slug, "source_url", meta.SourceURL, ) - o := orchestrator.New(oCfg, nf, log) + o := orchestrator.New(oCfg, nf, log, store) return o.Run(ctx) case "serve": @@ -164,7 +187,7 @@ func run(log *slog.Logger) error { "kokoro_url", kokoroURL, "kokoro_voice", kokoroVoice, ) - srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice) + srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice) return srv.ListenAndServe(ctx) default: @@ -192,6 +215,20 @@ func envOr(key, fallback string) string { return fallback } +// rankingCacheAdapter bridges storage.HybridStore (context-aware) to the +// context-free scraper.RankingPageCacher interface expected by novelfire.New. +type rankingCacheAdapter struct { + store *storage.HybridStore +} + +func (a *rankingCacheAdapter) WriteRankingPageCache(page int, html string) error { + return a.store.WriteRankingPageCache(context.Background(), page, html) +} + +func (a *rankingCacheAdapter) ReadRankingPageCache(page int) (string, error) { + return a.store.ReadRankingPageCache(context.Background(), page) +} + func printUsage() { fmt.Fprintf(os.Stderr, `libnovel scraper diff --git a/scraper/internal/orchestrator/orchestrator.go b/scraper/internal/orchestrator/orchestrator.go index d35a6c0..619c1b7 100644 --- a/scraper/internal/orchestrator/orchestrator.go +++ b/scraper/internal/orchestrator/orchestrator.go @@ -19,7 +19,7 @@ import ( "sync" "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/writer" + "github.com/libnovel/scraper/internal/storage" ) // Config holds tunable parameters for the orchestrator. @@ -28,7 +28,8 @@ type Config struct { // Defaults to runtime.NumCPU() when 0. Workers int - // StaticRoot is the path to the static/books output directory. + // StaticRoot is kept for backwards-compatibility but is no longer used + // when a Store is provided. StaticRoot string // SingleBookURL when non-empty causes the orchestrator to scrape only @@ -40,13 +41,13 @@ type Config struct { type Orchestrator struct { cfg Config novel scraper.NovelScraper - writer *writer.Writer + store storage.Store log *slog.Logger workers int } -// New returns a new Orchestrator. -func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator { +// New returns a new Orchestrator backed by the provided Store. +func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store) *Orchestrator { workers := cfg.Workers if workers <= 0 { workers = runtime.NumCPU() @@ -54,7 +55,7 @@ func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator return &Orchestrator{ cfg: cfg, novel: novel, - writer: writer.New(cfg.StaticRoot), + store: store, log: log, workers: workers, } @@ -66,7 +67,6 @@ func (o *Orchestrator) Run(ctx context.Context) error { o.log.Info("orchestrator starting", "source", o.novel.SourceName(), "workers", o.workers, - "static_root", o.cfg.StaticRoot, ) // chapterWork is the shared queue consumed by chapter worker goroutines. @@ -89,8 +89,8 @@ func (o *Orchestrator) Run(ctx context.Context) error { default: } - // Skip if already on disk. - if o.writer.ChapterExists(job.slug, job.ref) { + // Skip if already stored. + if o.store.ChapterExists(ctx, job.slug, job.ref) { o.log.Debug("chapter already exists, skipping", "book", job.slug, "chapter", job.ref.Number) continue @@ -107,7 +107,7 @@ func (o *Orchestrator) Run(ctx context.Context) error { continue } - if err := o.writer.WriteChapter(job.slug, chapter); err != nil { + if err := o.store.WriteChapter(ctx, job.slug, chapter); err != nil { o.log.Error("chapter write failed", "book", job.slug, "chapter", job.ref.Number, @@ -135,8 +135,8 @@ func (o *Orchestrator) Run(ctx context.Context) error { return } - // Persist / update metadata.yaml. - if err := o.writer.WriteMetadata(meta); err != nil { + // Persist / update metadata. + if err := o.store.WriteMetadata(ctx, meta); err != nil { o.log.Error("metadata write failed", "slug", meta.Slug, "err", err) // Continue — chapters can still be scraped. } diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index cc6a1e0..ab07ee9 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -22,7 +22,7 @@ import ( "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/writer" + "github.com/libnovel/scraper/internal/storage" ) // Server wraps an HTTP mux with the scraping endpoints. @@ -31,7 +31,7 @@ type Server struct { oCfg orchestrator.Config novel scraper.NovelScraper log *slog.Logger - writer *writer.Writer + store storage.Store mu sync.Mutex running bool rankingRunning bool @@ -42,26 +42,23 @@ type Server struct { voiceMu sync.RWMutex cachedVoices []string // populated on first request from Kokoro /v1/audio/voices - // audioMu guards audioCache and audioInFlight. - // audioCache maps a cache key to the Kokoro download filename returned by - // POST /v1/audio/speech with return_download_link=true. + // audioMu guards audioInFlight only. + // Completed audio filenames are persisted to the Store (PocketBase). // audioInFlight deduplicates concurrent generation requests for the same key. audioMu sync.Mutex - audioCache map[string]string // cacheKey → kokoro download filename audioInFlight map[string]chan struct{} // cacheKey → closed when done } // New creates a new Server. -func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server { +func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server { return &Server{ addr: addr, oCfg: oCfg, novel: novel, log: log, - writer: writer.New(oCfg.StaticRoot), + store: store, kokoroURL: kokoroURL, kokoroVoice: kokoroVoice, - audioCache: make(map[string]string), audioInFlight: make(map[string]chan struct{}), } } @@ -174,7 +171,7 @@ func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) return } - raw, err := s.writer.ReadChapter(slug, n) + raw, err := s.store.ReadChapter(r.Context(), slug, n) if err != nil { http.NotFound(w, r) return @@ -223,15 +220,14 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) - // Fast path: already generated this session. - s.audioMu.Lock() - if filename, ok := s.audioCache[cacheKey]; ok { - s.audioMu.Unlock() + // Fast path: already generated (check persistent store first). + if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok { s.writeAudioResponse(w, slug, n, voice, speed, filename) return } // Deduplicate concurrent generation for the same key. + s.audioMu.Lock() if ch, ok := s.audioInFlight[cacheKey]; ok { s.audioMu.Unlock() select { @@ -240,10 +236,8 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable) return } - s.audioMu.Lock() - filename, ok := s.audioCache[cacheKey] - s.audioMu.Unlock() - if ok { + // Check store again after waiting. + if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok { s.writeAudioResponse(w, slug, n, voice, speed, filename) } else { http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError) @@ -262,7 +256,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { }() // Load and validate chapter text. - raw, err := s.writer.ReadChapter(slug, n) + raw, err := s.store.ReadChapter(r.Context(), slug, n) if err != nil { http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound) return @@ -287,9 +281,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { return } - s.audioMu.Lock() - s.audioCache[cacheKey] = filename - s.audioMu.Unlock() + _ = s.store.SetAudioCache(r.Context(), cacheKey, filename) s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename) s.writeAudioResponse(w, slug, n, voice, speed, filename) @@ -378,10 +370,7 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) { } cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) - s.audioMu.Lock() - filename, ok := s.audioCache[cacheKey] - s.audioMu.Unlock() - + filename, ok := s.store.GetAudioCache(r.Context(), cacheKey) if !ok { http.Error(w, "audio not generated yet", http.StatusNotFound) return @@ -462,7 +451,7 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() - o := orchestrator.New(cfg, s.novel, s.log) + o := orchestrator.New(cfg, s.novel, s.log, s.store) if err := o.Run(ctx); err != nil { s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err)) } diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go index da600ba..12669e3 100644 --- a/scraper/internal/server/ui.go +++ b/scraper/internal/server/ui.go @@ -15,7 +15,7 @@ import ( "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/writer" + "github.com/libnovel/scraper/internal/storage" "github.com/yuin/goldmark" "github.com/yuin/goldmark/extension" goldhtml "github.com/yuin/goldmark/renderer/html" @@ -629,7 +629,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) { return } - books, err := s.writer.ListBooks() + books, err := s.store.ListBooks(r.Context()) if err != nil { http.Error(w, "failed to list books: "+err.Error(), http.StatusInternalServerError) return @@ -639,8 +639,8 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) { for i, b := range books { items[i] = homeBookItem{ BookMeta: b, - Downloaded: s.writer.CountChapters(b.Slug), - AddedAt: s.writer.MetadataMtime(b.Slug), + Downloaded: s.store.CountChapters(r.Context(), b.Slug), + AddedAt: s.store.MetadataMtime(r.Context(), b.Slug), } } @@ -877,7 +877,7 @@ const scrapeTmpl = ` ` func (s *Server) handleScrape(w http.ResponseWriter, r *http.Request) { - rankingItems, _ := s.writer.ReadRankingItems() + rankingItems, _ := s.store.ReadRankingItems(r.Context()) rankingJSON, _ := json.Marshal(rankingItems) t := template.Must(template.New("scrape").Parse(scrapeTmpl)) @@ -1324,12 +1324,12 @@ const rankingTmpl = ` // rankingViewItem enriches a RankingItem with whether it is present in the // local book library, so the template can highlight it differently. type rankingViewItem struct { - writer.RankingItem + storage.RankingItem Local bool } // toRankingViewItems annotates items with Local=true for slugs found in localSlugs. -func toRankingViewItems(items []writer.RankingItem, localSlugs map[string]bool) []rankingViewItem { +func toRankingViewItems(items []storage.RankingItem, localSlugs map[string]bool) []rankingViewItem { out := make([]rankingViewItem, len(items)) for i, it := range items { out[i] = rankingViewItem{ @@ -1403,13 +1403,13 @@ const rankingPageSize = 20 // It does NOT trigger a live scrape; use POST /ranking/refresh for that. // Supports ?page=N for browsing through cached items (20 per page). func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { - rankingItems, err := s.writer.ReadRankingItems() + rankingItems, err := s.store.ReadRankingItems(r.Context()) if err != nil { s.log.Error("failed to read cached ranking", "err", err) } cachedAt := "" - if info, statErr := s.writer.RankingFileInfo(); statErr == nil { + if info, statErr := s.store.RankingFileInfo(r.Context()); statErr == nil && info != nil { cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04") } @@ -1482,7 +1482,7 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { } // Encode full dataset + local slugs for client-side cross-page filtering. - localSlugs := s.writer.LocalSlugs() + localSlugs, _ := s.store.LocalSlugs(r.Context()) type rankingJSONItem struct { Rank int `json:"rank"` Slug string `json:"slug"` @@ -1577,14 +1577,14 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages) - var rankingItems []writer.RankingItem + var rankingItems []storage.RankingItem for rankingCh != nil || errCh != nil { select { case meta, ok := <-rankingCh: if !ok { rankingCh = nil } else { - rankingItems = append(rankingItems, writer.RankingItem{ + rankingItems = append(rankingItems, storage.RankingItem{ Rank: meta.Ranking, Slug: meta.Slug, Title: meta.Title, @@ -1605,7 +1605,7 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { } if len(rankingItems) > 0 { - if err := s.writer.WriteRanking(rankingItems); err != nil { + if err := s.store.WriteRanking(ctx, rankingItems); err != nil { s.log.Error("failed to save ranking", "err", err) } } @@ -1669,7 +1669,7 @@ const rankingViewTmpl = ` ` func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) { - items, err := s.writer.ReadRankingItems() + items, err := s.store.ReadRankingItems(r.Context()) if err != nil { http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError) return @@ -1927,7 +1927,7 @@ const bookTmpl = ` func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") - meta, ok, err := s.writer.ReadMetadata(slug) + meta, ok, err := s.store.ReadMetadata(r.Context(), slug) if err != nil { http.Error(w, "failed to read metadata: "+err.Error(), http.StatusInternalServerError) return @@ -1937,7 +1937,7 @@ func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) { return } - chapters, err := s.writer.ListChapters(slug) + chapters, err := s.store.ListChapters(r.Context(), slug) if err != nil { http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) return @@ -2054,7 +2054,7 @@ func (s *Server) handleBookChaptersPage(w http.ResponseWriter, r *http.Request) } } - chapters, err := s.writer.ListChapters(slug) + chapters, err := s.store.ListChapters(r.Context(), slug) if err != nil { http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) return @@ -3035,7 +3035,7 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { return } - raw, err := s.writer.ReadChapter(slug, n) + raw, err := s.store.ReadChapter(r.Context(), slug, n) if err != nil { http.NotFound(w, r) return @@ -3051,15 +3051,15 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { return } - chapters, _ := s.writer.ListChapters(slug) + chapters, _ := s.store.ListChapters(r.Context(), slug) prevN, nextN := adjacentChapters(chapters, n) title := firstHeading(raw, fmt.Sprintf("Chapter %d", n)) - chapterTitle, chapterDate := writer.SplitChapterTitle(title) + chapterTitle, chapterDate := splitChapterTitle(title) // Load cover URL for Media Session artwork (best-effort; ignore errors). var coverURL string - if meta, ok, err := s.writer.ReadMetadata(slug); err == nil && ok { + if meta, ok, err := s.store.ReadMetadata(r.Context(), slug); err == nil && ok { coverURL = meta.Cover } @@ -3096,6 +3096,36 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { // ─── helpers ────────────────────────────────────────────────────────────────── +// splitChapterTitle splits a raw chapter heading into a human-readable title +// and a trailing relative-date string (e.g. "1 year ago"). It mirrors the +// same logic in internal/writer and internal/storage/hybrid. +func splitChapterTitle(raw string) (title, date string) { + raw = strings.TrimSpace(raw) + // Strip leading numeric index. + if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 { + prefix := raw[:idx] + allDigit := true + for _, c := range prefix { + if c < '0' || c > '9' { + allDigit = false + break + } + } + if allDigit { + raw = strings.TrimSpace(raw[idx:]) + } + } + // Strip "Chapter N - N: " prefix. + chNumRe := regexp.MustCompile(`(?i)^chapter\s+\d+(?:\s*-\s*\d+)?\s*:\s*`) + raw = strings.TrimSpace(chNumRe.ReplaceAllString(raw, "")) + // Detect trailing relative date. + dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`) + if m := dateRe.FindStringSubmatchIndex(raw); m != nil { + return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]]) + } + return raw, "" +} + // sortedKeys returns the keys of a string-bool map in sorted order. func sortedKeys(m map[string]bool) []string { out := make([]string, 0, len(m)) @@ -3127,7 +3157,7 @@ func stripMarkdown(src string) string { // adjacentChapters returns the chapter numbers immediately before and after n // in the sorted chapters list. 0 means "does not exist". -func adjacentChapters(chapters []writer.ChapterInfo, n int) (prev, next int) { +func adjacentChapters(chapters []storage.ChapterInfo, n int) (prev, next int) { for i, ch := range chapters { if ch.Number == n { if i > 0 { @@ -3211,7 +3241,7 @@ func (s *Server) handleUIScrapeBook(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() - o := orchestrator.New(cfg, s.novel, s.log) + o := orchestrator.New(cfg, s.novel, s.log, s.store) if err := o.Run(ctx); err != nil { s.log.Error("UI scrape job failed", "url", bookURL, "err", err) } diff --git a/scraper/internal/storage/hybrid.go b/scraper/internal/storage/hybrid.go new file mode 100644 index 0000000..31184da --- /dev/null +++ b/scraper/internal/storage/hybrid.go @@ -0,0 +1,300 @@ +// hybrid.go implements the Store interface using PocketBase for structured data +// and MinIO for binary chapter/audio blobs. +package storage + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/libnovel/scraper/internal/scraper" +) + +// HybridStore satisfies Store by routing structured data to PocketBase and +// binary objects (chapters, audio) to MinIO. +type HybridStore struct { + pb *PocketBaseStore + minio *MinioClient +} + +// NewHybridStore constructs a HybridStore. It connects to both backends and +// calls EnsureCollections to bootstrap any missing PocketBase collections. +func NewHybridStore(ctx context.Context, pbCfg PocketBaseConfig, minioCfg MinioConfig) (*HybridStore, error) { + mc, err := NewMinioClient(ctx, minioCfg) + if err != nil { + return nil, fmt.Errorf("storage: minio: %w", err) + } + pb := NewPocketBaseStore(pbCfg) + if err := pb.EnsureCollections(ctx); err != nil { + // Log but don't fail — collection creation errors are often "already exists" + _ = err + } + return &HybridStore{pb: pb, minio: mc}, nil +} + +// ─── Book metadata ──────────────────────────────────────────────────────────── + +func (h *HybridStore) WriteMetadata(ctx context.Context, meta scraper.BookMeta) error { + return h.pb.UpsertBook(ctx, + meta.Slug, meta.Title, meta.Author, meta.Cover, + meta.Status, meta.Summary, meta.SourceURL, + meta.Genres, meta.TotalChapters, meta.Ranking, + ) +} + +func (h *HybridStore) ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) { + rec, found, err := h.pb.GetBook(ctx, slug) + if err != nil || !found { + return scraper.BookMeta{}, found, err + } + return recToBookMeta(rec), true, nil +} + +func (h *HybridStore) ListBooks(ctx context.Context) ([]scraper.BookMeta, error) { + rows, err := h.pb.ListBooks(ctx) + if err != nil { + return nil, err + } + books := make([]scraper.BookMeta, 0, len(rows)) + for _, r := range rows { + books = append(books, recToBookMeta(r)) + } + return books, nil +} + +func (h *HybridStore) LocalSlugs(ctx context.Context) (map[string]bool, error) { + books, err := h.ListBooks(ctx) + if err != nil { + return nil, err + } + slugs := make(map[string]bool, len(books)) + for _, b := range books { + slugs[b.Slug] = true + } + return slugs, nil +} + +func (h *HybridStore) MetadataMtime(ctx context.Context, slug string) int64 { + t, err := h.pb.BookMetaUpdated(ctx, slug) + if err != nil || t.IsZero() { + return 0 + } + return t.Unix() +} + +// ─── Chapters ───────────────────────────────────────────────────────────────── + +func (h *HybridStore) ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool { + return h.minio.ChapterExists(ctx, slug, ref.Volume, ref.Number) +} + +func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error { + content := "# " + chapter.Ref.Title + "\n\n" + chapter.Text + "\n" + if err := h.minio.PutChapter(ctx, slug, chapter.Ref.Volume, chapter.Ref.Number, content); err != nil { + return err + } + // Update chapter index in PocketBase. + title, dateLabel := splitChapterTitle(chapter.Ref.Title) + _ = h.pb.UpsertChapterIdx(ctx, slug, chapter.Ref.Number, title, dateLabel) + return nil +} + +func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) { + return h.minio.GetChapter(ctx, slug, 0, n) +} + +func (h *HybridStore) ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) { + rows, err := h.pb.ListChapterIdx(ctx, slug) + if err != nil { + return nil, err + } + infos := make([]ChapterInfo, 0, len(rows)) + for _, r := range rows { + n := int(floatVal(r, "number")) + title, _ := r["title"].(string) + date, _ := r["date_label"].(string) + infos = append(infos, ChapterInfo{Number: n, Title: title, Date: date}) + } + sort.Slice(infos, func(i, j int) bool { return infos[i].Number < infos[j].Number }) + return infos, nil +} + +func (h *HybridStore) CountChapters(ctx context.Context, slug string) int { + return h.pb.CountChapterIdx(ctx, slug) +} + +// ─── Ranking ───────────────────────────────────────────────────────────────── + +func (h *HybridStore) WriteRanking(ctx context.Context, items []RankingItem) error { + data, err := json.Marshal(items) + if err != nil { + return fmt.Errorf("storage: marshal ranking: %w", err) + } + return h.pb.SetRanking(ctx, string(data)) +} + +func (h *HybridStore) ReadRankingItems(ctx context.Context) ([]RankingItem, error) { + dataStr, _, err := h.pb.GetRanking(ctx) + if err != nil || dataStr == "" { + return nil, err + } + var items []RankingItem + if err := json.Unmarshal([]byte(dataStr), &items); err != nil { + return nil, fmt.Errorf("storage: unmarshal ranking: %w", err) + } + return items, nil +} + +func (h *HybridStore) RankingFileInfo(ctx context.Context) (os.FileInfo, error) { + return h.pb.RankingModTime(ctx) +} + +// ─── Ranking page HTML cache ────────────────────────────────────────────────── + +func (h *HybridStore) WriteRankingPageCache(ctx context.Context, page int, html string) error { + return h.pb.SetRankingPageHTML(ctx, page, html) +} + +func (h *HybridStore) ReadRankingPageCache(ctx context.Context, page int) (string, error) { + html, _, err := h.pb.GetRankingPageHTML(ctx, page) + return html, err +} + +func (h *HybridStore) RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error) { + return h.pb.RankingPageCacheModTime(ctx, page) +} + +// ─── Audio cache ────────────────────────────────────────────────────────────── + +func (h *HybridStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool) { + filename, ok, _ := h.pb.GetAudioCache(ctx, cacheKey) + return filename, ok +} + +func (h *HybridStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error { + return h.pb.SetAudioCache(ctx, cacheKey, filename) +} + +// ─── Reading progress ───────────────────────────────────────────────────────── + +func (h *HybridStore) GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) { + ch, updated, ok, err := h.pb.GetProgress(ctx, sessionID, slug) + if err != nil || !ok { + return ReadingProgress{}, false + } + return ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}, true +} + +func (h *HybridStore) SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error { + return h.pb.SetProgress(ctx, sessionID, p.Slug, p.Chapter) +} + +func (h *HybridStore) AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) { + rows, err := h.pb.AllProgress(ctx, sessionID) + if err != nil { + return nil, err + } + out := make([]ReadingProgress, 0, len(rows)) + for _, r := range rows { + slug, _ := r["slug"].(string) + ch := int(floatVal(r, "chapter")) + var updated time.Time + if ts, ok := r["updated"].(string); ok { + updated, _ = time.Parse(time.RFC3339, ts) + } + out = append(out, ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}) + } + return out, nil +} + +func (h *HybridStore) DeleteProgress(ctx context.Context, sessionID, slug string) error { + return h.pb.DeleteProgress(ctx, sessionID, slug) +} + +// ─── AudioObjectKey ─────────────────────────────────────────────────────────── + +func (h *HybridStore) AudioObjectKey(slug string, n int, voice string, speed float64) string { + return AudioObjectKey(slug, n, voice, speed) +} + +// ─── helpers ────────────────────────────────────────────────────────────────── + +func recToBookMeta(rec map[string]interface{}) scraper.BookMeta { + m := scraper.BookMeta{ + Slug: strVal(rec, "slug"), + Title: strVal(rec, "title"), + Author: strVal(rec, "author"), + Cover: strVal(rec, "cover"), + Status: strVal(rec, "status"), + Summary: strVal(rec, "summary"), + SourceURL: strVal(rec, "source_url"), + } + if tc := floatVal(rec, "total_chapters"); tc > 0 { + m.TotalChapters = int(tc) + } + if rk := floatVal(rec, "ranking"); rk > 0 { + m.Ranking = int(rk) + } + // Genres stored as JSON string or array. + switch v := rec["genres"].(type) { + case string: + _ = json.Unmarshal([]byte(v), &m.Genres) + case []interface{}: + for _, g := range v { + if s, ok := g.(string); ok { + m.Genres = append(m.Genres, s) + } + } + } + return m +} + +func strVal(m map[string]interface{}, key string) string { + if v, ok := m[key].(string); ok { + return v + } + return "" +} + +// splitChapterTitle mirrors writer.SplitChapterTitle logic (simplified). +func splitChapterTitle(raw string) (title, date string) { + raw = strings.TrimSpace(raw) + // Strip leading numeric index. + if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 { + prefix := raw[:idx] + allDigit := true + for _, c := range prefix { + if c < '0' || c > '9' { + allDigit = false + break + } + } + if allDigit { + raw = strings.TrimSpace(raw[idx:]) + } + } + // Detect trailing relative date. + units := []string{"second", "minute", "hour", "day", "week", "month", "year"} + lower := strings.ToLower(raw) + for _, u := range units { + for _, suffix := range []string{u + "s ago", u + " ago"} { + if idx := strings.LastIndex(lower, suffix); idx > 0 { + // Find start of date token (digit before the unit). + start := strings.LastIndex(raw[:idx], " ") + if start < 0 { + start = 0 + } + numPart := strings.TrimSpace(raw[start:idx]) + if _, err := strconv.Atoi(strings.Fields(numPart)[0]); err == nil { + return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)]) + } + } + } + } + return raw, "" +} -- 2.47.3 From da4a182f85bcaf9dd23b1d0d3b9fd305354a3f37 Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 14:47:58 +0500 Subject: [PATCH 005/231] Step 9: add session cookie + /api/progress GET/POST/DELETE endpoints --- scraper/internal/server/server.go | 123 ++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index ab07ee9..1cf54a7 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -10,6 +10,8 @@ package server import ( "bytes" "context" + "crypto/rand" + "encoding/hex" "encoding/json" "fmt" "io" @@ -109,6 +111,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error { mux.HandleFunc("GET /health", s.handleHealth) mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) + // Progress API + mux.HandleFunc("GET /api/progress", s.handleGetProgress) + mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress) + mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress) // UI routes mux.HandleFunc("GET /", s.handleHome) mux.HandleFunc("GET /scrape", s.handleScrape) @@ -162,6 +168,123 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) } +// ─── Session cookie helpers ─────────────────────────────────────────────────── + +const sessionCookieName = "libnovel_session" + +// sessionID returns the session ID from the request cookie, or "" if absent. +func sessionID(r *http.Request) string { + c, err := r.Cookie(sessionCookieName) + if err != nil { + return "" + } + return c.Value +} + +// newSessionID generates a random 16-byte hex session ID. +func newSessionID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +// ensureSession issues a new session cookie if the request does not already +// carry one, and returns the session ID (either existing or newly issued). +func ensureSession(w http.ResponseWriter, r *http.Request) string { + if id := sessionID(r); id != "" { + return id + } + id, err := newSessionID() + if err != nil { + // Very unlikely, but fall back to a timestamp-based ID. + id = fmt.Sprintf("fallback-%d", time.Now().UnixNano()) + } + http.SetCookie(w, &http.Cookie{ + Name: sessionCookieName, + Value: id, + Path: "/", + HttpOnly: true, + SameSite: http.SameSiteLaxMode, + MaxAge: 365 * 24 * 60 * 60, // 1 year + }) + return id +} + +// ─── Reading progress API ───────────────────────────────────────────────────── + +// handleGetProgress handles GET /api/progress. +// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...} +func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) { + sid := ensureSession(w, r) + entries, err := s.store.AllProgress(r.Context(), sid) + if err != nil { + s.log.Error("AllProgress failed", "err", err) + entries = nil + } + + progress := make(map[string]interface{}, len(entries)*2) + for _, p := range entries { + progress[p.Slug] = p.Chapter + progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli() + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(progress) +} + +// handleSetProgress handles POST /api/progress/{slug}. +// Body: {"chapter": N} +func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) { + sid := ensureSession(w, r) + slug := r.PathValue("slug") + if slug == "" { + http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest) + return + } + + var body struct { + Chapter int `json:"chapter"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 { + http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest) + return + } + + p := storage.ReadingProgress{ + Slug: slug, + Chapter: body.Chapter, + UpdatedAt: time.Now(), + } + if err := s.store.SetProgress(r.Context(), sid, p); err != nil { + s.log.Error("SetProgress failed", "slug", slug, "err", err) + http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{}) +} + +// handleDeleteProgress handles DELETE /api/progress/{slug}. +func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) { + sid := ensureSession(w, r) + slug := r.PathValue("slug") + if slug == "" { + http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest) + return + } + + if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil { + s.log.Error("DeleteProgress failed", "slug", slug, "err", err) + // Non-fatal — treat as success. + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{}) +} + // handleChapterText returns the plain text of a chapter (markdown stripped) // for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI. func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) { -- 2.47.3 From af86c6f96fd02d400109e69e761a5e6835968c62 Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 14:51:07 +0500 Subject: [PATCH 006/231] Step 10: migrate home page reading-progress JS from localStorage to /api/progress --- scraper/internal/server/ui.go | 322 +++++++++++++++++----------------- 1 file changed, 161 insertions(+), 161 deletions(-) diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go index 12669e3..daa318c 100644 --- a/scraper/internal/server/ui.go +++ b/scraper/internal/server/ui.go @@ -300,14 +300,9 @@ const homeTmpl = ` ` -- 2.47.3 From 5ac89da513ddd776a1a60b0184a9970c9f0157d0 Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 14:52:01 +0500 Subject: [PATCH 007/231] Steps 11-12: migrate chapter + book page reading-progress JS from localStorage to /api/progress --- scraper/internal/server/ui.go | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go index daa318c..f4e471c 100644 --- a/scraper/internal/server/ui.go +++ b/scraper/internal/server/ui.go @@ -1769,7 +1769,7 @@ const bookTmpl = ` - + -
+

Chapter {data.chapter.number}

{data.chapter.title || `Chapter ${data.chapter.number}`} @@ -51,13 +66,16 @@ {/if}

+ + + {#if !data.html}

Chapter content not available.

{:else} -
+
{@html data.html}
{/if} -- 2.47.3 From e4c4f8de66f63bc8b03a8fb274409f6031bee138 Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 21:43:32 +0500 Subject: [PATCH 017/231] feat(ui): wire SvelteKit into docker-compose with ui service and env vars --- .env.example | 18 ++++++++++++++++++ docker-compose.yml | 26 ++++++++++++++++++++++++++ ui/Dockerfile | 23 +++++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 ui/Dockerfile diff --git a/.env.example b/.env.example index d1e4792..0864f24 100644 --- a/.env.example +++ b/.env.example @@ -39,3 +39,21 @@ KOKORO_URL=http://kokoro:8880 # Single voices: af_bella, af_sky, af_heart, am_adam, … # Mixed voices: af_bella+af_sky or af_bella(2)+af_sky(1) (weighted blend) KOKORO_VOICE=af_bella + +# ── MinIO / S3 object storage ───────────────────────────────────────────────── +MINIO_ROOT_USER=admin +MINIO_ROOT_PASSWORD=changeme123 +MINIO_BUCKET_CHAPTERS=libnovel-chapters +MINIO_BUCKET_AUDIO=libnovel-audio + +# PocketBase admin credentials (used by scraper + UI server-side) +POCKETBASE_ADMIN_EMAIL=admin@libnovel.local +POCKETBASE_ADMIN_PASSWORD=changeme123 + +# ── SvelteKit UI ───────────────────────────────────────────────────────────── +# Port the UI container exposes on the host +UI_PORT=3000 + +# Public MinIO URL reachable from the browser (for audio/presigned URLs) +# In production, point this at your MinIO reverse-proxy or CDN domain. +PUBLIC_MINIO_PUBLIC_URL=http://localhost:9000 diff --git a/docker-compose.yml b/docker-compose.yml index 981700e..50dbdcc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -130,6 +130,32 @@ services: timeout: 5s retries: 3 + # ─── SvelteKit UI ──────────────────────────────────────────────────────────── + ui: + build: + context: ./ui + dockerfile: Dockerfile + container_name: libnovel-ui + restart: unless-stopped + depends_on: + scraper: + condition: service_healthy + pocketbase: + condition: service_healthy + environment: + SCRAPER_API_URL: "http://scraper:8080" + POCKETBASE_URL: "http://pocketbase:8090" + POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" + POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}" + PUBLIC_MINIO_PUBLIC_URL: "${PUBLIC_MINIO_PUBLIC_URL:-http://localhost:9000}" + ports: + - "${UI_PORT:-3000}:3000" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/"] + interval: 15s + timeout: 5s + retries: 3 + volumes: static_books: minio_data: diff --git a/ui/Dockerfile b/ui/Dockerfile new file mode 100644 index 0000000..797260b --- /dev/null +++ b/ui/Dockerfile @@ -0,0 +1,23 @@ +FROM node:22-alpine AS builder +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN npm ci + +COPY . . +RUN npm run build + +# ── Runtime image ────────────────────────────────────────────────────────────── +FROM node:22-alpine +WORKDIR /app + +# adapter-node produces a standalone build/ +COPY --from=builder /app/build ./build +COPY --from=builder /app/package.json ./ + +ENV NODE_ENV=production +ENV PORT=3000 +ENV HOST=0.0.0.0 + +EXPOSE 3000 +CMD ["node", "build"] -- 2.47.3 From f95ad3ed295072f05c961c957a0dcd06cb1045cc Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 22:00:13 +0500 Subject: [PATCH 018/231] feat(ui): proxy audio generation and streaming through SvelteKit, fix hardcoded scraper URL in AudioPlayer --- ui/src/lib/components/AudioPlayer.svelte | 10 +- ui/src/routes/api/audio/[slug]/[n]/+server.ts | 92 +++++++++++++++++++ 2 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 ui/src/routes/api/audio/[slug]/[n]/+server.ts diff --git a/ui/src/lib/components/AudioPlayer.svelte b/ui/src/lib/components/AudioPlayer.svelte index 187318c..992ea7a 100644 --- a/ui/src/lib/components/AudioPlayer.svelte +++ b/ui/src/lib/components/AudioPlayer.svelte @@ -55,11 +55,11 @@ status = 'generating'; errorMsg = ''; try { - const res = await fetch(`http://localhost:8080/ui/audio/${slug}/${chapter}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ voice, speed }) - }); + const res = await fetch(`/api/audio/${slug}/${chapter}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ voice, speed }) + }); if (!res.ok) throw new Error(`Generation failed: ${res.status}`); // After generation, fetch the presigned URL await loadAudio(); diff --git a/ui/src/routes/api/audio/[slug]/[n]/+server.ts b/ui/src/routes/api/audio/[slug]/[n]/+server.ts new file mode 100644 index 0000000..c196bbe --- /dev/null +++ b/ui/src/routes/api/audio/[slug]/[n]/+server.ts @@ -0,0 +1,92 @@ +import { error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { env } from '$env/dynamic/private'; + +const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080'; + +/** + * POST /api/audio/[slug]/[n] + * Proxies the audio generation request to the scraper's /api/audio endpoint. + * Keeps the scraper URL server-side — the browser never needs to know it. + * + * Body: { voice?: string, speed?: number } + * Response: { url: string, filename: string } + * where `url` is a relative path to GET /api/audio/[slug]/[n]?voice=...&speed=... + */ +export const POST: RequestHandler = async ({ params, request }) => { + const { slug, n } = params; + const chapter = parseInt(n, 10); + if (!slug || !chapter || chapter < 1) { + error(400, 'Invalid slug or chapter number'); + } + + let body: { voice?: string; speed?: number } = {}; + try { + body = await request.json(); + } catch { + // empty body is fine — scraper will use defaults + } + + const scraperRes = await fetch(`${SCRAPER_URL}/api/audio/${slug}/${chapter}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); + + if (!scraperRes.ok) { + const text = await scraperRes.text().catch(() => ''); + error(scraperRes.status as Parameters[0], text || 'Audio generation failed'); + } + + const data = (await scraperRes.json()) as { url: string; filename: string }; + + // Rewrite the proxy URL from the scraper's /ui/audio-proxy/... to our own + // /api/audio/[slug]/[n]?voice=...&speed=... so the browser never calls the scraper directly. + const voice = body.voice ?? ''; + const speed = body.speed ?? 1.0; + const qs = new URLSearchParams(); + if (voice) qs.set('voice', voice); + qs.set('speed', String(speed)); + + return new Response( + JSON.stringify({ + url: `/api/audio/${slug}/${chapter}?${qs.toString()}`, + filename: data.filename + }), + { headers: { 'Content-Type': 'application/json' } } + ); +}; + +/** + * GET /api/audio/[slug]/[n]?voice=...&speed=... + * Proxies the audio stream from the scraper's /api/audio-proxy endpoint. + * This is the URL the browser's