From 9add9033b9520f9f293e231c900dac0d181c6cd2 Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 14:34:25 +0500 Subject: [PATCH] feat(v2): add internal/storage package with MinIO + PocketBase clients and Store interface --- scraper/go.mod | 21 + scraper/go.sum | 39 ++ scraper/internal/storage/minio.go | 179 ++++++++ scraper/internal/storage/pocketbase.go | 560 +++++++++++++++++++++++++ scraper/internal/storage/store.go | 124 ++++++ 5 files changed, 923 insertions(+) create mode 100644 scraper/internal/storage/minio.go create mode 100644 scraper/internal/storage/pocketbase.go create mode 100644 scraper/internal/storage/store.go diff --git a/scraper/go.mod b/scraper/go.mod index ff9ea50..1ad5393 100644 --- a/scraper/go.mod +++ b/scraper/go.mod @@ -4,7 +4,28 @@ go 1.25.0 require ( github.com/gorilla/websocket v1.5.3 + github.com/minio/minio-go/v7 v7.0.98 github.com/yuin/goldmark v1.7.16 golang.org/x/net v0.51.0 gopkg.in/yaml.v3 v3.0.1 ) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect +) diff --git a/scraper/go.sum b/scraper/go.sum index 49bcf84..ebd83af 100644 --- a/scraper/go.sum +++ b/scraper/go.sum @@ -1,9 +1,48 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0= +github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE= github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/scraper/internal/storage/minio.go b/scraper/internal/storage/minio.go new file mode 100644 index 0000000..932011a --- /dev/null +++ b/scraper/internal/storage/minio.go @@ -0,0 +1,179 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// MinioConfig holds connection parameters for MinIO. +type MinioConfig struct { + Endpoint string // e.g. "minio:9000" + AccessKey string + SecretKey string + UseSSL bool + BucketChapters string // e.g. "libnovel-chapters" + BucketAudio string // e.g. "libnovel-audio" +} + +// MinioClient wraps a minio.Client and exposes object operations for +// chapters and audio files. +type MinioClient struct { + c *minio.Client + cfg MinioConfig +} + +// NewMinioClient creates a connected MinIO client and ensures the required +// buckets exist. +func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error) { + c, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: cfg.UseSSL, + }) + if err != nil { + return nil, fmt.Errorf("minio: new client: %w", err) + } + + mc := &MinioClient{c: c, cfg: cfg} + for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio} { + if err := mc.ensureBucket(ctx, bucket); err != nil { + return nil, err + } + } + return mc, nil +} + +// ensureBucket creates a bucket if it does not exist. +func (m *MinioClient) ensureBucket(ctx context.Context, bucket string) error { + exists, err := m.c.BucketExists(ctx, bucket) + if err != nil { + return fmt.Errorf("minio: bucket exists %q: %w", bucket, err) + } + if !exists { + if err := m.c.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil { + return fmt.Errorf("minio: make bucket %q: %w", bucket, err) + } + } + return nil +} + +// ─── Chapter objects ────────────────────────────────────────────────────────── + +// chapterKey returns the MinIO object key for a chapter. +// Layout: {slug}/vol-{vol}/{lo}-{hi}/chapter-{n}.md +func chapterKey(slug string, vol, n int) string { + const chaptersPerFolder = 50 + lo := ((n-1)/chaptersPerFolder)*chaptersPerFolder + 1 + hi := lo + chaptersPerFolder - 1 + return fmt.Sprintf("%s/vol-%d/%d-%d/chapter-%d.md", slug, vol, lo, hi, n) +} + +// PutChapter stores chapter markdown in MinIO. +func (m *MinioClient) PutChapter(ctx context.Context, slug string, vol, n int, content string) error { + key := chapterKey(slug, vol, n) + data := []byte(content) + _, err := m.c.PutObject(ctx, m.cfg.BucketChapters, key, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "text/markdown; charset=utf-8"}) + if err != nil { + return fmt.Errorf("minio: put chapter %s: %w", key, err) + } + return nil +} + +// GetChapter retrieves chapter markdown from MinIO. +func (m *MinioClient) GetChapter(ctx context.Context, slug string, vol, n int) (string, error) { + key := chapterKey(slug, vol, n) + obj, err := m.c.GetObject(ctx, m.cfg.BucketChapters, key, minio.GetObjectOptions{}) + if err != nil { + return "", fmt.Errorf("minio: get chapter %s: %w", key, err) + } + defer obj.Close() + data, err := io.ReadAll(obj) + if err != nil { + return "", fmt.Errorf("minio: read chapter %s: %w", key, err) + } + return string(data), nil +} + +// ChapterExists returns true if the object for this chapter is present. +func (m *MinioClient) ChapterExists(ctx context.Context, slug string, vol, n int) bool { + key := chapterKey(slug, vol, n) + _, err := m.c.StatObject(ctx, m.cfg.BucketChapters, key, minio.StatObjectOptions{}) + return err == nil +} + +// ListChapterKeys returns all object keys under slug/ in the chapters bucket, +// sorted lexicographically (MinIO returns them in order). +func (m *MinioClient) ListChapterKeys(ctx context.Context, slug string) ([]string, error) { + prefix := slug + "/" + var keys []string + for obj := range m.c.ListObjects(ctx, m.cfg.BucketChapters, + minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + if obj.Err != nil { + return nil, fmt.Errorf("minio: list chapters %s: %w", slug, obj.Err) + } + keys = append(keys, obj.Key) + } + return keys, nil +} + +// CountChapters returns the number of chapter objects for a slug. +func (m *MinioClient) CountChapters(ctx context.Context, slug string) int { + keys, _ := m.ListChapterKeys(ctx, slug) + return len(keys) +} + +// ─── Audio objects ──────────────────────────────────────────────────────────── + +// AudioObjectKey returns the MinIO key for a cached audio file. +// Key: {slug}/ch{n}-{voice}-{speed:.1f}.mp3 +func AudioObjectKey(slug string, n int, voice string, speed float64) string { + safe := sanitiseVoice(voice) + return fmt.Sprintf("%s/ch%d-%s-%.1f.mp3", slug, n, safe, speed) +} + +// PutAudio stores an audio file in the audio bucket. +func (m *MinioClient) PutAudio(ctx context.Context, key string, data []byte) error { + _, err := m.c.PutObject(ctx, m.cfg.BucketAudio, key, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "audio/mpeg"}) + if err != nil { + return fmt.Errorf("minio: put audio %s: %w", key, err) + } + return nil +} + +// GetAudio retrieves audio bytes from the audio bucket. +func (m *MinioClient) GetAudio(ctx context.Context, key string) ([]byte, error) { + obj, err := m.c.GetObject(ctx, m.cfg.BucketAudio, key, minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("minio: get audio %s: %w", key, err) + } + defer obj.Close() + return io.ReadAll(obj) +} + +// AudioExists returns true if the audio object is present in the bucket. +func (m *MinioClient) AudioExists(ctx context.Context, key string) bool { + _, err := m.c.StatObject(ctx, m.cfg.BucketAudio, key, minio.StatObjectOptions{}) + return err == nil +} + +// ─── helpers ────────────────────────────────────────────────────────────────── + +// sanitiseVoice converts a voice name to a filename-safe string. +func sanitiseVoice(voice string) string { + return strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || r == '_' || r == '-' { + return r + } + return '_' + }, voice) +} diff --git a/scraper/internal/storage/pocketbase.go b/scraper/internal/storage/pocketbase.go new file mode 100644 index 0000000..8a34ae3 --- /dev/null +++ b/scraper/internal/storage/pocketbase.go @@ -0,0 +1,560 @@ +// Package storage — PocketBase REST client. +// +// Collections expected in PocketBase: +// +// books — slug(text,unique), title, author, cover, status, genres(json), +// summary, total_chapters(number), source_url, ranking(number), updated(date) +// chapters_idx — slug(text), number(number), title, date_label, updated(date) +// ranking — data(json), updated(date) [single row, upserted by slug="_ranking_"] +// ranking_html — page(number,unique), html(text), updated(date) +// progress — session_id(text), slug(text), chapter(number), updated(date) +// audio_cache — cache_key(text,unique), filename(text), updated(date) +package storage + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" +) + +// PocketBaseConfig holds PocketBase connection settings. +type PocketBaseConfig struct { + BaseURL string // e.g. "http://pocketbase:8090" + AdminEmail string + AdminPassword string +} + +// pbClient is a minimal PocketBase admin REST client. +type pbClient struct { + cfg PocketBaseConfig + httpClient *http.Client + + tokenMu sync.RWMutex + token string + tokenExp time.Time +} + +// newPBClient creates a new PocketBase client. It does not authenticate yet; +// authentication happens lazily on the first API call. +func newPBClient(cfg PocketBaseConfig) *pbClient { + return &pbClient{ + cfg: cfg, + httpClient: &http.Client{Timeout: 15 * time.Second}, + } +} + +// ─── Auth ───────────────────────────────────────────────────────────────────── + +func (p *pbClient) authenticate(ctx context.Context) error { + body, _ := json.Marshal(map[string]string{ + "identity": p.cfg.AdminEmail, + "password": p.cfg.AdminPassword, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + p.cfg.BaseURL+"/api/admins/auth-with-password", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := p.httpClient.Do(req) + if err != nil { + return fmt.Errorf("pocketbase: auth: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pocketbase: auth status %d: %s", resp.StatusCode, b) + } + var result struct { + Token string `json:"token"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return fmt.Errorf("pocketbase: decode auth: %w", err) + } + p.tokenMu.Lock() + p.token = result.Token + p.tokenExp = time.Now().Add(12 * time.Hour) + p.tokenMu.Unlock() + return nil +} + +func (p *pbClient) authToken(ctx context.Context) (string, error) { + p.tokenMu.RLock() + tok, exp := p.token, p.tokenExp + p.tokenMu.RUnlock() + if tok != "" && time.Now().Before(exp) { + return tok, nil + } + if err := p.authenticate(ctx); err != nil { + return "", err + } + p.tokenMu.RLock() + defer p.tokenMu.RUnlock() + return p.token, nil +} + +// ─── Generic CRUD helpers ────────────────────────────────────────────────────── + +func (p *pbClient) do(ctx context.Context, method, path string, body interface{}) (*http.Response, error) { + tok, err := p.authToken(ctx) + if err != nil { + return nil, err + } + + var bodyReader io.Reader + if body != nil { + b, _ := json.Marshal(body) + bodyReader = bytes.NewReader(b) + } + + req, err := http.NewRequestWithContext(ctx, method, p.cfg.BaseURL+path, bodyReader) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", tok) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return p.httpClient.Do(req) +} + +// listOne fetches the first matching record from a collection. +func (p *pbClient) listOne(ctx context.Context, collection, filter string) (map[string]interface{}, error) { + q := url.Values{} + q.Set("filter", filter) + q.Set("perPage", "1") + path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode()) + resp, err := p.do(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, nil + } + var result struct { + Items []map[string]interface{} `json:"items"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + if len(result.Items) == 0 { + return nil, nil + } + return result.Items[0], nil +} + +// listAll returns all records (up to 500) from a collection matching filter. +func (p *pbClient) listAll(ctx context.Context, collection, filter, sort string) ([]map[string]interface{}, error) { + q := url.Values{} + if filter != "" { + q.Set("filter", filter) + } + if sort != "" { + q.Set("sort", sort) + } + q.Set("perPage", "500") + path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode()) + resp, err := p.do(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + var result struct { + Items []map[string]interface{} `json:"items"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, err + } + return result.Items, nil +} + +// upsert creates a record; if one matching filter already exists it updates it. +func (p *pbClient) upsert(ctx context.Context, collection, filter string, data map[string]interface{}) error { + existing, err := p.listOne(ctx, collection, filter) + if err != nil { + return err + } + if existing != nil { + id := existing["id"].(string) + resp, err := p.do(ctx, http.MethodPatch, + fmt.Sprintf("/api/collections/%s/records/%s", collection, id), data) + if err != nil { + return err + } + resp.Body.Close() + return nil + } + resp, err := p.do(ctx, http.MethodPost, + fmt.Sprintf("/api/collections/%s/records", collection), data) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +// deleteWhere deletes all records matching filter in collection. +func (p *pbClient) deleteWhere(ctx context.Context, collection, filter string) error { + items, err := p.listAll(ctx, collection, filter, "") + if err != nil { + return err + } + for _, item := range items { + id, _ := item["id"].(string) + resp, err := p.do(ctx, http.MethodDelete, + fmt.Sprintf("/api/collections/%s/records/%s", collection, id), nil) + if err != nil { + return err + } + resp.Body.Close() + } + return nil +} + +// ─── PocketBaseStore ────────────────────────────────────────────────────────── + +// PocketBaseStore implements the structured-data portion of the Store interface +// backed by PocketBase REST API. +type PocketBaseStore struct { + pb *pbClient +} + +// NewPocketBaseStore returns a connected PocketBaseStore. +func NewPocketBaseStore(cfg PocketBaseConfig) *PocketBaseStore { + return &PocketBaseStore{pb: newPBClient(cfg)} +} + +// Ping verifies connectivity by authenticating. +func (s *PocketBaseStore) Ping(ctx context.Context) error { + _, err := s.pb.authToken(ctx) + return err +} + +// ─── Collections schema bootstrap ──────────────────────────────────────────── +// CollectionDef maps a collection name to its fields for auto-creation. + +// EnsureCollections creates missing collections via the PocketBase API. +// Safe to call on every startup — existing collections are skipped. +func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error { + // We just attempt to create each collection; 400/422 errors for "already + // exists" are silently ignored. + collections := []map[string]interface{}{ + { + "name": "books", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "slug", "type": "text", "required": true, "options": map[string]interface{}{"min": 1}}, + {"name": "title", "type": "text", "required": true}, + {"name": "author", "type": "text"}, + {"name": "cover", "type": "url"}, + {"name": "status", "type": "text"}, + {"name": "genres", "type": "json"}, + {"name": "summary", "type": "text"}, + {"name": "total_chapters", "type": "number"}, + {"name": "source_url", "type": "url"}, + {"name": "ranking", "type": "number"}, + {"name": "meta_updated", "type": "date"}, + }, + }, + { + "name": "chapters_idx", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "slug", "type": "text", "required": true}, + {"name": "number", "type": "number", "required": true}, + {"name": "title", "type": "text"}, + {"name": "date_label", "type": "text"}, + }, + }, + { + "name": "ranking", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "key", "type": "text", "required": true}, + {"name": "data", "type": "json"}, + {"name": "updated", "type": "date"}, + }, + }, + { + "name": "ranking_html", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "page", "type": "number", "required": true}, + {"name": "html", "type": "text"}, + {"name": "updated", "type": "date"}, + }, + }, + { + "name": "progress", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "session_id", "type": "text", "required": true}, + {"name": "slug", "type": "text", "required": true}, + {"name": "chapter", "type": "number"}, + {"name": "updated", "type": "date"}, + }, + }, + { + "name": "audio_cache", + "type": "base", + "schema": []map[string]interface{}{ + {"name": "cache_key", "type": "text", "required": true}, + {"name": "filename", "type": "text"}, + {"name": "updated", "type": "date"}, + }, + }, + } + for _, col := range collections { + resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections", col) + if err != nil { + return fmt.Errorf("pocketbase: ensure collection %v: %w", col["name"], err) + } + resp.Body.Close() + // 400/422 = already exists or schema mismatch — ignore + } + return nil +} + +// ─── Book metadata ──────────────────────────────────────────────────────────── + +func (s *PocketBaseStore) UpsertBook(ctx context.Context, slug, title, author, cover, status, summary, sourceURL string, genres []string, totalChapters, ranking int) error { + genresJSON, _ := json.Marshal(genres) + return s.pb.upsert(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug)), map[string]interface{}{ + "slug": slug, + "title": title, + "author": author, + "cover": cover, + "status": status, + "genres": string(genresJSON), + "summary": summary, + "total_chapters": totalChapters, + "source_url": sourceURL, + "ranking": ranking, + "meta_updated": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *PocketBaseStore) GetBook(ctx context.Context, slug string) (map[string]interface{}, bool, error) { + rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug))) + if err != nil { + return nil, false, err + } + if rec == nil { + return nil, false, nil + } + return rec, true, nil +} + +func (s *PocketBaseStore) ListBooks(ctx context.Context) ([]map[string]interface{}, error) { + return s.pb.listAll(ctx, "books", "", "+title") +} + +func (s *PocketBaseStore) BookMetaUpdated(ctx context.Context, slug string) (time.Time, error) { + rec, err := s.pb.listOne(ctx, "books", fmt.Sprintf(`slug="%s"`, pbEsc(slug))) + if err != nil || rec == nil { + return time.Time{}, err + } + if ts, ok := rec["meta_updated"].(string); ok { + t, err := time.Parse(time.RFC3339, ts) + if err == nil { + return t, nil + } + } + return time.Time{}, nil +} + +// ─── Chapter index ──────────────────────────────────────────────────────────── + +func (s *PocketBaseStore) UpsertChapterIdx(ctx context.Context, slug string, number int, title, dateLabel string) error { + return s.pb.upsert(ctx, "chapters_idx", + fmt.Sprintf(`slug="%s"&&number=%d`, pbEsc(slug), number), + map[string]interface{}{ + "slug": slug, + "number": number, + "title": title, + "date_label": dateLabel, + }) +} + +func (s *PocketBaseStore) ListChapterIdx(ctx context.Context, slug string) ([]map[string]interface{}, error) { + return s.pb.listAll(ctx, "chapters_idx", + fmt.Sprintf(`slug="%s"`, pbEsc(slug)), "+number") +} + +func (s *PocketBaseStore) CountChapterIdx(ctx context.Context, slug string) int { + rows, _ := s.ListChapterIdx(ctx, slug) + return len(rows) +} + +// ─── Ranking ────────────────────────────────────────────────────────────────── + +func (s *PocketBaseStore) SetRanking(ctx context.Context, dataJSON string) error { + return s.pb.upsert(ctx, "ranking", `key="_ranking_"`, map[string]interface{}{ + "key": "_ranking_", + "data": dataJSON, + "updated": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *PocketBaseStore) GetRanking(ctx context.Context) (string, time.Time, error) { + rec, err := s.pb.listOne(ctx, "ranking", `key="_ranking_"`) + if err != nil || rec == nil { + return "", time.Time{}, err + } + data, _ := rec["data"].(string) + var updated time.Time + if ts, ok := rec["updated"].(string); ok { + updated, _ = time.Parse(time.RFC3339, ts) + } + return data, updated, nil +} + +// ─── Ranking page HTML cache ────────────────────────────────────────────────── + +func (s *PocketBaseStore) SetRankingPageHTML(ctx context.Context, page int, html string) error { + return s.pb.upsert(ctx, "ranking_html", + fmt.Sprintf(`page=%d`, page), + map[string]interface{}{ + "page": page, + "html": html, + "updated": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *PocketBaseStore) GetRankingPageHTML(ctx context.Context, page int) (string, time.Time, error) { + rec, err := s.pb.listOne(ctx, "ranking_html", fmt.Sprintf(`page=%d`, page)) + if err != nil || rec == nil { + return "", time.Time{}, err + } + html, _ := rec["html"].(string) + var updated time.Time + if ts, ok := rec["updated"].(string); ok { + updated, _ = time.Parse(time.RFC3339, ts) + } + return html, updated, nil +} + +// ─── Reading progress ───────────────────────────────────────────────────────── + +func (s *PocketBaseStore) SetProgress(ctx context.Context, sessionID, slug string, chapter int) error { + return s.pb.upsert(ctx, "progress", + fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug)), + map[string]interface{}{ + "session_id": sessionID, + "slug": slug, + "chapter": chapter, + "updated": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *PocketBaseStore) GetProgress(ctx context.Context, sessionID, slug string) (int, time.Time, bool, error) { + rec, err := s.pb.listOne(ctx, "progress", + fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug))) + if err != nil { + return 0, time.Time{}, false, err + } + if rec == nil { + return 0, time.Time{}, false, nil + } + ch := int(floatVal(rec, "chapter")) + var updated time.Time + if ts, ok := rec["updated"].(string); ok { + updated, _ = time.Parse(time.RFC3339, ts) + } + return ch, updated, true, nil +} + +func (s *PocketBaseStore) AllProgress(ctx context.Context, sessionID string) ([]map[string]interface{}, error) { + return s.pb.listAll(ctx, "progress", + fmt.Sprintf(`session_id="%s"`, pbEsc(sessionID)), "-updated") +} + +func (s *PocketBaseStore) DeleteProgress(ctx context.Context, sessionID, slug string) error { + return s.pb.deleteWhere(ctx, "progress", + fmt.Sprintf(`session_id="%s"&&slug="%s"`, pbEsc(sessionID), pbEsc(slug))) +} + +// ─── Audio cache ────────────────────────────────────────────────────────────── + +func (s *PocketBaseStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error { + return s.pb.upsert(ctx, "audio_cache", + fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey)), + map[string]interface{}{ + "cache_key": cacheKey, + "filename": filename, + "updated": time.Now().UTC().Format(time.RFC3339), + }) +} + +func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool, error) { + rec, err := s.pb.listOne(ctx, "audio_cache", + fmt.Sprintf(`cache_key="%s"`, pbEsc(cacheKey))) + if err != nil || rec == nil { + return "", false, err + } + filename, _ := rec["filename"].(string) + return filename, filename != "", nil +} + +// ─── rankingFileInfo is a minimal os.FileInfo implementation ───────────────── + +type rankingFileInfo struct { + modTime time.Time +} + +func (r rankingFileInfo) Name() string { return "ranking" } +func (r rankingFileInfo) Size() int64 { return 0 } +func (r rankingFileInfo) Mode() os.FileMode { return 0o444 } +func (r rankingFileInfo) ModTime() time.Time { return r.modTime } +func (r rankingFileInfo) IsDir() bool { return false } +func (r rankingFileInfo) Sys() interface{} { return nil } + +var _ os.FileInfo = rankingFileInfo{} + +// RankingModTime returns file-info-compatible data for the ranking record. +func (s *PocketBaseStore) RankingModTime(ctx context.Context) (os.FileInfo, error) { + _, updated, err := s.GetRanking(ctx) + if err != nil { + return nil, err + } + if updated.IsZero() { + return nil, nil + } + return rankingFileInfo{modTime: updated}, nil +} + +// RankingPageCacheModTime returns file-info for a cached ranking page. +func (s *PocketBaseStore) RankingPageCacheModTime(ctx context.Context, page int) (os.FileInfo, error) { + _, updated, err := s.GetRankingPageHTML(ctx, page) + if err != nil { + return nil, err + } + if updated.IsZero() { + return nil, nil + } + return rankingFileInfo{modTime: updated}, nil +} + +// ─── helpers ────────────────────────────────────────────────────────────────── + +// pbEsc escapes a string for use in a PocketBase filter expression. +// Only escapes double-quotes to prevent injection. +func pbEsc(s string) string { + return strings.ReplaceAll(s, `"`, `\"`) +} + +func floatVal(m map[string]interface{}, key string) float64 { + if v, ok := m[key].(float64); ok { + return v + } + return 0 +} diff --git a/scraper/internal/storage/store.go b/scraper/internal/storage/store.go new file mode 100644 index 0000000..6868594 --- /dev/null +++ b/scraper/internal/storage/store.go @@ -0,0 +1,124 @@ +// Package storage defines the unified Store interface and helper types used by +// the server and orchestrator. Concrete implementations back the interface +// with PocketBase (structured data) and MinIO (binary objects). +package storage + +import ( + "context" + "os" + "time" + + "github.com/libnovel/scraper/internal/scraper" +) + +// ─── Shared types ───────────────────────────────────────────────────────────── + +// ChapterInfo is a lightweight chapter descriptor (mirrors writer.ChapterInfo). +type ChapterInfo struct { + Number int + Title string + Date string +} + +// RankingItem represents a single entry in the novel ranking list. +type RankingItem struct { + Rank int `json:"rank"` + Slug string `json:"slug"` + Title string `json:"title"` + Author string `json:"author,omitempty"` + Cover string `json:"cover,omitempty"` + Status string `json:"status,omitempty"` + Genres []string `json:"genres,omitempty"` + SourceURL string `json:"source_url,omitempty"` +} + +// ReadingProgress holds a single user's reading position for one book. +type ReadingProgress struct { + Slug string `json:"slug"` + Chapter int `json:"chapter"` + UpdatedAt time.Time `json:"updated_at"` +} + +// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro +// download filename so audio is not re-generated after a server restart. +type AudioCacheEntry struct { + CacheKey string `json:"cache_key"` + Filename string `json:"filename"` +} + +// ─── Store interface ────────────────────────────────────────────────────────── + +// Store is the single persistence abstraction consumed by the server and the +// orchestrator. Implementations may route calls to different backends +// (PocketBase for structured records, MinIO for binary blobs). +type Store interface { + // ── Book metadata ────────────────────────────────────────────────────── + + // WriteMetadata upserts book metadata. + WriteMetadata(ctx context.Context, meta scraper.BookMeta) error + // ReadMetadata returns the metadata for slug. Returns (zero, false, nil) + // when the book is not found. + ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) + // ListBooks returns all books, sorted alphabetically by title. + ListBooks(ctx context.Context) ([]scraper.BookMeta, error) + // LocalSlugs returns the set of slugs that have metadata stored. + LocalSlugs(ctx context.Context) (map[string]bool, error) + // MetadataMtime returns the Unix-second mtime of the metadata record, or 0. + MetadataMtime(ctx context.Context, slug string) int64 + + // ── Chapters (binary blobs in MinIO) ─────────────────────────────────── + + // ChapterExists returns true if the markdown file for the given ref exists. + ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool + // WriteChapter stores the chapter markdown. + WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error + // ReadChapter returns the raw markdown for chapter number n. + ReadChapter(ctx context.Context, slug string, n int) (string, error) + // ListChapters returns all stored chapters for slug, sorted by number. + ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) + // CountChapters returns the number of stored chapters for slug. + CountChapters(ctx context.Context, slug string) int + + // ── Ranking ──────────────────────────────────────────────────────────── + + // WriteRanking persists the ranking list. + WriteRanking(ctx context.Context, items []RankingItem) error + // ReadRankingItems returns the stored ranking items. + ReadRankingItems(ctx context.Context) ([]RankingItem, error) + // RankingFileInfo returns os.FileInfo-like data for the ranking record. + // Returns (nil, nil) when no ranking has been stored yet. + RankingFileInfo(ctx context.Context) (os.FileInfo, error) + + // ── Ranking page HTML cache ──────────────────────────────────────────── + + // WriteRankingPageCache stores raw HTML for a ranking page. + WriteRankingPageCache(ctx context.Context, page int, html string) error + // ReadRankingPageCache returns cached HTML for a ranking page, or "" on miss. + ReadRankingPageCache(ctx context.Context, page int) (string, error) + // RankingPageCacheInfo returns file-like info for a cached ranking page. + RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error) + + // ── Audio cache ──────────────────────────────────────────────────────── + + // GetAudioCache returns the Kokoro filename for cacheKey, or ("", false). + GetAudioCache(ctx context.Context, cacheKey string) (string, bool) + // SetAudioCache persists a Kokoro filename for cacheKey. + SetAudioCache(ctx context.Context, cacheKey, filename string) error + + // ── Reading progress ─────────────────────────────────────────────────── + + // GetProgress returns the reading progress for the given session ID and slug. + // Returns (zero, false) if no progress is recorded. + GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) + // SetProgress saves or updates reading progress. + SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error + // AllProgress returns all progress entries for a session. + AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) + // DeleteProgress removes progress for a specific slug. + DeleteProgress(ctx context.Context, sessionID, slug string) error + + // ── Audio object paths (MinIO) ───────────────────────────────────────── + + // AudioObjectKey returns the MinIO object key for a cached audio file. + AudioObjectKey(slug string, n int, voice string, speed float64) string +}