Files
libnovel/backend/internal/storage/minio.go
Admin 3918bc8dc3
All checks were successful
Release / Scraper / Test (push) Successful in 20s
Release / UI / Build (push) Successful in 25s
CI / Scraper / Lint (pull_request) Successful in 10s
Release / v2 / Build ui-v2 (push) Successful in 28s
CI / Scraper / Test (pull_request) Successful in 15s
CI / UI / Build (pull_request) Successful in 25s
Release / Scraper / Docker (push) Successful in 55s
Release / UI / Docker (push) Successful in 43s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 36s
Release / v2 / Test backend (push) Successful in 3m51s
Release / v2 / Docker / runner (push) Successful in 36s
Release / v2 / Docker / backend (push) Successful in 1m34s
iOS CI / Build (pull_request) Successful in 5m33s
iOS CI / Test (pull_request) Successful in 11m25s
fix: serve browse pages from MinIO cache; fix ReapStaleTasks PocketBase filter
- Runner fetches 9 browse combos (genre×sort×status) every 6h and stores
  JSON snapshots in MinIO libnovel-browse bucket (browse_refresh.go)
- Backend handleBrowse reads page-1 results from MinIO first; falls back
  to live novelfire.net fetch; returns empty+cached:false on total failure
  instead of 502
- Add BrowseStore interface (bookstore.go), MinIO put/get helpers (minio.go),
  Store methods + compile-time assertion (store.go), BucketBrowse config,
  wiring in cmd/backend and cmd/runner, docker-compose-new bucket init
- Fix ReapStaleTasks: PocketBase datetime fields require heartbeat_at=null
  (not heartbeat_at="") in filter expressions, and nil (not "") in patch
  payload — was causing 400 errors on every reap cycle
2026-03-15 21:19:28 +05:00

223 lines
7.8 KiB
Go

package storage
import (
"context"
"fmt"
"io"
"net/url"
"path"
"strings"
"time"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/libnovel/backend/internal/config"
)
// minioClient wraps the official minio-go client with bucket names.
type minioClient struct {
client *minio.Client // internal — all read/write operations
pubClient *minio.Client // presign-only — initialised against the public endpoint
bucketChapters string
bucketAudio string
bucketAvatars string
bucketBrowse string
}
func newMinioClient(cfg config.MinIO) (*minioClient, error) {
creds := credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, "")
internal, err := minio.New(cfg.Endpoint, &minio.Options{
Creds: creds,
Secure: cfg.UseSSL,
})
if err != nil {
return nil, fmt.Errorf("minio: init internal client: %w", err)
}
// Presigned URLs must be signed with the hostname the browser will use
// (PUBLIC_MINIO_PUBLIC_URL), because AWS Signature V4 includes the Host
// header in the canonical request — a URL signed against "minio:9000" will
// return SignatureDoesNotMatch when the browser fetches it from
// "localhost:9000".
//
// However, minio-go normally makes a live BucketLocation HTTP call before
// signing, which would fail from inside the container when the public
// endpoint is externally-facing (e.g. "localhost:9000" is unreachable from
// within Docker). We prevent this by:
// 1. Setting Region: "us-east-1" — minio-go skips getBucketLocation when
// the region is already known (bucket-cache.go:49).
// 2. Setting BucketLookup: BucketLookupPath — forces path-style URLs
// (e.g. host/bucket/key), matching MinIO's default behaviour and
// avoiding any virtual-host DNS probing.
//
// When no public endpoint is configured (or it equals the internal one),
// fall back to the internal client so presigning still works.
publicEndpoint := cfg.PublicEndpoint
if u, err2 := url.Parse(publicEndpoint); err2 == nil && u.Host != "" {
publicEndpoint = u.Host // strip scheme so minio.New is happy
}
pubUseSSL := cfg.PublicUseSSL
if publicEndpoint == "" || publicEndpoint == cfg.Endpoint {
publicEndpoint = cfg.Endpoint
pubUseSSL = cfg.UseSSL
}
pub, err := minio.New(publicEndpoint, &minio.Options{
Creds: creds,
Secure: pubUseSSL,
Region: "us-east-1", // skip live BucketLocation preflight
BucketLookup: minio.BucketLookupPath,
})
if err != nil {
return nil, fmt.Errorf("minio: init public client: %w", err)
}
return &minioClient{
client: internal,
pubClient: pub,
bucketChapters: cfg.BucketChapters,
bucketAudio: cfg.BucketAudio,
bucketAvatars: cfg.BucketAvatars,
bucketBrowse: cfg.BucketBrowse,
}, nil
}
// ensureBuckets creates all required buckets if they don't already exist.
func (m *minioClient) ensureBuckets(ctx context.Context) error {
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse} {
exists, err := m.client.BucketExists(ctx, bucket)
if err != nil {
return fmt.Errorf("minio: check bucket %q: %w", bucket, err)
}
if !exists {
if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
return fmt.Errorf("minio: create bucket %q: %w", bucket, err)
}
}
}
return nil
}
// ── Key helpers ───────────────────────────────────────────────────────────────
// ChapterObjectKey returns the MinIO object key for a chapter markdown file.
// Format: {slug}/chapter-{n:06d}.md
func ChapterObjectKey(slug string, n int) string {
return fmt.Sprintf("%s/chapter-%06d.md", slug, n)
}
// AudioObjectKey returns the MinIO object key for a cached audio file.
// Format: {slug}/{n}/{voice}.mp3
func AudioObjectKey(slug string, n int, voice string) string {
return fmt.Sprintf("%s/%d/%s.mp3", slug, n, voice)
}
// AvatarObjectKey returns the MinIO object key for a user avatar image.
// Format: {userID}/{ext}.{ext}
func AvatarObjectKey(userID, ext string) string {
return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
}
// BrowseObjectKey returns the MinIO object key for a cached browse page snapshot.
// Format: browse/{genre}/{sort}/{status}/{type}/page-{n}.json
func BrowseObjectKey(genre, sort, status, novelType string, page int) string {
return fmt.Sprintf("browse/%s/%s/%s/%s/page-%d.json", genre, sort, status, novelType, page)
}
// chapterNumberFromKey extracts the chapter number from a MinIO object key.
// e.g. "my-book/chapter-000042.md" → 42
func chapterNumberFromKey(key string) int {
base := path.Base(key)
base = strings.TrimPrefix(base, "chapter-")
base = strings.TrimSuffix(base, ".md")
var n int
fmt.Sscanf(base, "%d", &n)
return n
}
// ── Object operations ─────────────────────────────────────────────────────────
func (m *minioClient) putObject(ctx context.Context, bucket, key, contentType string, data []byte) error {
_, err := m.client.PutObject(ctx, bucket, key,
strings.NewReader(string(data)),
int64(len(data)),
minio.PutObjectOptions{ContentType: contentType},
)
return err
}
func (m *minioClient) getObject(ctx context.Context, bucket, key string) ([]byte, error) {
obj, err := m.client.GetObject(ctx, bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
defer obj.Close()
return io.ReadAll(obj)
}
func (m *minioClient) objectExists(ctx context.Context, bucket, key string) bool {
_, err := m.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{})
return err == nil
}
func (m *minioClient) presignGet(ctx context.Context, bucket, key string, expires time.Duration) (string, error) {
u, err := m.pubClient.PresignedGetObject(ctx, bucket, key, expires, nil)
if err != nil {
return "", fmt.Errorf("minio presign %s/%s: %w", bucket, key, err)
}
return u.String(), nil
}
func (m *minioClient) presignPut(ctx context.Context, bucket, key string, expires time.Duration) (string, error) {
u, err := m.pubClient.PresignedPutObject(ctx, bucket, key, expires)
if err != nil {
return "", fmt.Errorf("minio presign PUT %s/%s: %w", bucket, key, err)
}
return u.String(), nil
}
func (m *minioClient) deleteObjects(ctx context.Context, bucket, prefix string) error {
objCh := m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix})
for obj := range objCh {
if obj.Err != nil {
return obj.Err
}
if err := m.client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}); err != nil {
return err
}
}
return nil
}
func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string) ([]string, error) {
var keys []string
for obj := range m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) {
if obj.Err != nil {
return nil, obj.Err
}
keys = append(keys, obj.Key)
}
return keys, nil
}
// ── Browse operations ─────────────────────────────────────────────────────────
// putBrowse stores raw JSON bytes for a browse page snapshot.
func (m *minioClient) putBrowse(ctx context.Context, key string, data []byte) error {
return m.putObject(ctx, m.bucketBrowse, key, "application/json", data)
}
// getBrowse retrieves a browse page snapshot. Returns (nil, false, nil) when
// the object does not exist.
func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool, error) {
if !m.objectExists(ctx, m.bucketBrowse, key) {
return nil, false, nil
}
data, err := m.getObject(ctx, m.bucketBrowse, key)
if err != nil {
return nil, false, err
}
return data, true, nil
}