fix: serve browse pages from MinIO cache; fix ReapStaleTasks PocketBase filter
All checks were successful
Release / Scraper / Test (push) Successful in 20s
Release / UI / Build (push) Successful in 25s
CI / Scraper / Lint (pull_request) Successful in 10s
Release / v2 / Build ui-v2 (push) Successful in 28s
CI / Scraper / Test (pull_request) Successful in 15s
CI / UI / Build (pull_request) Successful in 25s
Release / Scraper / Docker (push) Successful in 55s
Release / UI / Docker (push) Successful in 43s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 36s
Release / v2 / Test backend (push) Successful in 3m51s
Release / v2 / Docker / runner (push) Successful in 36s
Release / v2 / Docker / backend (push) Successful in 1m34s
iOS CI / Build (pull_request) Successful in 5m33s
iOS CI / Test (pull_request) Successful in 11m25s
All checks were successful
Release / Scraper / Test (push) Successful in 20s
Release / UI / Build (push) Successful in 25s
CI / Scraper / Lint (pull_request) Successful in 10s
Release / v2 / Build ui-v2 (push) Successful in 28s
CI / Scraper / Test (pull_request) Successful in 15s
CI / UI / Build (pull_request) Successful in 25s
Release / Scraper / Docker (push) Successful in 55s
Release / UI / Docker (push) Successful in 43s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 36s
Release / v2 / Test backend (push) Successful in 3m51s
Release / v2 / Docker / runner (push) Successful in 36s
Release / v2 / Docker / backend (push) Successful in 1m34s
iOS CI / Build (pull_request) Successful in 5m33s
iOS CI / Test (pull_request) Successful in 11m25s
- Runner fetches 9 browse combos (genre×sort×status) every 6h and stores JSON snapshots in MinIO libnovel-browse bucket (browse_refresh.go) - Backend handleBrowse reads page-1 results from MinIO first; falls back to live novelfire.net fetch; returns empty+cached:false on total failure instead of 502 - Add BrowseStore interface (bookstore.go), MinIO put/get helpers (minio.go), Store methods + compile-time assertion (store.go), BucketBrowse config, wiring in cmd/backend and cmd/runner, docker-compose-new bucket init - Fix ReapStaleTasks: PocketBase datetime fields require heartbeat_at=null (not heartbeat_at="") in filter expressions, and nil (not "") in patch payload — was causing 400 errors on every reap cycle
This commit is contained in:
@@ -22,6 +22,7 @@ type minioClient struct {
|
||||
bucketChapters string
|
||||
bucketAudio string
|
||||
bucketAvatars string
|
||||
bucketBrowse string
|
||||
}
|
||||
|
||||
func newMinioClient(cfg config.MinIO) (*minioClient, error) {
|
||||
@@ -78,12 +79,13 @@ func newMinioClient(cfg config.MinIO) (*minioClient, error) {
|
||||
bucketChapters: cfg.BucketChapters,
|
||||
bucketAudio: cfg.BucketAudio,
|
||||
bucketAvatars: cfg.BucketAvatars,
|
||||
bucketBrowse: cfg.BucketBrowse,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ensureBuckets creates all required buckets if they don't already exist.
|
||||
func (m *minioClient) ensureBuckets(ctx context.Context) error {
|
||||
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars} {
|
||||
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse} {
|
||||
exists, err := m.client.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: check bucket %q: %w", bucket, err)
|
||||
@@ -117,6 +119,12 @@ func AvatarObjectKey(userID, ext string) string {
|
||||
return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
|
||||
}
|
||||
|
||||
// BrowseObjectKey returns the MinIO object key for a cached browse page snapshot.
|
||||
// Format: browse/{genre}/{sort}/{status}/{type}/page-{n}.json
|
||||
func BrowseObjectKey(genre, sort, status, novelType string, page int) string {
|
||||
return fmt.Sprintf("browse/%s/%s/%s/%s/page-%d.json", genre, sort, status, novelType, page)
|
||||
}
|
||||
|
||||
// chapterNumberFromKey extracts the chapter number from a MinIO object key.
|
||||
// e.g. "my-book/chapter-000042.md" → 42
|
||||
func chapterNumberFromKey(key string) int {
|
||||
@@ -192,3 +200,23 @@ func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// ── Browse operations ─────────────────────────────────────────────────────────
|
||||
|
||||
// putBrowse stores raw JSON bytes for a browse page snapshot.
|
||||
func (m *minioClient) putBrowse(ctx context.Context, key string, data []byte) error {
|
||||
return m.putObject(ctx, m.bucketBrowse, key, "application/json", data)
|
||||
}
|
||||
|
||||
// getBrowse retrieves a browse page snapshot. Returns (nil, false, nil) when
|
||||
// the object does not exist.
|
||||
func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool, error) {
|
||||
if !m.objectExists(ctx, m.bucketBrowse, key) {
|
||||
return nil, false, nil
|
||||
}
|
||||
data, err := m.getObject(ctx, m.bucketBrowse, key)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return data, true, nil
|
||||
}
|
||||
|
||||
@@ -50,6 +50,7 @@ var _ bookstore.RankingStore = (*Store)(nil)
|
||||
var _ bookstore.AudioStore = (*Store)(nil)
|
||||
var _ bookstore.PresignStore = (*Store)(nil)
|
||||
var _ bookstore.ProgressStore = (*Store)(nil)
|
||||
var _ bookstore.BrowseStore = (*Store)(nil)
|
||||
var _ taskqueue.Producer = (*Store)(nil)
|
||||
var _ taskqueue.Consumer = (*Store)(nil)
|
||||
var _ taskqueue.Reader = (*Store)(nil)
|
||||
@@ -608,12 +609,13 @@ func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
|
||||
// re-claimed. Returns the number of tasks reaped.
|
||||
func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) {
|
||||
threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339)
|
||||
// Match tasks that are running AND (heartbeat_at is empty OR heartbeat_at < threshold).
|
||||
filter := fmt.Sprintf(`status="running"&&(heartbeat_at=""||heartbeat_at<"%s")`, threshold)
|
||||
// Match tasks that are running AND (heartbeat_at is null OR heartbeat_at < threshold).
|
||||
// PocketBase datetime fields require `=null` not `=""` in filter expressions.
|
||||
filter := fmt.Sprintf(`status="running"&&(heartbeat_at=null||heartbeat_at<"%s")`, threshold)
|
||||
resetPayload := map[string]any{
|
||||
"status": string(domain.TaskStatusPending),
|
||||
"worker_id": "",
|
||||
"heartbeat_at": "",
|
||||
"heartbeat_at": nil,
|
||||
}
|
||||
|
||||
total := 0
|
||||
@@ -767,3 +769,22 @@ func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
|
||||
Finished: finished,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ── BrowseStore ────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error {
|
||||
key := BrowseObjectKey(genre, sort, status, novelType, page)
|
||||
if err := s.mc.putBrowse(ctx, key, data); err != nil {
|
||||
return fmt.Errorf("PutBrowsePage: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) {
|
||||
key := BrowseObjectKey(genre, sort, status, novelType, page)
|
||||
data, ok, err := s.mc.getBrowse(ctx, key)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("GetBrowsePage: %w", err)
|
||||
}
|
||||
return data, ok, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user