Files
libnovel/backend/internal/storage/store.go
Admin 3918bc8dc3
All checks were successful
Release / Scraper / Test (push) Successful in 20s
Release / UI / Build (push) Successful in 25s
CI / Scraper / Lint (pull_request) Successful in 10s
Release / v2 / Build ui-v2 (push) Successful in 28s
CI / Scraper / Test (pull_request) Successful in 15s
CI / UI / Build (pull_request) Successful in 25s
Release / Scraper / Docker (push) Successful in 55s
Release / UI / Docker (push) Successful in 43s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 36s
Release / v2 / Test backend (push) Successful in 3m51s
Release / v2 / Docker / runner (push) Successful in 36s
Release / v2 / Docker / backend (push) Successful in 1m34s
iOS CI / Build (pull_request) Successful in 5m33s
iOS CI / Test (pull_request) Successful in 11m25s
fix: serve browse pages from MinIO cache; fix ReapStaleTasks PocketBase filter
- Runner fetches 9 browse combos (genre×sort×status) every 6h and stores
  JSON snapshots in MinIO libnovel-browse bucket (browse_refresh.go)
- Backend handleBrowse reads page-1 results from MinIO first; falls back
  to live novelfire.net fetch; returns empty+cached:false on total failure
  instead of 502
- Add BrowseStore interface (bookstore.go), MinIO put/get helpers (minio.go),
  Store methods + compile-time assertion (store.go), BucketBrowse config,
  wiring in cmd/backend and cmd/runner, docker-compose-new bucket init
- Fix ReapStaleTasks: PocketBase datetime fields require heartbeat_at=null
  (not heartbeat_at="") in filter expressions, and nil (not "") in patch
  payload — was causing 400 errors on every reap cycle
2026-03-15 21:19:28 +05:00

791 lines
26 KiB
Go

package storage
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"strings"
"time"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/config"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/taskqueue"
)
// Store is the unified persistence implementation that satisfies all bookstore
// and taskqueue interfaces. It routes structured data to PocketBase and binary
// blobs to MinIO.
type Store struct {
pb *pbClient
mc *minioClient
log *slog.Logger
}
// NewStore initialises PocketBase and MinIO connections and ensures all MinIO
// buckets exist. Returns a ready-to-use Store.
func NewStore(ctx context.Context, cfg config.Config, log *slog.Logger) (*Store, error) {
pb := newPBClient(cfg.PocketBase, log)
// Validate PocketBase connectivity by fetching an auth token.
if _, err := pb.authToken(ctx); err != nil {
return nil, fmt.Errorf("pocketbase: %w", err)
}
mc, err := newMinioClient(cfg.MinIO)
if err != nil {
return nil, fmt.Errorf("minio: %w", err)
}
if err := mc.ensureBuckets(ctx); err != nil {
return nil, fmt.Errorf("minio: ensure buckets: %w", err)
}
return &Store{pb: pb, mc: mc, log: log}, nil
}
// Compile-time interface satisfaction.
var _ bookstore.BookWriter = (*Store)(nil)
var _ bookstore.BookReader = (*Store)(nil)
var _ bookstore.RankingStore = (*Store)(nil)
var _ bookstore.AudioStore = (*Store)(nil)
var _ bookstore.PresignStore = (*Store)(nil)
var _ bookstore.ProgressStore = (*Store)(nil)
var _ bookstore.BrowseStore = (*Store)(nil)
var _ taskqueue.Producer = (*Store)(nil)
var _ taskqueue.Consumer = (*Store)(nil)
var _ taskqueue.Reader = (*Store)(nil)
// ── BookWriter ────────────────────────────────────────────────────────────────
func (s *Store) WriteMetadata(ctx context.Context, meta domain.BookMeta) error {
payload := map[string]any{
"slug": meta.Slug,
"title": meta.Title,
"author": meta.Author,
"cover": meta.Cover,
"status": meta.Status,
"genres": meta.Genres,
"summary": meta.Summary,
"total_chapters": meta.TotalChapters,
"source_url": meta.SourceURL,
"ranking": meta.Ranking,
}
// Upsert via filter: if exists PATCH, otherwise POST.
existing, err := s.getBookBySlug(ctx, meta.Slug)
if err != nil && err != ErrNotFound {
return fmt.Errorf("WriteMetadata: %w", err)
}
if err == ErrNotFound {
return s.pb.post(ctx, "/api/collections/books/records", payload, nil)
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", existing.ID), payload)
}
func (s *Store) WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error {
key := ChapterObjectKey(slug, chapter.Ref.Number)
if err := s.mc.putObject(ctx, s.mc.bucketChapters, key, "text/markdown", []byte(chapter.Text)); err != nil {
return fmt.Errorf("WriteChapter: minio: %w", err)
}
// Upsert the chapters_idx record in PocketBase.
return s.upsertChapterIdx(ctx, slug, chapter.Ref)
}
func (s *Store) WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error {
for _, ref := range refs {
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
s.log.Warn("WriteChapterRefs: upsert failed", "slug", slug, "chapter", ref.Number, "err", err)
}
}
return nil
}
func (s *Store) ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool {
return s.mc.objectExists(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, ref.Number))
}
func (s *Store) upsertChapterIdx(ctx context.Context, slug string, ref domain.ChapterRef) error {
payload := map[string]any{
"slug": slug,
"number": ref.Number,
"title": ref.Title,
}
filter := fmt.Sprintf(`slug=%q&&number=%d`, slug, ref.Number)
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "")
if err != nil && err != ErrNotFound {
return err
}
if len(items) == 0 {
return s.pb.post(ctx, "/api/collections/chapters_idx/records", payload, nil)
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/chapters_idx/records/%s", rec.ID), payload)
}
// ── BookReader ────────────────────────────────────────────────────────────────
type pbBook struct {
ID string `json:"id"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author"`
Cover string `json:"cover"`
Status string `json:"status"`
Genres []string `json:"genres"`
Summary string `json:"summary"`
TotalChapters int `json:"total_chapters"`
SourceURL string `json:"source_url"`
Ranking int `json:"ranking"`
Updated string `json:"updated"`
}
func (b pbBook) toDomain() domain.BookMeta {
return domain.BookMeta{
Slug: b.Slug,
Title: b.Title,
Author: b.Author,
Cover: b.Cover,
Status: b.Status,
Genres: b.Genres,
Summary: b.Summary,
TotalChapters: b.TotalChapters,
SourceURL: b.SourceURL,
Ranking: b.Ranking,
}
}
func (s *Store) getBookBySlug(ctx context.Context, slug string) (pbBook, error) {
filter := fmt.Sprintf(`slug=%q`, slug)
items, err := s.pb.listAll(ctx, "books", filter, "")
if err != nil {
return pbBook{}, err
}
if len(items) == 0 {
return pbBook{}, ErrNotFound
}
var b pbBook
json.Unmarshal(items[0], &b)
return b, nil
}
func (s *Store) ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error) {
b, err := s.getBookBySlug(ctx, slug)
if err == ErrNotFound {
return domain.BookMeta{}, false, nil
}
if err != nil {
return domain.BookMeta{}, false, err
}
return b.toDomain(), true, nil
}
func (s *Store) ListBooks(ctx context.Context) ([]domain.BookMeta, error) {
items, err := s.pb.listAll(ctx, "books", "", "title")
if err != nil {
return nil, err
}
books := make([]domain.BookMeta, 0, len(items))
for _, raw := range items {
var b pbBook
json.Unmarshal(raw, &b)
books = append(books, b.toDomain())
}
return books, nil
}
func (s *Store) LocalSlugs(ctx context.Context) (map[string]bool, error) {
items, err := s.pb.listAll(ctx, "books", "", "")
if err != nil {
return nil, err
}
slugs := make(map[string]bool, len(items))
for _, raw := range items {
var b struct {
Slug string `json:"slug"`
}
json.Unmarshal(raw, &b)
if b.Slug != "" {
slugs[b.Slug] = true
}
}
return slugs, nil
}
func (s *Store) MetadataMtime(ctx context.Context, slug string) int64 {
b, err := s.getBookBySlug(ctx, slug)
if err != nil {
return 0
}
t, err := time.Parse(time.RFC3339, b.Updated)
if err != nil {
return 0
}
return t.Unix()
}
func (s *Store) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
data, err := s.mc.getObject(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n))
if err != nil {
return "", fmt.Errorf("ReadChapter: %w", err)
}
return string(data), nil
}
func (s *Store) ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error) {
filter := fmt.Sprintf(`slug=%q`, slug)
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "number")
if err != nil {
return nil, err
}
chapters := make([]domain.ChapterInfo, 0, len(items))
for _, raw := range items {
var rec struct {
Number int `json:"number"`
Title string `json:"title"`
}
json.Unmarshal(raw, &rec)
chapters = append(chapters, domain.ChapterInfo{Number: rec.Number, Title: rec.Title})
}
return chapters, nil
}
func (s *Store) CountChapters(ctx context.Context, slug string) int {
chapters, err := s.ListChapters(ctx, slug)
if err != nil {
return 0
}
return len(chapters)
}
func (s *Store) ReindexChapters(ctx context.Context, slug string) (int, error) {
keys, err := s.mc.listObjectKeys(ctx, s.mc.bucketChapters, slug+"/")
if err != nil {
return 0, fmt.Errorf("ReindexChapters: list objects: %w", err)
}
count := 0
for _, key := range keys {
if !strings.HasSuffix(key, ".md") {
continue
}
n := chapterNumberFromKey(key)
if n == 0 {
continue
}
ref := domain.ChapterRef{Number: n}
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
s.log.Warn("ReindexChapters: upsert failed", "key", key, "err", err)
continue
}
count++
}
return count, nil
}
// ── RankingStore ──────────────────────────────────────────────────────────────
func (s *Store) WriteRankingItem(ctx context.Context, item domain.RankingItem) error {
payload := map[string]any{
"rank": item.Rank,
"slug": item.Slug,
"title": item.Title,
"author": item.Author,
"cover": item.Cover,
"status": item.Status,
"genres": item.Genres,
"source_url": item.SourceURL,
}
filter := fmt.Sprintf(`slug=%q`, item.Slug)
items, err := s.pb.listAll(ctx, "ranking", filter, "")
if err != nil && err != ErrNotFound {
return err
}
if len(items) == 0 {
return s.pb.post(ctx, "/api/collections/ranking/records", payload, nil)
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/ranking/records/%s", rec.ID), payload)
}
func (s *Store) ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error) {
items, err := s.pb.listAll(ctx, "ranking", "", "rank")
if err != nil {
return nil, err
}
result := make([]domain.RankingItem, 0, len(items))
for _, raw := range items {
var rec struct {
Rank int `json:"rank"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author"`
Cover string `json:"cover"`
Status string `json:"status"`
Genres []string `json:"genres"`
SourceURL string `json:"source_url"`
Updated string `json:"updated"`
}
json.Unmarshal(raw, &rec)
t, _ := time.Parse(time.RFC3339, rec.Updated)
result = append(result, domain.RankingItem{
Rank: rec.Rank,
Slug: rec.Slug,
Title: rec.Title,
Author: rec.Author,
Cover: rec.Cover,
Status: rec.Status,
Genres: rec.Genres,
SourceURL: rec.SourceURL,
Updated: t,
})
}
return result, nil
}
func (s *Store) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) {
items, err := s.ReadRankingItems(ctx)
if err != nil || len(items) == 0 {
return false, err
}
var latest time.Time
for _, item := range items {
if item.Updated.After(latest) {
latest = item.Updated
}
}
return time.Since(latest) < maxAge, nil
}
// ── AudioStore ────────────────────────────────────────────────────────────────
func (s *Store) AudioObjectKey(slug string, n int, voice string) string {
return AudioObjectKey(slug, n, voice)
}
func (s *Store) AudioExists(ctx context.Context, key string) bool {
return s.mc.objectExists(ctx, s.mc.bucketAudio, key)
}
func (s *Store) PutAudio(ctx context.Context, key string, data []byte) error {
return s.mc.putObject(ctx, s.mc.bucketAudio, key, "audio/mpeg", data)
}
// ── PresignStore ──────────────────────────────────────────────────────────────
func (s *Store) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) {
return s.mc.presignGet(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n), expires)
}
func (s *Store) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
return s.mc.presignGet(ctx, s.mc.bucketAudio, key, expires)
}
func (s *Store) PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error) {
key = AvatarObjectKey(userID, ext)
uploadURL, err = s.mc.presignPut(ctx, s.mc.bucketAvatars, key, 15*time.Minute)
return
}
func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) {
for _, ext := range []string{"jpg", "png", "webp"} {
key := AvatarObjectKey(userID, ext)
if s.mc.objectExists(ctx, s.mc.bucketAvatars, key) {
u, err := s.mc.presignGet(ctx, s.mc.bucketAvatars, key, 1*time.Hour)
return u, true, err
}
}
return "", false, nil
}
func (s *Store) DeleteAvatar(ctx context.Context, userID string) error {
return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
}
// ── ProgressStore ─────────────────────────────────────────────────────────────
func (s *Store) GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool) {
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
items, err := s.pb.listAll(ctx, "progress", filter, "")
if err != nil || len(items) == 0 {
return domain.ReadingProgress{}, false
}
var rec struct {
Slug string `json:"slug"`
Chapter int `json:"chapter"`
UpdatedAt string `json:"updated"`
}
json.Unmarshal(items[0], &rec)
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
return domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t}, true
}
func (s *Store) SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error {
payload := map[string]any{
"session_id": sessionID,
"slug": p.Slug,
"chapter": p.Chapter,
}
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, p.Slug)
items, err := s.pb.listAll(ctx, "progress", filter, "")
if err != nil && err != ErrNotFound {
return err
}
if len(items) == 0 {
return s.pb.post(ctx, "/api/collections/progress/records", payload, nil)
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID), payload)
}
func (s *Store) AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error) {
filter := fmt.Sprintf(`session_id=%q`, sessionID)
items, err := s.pb.listAll(ctx, "progress", filter, "-updated")
if err != nil {
return nil, err
}
result := make([]domain.ReadingProgress, 0, len(items))
for _, raw := range items {
var rec struct {
Slug string `json:"slug"`
Chapter int `json:"chapter"`
UpdatedAt string `json:"updated"`
}
json.Unmarshal(raw, &rec)
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
result = append(result, domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t})
}
return result, nil
}
func (s *Store) DeleteProgress(ctx context.Context, sessionID, slug string) error {
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
items, err := s.pb.listAll(ctx, "progress", filter, "")
if err != nil || len(items) == 0 {
return nil
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.delete(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID))
}
// ── taskqueue.Producer ────────────────────────────────────────────────────────
func (s *Store) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) {
payload := map[string]any{
"kind": kind,
"target_url": targetURL,
"from_chapter": fromChapter,
"to_chapter": toChapter,
"status": string(domain.TaskStatusPending),
"started": time.Now().UTC().Format(time.RFC3339),
}
var rec struct {
ID string `json:"id"`
}
if err := s.pb.post(ctx, "/api/collections/scraping_tasks/records", payload, &rec); err != nil {
return "", err
}
return rec.ID, nil
}
func (s *Store) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) {
cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, voice)
payload := map[string]any{
"cache_key": cacheKey,
"slug": slug,
"chapter": chapter,
"voice": voice,
"status": string(domain.TaskStatusPending),
"started": time.Now().UTC().Format(time.RFC3339),
}
var rec struct {
ID string `json:"id"`
}
if err := s.pb.post(ctx, "/api/collections/audio_jobs/records", payload, &rec); err != nil {
return "", err
}
return rec.ID, nil
}
func (s *Store) CancelTask(ctx context.Context, id string) error {
// Try scraping_tasks first, then audio_jobs.
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),
map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id),
map[string]string{"status": string(domain.TaskStatusCancelled)})
}
// ── taskqueue.Consumer ────────────────────────────────────────────────────────
func (s *Store) ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error) {
raw, err := s.pb.claimRecord(ctx, "scraping_tasks", workerID, nil)
if err != nil {
return domain.ScrapeTask{}, false, err
}
if raw == nil {
return domain.ScrapeTask{}, false, nil
}
task, err := parseScrapeTask(raw)
return task, err == nil, err
}
func (s *Store) ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error) {
raw, err := s.pb.claimRecord(ctx, "audio_jobs", workerID, nil)
if err != nil {
return domain.AudioTask{}, false, err
}
if raw == nil {
return domain.AudioTask{}, false, nil
}
task, err := parseAudioTask(raw)
return task, err == nil, err
}
func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
status := string(domain.TaskStatusDone)
if result.ErrorMessage != "" {
status = string(domain.TaskStatusFailed)
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), map[string]any{
"status": status,
"books_found": result.BooksFound,
"chapters_scraped": result.ChaptersScraped,
"chapters_skipped": result.ChaptersSkipped,
"errors": result.Errors,
"error_message": result.ErrorMessage,
"finished": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *Store) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error {
status := string(domain.TaskStatusDone)
if result.ErrorMessage != "" {
status = string(domain.TaskStatusFailed)
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), map[string]any{
"status": status,
"error_message": result.ErrorMessage,
"finished": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
payload := map[string]any{
"status": string(domain.TaskStatusFailed),
"error_message": errMsg,
"finished": time.Now().UTC().Format(time.RFC3339),
}
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
}
// HeartbeatTask updates the heartbeat_at field on a running task.
// Tries scraping_tasks first, then audio_jobs (same pattern as FailTask).
func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
payload := map[string]any{
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
}
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
}
// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing
// or older than staleAfter, and resets them to pending so they can be
// re-claimed. Returns the number of tasks reaped.
func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) {
threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339)
// Match tasks that are running AND (heartbeat_at is null OR heartbeat_at < threshold).
// PocketBase datetime fields require `=null` not `=""` in filter expressions.
filter := fmt.Sprintf(`status="running"&&(heartbeat_at=null||heartbeat_at<"%s")`, threshold)
resetPayload := map[string]any{
"status": string(domain.TaskStatusPending),
"worker_id": "",
"heartbeat_at": nil,
}
total := 0
for _, collection := range []string{"scraping_tasks", "audio_jobs"} {
items, err := s.pb.listAll(ctx, collection, filter, "")
if err != nil {
return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err)
}
for _, raw := range items {
var rec struct {
ID string `json:"id"`
}
if err := json.Unmarshal(raw, &rec); err != nil || rec.ID == "" {
continue
}
path := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID)
if err := s.pb.patch(ctx, path, resetPayload); err != nil {
s.log.Warn("ReapStaleTasks: patch failed", "collection", collection, "id", rec.ID, "err", err)
continue
}
total++
}
}
return total, nil
}
// ── taskqueue.Reader ──────────────────────────────────────────────────────────
func (s *Store) ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error) {
items, err := s.pb.listAll(ctx, "scraping_tasks", "", "-started")
if err != nil {
return nil, err
}
tasks := make([]domain.ScrapeTask, 0, len(items))
for _, raw := range items {
t, err := parseScrapeTask(raw)
if err == nil {
tasks = append(tasks, t)
}
}
return tasks, nil
}
func (s *Store) GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error) {
var raw json.RawMessage
if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), &raw); err != nil {
if err == ErrNotFound {
return domain.ScrapeTask{}, false, nil
}
return domain.ScrapeTask{}, false, err
}
t, err := parseScrapeTask(raw)
return t, err == nil, err
}
func (s *Store) ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error) {
items, err := s.pb.listAll(ctx, "audio_jobs", "", "-started")
if err != nil {
return nil, err
}
tasks := make([]domain.AudioTask, 0, len(items))
for _, raw := range items {
t, err := parseAudioTask(raw)
if err == nil {
tasks = append(tasks, t)
}
}
return tasks, nil
}
func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) {
filter := fmt.Sprintf(`cache_key=%q`, cacheKey)
items, err := s.pb.listAll(ctx, "audio_jobs", filter, "-started")
if err != nil || len(items) == 0 {
return domain.AudioTask{}, false, err
}
t, err := parseAudioTask(items[0])
return t, err == nil, err
}
// ── Parsers ───────────────────────────────────────────────────────────────────
func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) {
var rec struct {
ID string `json:"id"`
Kind string `json:"kind"`
TargetURL string `json:"target_url"`
FromChapter int `json:"from_chapter"`
ToChapter int `json:"to_chapter"`
WorkerID string `json:"worker_id"`
Status string `json:"status"`
BooksFound int `json:"books_found"`
ChaptersScraped int `json:"chapters_scraped"`
ChaptersSkipped int `json:"chapters_skipped"`
Errors int `json:"errors"`
Started string `json:"started"`
Finished string `json:"finished"`
ErrorMessage string `json:"error_message"`
}
if err := json.Unmarshal(raw, &rec); err != nil {
return domain.ScrapeTask{}, err
}
started, _ := time.Parse(time.RFC3339, rec.Started)
finished, _ := time.Parse(time.RFC3339, rec.Finished)
return domain.ScrapeTask{
ID: rec.ID,
Kind: rec.Kind,
TargetURL: rec.TargetURL,
FromChapter: rec.FromChapter,
ToChapter: rec.ToChapter,
WorkerID: rec.WorkerID,
Status: domain.TaskStatus(rec.Status),
BooksFound: rec.BooksFound,
ChaptersScraped: rec.ChaptersScraped,
ChaptersSkipped: rec.ChaptersSkipped,
Errors: rec.Errors,
Started: started,
Finished: finished,
ErrorMessage: rec.ErrorMessage,
}, nil
}
func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
var rec struct {
ID string `json:"id"`
CacheKey string `json:"cache_key"`
Slug string `json:"slug"`
Chapter int `json:"chapter"`
Voice string `json:"voice"`
WorkerID string `json:"worker_id"`
Status string `json:"status"`
ErrorMessage string `json:"error_message"`
Started string `json:"started"`
Finished string `json:"finished"`
}
if err := json.Unmarshal(raw, &rec); err != nil {
return domain.AudioTask{}, err
}
started, _ := time.Parse(time.RFC3339, rec.Started)
finished, _ := time.Parse(time.RFC3339, rec.Finished)
return domain.AudioTask{
ID: rec.ID,
CacheKey: rec.CacheKey,
Slug: rec.Slug,
Chapter: rec.Chapter,
Voice: rec.Voice,
WorkerID: rec.WorkerID,
Status: domain.TaskStatus(rec.Status),
ErrorMessage: rec.ErrorMessage,
Started: started,
Finished: finished,
}, nil
}
// ── BrowseStore ────────────────────────────────────────────────────────────────
func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error {
key := BrowseObjectKey(genre, sort, status, novelType, page)
if err := s.mc.putBrowse(ctx, key, data); err != nil {
return fmt.Errorf("PutBrowsePage: %w", err)
}
return nil
}
func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) {
key := BrowseObjectKey(genre, sort, status, novelType, page)
data, ok, err := s.mc.getBrowse(ctx, key)
if err != nil {
return nil, false, fmt.Errorf("GetBrowsePage: %w", err)
}
return data, ok, nil
}