All checks were successful
Release / Scraper / Test (push) Successful in 20s
Release / UI / Build (push) Successful in 25s
CI / Scraper / Lint (pull_request) Successful in 10s
Release / v2 / Build ui-v2 (push) Successful in 28s
CI / Scraper / Test (pull_request) Successful in 15s
CI / UI / Build (pull_request) Successful in 25s
Release / Scraper / Docker (push) Successful in 55s
Release / UI / Docker (push) Successful in 43s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 36s
Release / v2 / Test backend (push) Successful in 3m51s
Release / v2 / Docker / runner (push) Successful in 36s
Release / v2 / Docker / backend (push) Successful in 1m34s
iOS CI / Build (pull_request) Successful in 5m33s
iOS CI / Test (pull_request) Successful in 11m25s
- Runner fetches 9 browse combos (genre×sort×status) every 6h and stores JSON snapshots in MinIO libnovel-browse bucket (browse_refresh.go) - Backend handleBrowse reads page-1 results from MinIO first; falls back to live novelfire.net fetch; returns empty+cached:false on total failure instead of 502 - Add BrowseStore interface (bookstore.go), MinIO put/get helpers (minio.go), Store methods + compile-time assertion (store.go), BucketBrowse config, wiring in cmd/backend and cmd/runner, docker-compose-new bucket init - Fix ReapStaleTasks: PocketBase datetime fields require heartbeat_at=null (not heartbeat_at="") in filter expressions, and nil (not "") in patch payload — was causing 400 errors on every reap cycle
791 lines
26 KiB
Go
791 lines
26 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/libnovel/backend/internal/bookstore"
|
|
"github.com/libnovel/backend/internal/config"
|
|
"github.com/libnovel/backend/internal/domain"
|
|
"github.com/libnovel/backend/internal/taskqueue"
|
|
)
|
|
|
|
// Store is the unified persistence implementation that satisfies all bookstore
|
|
// and taskqueue interfaces. It routes structured data to PocketBase and binary
|
|
// blobs to MinIO.
|
|
type Store struct {
|
|
pb *pbClient
|
|
mc *minioClient
|
|
log *slog.Logger
|
|
}
|
|
|
|
// NewStore initialises PocketBase and MinIO connections and ensures all MinIO
|
|
// buckets exist. Returns a ready-to-use Store.
|
|
func NewStore(ctx context.Context, cfg config.Config, log *slog.Logger) (*Store, error) {
|
|
pb := newPBClient(cfg.PocketBase, log)
|
|
// Validate PocketBase connectivity by fetching an auth token.
|
|
if _, err := pb.authToken(ctx); err != nil {
|
|
return nil, fmt.Errorf("pocketbase: %w", err)
|
|
}
|
|
|
|
mc, err := newMinioClient(cfg.MinIO)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("minio: %w", err)
|
|
}
|
|
if err := mc.ensureBuckets(ctx); err != nil {
|
|
return nil, fmt.Errorf("minio: ensure buckets: %w", err)
|
|
}
|
|
|
|
return &Store{pb: pb, mc: mc, log: log}, nil
|
|
}
|
|
|
|
// Compile-time interface satisfaction.
|
|
var _ bookstore.BookWriter = (*Store)(nil)
|
|
var _ bookstore.BookReader = (*Store)(nil)
|
|
var _ bookstore.RankingStore = (*Store)(nil)
|
|
var _ bookstore.AudioStore = (*Store)(nil)
|
|
var _ bookstore.PresignStore = (*Store)(nil)
|
|
var _ bookstore.ProgressStore = (*Store)(nil)
|
|
var _ bookstore.BrowseStore = (*Store)(nil)
|
|
var _ taskqueue.Producer = (*Store)(nil)
|
|
var _ taskqueue.Consumer = (*Store)(nil)
|
|
var _ taskqueue.Reader = (*Store)(nil)
|
|
|
|
// ── BookWriter ────────────────────────────────────────────────────────────────
|
|
|
|
func (s *Store) WriteMetadata(ctx context.Context, meta domain.BookMeta) error {
|
|
payload := map[string]any{
|
|
"slug": meta.Slug,
|
|
"title": meta.Title,
|
|
"author": meta.Author,
|
|
"cover": meta.Cover,
|
|
"status": meta.Status,
|
|
"genres": meta.Genres,
|
|
"summary": meta.Summary,
|
|
"total_chapters": meta.TotalChapters,
|
|
"source_url": meta.SourceURL,
|
|
"ranking": meta.Ranking,
|
|
}
|
|
// Upsert via filter: if exists PATCH, otherwise POST.
|
|
existing, err := s.getBookBySlug(ctx, meta.Slug)
|
|
if err != nil && err != ErrNotFound {
|
|
return fmt.Errorf("WriteMetadata: %w", err)
|
|
}
|
|
if err == ErrNotFound {
|
|
return s.pb.post(ctx, "/api/collections/books/records", payload, nil)
|
|
}
|
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", existing.ID), payload)
|
|
}
|
|
|
|
func (s *Store) WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error {
|
|
key := ChapterObjectKey(slug, chapter.Ref.Number)
|
|
if err := s.mc.putObject(ctx, s.mc.bucketChapters, key, "text/markdown", []byte(chapter.Text)); err != nil {
|
|
return fmt.Errorf("WriteChapter: minio: %w", err)
|
|
}
|
|
// Upsert the chapters_idx record in PocketBase.
|
|
return s.upsertChapterIdx(ctx, slug, chapter.Ref)
|
|
}
|
|
|
|
func (s *Store) WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error {
|
|
for _, ref := range refs {
|
|
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
|
|
s.log.Warn("WriteChapterRefs: upsert failed", "slug", slug, "chapter", ref.Number, "err", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool {
|
|
return s.mc.objectExists(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, ref.Number))
|
|
}
|
|
|
|
func (s *Store) upsertChapterIdx(ctx context.Context, slug string, ref domain.ChapterRef) error {
|
|
payload := map[string]any{
|
|
"slug": slug,
|
|
"number": ref.Number,
|
|
"title": ref.Title,
|
|
}
|
|
filter := fmt.Sprintf(`slug=%q&&number=%d`, slug, ref.Number)
|
|
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "")
|
|
if err != nil && err != ErrNotFound {
|
|
return err
|
|
}
|
|
if len(items) == 0 {
|
|
return s.pb.post(ctx, "/api/collections/chapters_idx/records", payload, nil)
|
|
}
|
|
var rec struct {
|
|
ID string `json:"id"`
|
|
}
|
|
json.Unmarshal(items[0], &rec)
|
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/chapters_idx/records/%s", rec.ID), payload)
|
|
}
|
|
|
|
// ── BookReader ────────────────────────────────────────────────────────────────
|
|
|
|
type pbBook struct {
|
|
ID string `json:"id"`
|
|
Slug string `json:"slug"`
|
|
Title string `json:"title"`
|
|
Author string `json:"author"`
|
|
Cover string `json:"cover"`
|
|
Status string `json:"status"`
|
|
Genres []string `json:"genres"`
|
|
Summary string `json:"summary"`
|
|
TotalChapters int `json:"total_chapters"`
|
|
SourceURL string `json:"source_url"`
|
|
Ranking int `json:"ranking"`
|
|
Updated string `json:"updated"`
|
|
}
|
|
|
|
func (b pbBook) toDomain() domain.BookMeta {
|
|
return domain.BookMeta{
|
|
Slug: b.Slug,
|
|
Title: b.Title,
|
|
Author: b.Author,
|
|
Cover: b.Cover,
|
|
Status: b.Status,
|
|
Genres: b.Genres,
|
|
Summary: b.Summary,
|
|
TotalChapters: b.TotalChapters,
|
|
SourceURL: b.SourceURL,
|
|
Ranking: b.Ranking,
|
|
}
|
|
}
|
|
|
|
func (s *Store) getBookBySlug(ctx context.Context, slug string) (pbBook, error) {
|
|
filter := fmt.Sprintf(`slug=%q`, slug)
|
|
items, err := s.pb.listAll(ctx, "books", filter, "")
|
|
if err != nil {
|
|
return pbBook{}, err
|
|
}
|
|
if len(items) == 0 {
|
|
return pbBook{}, ErrNotFound
|
|
}
|
|
var b pbBook
|
|
json.Unmarshal(items[0], &b)
|
|
return b, nil
|
|
}
|
|
|
|
func (s *Store) ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error) {
|
|
b, err := s.getBookBySlug(ctx, slug)
|
|
if err == ErrNotFound {
|
|
return domain.BookMeta{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return domain.BookMeta{}, false, err
|
|
}
|
|
return b.toDomain(), true, nil
|
|
}
|
|
|
|
func (s *Store) ListBooks(ctx context.Context) ([]domain.BookMeta, error) {
|
|
items, err := s.pb.listAll(ctx, "books", "", "title")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
books := make([]domain.BookMeta, 0, len(items))
|
|
for _, raw := range items {
|
|
var b pbBook
|
|
json.Unmarshal(raw, &b)
|
|
books = append(books, b.toDomain())
|
|
}
|
|
return books, nil
|
|
}
|
|
|
|
func (s *Store) LocalSlugs(ctx context.Context) (map[string]bool, error) {
|
|
items, err := s.pb.listAll(ctx, "books", "", "")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
slugs := make(map[string]bool, len(items))
|
|
for _, raw := range items {
|
|
var b struct {
|
|
Slug string `json:"slug"`
|
|
}
|
|
json.Unmarshal(raw, &b)
|
|
if b.Slug != "" {
|
|
slugs[b.Slug] = true
|
|
}
|
|
}
|
|
return slugs, nil
|
|
}
|
|
|
|
func (s *Store) MetadataMtime(ctx context.Context, slug string) int64 {
|
|
b, err := s.getBookBySlug(ctx, slug)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
t, err := time.Parse(time.RFC3339, b.Updated)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
return t.Unix()
|
|
}
|
|
|
|
func (s *Store) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
|
|
data, err := s.mc.getObject(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n))
|
|
if err != nil {
|
|
return "", fmt.Errorf("ReadChapter: %w", err)
|
|
}
|
|
return string(data), nil
|
|
}
|
|
|
|
func (s *Store) ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error) {
|
|
filter := fmt.Sprintf(`slug=%q`, slug)
|
|
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "number")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
chapters := make([]domain.ChapterInfo, 0, len(items))
|
|
for _, raw := range items {
|
|
var rec struct {
|
|
Number int `json:"number"`
|
|
Title string `json:"title"`
|
|
}
|
|
json.Unmarshal(raw, &rec)
|
|
chapters = append(chapters, domain.ChapterInfo{Number: rec.Number, Title: rec.Title})
|
|
}
|
|
return chapters, nil
|
|
}
|
|
|
|
func (s *Store) CountChapters(ctx context.Context, slug string) int {
|
|
chapters, err := s.ListChapters(ctx, slug)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
return len(chapters)
|
|
}
|
|
|
|
func (s *Store) ReindexChapters(ctx context.Context, slug string) (int, error) {
|
|
keys, err := s.mc.listObjectKeys(ctx, s.mc.bucketChapters, slug+"/")
|
|
if err != nil {
|
|
return 0, fmt.Errorf("ReindexChapters: list objects: %w", err)
|
|
}
|
|
count := 0
|
|
for _, key := range keys {
|
|
if !strings.HasSuffix(key, ".md") {
|
|
continue
|
|
}
|
|
n := chapterNumberFromKey(key)
|
|
if n == 0 {
|
|
continue
|
|
}
|
|
ref := domain.ChapterRef{Number: n}
|
|
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
|
|
s.log.Warn("ReindexChapters: upsert failed", "key", key, "err", err)
|
|
continue
|
|
}
|
|
count++
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
// ── RankingStore ──────────────────────────────────────────────────────────────
|
|
|
|
func (s *Store) WriteRankingItem(ctx context.Context, item domain.RankingItem) error {
|
|
payload := map[string]any{
|
|
"rank": item.Rank,
|
|
"slug": item.Slug,
|
|
"title": item.Title,
|
|
"author": item.Author,
|
|
"cover": item.Cover,
|
|
"status": item.Status,
|
|
"genres": item.Genres,
|
|
"source_url": item.SourceURL,
|
|
}
|
|
filter := fmt.Sprintf(`slug=%q`, item.Slug)
|
|
items, err := s.pb.listAll(ctx, "ranking", filter, "")
|
|
if err != nil && err != ErrNotFound {
|
|
return err
|
|
}
|
|
if len(items) == 0 {
|
|
return s.pb.post(ctx, "/api/collections/ranking/records", payload, nil)
|
|
}
|
|
var rec struct {
|
|
ID string `json:"id"`
|
|
}
|
|
json.Unmarshal(items[0], &rec)
|
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/ranking/records/%s", rec.ID), payload)
|
|
}
|
|
|
|
func (s *Store) ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error) {
|
|
items, err := s.pb.listAll(ctx, "ranking", "", "rank")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result := make([]domain.RankingItem, 0, len(items))
|
|
for _, raw := range items {
|
|
var rec struct {
|
|
Rank int `json:"rank"`
|
|
Slug string `json:"slug"`
|
|
Title string `json:"title"`
|
|
Author string `json:"author"`
|
|
Cover string `json:"cover"`
|
|
Status string `json:"status"`
|
|
Genres []string `json:"genres"`
|
|
SourceURL string `json:"source_url"`
|
|
Updated string `json:"updated"`
|
|
}
|
|
json.Unmarshal(raw, &rec)
|
|
t, _ := time.Parse(time.RFC3339, rec.Updated)
|
|
result = append(result, domain.RankingItem{
|
|
Rank: rec.Rank,
|
|
Slug: rec.Slug,
|
|
Title: rec.Title,
|
|
Author: rec.Author,
|
|
Cover: rec.Cover,
|
|
Status: rec.Status,
|
|
Genres: rec.Genres,
|
|
SourceURL: rec.SourceURL,
|
|
Updated: t,
|
|
})
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Store) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) {
|
|
items, err := s.ReadRankingItems(ctx)
|
|
if err != nil || len(items) == 0 {
|
|
return false, err
|
|
}
|
|
var latest time.Time
|
|
for _, item := range items {
|
|
if item.Updated.After(latest) {
|
|
latest = item.Updated
|
|
}
|
|
}
|
|
return time.Since(latest) < maxAge, nil
|
|
}
|
|
|
|
// ── AudioStore ────────────────────────────────────────────────────────────────
|
|
|
|
func (s *Store) AudioObjectKey(slug string, n int, voice string) string {
|
|
return AudioObjectKey(slug, n, voice)
|
|
}
|
|
|
|
func (s *Store) AudioExists(ctx context.Context, key string) bool {
|
|
return s.mc.objectExists(ctx, s.mc.bucketAudio, key)
|
|
}
|
|
|
|
func (s *Store) PutAudio(ctx context.Context, key string, data []byte) error {
|
|
return s.mc.putObject(ctx, s.mc.bucketAudio, key, "audio/mpeg", data)
|
|
}
|
|
|
|
// ── PresignStore ──────────────────────────────────────────────────────────────
|
|
|
|
func (s *Store) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) {
|
|
return s.mc.presignGet(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n), expires)
|
|
}
|
|
|
|
func (s *Store) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
|
|
return s.mc.presignGet(ctx, s.mc.bucketAudio, key, expires)
|
|
}
|
|
|
|
func (s *Store) PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error) {
|
|
key = AvatarObjectKey(userID, ext)
|
|
uploadURL, err = s.mc.presignPut(ctx, s.mc.bucketAvatars, key, 15*time.Minute)
|
|
return
|
|
}
|
|
|
|
func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) {
|
|
for _, ext := range []string{"jpg", "png", "webp"} {
|
|
key := AvatarObjectKey(userID, ext)
|
|
if s.mc.objectExists(ctx, s.mc.bucketAvatars, key) {
|
|
u, err := s.mc.presignGet(ctx, s.mc.bucketAvatars, key, 1*time.Hour)
|
|
return u, true, err
|
|
}
|
|
}
|
|
return "", false, nil
|
|
}
|
|
|
|
func (s *Store) DeleteAvatar(ctx context.Context, userID string) error {
|
|
return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
|
|
}
|
|
|
|
// ── ProgressStore ─────────────────────────────────────────────────────────────
|
|
|
|
func (s *Store) GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool) {
|
|
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
|
|
items, err := s.pb.listAll(ctx, "progress", filter, "")
|
|
if err != nil || len(items) == 0 {
|
|
return domain.ReadingProgress{}, false
|
|
}
|
|
var rec struct {
|
|
Slug string `json:"slug"`
|
|
Chapter int `json:"chapter"`
|
|
UpdatedAt string `json:"updated"`
|
|
}
|
|
json.Unmarshal(items[0], &rec)
|
|
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
|
|
return domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t}, true
|
|
}
|
|
|
|
func (s *Store) SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error {
|
|
payload := map[string]any{
|
|
"session_id": sessionID,
|
|
"slug": p.Slug,
|
|
"chapter": p.Chapter,
|
|
}
|
|
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, p.Slug)
|
|
items, err := s.pb.listAll(ctx, "progress", filter, "")
|
|
if err != nil && err != ErrNotFound {
|
|
return err
|
|
}
|
|
if len(items) == 0 {
|
|
return s.pb.post(ctx, "/api/collections/progress/records", payload, nil)
|
|
}
|
|
var rec struct {
|
|
ID string `json:"id"`
|
|
}
|
|
json.Unmarshal(items[0], &rec)
|
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID), payload)
|
|
}
|
|
|
|
func (s *Store) AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error) {
|
|
filter := fmt.Sprintf(`session_id=%q`, sessionID)
|
|
items, err := s.pb.listAll(ctx, "progress", filter, "-updated")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result := make([]domain.ReadingProgress, 0, len(items))
|
|
for _, raw := range items {
|
|
var rec struct {
|
|
Slug string `json:"slug"`
|
|
Chapter int `json:"chapter"`
|
|
UpdatedAt string `json:"updated"`
|
|
}
|
|
json.Unmarshal(raw, &rec)
|
|
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
|
|
result = append(result, domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t})
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (s *Store) DeleteProgress(ctx context.Context, sessionID, slug string) error {
|
|
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
|
|
items, err := s.pb.listAll(ctx, "progress", filter, "")
|
|
if err != nil || len(items) == 0 {
|
|
return nil
|
|
}
|
|
var rec struct {
|
|
ID string `json:"id"`
|
|
}
|
|
json.Unmarshal(items[0], &rec)
|
|
return s.pb.delete(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID))
|
|
}
|
|
|
|
// ── taskqueue.Producer ────────────────────────────────────────────────────────
|
|
|
|
func (s *Store) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) {
|
|
payload := map[string]any{
|
|
"kind": kind,
|
|
"target_url": targetURL,
|
|
"from_chapter": fromChapter,
|
|
"to_chapter": toChapter,
|
|
"status": string(domain.TaskStatusPending),
|
|
"started": time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
var rec struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err := s.pb.post(ctx, "/api/collections/scraping_tasks/records", payload, &rec); err != nil {
|
|
return "", err
|
|
}
|
|
return rec.ID, nil
|
|
}
|
|
|
|
func (s *Store) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) {
|
|
cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, voice)
|
|
payload := map[string]any{
|
|
"cache_key": cacheKey,
|
|
"slug": slug,
|
|
"chapter": chapter,
|
|
"voice": voice,
|
|
"status": string(domain.TaskStatusPending),
|
|
"started": time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
var rec struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err := s.pb.post(ctx, "/api/collections/audio_jobs/records", payload, &rec); err != nil {
|
|
return "", err
|
|
}
|
|
return rec.ID, nil
|
|
}
|
|
|
|
func (s *Store) CancelTask(ctx context.Context, id string) error {
|
|
// Try scraping_tasks first, then audio_jobs.
|
|
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),
|
|
map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil {
|
|
return nil
|
|
}
|
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id),
|
|
map[string]string{"status": string(domain.TaskStatusCancelled)})
|
|
}
|
|
|
|
// ── taskqueue.Consumer ────────────────────────────────────────────────────────
|
|
|
|
func (s *Store) ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error) {
|
|
raw, err := s.pb.claimRecord(ctx, "scraping_tasks", workerID, nil)
|
|
if err != nil {
|
|
return domain.ScrapeTask{}, false, err
|
|
}
|
|
if raw == nil {
|
|
return domain.ScrapeTask{}, false, nil
|
|
}
|
|
task, err := parseScrapeTask(raw)
|
|
return task, err == nil, err
|
|
}
|
|
|
|
func (s *Store) ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error) {
|
|
raw, err := s.pb.claimRecord(ctx, "audio_jobs", workerID, nil)
|
|
if err != nil {
|
|
return domain.AudioTask{}, false, err
|
|
}
|
|
if raw == nil {
|
|
return domain.AudioTask{}, false, nil
|
|
}
|
|
task, err := parseAudioTask(raw)
|
|
return task, err == nil, err
|
|
}
|
|
|
|
func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
|
|
status := string(domain.TaskStatusDone)
|
|
if result.ErrorMessage != "" {
|
|
status = string(domain.TaskStatusFailed)
|
|
}
|
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), map[string]any{
|
|
"status": status,
|
|
"books_found": result.BooksFound,
|
|
"chapters_scraped": result.ChaptersScraped,
|
|
"chapters_skipped": result.ChaptersSkipped,
|
|
"errors": result.Errors,
|
|
"error_message": result.ErrorMessage,
|
|
"finished": time.Now().UTC().Format(time.RFC3339),
|
|
})
|
|
}
|
|
|
|
func (s *Store) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error {
|
|
status := string(domain.TaskStatusDone)
|
|
if result.ErrorMessage != "" {
|
|
status = string(domain.TaskStatusFailed)
|
|
}
|
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), map[string]any{
|
|
"status": status,
|
|
"error_message": result.ErrorMessage,
|
|
"finished": time.Now().UTC().Format(time.RFC3339),
|
|
})
|
|
}
|
|
|
|
func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
|
|
payload := map[string]any{
|
|
"status": string(domain.TaskStatusFailed),
|
|
"error_message": errMsg,
|
|
"finished": time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
|
|
return nil
|
|
}
|
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
|
|
}
|
|
|
|
// HeartbeatTask updates the heartbeat_at field on a running task.
|
|
// Tries scraping_tasks first, then audio_jobs (same pattern as FailTask).
|
|
func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
|
|
payload := map[string]any{
|
|
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
|
|
return nil
|
|
}
|
|
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
|
|
}
|
|
|
|
// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing
|
|
// or older than staleAfter, and resets them to pending so they can be
|
|
// re-claimed. Returns the number of tasks reaped.
|
|
func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) {
|
|
threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339)
|
|
// Match tasks that are running AND (heartbeat_at is null OR heartbeat_at < threshold).
|
|
// PocketBase datetime fields require `=null` not `=""` in filter expressions.
|
|
filter := fmt.Sprintf(`status="running"&&(heartbeat_at=null||heartbeat_at<"%s")`, threshold)
|
|
resetPayload := map[string]any{
|
|
"status": string(domain.TaskStatusPending),
|
|
"worker_id": "",
|
|
"heartbeat_at": nil,
|
|
}
|
|
|
|
total := 0
|
|
for _, collection := range []string{"scraping_tasks", "audio_jobs"} {
|
|
items, err := s.pb.listAll(ctx, collection, filter, "")
|
|
if err != nil {
|
|
return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err)
|
|
}
|
|
for _, raw := range items {
|
|
var rec struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err := json.Unmarshal(raw, &rec); err != nil || rec.ID == "" {
|
|
continue
|
|
}
|
|
path := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID)
|
|
if err := s.pb.patch(ctx, path, resetPayload); err != nil {
|
|
s.log.Warn("ReapStaleTasks: patch failed", "collection", collection, "id", rec.ID, "err", err)
|
|
continue
|
|
}
|
|
total++
|
|
}
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
// ── taskqueue.Reader ──────────────────────────────────────────────────────────
|
|
|
|
func (s *Store) ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error) {
|
|
items, err := s.pb.listAll(ctx, "scraping_tasks", "", "-started")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tasks := make([]domain.ScrapeTask, 0, len(items))
|
|
for _, raw := range items {
|
|
t, err := parseScrapeTask(raw)
|
|
if err == nil {
|
|
tasks = append(tasks, t)
|
|
}
|
|
}
|
|
return tasks, nil
|
|
}
|
|
|
|
func (s *Store) GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error) {
|
|
var raw json.RawMessage
|
|
if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), &raw); err != nil {
|
|
if err == ErrNotFound {
|
|
return domain.ScrapeTask{}, false, nil
|
|
}
|
|
return domain.ScrapeTask{}, false, err
|
|
}
|
|
t, err := parseScrapeTask(raw)
|
|
return t, err == nil, err
|
|
}
|
|
|
|
func (s *Store) ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error) {
|
|
items, err := s.pb.listAll(ctx, "audio_jobs", "", "-started")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tasks := make([]domain.AudioTask, 0, len(items))
|
|
for _, raw := range items {
|
|
t, err := parseAudioTask(raw)
|
|
if err == nil {
|
|
tasks = append(tasks, t)
|
|
}
|
|
}
|
|
return tasks, nil
|
|
}
|
|
|
|
func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) {
|
|
filter := fmt.Sprintf(`cache_key=%q`, cacheKey)
|
|
items, err := s.pb.listAll(ctx, "audio_jobs", filter, "-started")
|
|
if err != nil || len(items) == 0 {
|
|
return domain.AudioTask{}, false, err
|
|
}
|
|
t, err := parseAudioTask(items[0])
|
|
return t, err == nil, err
|
|
}
|
|
|
|
// ── Parsers ───────────────────────────────────────────────────────────────────
|
|
|
|
func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) {
|
|
var rec struct {
|
|
ID string `json:"id"`
|
|
Kind string `json:"kind"`
|
|
TargetURL string `json:"target_url"`
|
|
FromChapter int `json:"from_chapter"`
|
|
ToChapter int `json:"to_chapter"`
|
|
WorkerID string `json:"worker_id"`
|
|
Status string `json:"status"`
|
|
BooksFound int `json:"books_found"`
|
|
ChaptersScraped int `json:"chapters_scraped"`
|
|
ChaptersSkipped int `json:"chapters_skipped"`
|
|
Errors int `json:"errors"`
|
|
Started string `json:"started"`
|
|
Finished string `json:"finished"`
|
|
ErrorMessage string `json:"error_message"`
|
|
}
|
|
if err := json.Unmarshal(raw, &rec); err != nil {
|
|
return domain.ScrapeTask{}, err
|
|
}
|
|
started, _ := time.Parse(time.RFC3339, rec.Started)
|
|
finished, _ := time.Parse(time.RFC3339, rec.Finished)
|
|
return domain.ScrapeTask{
|
|
ID: rec.ID,
|
|
Kind: rec.Kind,
|
|
TargetURL: rec.TargetURL,
|
|
FromChapter: rec.FromChapter,
|
|
ToChapter: rec.ToChapter,
|
|
WorkerID: rec.WorkerID,
|
|
Status: domain.TaskStatus(rec.Status),
|
|
BooksFound: rec.BooksFound,
|
|
ChaptersScraped: rec.ChaptersScraped,
|
|
ChaptersSkipped: rec.ChaptersSkipped,
|
|
Errors: rec.Errors,
|
|
Started: started,
|
|
Finished: finished,
|
|
ErrorMessage: rec.ErrorMessage,
|
|
}, nil
|
|
}
|
|
|
|
func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
|
|
var rec struct {
|
|
ID string `json:"id"`
|
|
CacheKey string `json:"cache_key"`
|
|
Slug string `json:"slug"`
|
|
Chapter int `json:"chapter"`
|
|
Voice string `json:"voice"`
|
|
WorkerID string `json:"worker_id"`
|
|
Status string `json:"status"`
|
|
ErrorMessage string `json:"error_message"`
|
|
Started string `json:"started"`
|
|
Finished string `json:"finished"`
|
|
}
|
|
if err := json.Unmarshal(raw, &rec); err != nil {
|
|
return domain.AudioTask{}, err
|
|
}
|
|
started, _ := time.Parse(time.RFC3339, rec.Started)
|
|
finished, _ := time.Parse(time.RFC3339, rec.Finished)
|
|
return domain.AudioTask{
|
|
ID: rec.ID,
|
|
CacheKey: rec.CacheKey,
|
|
Slug: rec.Slug,
|
|
Chapter: rec.Chapter,
|
|
Voice: rec.Voice,
|
|
WorkerID: rec.WorkerID,
|
|
Status: domain.TaskStatus(rec.Status),
|
|
ErrorMessage: rec.ErrorMessage,
|
|
Started: started,
|
|
Finished: finished,
|
|
}, nil
|
|
}
|
|
|
|
// ── BrowseStore ────────────────────────────────────────────────────────────────
|
|
|
|
func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error {
|
|
key := BrowseObjectKey(genre, sort, status, novelType, page)
|
|
if err := s.mc.putBrowse(ctx, key, data); err != nil {
|
|
return fmt.Errorf("PutBrowsePage: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) {
|
|
key := BrowseObjectKey(genre, sort, status, novelType, page)
|
|
data, ok, err := s.mc.getBrowse(ctx, key)
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("GetBrowsePage: %w", err)
|
|
}
|
|
return data, ok, nil
|
|
}
|