feat: add v2 stack (backend, runner, ui-v2) with release workflow
All checks were successful
Release / Scraper / Test (push) Successful in 10s
Release / UI / Build (push) Successful in 26s
Release / v2 / Build ui-v2 (push) Successful in 17s
Release / Scraper / Docker (push) Successful in 47s
Release / UI / Docker (push) Successful in 56s
CI / Scraper / Lint (pull_request) Successful in 7s
CI / Scraper / Test (pull_request) Successful in 8s
CI / UI / Build (pull_request) Successful in 16s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 56s
Release / v2 / Test backend (push) Successful in 4m35s
iOS CI / Build (pull_request) Successful in 4m28s
Release / v2 / Docker / backend (push) Successful in 1m29s
Release / v2 / Docker / runner (push) Successful in 1m39s
iOS CI / Test (pull_request) Successful in 9m51s
All checks were successful
Release / Scraper / Test (push) Successful in 10s
Release / UI / Build (push) Successful in 26s
Release / v2 / Build ui-v2 (push) Successful in 17s
Release / Scraper / Docker (push) Successful in 47s
Release / UI / Docker (push) Successful in 56s
CI / Scraper / Lint (pull_request) Successful in 7s
CI / Scraper / Test (pull_request) Successful in 8s
CI / UI / Build (pull_request) Successful in 16s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 56s
Release / v2 / Test backend (push) Successful in 4m35s
iOS CI / Build (pull_request) Successful in 4m28s
Release / v2 / Docker / backend (push) Successful in 1m29s
Release / v2 / Docker / runner (push) Successful in 1m39s
iOS CI / Test (pull_request) Successful in 9m51s
- backend/: Go API server and runner binaries with PocketBase + MinIO storage - ui-v2/: SvelteKit frontend rewrite - docker-compose-new.yml: compose file for the v2 stack - .gitea/workflows/release-v2.yaml: CI/CD for backend, runner, and ui-v2 Docker Hub images - scripts/pb-init.sh: migrate from wget to curl, add superuser bootstrap for fresh installs - .env.example: document DOCKER_BUILDKIT=1 for Colima users
This commit is contained in:
194
backend/internal/storage/minio.go
Normal file
194
backend/internal/storage/minio.go
Normal file
@@ -0,0 +1,194 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
|
||||
"github.com/libnovel/backend/internal/config"
|
||||
)
|
||||
|
||||
// minioClient wraps the official minio-go client with bucket names.
|
||||
type minioClient struct {
|
||||
client *minio.Client // internal — all read/write operations
|
||||
pubClient *minio.Client // presign-only — initialised against the public endpoint
|
||||
bucketChapters string
|
||||
bucketAudio string
|
||||
bucketAvatars string
|
||||
}
|
||||
|
||||
func newMinioClient(cfg config.MinIO) (*minioClient, error) {
|
||||
creds := credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, "")
|
||||
|
||||
internal, err := minio.New(cfg.Endpoint, &minio.Options{
|
||||
Creds: creds,
|
||||
Secure: cfg.UseSSL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio: init internal client: %w", err)
|
||||
}
|
||||
|
||||
// Presigned URLs must be signed with the hostname the browser will use
|
||||
// (PUBLIC_MINIO_PUBLIC_URL), because AWS Signature V4 includes the Host
|
||||
// header in the canonical request — a URL signed against "minio:9000" will
|
||||
// return SignatureDoesNotMatch when the browser fetches it from
|
||||
// "localhost:9000".
|
||||
//
|
||||
// However, minio-go normally makes a live BucketLocation HTTP call before
|
||||
// signing, which would fail from inside the container when the public
|
||||
// endpoint is externally-facing (e.g. "localhost:9000" is unreachable from
|
||||
// within Docker). We prevent this by:
|
||||
// 1. Setting Region: "us-east-1" — minio-go skips getBucketLocation when
|
||||
// the region is already known (bucket-cache.go:49).
|
||||
// 2. Setting BucketLookup: BucketLookupPath — forces path-style URLs
|
||||
// (e.g. host/bucket/key), matching MinIO's default behaviour and
|
||||
// avoiding any virtual-host DNS probing.
|
||||
//
|
||||
// When no public endpoint is configured (or it equals the internal one),
|
||||
// fall back to the internal client so presigning still works.
|
||||
publicEndpoint := cfg.PublicEndpoint
|
||||
if u, err2 := url.Parse(publicEndpoint); err2 == nil && u.Host != "" {
|
||||
publicEndpoint = u.Host // strip scheme so minio.New is happy
|
||||
}
|
||||
pubUseSSL := cfg.PublicUseSSL
|
||||
if publicEndpoint == "" || publicEndpoint == cfg.Endpoint {
|
||||
publicEndpoint = cfg.Endpoint
|
||||
pubUseSSL = cfg.UseSSL
|
||||
}
|
||||
pub, err := minio.New(publicEndpoint, &minio.Options{
|
||||
Creds: creds,
|
||||
Secure: pubUseSSL,
|
||||
Region: "us-east-1", // skip live BucketLocation preflight
|
||||
BucketLookup: minio.BucketLookupPath,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio: init public client: %w", err)
|
||||
}
|
||||
|
||||
return &minioClient{
|
||||
client: internal,
|
||||
pubClient: pub,
|
||||
bucketChapters: cfg.BucketChapters,
|
||||
bucketAudio: cfg.BucketAudio,
|
||||
bucketAvatars: cfg.BucketAvatars,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ensureBuckets creates all required buckets if they don't already exist.
|
||||
func (m *minioClient) ensureBuckets(ctx context.Context) error {
|
||||
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars} {
|
||||
exists, err := m.client.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: check bucket %q: %w", bucket, err)
|
||||
}
|
||||
if !exists {
|
||||
if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
return fmt.Errorf("minio: create bucket %q: %w", bucket, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ── Key helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
// ChapterObjectKey returns the MinIO object key for a chapter markdown file.
|
||||
// Format: {slug}/chapter-{n:06d}.md
|
||||
func ChapterObjectKey(slug string, n int) string {
|
||||
return fmt.Sprintf("%s/chapter-%06d.md", slug, n)
|
||||
}
|
||||
|
||||
// AudioObjectKey returns the MinIO object key for a cached audio file.
|
||||
// Format: {slug}/{n}/{voice}.mp3
|
||||
func AudioObjectKey(slug string, n int, voice string) string {
|
||||
return fmt.Sprintf("%s/%d/%s.mp3", slug, n, voice)
|
||||
}
|
||||
|
||||
// AvatarObjectKey returns the MinIO object key for a user avatar image.
|
||||
// Format: {userID}/{ext}.{ext}
|
||||
func AvatarObjectKey(userID, ext string) string {
|
||||
return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
|
||||
}
|
||||
|
||||
// chapterNumberFromKey extracts the chapter number from a MinIO object key.
|
||||
// e.g. "my-book/chapter-000042.md" → 42
|
||||
func chapterNumberFromKey(key string) int {
|
||||
base := path.Base(key)
|
||||
base = strings.TrimPrefix(base, "chapter-")
|
||||
base = strings.TrimSuffix(base, ".md")
|
||||
var n int
|
||||
fmt.Sscanf(base, "%d", &n)
|
||||
return n
|
||||
}
|
||||
|
||||
// ── Object operations ─────────────────────────────────────────────────────────
|
||||
|
||||
func (m *minioClient) putObject(ctx context.Context, bucket, key, contentType string, data []byte) error {
|
||||
_, err := m.client.PutObject(ctx, bucket, key,
|
||||
strings.NewReader(string(data)),
|
||||
int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: contentType},
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *minioClient) getObject(ctx context.Context, bucket, key string) ([]byte, error) {
|
||||
obj, err := m.client.GetObject(ctx, bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer obj.Close()
|
||||
return io.ReadAll(obj)
|
||||
}
|
||||
|
||||
func (m *minioClient) objectExists(ctx context.Context, bucket, key string) bool {
|
||||
_, err := m.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (m *minioClient) presignGet(ctx context.Context, bucket, key string, expires time.Duration) (string, error) {
|
||||
u, err := m.pubClient.PresignedGetObject(ctx, bucket, key, expires, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio presign %s/%s: %w", bucket, key, err)
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func (m *minioClient) presignPut(ctx context.Context, bucket, key string, expires time.Duration) (string, error) {
|
||||
u, err := m.pubClient.PresignedPutObject(ctx, bucket, key, expires)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("minio presign PUT %s/%s: %w", bucket, key, err)
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func (m *minioClient) deleteObjects(ctx context.Context, bucket, prefix string) error {
|
||||
objCh := m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix})
|
||||
for obj := range objCh {
|
||||
if obj.Err != nil {
|
||||
return obj.Err
|
||||
}
|
||||
if err := m.client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string) ([]string, error) {
|
||||
var keys []string
|
||||
for obj := range m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) {
|
||||
if obj.Err != nil {
|
||||
return nil, obj.Err
|
||||
}
|
||||
keys = append(keys, obj.Key)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
268
backend/internal/storage/pocketbase.go
Normal file
268
backend/internal/storage/pocketbase.go
Normal file
@@ -0,0 +1,268 @@
|
||||
// Package storage provides the concrete implementations of all bookstore and
|
||||
// taskqueue interfaces backed by PocketBase (structured data) and MinIO (blobs).
|
||||
//
|
||||
// Entry point: NewStore(ctx, cfg, log) returns a *Store that satisfies every
|
||||
// interface defined in bookstore and taskqueue.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/backend/internal/config"
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
)
|
||||
|
||||
// ErrNotFound is returned by single-record lookups when no record exists.
|
||||
var ErrNotFound = errors.New("storage: record not found")
|
||||
|
||||
// pbClient is the internal PocketBase REST admin client.
|
||||
type pbClient struct {
|
||||
baseURL string
|
||||
email string
|
||||
password string
|
||||
log *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
token string
|
||||
exp time.Time
|
||||
}
|
||||
|
||||
func newPBClient(cfg config.PocketBase, log *slog.Logger) *pbClient {
|
||||
return &pbClient{
|
||||
baseURL: strings.TrimRight(cfg.URL, "/"),
|
||||
email: cfg.AdminEmail,
|
||||
password: cfg.AdminPassword,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
// authToken returns a valid admin auth token, refreshing it when expired.
|
||||
func (c *pbClient) authToken(ctx context.Context) (string, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.token != "" && time.Now().Before(c.exp) {
|
||||
return c.token, nil
|
||||
}
|
||||
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
"identity": c.email,
|
||||
"password": c.password,
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
c.baseURL+"/api/collections/_superusers/auth-with-password", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pb auth: build request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pb auth: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
return "", fmt.Errorf("pb auth: status %d: %s", resp.StatusCode, string(raw))
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
|
||||
return "", fmt.Errorf("pb auth: decode: %w", err)
|
||||
}
|
||||
c.token = payload.Token
|
||||
c.exp = time.Now().Add(30 * time.Minute)
|
||||
return c.token, nil
|
||||
}
|
||||
|
||||
// do executes an authenticated PocketBase REST request.
|
||||
func (c *pbClient) do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
|
||||
tok, err := c.authToken(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pb: build request %s %s: %w", method, path, err)
|
||||
}
|
||||
req.Header.Set("Authorization", tok)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pb: %s %s: %w", method, path, err)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// get is a convenience wrapper that decodes a JSON response into v.
|
||||
func (c *pbClient) get(ctx context.Context, path string, v any) error {
|
||||
resp, err := c.do(ctx, http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return ErrNotFound
|
||||
}
|
||||
if resp.StatusCode >= 400 {
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pb GET %s: status %d: %s", path, resp.StatusCode, string(raw))
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(v)
|
||||
}
|
||||
|
||||
// post creates a record and decodes the created record into v.
|
||||
func (c *pbClient) post(ctx context.Context, path string, payload, v any) error {
|
||||
b, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pb: marshal: %w", err)
|
||||
}
|
||||
resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pb POST %s: status %d: %s", path, resp.StatusCode, string(raw))
|
||||
}
|
||||
if v != nil {
|
||||
return json.NewDecoder(resp.Body).Decode(v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// patch updates a record.
|
||||
func (c *pbClient) patch(ctx context.Context, path string, payload any) error {
|
||||
b, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pb: marshal: %w", err)
|
||||
}
|
||||
resp, err := c.do(ctx, http.MethodPatch, path, bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pb PATCH %s: status %d: %s", path, resp.StatusCode, string(raw))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// delete removes a record.
|
||||
func (c *pbClient) delete(ctx context.Context, path string) error {
|
||||
resp, err := c.do(ctx, http.MethodDelete, path, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return ErrNotFound
|
||||
}
|
||||
if resp.StatusCode >= 400 {
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("pb DELETE %s: status %d: %s", path, resp.StatusCode, string(raw))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// listAll fetches all pages of a collection. PocketBase returns at most 200
|
||||
// records per page; we paginate until empty.
|
||||
func (c *pbClient) listAll(ctx context.Context, collection string, filter, sort string) ([]json.RawMessage, error) {
|
||||
var all []json.RawMessage
|
||||
page := 1
|
||||
for {
|
||||
q := url.Values{
|
||||
"page": {fmt.Sprintf("%d", page)},
|
||||
"perPage": {"200"},
|
||||
}
|
||||
if filter != "" {
|
||||
q.Set("filter", filter)
|
||||
}
|
||||
if sort != "" {
|
||||
q.Set("sort", sort)
|
||||
}
|
||||
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||
|
||||
var result struct {
|
||||
Items []json.RawMessage `json:"items"`
|
||||
Page int `json:"page"`
|
||||
Pages int `json:"totalPages"`
|
||||
}
|
||||
if err := c.get(ctx, path, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all = append(all, result.Items...)
|
||||
if result.Page >= result.Pages {
|
||||
break
|
||||
}
|
||||
page++
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// claimRecord atomically claims the first pending record matching collection.
|
||||
// It fetches the oldest pending record (filter + sort), then PATCHes it with
|
||||
// the claim payload. Returns (nil, nil) when the queue is empty.
|
||||
func (c *pbClient) claimRecord(ctx context.Context, collection, workerID string, extraClaim map[string]any) (json.RawMessage, error) {
|
||||
q := url.Values{}
|
||||
q.Set("filter", `status="pending"`)
|
||||
q.Set("sort", "+started")
|
||||
q.Set("perPage", "1")
|
||||
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
|
||||
|
||||
var result struct {
|
||||
Items []json.RawMessage `json:"items"`
|
||||
}
|
||||
if err := c.get(ctx, path, &result); err != nil {
|
||||
return nil, fmt.Errorf("claimRecord list: %w", err)
|
||||
}
|
||||
if len(result.Items) == 0 {
|
||||
return nil, nil // queue empty
|
||||
}
|
||||
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(result.Items[0], &rec); err != nil {
|
||||
return nil, fmt.Errorf("claimRecord parse id: %w", err)
|
||||
}
|
||||
|
||||
claim := map[string]any{
|
||||
"status": string(domain.TaskStatusRunning),
|
||||
"worker_id": workerID,
|
||||
}
|
||||
for k, v := range extraClaim {
|
||||
claim[k] = v
|
||||
}
|
||||
|
||||
claimPath := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID)
|
||||
if err := c.patch(ctx, claimPath, claim); err != nil {
|
||||
return nil, fmt.Errorf("claimRecord patch: %w", err)
|
||||
}
|
||||
|
||||
// Re-fetch the updated record so caller has current state.
|
||||
var updated json.RawMessage
|
||||
if err := c.get(ctx, claimPath, &updated); err != nil {
|
||||
return nil, fmt.Errorf("claimRecord re-fetch: %w", err)
|
||||
}
|
||||
return updated, nil
|
||||
}
|
||||
769
backend/internal/storage/store.go
Normal file
769
backend/internal/storage/store.go
Normal file
@@ -0,0 +1,769 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/backend/internal/bookstore"
|
||||
"github.com/libnovel/backend/internal/config"
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
)
|
||||
|
||||
// Store is the unified persistence implementation that satisfies all bookstore
|
||||
// and taskqueue interfaces. It routes structured data to PocketBase and binary
|
||||
// blobs to MinIO.
|
||||
type Store struct {
|
||||
pb *pbClient
|
||||
mc *minioClient
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewStore initialises PocketBase and MinIO connections and ensures all MinIO
|
||||
// buckets exist. Returns a ready-to-use Store.
|
||||
func NewStore(ctx context.Context, cfg config.Config, log *slog.Logger) (*Store, error) {
|
||||
pb := newPBClient(cfg.PocketBase, log)
|
||||
// Validate PocketBase connectivity by fetching an auth token.
|
||||
if _, err := pb.authToken(ctx); err != nil {
|
||||
return nil, fmt.Errorf("pocketbase: %w", err)
|
||||
}
|
||||
|
||||
mc, err := newMinioClient(cfg.MinIO)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minio: %w", err)
|
||||
}
|
||||
if err := mc.ensureBuckets(ctx); err != nil {
|
||||
return nil, fmt.Errorf("minio: ensure buckets: %w", err)
|
||||
}
|
||||
|
||||
return &Store{pb: pb, mc: mc, log: log}, nil
|
||||
}
|
||||
|
||||
// Compile-time interface satisfaction.
|
||||
var _ bookstore.BookWriter = (*Store)(nil)
|
||||
var _ bookstore.BookReader = (*Store)(nil)
|
||||
var _ bookstore.RankingStore = (*Store)(nil)
|
||||
var _ bookstore.AudioStore = (*Store)(nil)
|
||||
var _ bookstore.PresignStore = (*Store)(nil)
|
||||
var _ bookstore.ProgressStore = (*Store)(nil)
|
||||
var _ taskqueue.Producer = (*Store)(nil)
|
||||
var _ taskqueue.Consumer = (*Store)(nil)
|
||||
var _ taskqueue.Reader = (*Store)(nil)
|
||||
|
||||
// ── BookWriter ────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) WriteMetadata(ctx context.Context, meta domain.BookMeta) error {
|
||||
payload := map[string]any{
|
||||
"slug": meta.Slug,
|
||||
"title": meta.Title,
|
||||
"author": meta.Author,
|
||||
"cover": meta.Cover,
|
||||
"status": meta.Status,
|
||||
"genres": meta.Genres,
|
||||
"summary": meta.Summary,
|
||||
"total_chapters": meta.TotalChapters,
|
||||
"source_url": meta.SourceURL,
|
||||
"ranking": meta.Ranking,
|
||||
}
|
||||
// Upsert via filter: if exists PATCH, otherwise POST.
|
||||
existing, err := s.getBookBySlug(ctx, meta.Slug)
|
||||
if err != nil && err != ErrNotFound {
|
||||
return fmt.Errorf("WriteMetadata: %w", err)
|
||||
}
|
||||
if err == ErrNotFound {
|
||||
return s.pb.post(ctx, "/api/collections/books/records", payload, nil)
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", existing.ID), payload)
|
||||
}
|
||||
|
||||
func (s *Store) WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error {
|
||||
key := ChapterObjectKey(slug, chapter.Ref.Number)
|
||||
if err := s.mc.putObject(ctx, s.mc.bucketChapters, key, "text/markdown", []byte(chapter.Text)); err != nil {
|
||||
return fmt.Errorf("WriteChapter: minio: %w", err)
|
||||
}
|
||||
// Upsert the chapters_idx record in PocketBase.
|
||||
return s.upsertChapterIdx(ctx, slug, chapter.Ref)
|
||||
}
|
||||
|
||||
func (s *Store) WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error {
|
||||
for _, ref := range refs {
|
||||
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
|
||||
s.log.Warn("WriteChapterRefs: upsert failed", "slug", slug, "chapter", ref.Number, "err", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool {
|
||||
return s.mc.objectExists(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, ref.Number))
|
||||
}
|
||||
|
||||
func (s *Store) upsertChapterIdx(ctx context.Context, slug string, ref domain.ChapterRef) error {
|
||||
payload := map[string]any{
|
||||
"slug": slug,
|
||||
"number": ref.Number,
|
||||
"title": ref.Title,
|
||||
}
|
||||
filter := fmt.Sprintf(`slug=%q&&number=%d`, slug, ref.Number)
|
||||
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "")
|
||||
if err != nil && err != ErrNotFound {
|
||||
return err
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return s.pb.post(ctx, "/api/collections/chapters_idx/records", payload, nil)
|
||||
}
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
json.Unmarshal(items[0], &rec)
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/chapters_idx/records/%s", rec.ID), payload)
|
||||
}
|
||||
|
||||
// ── BookReader ────────────────────────────────────────────────────────────────
|
||||
|
||||
type pbBook struct {
|
||||
ID string `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Cover string `json:"cover"`
|
||||
Status string `json:"status"`
|
||||
Genres []string `json:"genres"`
|
||||
Summary string `json:"summary"`
|
||||
TotalChapters int `json:"total_chapters"`
|
||||
SourceURL string `json:"source_url"`
|
||||
Ranking int `json:"ranking"`
|
||||
Updated string `json:"updated"`
|
||||
}
|
||||
|
||||
func (b pbBook) toDomain() domain.BookMeta {
|
||||
return domain.BookMeta{
|
||||
Slug: b.Slug,
|
||||
Title: b.Title,
|
||||
Author: b.Author,
|
||||
Cover: b.Cover,
|
||||
Status: b.Status,
|
||||
Genres: b.Genres,
|
||||
Summary: b.Summary,
|
||||
TotalChapters: b.TotalChapters,
|
||||
SourceURL: b.SourceURL,
|
||||
Ranking: b.Ranking,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) getBookBySlug(ctx context.Context, slug string) (pbBook, error) {
|
||||
filter := fmt.Sprintf(`slug=%q`, slug)
|
||||
items, err := s.pb.listAll(ctx, "books", filter, "")
|
||||
if err != nil {
|
||||
return pbBook{}, err
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return pbBook{}, ErrNotFound
|
||||
}
|
||||
var b pbBook
|
||||
json.Unmarshal(items[0], &b)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (s *Store) ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error) {
|
||||
b, err := s.getBookBySlug(ctx, slug)
|
||||
if err == ErrNotFound {
|
||||
return domain.BookMeta{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return domain.BookMeta{}, false, err
|
||||
}
|
||||
return b.toDomain(), true, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListBooks(ctx context.Context) ([]domain.BookMeta, error) {
|
||||
items, err := s.pb.listAll(ctx, "books", "", "title")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
books := make([]domain.BookMeta, 0, len(items))
|
||||
for _, raw := range items {
|
||||
var b pbBook
|
||||
json.Unmarshal(raw, &b)
|
||||
books = append(books, b.toDomain())
|
||||
}
|
||||
return books, nil
|
||||
}
|
||||
|
||||
func (s *Store) LocalSlugs(ctx context.Context) (map[string]bool, error) {
|
||||
items, err := s.pb.listAll(ctx, "books", "", "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slugs := make(map[string]bool, len(items))
|
||||
for _, raw := range items {
|
||||
var b struct {
|
||||
Slug string `json:"slug"`
|
||||
}
|
||||
json.Unmarshal(raw, &b)
|
||||
if b.Slug != "" {
|
||||
slugs[b.Slug] = true
|
||||
}
|
||||
}
|
||||
return slugs, nil
|
||||
}
|
||||
|
||||
func (s *Store) MetadataMtime(ctx context.Context, slug string) int64 {
|
||||
b, err := s.getBookBySlug(ctx, slug)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, b.Updated)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
func (s *Store) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
|
||||
data, err := s.mc.getObject(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ReadChapter: %w", err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func (s *Store) ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error) {
|
||||
filter := fmt.Sprintf(`slug=%q`, slug)
|
||||
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "number")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chapters := make([]domain.ChapterInfo, 0, len(items))
|
||||
for _, raw := range items {
|
||||
var rec struct {
|
||||
Number int `json:"number"`
|
||||
Title string `json:"title"`
|
||||
}
|
||||
json.Unmarshal(raw, &rec)
|
||||
chapters = append(chapters, domain.ChapterInfo{Number: rec.Number, Title: rec.Title})
|
||||
}
|
||||
return chapters, nil
|
||||
}
|
||||
|
||||
func (s *Store) CountChapters(ctx context.Context, slug string) int {
|
||||
chapters, err := s.ListChapters(ctx, slug)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return len(chapters)
|
||||
}
|
||||
|
||||
func (s *Store) ReindexChapters(ctx context.Context, slug string) (int, error) {
|
||||
keys, err := s.mc.listObjectKeys(ctx, s.mc.bucketChapters, slug+"/")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("ReindexChapters: list objects: %w", err)
|
||||
}
|
||||
count := 0
|
||||
for _, key := range keys {
|
||||
if !strings.HasSuffix(key, ".md") {
|
||||
continue
|
||||
}
|
||||
n := chapterNumberFromKey(key)
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
ref := domain.ChapterRef{Number: n}
|
||||
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
|
||||
s.log.Warn("ReindexChapters: upsert failed", "key", key, "err", err)
|
||||
continue
|
||||
}
|
||||
count++
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// ── RankingStore ──────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) WriteRankingItem(ctx context.Context, item domain.RankingItem) error {
|
||||
payload := map[string]any{
|
||||
"rank": item.Rank,
|
||||
"slug": item.Slug,
|
||||
"title": item.Title,
|
||||
"author": item.Author,
|
||||
"cover": item.Cover,
|
||||
"status": item.Status,
|
||||
"genres": item.Genres,
|
||||
"source_url": item.SourceURL,
|
||||
}
|
||||
filter := fmt.Sprintf(`slug=%q`, item.Slug)
|
||||
items, err := s.pb.listAll(ctx, "ranking", filter, "")
|
||||
if err != nil && err != ErrNotFound {
|
||||
return err
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return s.pb.post(ctx, "/api/collections/ranking/records", payload, nil)
|
||||
}
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
json.Unmarshal(items[0], &rec)
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/ranking/records/%s", rec.ID), payload)
|
||||
}
|
||||
|
||||
func (s *Store) ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error) {
|
||||
items, err := s.pb.listAll(ctx, "ranking", "", "rank")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]domain.RankingItem, 0, len(items))
|
||||
for _, raw := range items {
|
||||
var rec struct {
|
||||
Rank int `json:"rank"`
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Author string `json:"author"`
|
||||
Cover string `json:"cover"`
|
||||
Status string `json:"status"`
|
||||
Genres []string `json:"genres"`
|
||||
SourceURL string `json:"source_url"`
|
||||
Updated string `json:"updated"`
|
||||
}
|
||||
json.Unmarshal(raw, &rec)
|
||||
t, _ := time.Parse(time.RFC3339, rec.Updated)
|
||||
result = append(result, domain.RankingItem{
|
||||
Rank: rec.Rank,
|
||||
Slug: rec.Slug,
|
||||
Title: rec.Title,
|
||||
Author: rec.Author,
|
||||
Cover: rec.Cover,
|
||||
Status: rec.Status,
|
||||
Genres: rec.Genres,
|
||||
SourceURL: rec.SourceURL,
|
||||
Updated: t,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) {
|
||||
items, err := s.ReadRankingItems(ctx)
|
||||
if err != nil || len(items) == 0 {
|
||||
return false, err
|
||||
}
|
||||
var latest time.Time
|
||||
for _, item := range items {
|
||||
if item.Updated.After(latest) {
|
||||
latest = item.Updated
|
||||
}
|
||||
}
|
||||
return time.Since(latest) < maxAge, nil
|
||||
}
|
||||
|
||||
// ── AudioStore ────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) AudioObjectKey(slug string, n int, voice string) string {
|
||||
return AudioObjectKey(slug, n, voice)
|
||||
}
|
||||
|
||||
func (s *Store) AudioExists(ctx context.Context, key string) bool {
|
||||
return s.mc.objectExists(ctx, s.mc.bucketAudio, key)
|
||||
}
|
||||
|
||||
func (s *Store) PutAudio(ctx context.Context, key string, data []byte) error {
|
||||
return s.mc.putObject(ctx, s.mc.bucketAudio, key, "audio/mpeg", data)
|
||||
}
|
||||
|
||||
// ── PresignStore ──────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) {
|
||||
return s.mc.presignGet(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n), expires)
|
||||
}
|
||||
|
||||
func (s *Store) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
|
||||
return s.mc.presignGet(ctx, s.mc.bucketAudio, key, expires)
|
||||
}
|
||||
|
||||
func (s *Store) PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error) {
|
||||
key = AvatarObjectKey(userID, ext)
|
||||
uploadURL, err = s.mc.presignPut(ctx, s.mc.bucketAvatars, key, 15*time.Minute)
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) {
|
||||
for _, ext := range []string{"jpg", "png", "webp"} {
|
||||
key := AvatarObjectKey(userID, ext)
|
||||
if s.mc.objectExists(ctx, s.mc.bucketAvatars, key) {
|
||||
u, err := s.mc.presignGet(ctx, s.mc.bucketAvatars, key, 1*time.Hour)
|
||||
return u, true, err
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
func (s *Store) DeleteAvatar(ctx context.Context, userID string) error {
|
||||
return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
|
||||
}
|
||||
|
||||
// ── ProgressStore ─────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool) {
|
||||
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
|
||||
items, err := s.pb.listAll(ctx, "progress", filter, "")
|
||||
if err != nil || len(items) == 0 {
|
||||
return domain.ReadingProgress{}, false
|
||||
}
|
||||
var rec struct {
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
UpdatedAt string `json:"updated"`
|
||||
}
|
||||
json.Unmarshal(items[0], &rec)
|
||||
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
|
||||
return domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t}, true
|
||||
}
|
||||
|
||||
func (s *Store) SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error {
|
||||
payload := map[string]any{
|
||||
"session_id": sessionID,
|
||||
"slug": p.Slug,
|
||||
"chapter": p.Chapter,
|
||||
}
|
||||
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, p.Slug)
|
||||
items, err := s.pb.listAll(ctx, "progress", filter, "")
|
||||
if err != nil && err != ErrNotFound {
|
||||
return err
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return s.pb.post(ctx, "/api/collections/progress/records", payload, nil)
|
||||
}
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
json.Unmarshal(items[0], &rec)
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID), payload)
|
||||
}
|
||||
|
||||
func (s *Store) AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error) {
|
||||
filter := fmt.Sprintf(`session_id=%q`, sessionID)
|
||||
items, err := s.pb.listAll(ctx, "progress", filter, "-updated")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := make([]domain.ReadingProgress, 0, len(items))
|
||||
for _, raw := range items {
|
||||
var rec struct {
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
UpdatedAt string `json:"updated"`
|
||||
}
|
||||
json.Unmarshal(raw, &rec)
|
||||
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
|
||||
result = append(result, domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) DeleteProgress(ctx context.Context, sessionID, slug string) error {
|
||||
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
|
||||
items, err := s.pb.listAll(ctx, "progress", filter, "")
|
||||
if err != nil || len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
json.Unmarshal(items[0], &rec)
|
||||
return s.pb.delete(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID))
|
||||
}
|
||||
|
||||
// ── taskqueue.Producer ────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) {
|
||||
payload := map[string]any{
|
||||
"kind": kind,
|
||||
"target_url": targetURL,
|
||||
"from_chapter": fromChapter,
|
||||
"to_chapter": toChapter,
|
||||
"status": string(domain.TaskStatusPending),
|
||||
"started": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := s.pb.post(ctx, "/api/collections/scraping_tasks/records", payload, &rec); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rec.ID, nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) {
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, voice)
|
||||
payload := map[string]any{
|
||||
"cache_key": cacheKey,
|
||||
"slug": slug,
|
||||
"chapter": chapter,
|
||||
"voice": voice,
|
||||
"status": string(domain.TaskStatusPending),
|
||||
"started": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := s.pb.post(ctx, "/api/collections/audio_jobs/records", payload, &rec); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rec.ID, nil
|
||||
}
|
||||
|
||||
func (s *Store) CancelTask(ctx context.Context, id string) error {
|
||||
// Try scraping_tasks first, then audio_jobs.
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),
|
||||
map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id),
|
||||
map[string]string{"status": string(domain.TaskStatusCancelled)})
|
||||
}
|
||||
|
||||
// ── taskqueue.Consumer ────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error) {
|
||||
raw, err := s.pb.claimRecord(ctx, "scraping_tasks", workerID, nil)
|
||||
if err != nil {
|
||||
return domain.ScrapeTask{}, false, err
|
||||
}
|
||||
if raw == nil {
|
||||
return domain.ScrapeTask{}, false, nil
|
||||
}
|
||||
task, err := parseScrapeTask(raw)
|
||||
return task, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error) {
|
||||
raw, err := s.pb.claimRecord(ctx, "audio_jobs", workerID, nil)
|
||||
if err != nil {
|
||||
return domain.AudioTask{}, false, err
|
||||
}
|
||||
if raw == nil {
|
||||
return domain.AudioTask{}, false, nil
|
||||
}
|
||||
task, err := parseAudioTask(raw)
|
||||
return task, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
|
||||
status := string(domain.TaskStatusDone)
|
||||
if result.ErrorMessage != "" {
|
||||
status = string(domain.TaskStatusFailed)
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), map[string]any{
|
||||
"status": status,
|
||||
"books_found": result.BooksFound,
|
||||
"chapters_scraped": result.ChaptersScraped,
|
||||
"chapters_skipped": result.ChaptersSkipped,
|
||||
"errors": result.Errors,
|
||||
"error_message": result.ErrorMessage,
|
||||
"finished": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error {
|
||||
status := string(domain.TaskStatusDone)
|
||||
if result.ErrorMessage != "" {
|
||||
status = string(domain.TaskStatusFailed)
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), map[string]any{
|
||||
"status": status,
|
||||
"error_message": result.ErrorMessage,
|
||||
"finished": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
|
||||
payload := map[string]any{
|
||||
"status": string(domain.TaskStatusFailed),
|
||||
"error_message": errMsg,
|
||||
"finished": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
|
||||
}
|
||||
|
||||
// HeartbeatTask updates the heartbeat_at field on a running task.
|
||||
// Tries scraping_tasks first, then audio_jobs (same pattern as FailTask).
|
||||
func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
|
||||
payload := map[string]any{
|
||||
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
|
||||
}
|
||||
|
||||
// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing
|
||||
// or older than staleAfter, and resets them to pending so they can be
|
||||
// re-claimed. Returns the number of tasks reaped.
|
||||
func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) {
|
||||
threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339)
|
||||
// Match tasks that are running AND (heartbeat_at is empty OR heartbeat_at < threshold).
|
||||
filter := fmt.Sprintf(`status="running"&&(heartbeat_at=""||heartbeat_at<"%s")`, threshold)
|
||||
resetPayload := map[string]any{
|
||||
"status": string(domain.TaskStatusPending),
|
||||
"worker_id": "",
|
||||
"heartbeat_at": "",
|
||||
}
|
||||
|
||||
total := 0
|
||||
for _, collection := range []string{"scraping_tasks", "audio_jobs"} {
|
||||
items, err := s.pb.listAll(ctx, collection, filter, "")
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err)
|
||||
}
|
||||
for _, raw := range items {
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &rec); err != nil || rec.ID == "" {
|
||||
continue
|
||||
}
|
||||
path := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID)
|
||||
if err := s.pb.patch(ctx, path, resetPayload); err != nil {
|
||||
s.log.Warn("ReapStaleTasks: patch failed", "collection", collection, "id", rec.ID, "err", err)
|
||||
continue
|
||||
}
|
||||
total++
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// ── taskqueue.Reader ──────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error) {
|
||||
items, err := s.pb.listAll(ctx, "scraping_tasks", "", "-started")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks := make([]domain.ScrapeTask, 0, len(items))
|
||||
for _, raw := range items {
|
||||
t, err := parseScrapeTask(raw)
|
||||
if err == nil {
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error) {
|
||||
var raw json.RawMessage
|
||||
if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), &raw); err != nil {
|
||||
if err == ErrNotFound {
|
||||
return domain.ScrapeTask{}, false, nil
|
||||
}
|
||||
return domain.ScrapeTask{}, false, err
|
||||
}
|
||||
t, err := parseScrapeTask(raw)
|
||||
return t, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error) {
|
||||
items, err := s.pb.listAll(ctx, "audio_jobs", "", "-started")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks := make([]domain.AudioTask, 0, len(items))
|
||||
for _, raw := range items {
|
||||
t, err := parseAudioTask(raw)
|
||||
if err == nil {
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) {
|
||||
filter := fmt.Sprintf(`cache_key=%q`, cacheKey)
|
||||
items, err := s.pb.listAll(ctx, "audio_jobs", filter, "-started")
|
||||
if err != nil || len(items) == 0 {
|
||||
return domain.AudioTask{}, false, err
|
||||
}
|
||||
t, err := parseAudioTask(items[0])
|
||||
return t, err == nil, err
|
||||
}
|
||||
|
||||
// ── Parsers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) {
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
TargetURL string `json:"target_url"`
|
||||
FromChapter int `json:"from_chapter"`
|
||||
ToChapter int `json:"to_chapter"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
Status string `json:"status"`
|
||||
BooksFound int `json:"books_found"`
|
||||
ChaptersScraped int `json:"chapters_scraped"`
|
||||
ChaptersSkipped int `json:"chapters_skipped"`
|
||||
Errors int `json:"errors"`
|
||||
Started string `json:"started"`
|
||||
Finished string `json:"finished"`
|
||||
ErrorMessage string `json:"error_message"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &rec); err != nil {
|
||||
return domain.ScrapeTask{}, err
|
||||
}
|
||||
started, _ := time.Parse(time.RFC3339, rec.Started)
|
||||
finished, _ := time.Parse(time.RFC3339, rec.Finished)
|
||||
return domain.ScrapeTask{
|
||||
ID: rec.ID,
|
||||
Kind: rec.Kind,
|
||||
TargetURL: rec.TargetURL,
|
||||
FromChapter: rec.FromChapter,
|
||||
ToChapter: rec.ToChapter,
|
||||
WorkerID: rec.WorkerID,
|
||||
Status: domain.TaskStatus(rec.Status),
|
||||
BooksFound: rec.BooksFound,
|
||||
ChaptersScraped: rec.ChaptersScraped,
|
||||
ChaptersSkipped: rec.ChaptersSkipped,
|
||||
Errors: rec.Errors,
|
||||
Started: started,
|
||||
Finished: finished,
|
||||
ErrorMessage: rec.ErrorMessage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
CacheKey string `json:"cache_key"`
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
Voice string `json:"voice"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
Status string `json:"status"`
|
||||
ErrorMessage string `json:"error_message"`
|
||||
Started string `json:"started"`
|
||||
Finished string `json:"finished"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &rec); err != nil {
|
||||
return domain.AudioTask{}, err
|
||||
}
|
||||
started, _ := time.Parse(time.RFC3339, rec.Started)
|
||||
finished, _ := time.Parse(time.RFC3339, rec.Finished)
|
||||
return domain.AudioTask{
|
||||
ID: rec.ID,
|
||||
CacheKey: rec.CacheKey,
|
||||
Slug: rec.Slug,
|
||||
Chapter: rec.Chapter,
|
||||
Voice: rec.Voice,
|
||||
WorkerID: rec.WorkerID,
|
||||
Status: domain.TaskStatus(rec.Status),
|
||||
ErrorMessage: rec.ErrorMessage,
|
||||
Started: started,
|
||||
Finished: finished,
|
||||
}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user