feat: add v2 stack (backend, runner, ui-v2) with release workflow
All checks were successful
Release / Scraper / Test (push) Successful in 10s
Release / UI / Build (push) Successful in 26s
Release / v2 / Build ui-v2 (push) Successful in 17s
Release / Scraper / Docker (push) Successful in 47s
Release / UI / Docker (push) Successful in 56s
CI / Scraper / Lint (pull_request) Successful in 7s
CI / Scraper / Test (pull_request) Successful in 8s
CI / UI / Build (pull_request) Successful in 16s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 56s
Release / v2 / Test backend (push) Successful in 4m35s
iOS CI / Build (pull_request) Successful in 4m28s
Release / v2 / Docker / backend (push) Successful in 1m29s
Release / v2 / Docker / runner (push) Successful in 1m39s
iOS CI / Test (pull_request) Successful in 9m51s

- backend/: Go API server and runner binaries with PocketBase + MinIO storage
- ui-v2/: SvelteKit frontend rewrite
- docker-compose-new.yml: compose file for the v2 stack
- .gitea/workflows/release-v2.yaml: CI/CD for backend, runner, and ui-v2 Docker Hub images
- scripts/pb-init.sh: migrate from wget to curl, add superuser bootstrap for fresh installs
- .env.example: document DOCKER_BUILDKIT=1 for Colima users
This commit is contained in:
Admin
2026-03-15 19:32:40 +05:00
parent 1642434a79
commit 5825b859b7
142 changed files with 22768 additions and 26 deletions

View File

@@ -0,0 +1,194 @@
package storage
import (
"context"
"fmt"
"io"
"net/url"
"path"
"strings"
"time"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/libnovel/backend/internal/config"
)
// minioClient wraps the official minio-go client with bucket names.
type minioClient struct {
client *minio.Client // internal — all read/write operations
pubClient *minio.Client // presign-only — initialised against the public endpoint
bucketChapters string
bucketAudio string
bucketAvatars string
}
func newMinioClient(cfg config.MinIO) (*minioClient, error) {
creds := credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, "")
internal, err := minio.New(cfg.Endpoint, &minio.Options{
Creds: creds,
Secure: cfg.UseSSL,
})
if err != nil {
return nil, fmt.Errorf("minio: init internal client: %w", err)
}
// Presigned URLs must be signed with the hostname the browser will use
// (PUBLIC_MINIO_PUBLIC_URL), because AWS Signature V4 includes the Host
// header in the canonical request — a URL signed against "minio:9000" will
// return SignatureDoesNotMatch when the browser fetches it from
// "localhost:9000".
//
// However, minio-go normally makes a live BucketLocation HTTP call before
// signing, which would fail from inside the container when the public
// endpoint is externally-facing (e.g. "localhost:9000" is unreachable from
// within Docker). We prevent this by:
// 1. Setting Region: "us-east-1" — minio-go skips getBucketLocation when
// the region is already known (bucket-cache.go:49).
// 2. Setting BucketLookup: BucketLookupPath — forces path-style URLs
// (e.g. host/bucket/key), matching MinIO's default behaviour and
// avoiding any virtual-host DNS probing.
//
// When no public endpoint is configured (or it equals the internal one),
// fall back to the internal client so presigning still works.
publicEndpoint := cfg.PublicEndpoint
if u, err2 := url.Parse(publicEndpoint); err2 == nil && u.Host != "" {
publicEndpoint = u.Host // strip scheme so minio.New is happy
}
pubUseSSL := cfg.PublicUseSSL
if publicEndpoint == "" || publicEndpoint == cfg.Endpoint {
publicEndpoint = cfg.Endpoint
pubUseSSL = cfg.UseSSL
}
pub, err := minio.New(publicEndpoint, &minio.Options{
Creds: creds,
Secure: pubUseSSL,
Region: "us-east-1", // skip live BucketLocation preflight
BucketLookup: minio.BucketLookupPath,
})
if err != nil {
return nil, fmt.Errorf("minio: init public client: %w", err)
}
return &minioClient{
client: internal,
pubClient: pub,
bucketChapters: cfg.BucketChapters,
bucketAudio: cfg.BucketAudio,
bucketAvatars: cfg.BucketAvatars,
}, nil
}
// ensureBuckets creates all required buckets if they don't already exist.
func (m *minioClient) ensureBuckets(ctx context.Context) error {
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars} {
exists, err := m.client.BucketExists(ctx, bucket)
if err != nil {
return fmt.Errorf("minio: check bucket %q: %w", bucket, err)
}
if !exists {
if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
return fmt.Errorf("minio: create bucket %q: %w", bucket, err)
}
}
}
return nil
}
// ── Key helpers ───────────────────────────────────────────────────────────────
// ChapterObjectKey returns the MinIO object key for a chapter markdown file.
// Format: {slug}/chapter-{n:06d}.md
func ChapterObjectKey(slug string, n int) string {
return fmt.Sprintf("%s/chapter-%06d.md", slug, n)
}
// AudioObjectKey returns the MinIO object key for a cached audio file.
// Format: {slug}/{n}/{voice}.mp3
func AudioObjectKey(slug string, n int, voice string) string {
return fmt.Sprintf("%s/%d/%s.mp3", slug, n, voice)
}
// AvatarObjectKey returns the MinIO object key for a user avatar image.
// Format: {userID}/{ext}.{ext}
func AvatarObjectKey(userID, ext string) string {
return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
}
// chapterNumberFromKey extracts the chapter number from a MinIO object key.
// e.g. "my-book/chapter-000042.md" → 42
func chapterNumberFromKey(key string) int {
base := path.Base(key)
base = strings.TrimPrefix(base, "chapter-")
base = strings.TrimSuffix(base, ".md")
var n int
fmt.Sscanf(base, "%d", &n)
return n
}
// ── Object operations ─────────────────────────────────────────────────────────
func (m *minioClient) putObject(ctx context.Context, bucket, key, contentType string, data []byte) error {
_, err := m.client.PutObject(ctx, bucket, key,
strings.NewReader(string(data)),
int64(len(data)),
minio.PutObjectOptions{ContentType: contentType},
)
return err
}
func (m *minioClient) getObject(ctx context.Context, bucket, key string) ([]byte, error) {
obj, err := m.client.GetObject(ctx, bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
defer obj.Close()
return io.ReadAll(obj)
}
func (m *minioClient) objectExists(ctx context.Context, bucket, key string) bool {
_, err := m.client.StatObject(ctx, bucket, key, minio.StatObjectOptions{})
return err == nil
}
func (m *minioClient) presignGet(ctx context.Context, bucket, key string, expires time.Duration) (string, error) {
u, err := m.pubClient.PresignedGetObject(ctx, bucket, key, expires, nil)
if err != nil {
return "", fmt.Errorf("minio presign %s/%s: %w", bucket, key, err)
}
return u.String(), nil
}
func (m *minioClient) presignPut(ctx context.Context, bucket, key string, expires time.Duration) (string, error) {
u, err := m.pubClient.PresignedPutObject(ctx, bucket, key, expires)
if err != nil {
return "", fmt.Errorf("minio presign PUT %s/%s: %w", bucket, key, err)
}
return u.String(), nil
}
func (m *minioClient) deleteObjects(ctx context.Context, bucket, prefix string) error {
objCh := m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix})
for obj := range objCh {
if obj.Err != nil {
return obj.Err
}
if err := m.client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}); err != nil {
return err
}
}
return nil
}
func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string) ([]string, error) {
var keys []string
for obj := range m.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix}) {
if obj.Err != nil {
return nil, obj.Err
}
keys = append(keys, obj.Key)
}
return keys, nil
}

View File

@@ -0,0 +1,268 @@
// Package storage provides the concrete implementations of all bookstore and
// taskqueue interfaces backed by PocketBase (structured data) and MinIO (blobs).
//
// Entry point: NewStore(ctx, cfg, log) returns a *Store that satisfies every
// interface defined in bookstore and taskqueue.
package storage
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/libnovel/backend/internal/config"
"github.com/libnovel/backend/internal/domain"
)
// ErrNotFound is returned by single-record lookups when no record exists.
var ErrNotFound = errors.New("storage: record not found")
// pbClient is the internal PocketBase REST admin client.
type pbClient struct {
baseURL string
email string
password string
log *slog.Logger
mu sync.Mutex
token string
exp time.Time
}
func newPBClient(cfg config.PocketBase, log *slog.Logger) *pbClient {
return &pbClient{
baseURL: strings.TrimRight(cfg.URL, "/"),
email: cfg.AdminEmail,
password: cfg.AdminPassword,
log: log,
}
}
// authToken returns a valid admin auth token, refreshing it when expired.
func (c *pbClient) authToken(ctx context.Context) (string, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.token != "" && time.Now().Before(c.exp) {
return c.token, nil
}
body, _ := json.Marshal(map[string]string{
"identity": c.email,
"password": c.password,
})
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
c.baseURL+"/api/collections/_superusers/auth-with-password", bytes.NewReader(body))
if err != nil {
return "", fmt.Errorf("pb auth: build request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", fmt.Errorf("pb auth: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
raw, _ := io.ReadAll(resp.Body)
return "", fmt.Errorf("pb auth: status %d: %s", resp.StatusCode, string(raw))
}
var payload struct {
Token string `json:"token"`
}
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
return "", fmt.Errorf("pb auth: decode: %w", err)
}
c.token = payload.Token
c.exp = time.Now().Add(30 * time.Minute)
return c.token, nil
}
// do executes an authenticated PocketBase REST request.
func (c *pbClient) do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
tok, err := c.authToken(ctx)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
if err != nil {
return nil, fmt.Errorf("pb: build request %s %s: %w", method, path, err)
}
req.Header.Set("Authorization", tok)
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("pb: %s %s: %w", method, path, err)
}
return resp, nil
}
// get is a convenience wrapper that decodes a JSON response into v.
func (c *pbClient) get(ctx context.Context, path string, v any) error {
resp, err := c.do(ctx, http.MethodGet, path, nil)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return ErrNotFound
}
if resp.StatusCode >= 400 {
raw, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pb GET %s: status %d: %s", path, resp.StatusCode, string(raw))
}
return json.NewDecoder(resp.Body).Decode(v)
}
// post creates a record and decodes the created record into v.
func (c *pbClient) post(ctx context.Context, path string, payload, v any) error {
b, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("pb: marshal: %w", err)
}
resp, err := c.do(ctx, http.MethodPost, path, bytes.NewReader(b))
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
raw, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pb POST %s: status %d: %s", path, resp.StatusCode, string(raw))
}
if v != nil {
return json.NewDecoder(resp.Body).Decode(v)
}
return nil
}
// patch updates a record.
func (c *pbClient) patch(ctx context.Context, path string, payload any) error {
b, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("pb: marshal: %w", err)
}
resp, err := c.do(ctx, http.MethodPatch, path, bytes.NewReader(b))
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
raw, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pb PATCH %s: status %d: %s", path, resp.StatusCode, string(raw))
}
return nil
}
// delete removes a record.
func (c *pbClient) delete(ctx context.Context, path string) error {
resp, err := c.do(ctx, http.MethodDelete, path, nil)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
return ErrNotFound
}
if resp.StatusCode >= 400 {
raw, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pb DELETE %s: status %d: %s", path, resp.StatusCode, string(raw))
}
return nil
}
// listAll fetches all pages of a collection. PocketBase returns at most 200
// records per page; we paginate until empty.
func (c *pbClient) listAll(ctx context.Context, collection string, filter, sort string) ([]json.RawMessage, error) {
var all []json.RawMessage
page := 1
for {
q := url.Values{
"page": {fmt.Sprintf("%d", page)},
"perPage": {"200"},
}
if filter != "" {
q.Set("filter", filter)
}
if sort != "" {
q.Set("sort", sort)
}
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
var result struct {
Items []json.RawMessage `json:"items"`
Page int `json:"page"`
Pages int `json:"totalPages"`
}
if err := c.get(ctx, path, &result); err != nil {
return nil, err
}
all = append(all, result.Items...)
if result.Page >= result.Pages {
break
}
page++
}
return all, nil
}
// claimRecord atomically claims the first pending record matching collection.
// It fetches the oldest pending record (filter + sort), then PATCHes it with
// the claim payload. Returns (nil, nil) when the queue is empty.
func (c *pbClient) claimRecord(ctx context.Context, collection, workerID string, extraClaim map[string]any) (json.RawMessage, error) {
q := url.Values{}
q.Set("filter", `status="pending"`)
q.Set("sort", "+started")
q.Set("perPage", "1")
path := fmt.Sprintf("/api/collections/%s/records?%s", collection, q.Encode())
var result struct {
Items []json.RawMessage `json:"items"`
}
if err := c.get(ctx, path, &result); err != nil {
return nil, fmt.Errorf("claimRecord list: %w", err)
}
if len(result.Items) == 0 {
return nil, nil // queue empty
}
var rec struct {
ID string `json:"id"`
}
if err := json.Unmarshal(result.Items[0], &rec); err != nil {
return nil, fmt.Errorf("claimRecord parse id: %w", err)
}
claim := map[string]any{
"status": string(domain.TaskStatusRunning),
"worker_id": workerID,
}
for k, v := range extraClaim {
claim[k] = v
}
claimPath := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID)
if err := c.patch(ctx, claimPath, claim); err != nil {
return nil, fmt.Errorf("claimRecord patch: %w", err)
}
// Re-fetch the updated record so caller has current state.
var updated json.RawMessage
if err := c.get(ctx, claimPath, &updated); err != nil {
return nil, fmt.Errorf("claimRecord re-fetch: %w", err)
}
return updated, nil
}

View File

@@ -0,0 +1,769 @@
package storage
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"strings"
"time"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/config"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/taskqueue"
)
// Store is the unified persistence implementation that satisfies all bookstore
// and taskqueue interfaces. It routes structured data to PocketBase and binary
// blobs to MinIO.
type Store struct {
pb *pbClient
mc *minioClient
log *slog.Logger
}
// NewStore initialises PocketBase and MinIO connections and ensures all MinIO
// buckets exist. Returns a ready-to-use Store.
func NewStore(ctx context.Context, cfg config.Config, log *slog.Logger) (*Store, error) {
pb := newPBClient(cfg.PocketBase, log)
// Validate PocketBase connectivity by fetching an auth token.
if _, err := pb.authToken(ctx); err != nil {
return nil, fmt.Errorf("pocketbase: %w", err)
}
mc, err := newMinioClient(cfg.MinIO)
if err != nil {
return nil, fmt.Errorf("minio: %w", err)
}
if err := mc.ensureBuckets(ctx); err != nil {
return nil, fmt.Errorf("minio: ensure buckets: %w", err)
}
return &Store{pb: pb, mc: mc, log: log}, nil
}
// Compile-time interface satisfaction.
var _ bookstore.BookWriter = (*Store)(nil)
var _ bookstore.BookReader = (*Store)(nil)
var _ bookstore.RankingStore = (*Store)(nil)
var _ bookstore.AudioStore = (*Store)(nil)
var _ bookstore.PresignStore = (*Store)(nil)
var _ bookstore.ProgressStore = (*Store)(nil)
var _ taskqueue.Producer = (*Store)(nil)
var _ taskqueue.Consumer = (*Store)(nil)
var _ taskqueue.Reader = (*Store)(nil)
// ── BookWriter ────────────────────────────────────────────────────────────────
func (s *Store) WriteMetadata(ctx context.Context, meta domain.BookMeta) error {
payload := map[string]any{
"slug": meta.Slug,
"title": meta.Title,
"author": meta.Author,
"cover": meta.Cover,
"status": meta.Status,
"genres": meta.Genres,
"summary": meta.Summary,
"total_chapters": meta.TotalChapters,
"source_url": meta.SourceURL,
"ranking": meta.Ranking,
}
// Upsert via filter: if exists PATCH, otherwise POST.
existing, err := s.getBookBySlug(ctx, meta.Slug)
if err != nil && err != ErrNotFound {
return fmt.Errorf("WriteMetadata: %w", err)
}
if err == ErrNotFound {
return s.pb.post(ctx, "/api/collections/books/records", payload, nil)
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/books/records/%s", existing.ID), payload)
}
func (s *Store) WriteChapter(ctx context.Context, slug string, chapter domain.Chapter) error {
key := ChapterObjectKey(slug, chapter.Ref.Number)
if err := s.mc.putObject(ctx, s.mc.bucketChapters, key, "text/markdown", []byte(chapter.Text)); err != nil {
return fmt.Errorf("WriteChapter: minio: %w", err)
}
// Upsert the chapters_idx record in PocketBase.
return s.upsertChapterIdx(ctx, slug, chapter.Ref)
}
func (s *Store) WriteChapterRefs(ctx context.Context, slug string, refs []domain.ChapterRef) error {
for _, ref := range refs {
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
s.log.Warn("WriteChapterRefs: upsert failed", "slug", slug, "chapter", ref.Number, "err", err)
}
}
return nil
}
func (s *Store) ChapterExists(ctx context.Context, slug string, ref domain.ChapterRef) bool {
return s.mc.objectExists(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, ref.Number))
}
func (s *Store) upsertChapterIdx(ctx context.Context, slug string, ref domain.ChapterRef) error {
payload := map[string]any{
"slug": slug,
"number": ref.Number,
"title": ref.Title,
}
filter := fmt.Sprintf(`slug=%q&&number=%d`, slug, ref.Number)
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "")
if err != nil && err != ErrNotFound {
return err
}
if len(items) == 0 {
return s.pb.post(ctx, "/api/collections/chapters_idx/records", payload, nil)
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/chapters_idx/records/%s", rec.ID), payload)
}
// ── BookReader ────────────────────────────────────────────────────────────────
type pbBook struct {
ID string `json:"id"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author"`
Cover string `json:"cover"`
Status string `json:"status"`
Genres []string `json:"genres"`
Summary string `json:"summary"`
TotalChapters int `json:"total_chapters"`
SourceURL string `json:"source_url"`
Ranking int `json:"ranking"`
Updated string `json:"updated"`
}
func (b pbBook) toDomain() domain.BookMeta {
return domain.BookMeta{
Slug: b.Slug,
Title: b.Title,
Author: b.Author,
Cover: b.Cover,
Status: b.Status,
Genres: b.Genres,
Summary: b.Summary,
TotalChapters: b.TotalChapters,
SourceURL: b.SourceURL,
Ranking: b.Ranking,
}
}
func (s *Store) getBookBySlug(ctx context.Context, slug string) (pbBook, error) {
filter := fmt.Sprintf(`slug=%q`, slug)
items, err := s.pb.listAll(ctx, "books", filter, "")
if err != nil {
return pbBook{}, err
}
if len(items) == 0 {
return pbBook{}, ErrNotFound
}
var b pbBook
json.Unmarshal(items[0], &b)
return b, nil
}
func (s *Store) ReadMetadata(ctx context.Context, slug string) (domain.BookMeta, bool, error) {
b, err := s.getBookBySlug(ctx, slug)
if err == ErrNotFound {
return domain.BookMeta{}, false, nil
}
if err != nil {
return domain.BookMeta{}, false, err
}
return b.toDomain(), true, nil
}
func (s *Store) ListBooks(ctx context.Context) ([]domain.BookMeta, error) {
items, err := s.pb.listAll(ctx, "books", "", "title")
if err != nil {
return nil, err
}
books := make([]domain.BookMeta, 0, len(items))
for _, raw := range items {
var b pbBook
json.Unmarshal(raw, &b)
books = append(books, b.toDomain())
}
return books, nil
}
func (s *Store) LocalSlugs(ctx context.Context) (map[string]bool, error) {
items, err := s.pb.listAll(ctx, "books", "", "")
if err != nil {
return nil, err
}
slugs := make(map[string]bool, len(items))
for _, raw := range items {
var b struct {
Slug string `json:"slug"`
}
json.Unmarshal(raw, &b)
if b.Slug != "" {
slugs[b.Slug] = true
}
}
return slugs, nil
}
func (s *Store) MetadataMtime(ctx context.Context, slug string) int64 {
b, err := s.getBookBySlug(ctx, slug)
if err != nil {
return 0
}
t, err := time.Parse(time.RFC3339, b.Updated)
if err != nil {
return 0
}
return t.Unix()
}
func (s *Store) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
data, err := s.mc.getObject(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n))
if err != nil {
return "", fmt.Errorf("ReadChapter: %w", err)
}
return string(data), nil
}
func (s *Store) ListChapters(ctx context.Context, slug string) ([]domain.ChapterInfo, error) {
filter := fmt.Sprintf(`slug=%q`, slug)
items, err := s.pb.listAll(ctx, "chapters_idx", filter, "number")
if err != nil {
return nil, err
}
chapters := make([]domain.ChapterInfo, 0, len(items))
for _, raw := range items {
var rec struct {
Number int `json:"number"`
Title string `json:"title"`
}
json.Unmarshal(raw, &rec)
chapters = append(chapters, domain.ChapterInfo{Number: rec.Number, Title: rec.Title})
}
return chapters, nil
}
func (s *Store) CountChapters(ctx context.Context, slug string) int {
chapters, err := s.ListChapters(ctx, slug)
if err != nil {
return 0
}
return len(chapters)
}
func (s *Store) ReindexChapters(ctx context.Context, slug string) (int, error) {
keys, err := s.mc.listObjectKeys(ctx, s.mc.bucketChapters, slug+"/")
if err != nil {
return 0, fmt.Errorf("ReindexChapters: list objects: %w", err)
}
count := 0
for _, key := range keys {
if !strings.HasSuffix(key, ".md") {
continue
}
n := chapterNumberFromKey(key)
if n == 0 {
continue
}
ref := domain.ChapterRef{Number: n}
if err := s.upsertChapterIdx(ctx, slug, ref); err != nil {
s.log.Warn("ReindexChapters: upsert failed", "key", key, "err", err)
continue
}
count++
}
return count, nil
}
// ── RankingStore ──────────────────────────────────────────────────────────────
func (s *Store) WriteRankingItem(ctx context.Context, item domain.RankingItem) error {
payload := map[string]any{
"rank": item.Rank,
"slug": item.Slug,
"title": item.Title,
"author": item.Author,
"cover": item.Cover,
"status": item.Status,
"genres": item.Genres,
"source_url": item.SourceURL,
}
filter := fmt.Sprintf(`slug=%q`, item.Slug)
items, err := s.pb.listAll(ctx, "ranking", filter, "")
if err != nil && err != ErrNotFound {
return err
}
if len(items) == 0 {
return s.pb.post(ctx, "/api/collections/ranking/records", payload, nil)
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/ranking/records/%s", rec.ID), payload)
}
func (s *Store) ReadRankingItems(ctx context.Context) ([]domain.RankingItem, error) {
items, err := s.pb.listAll(ctx, "ranking", "", "rank")
if err != nil {
return nil, err
}
result := make([]domain.RankingItem, 0, len(items))
for _, raw := range items {
var rec struct {
Rank int `json:"rank"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author"`
Cover string `json:"cover"`
Status string `json:"status"`
Genres []string `json:"genres"`
SourceURL string `json:"source_url"`
Updated string `json:"updated"`
}
json.Unmarshal(raw, &rec)
t, _ := time.Parse(time.RFC3339, rec.Updated)
result = append(result, domain.RankingItem{
Rank: rec.Rank,
Slug: rec.Slug,
Title: rec.Title,
Author: rec.Author,
Cover: rec.Cover,
Status: rec.Status,
Genres: rec.Genres,
SourceURL: rec.SourceURL,
Updated: t,
})
}
return result, nil
}
func (s *Store) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) {
items, err := s.ReadRankingItems(ctx)
if err != nil || len(items) == 0 {
return false, err
}
var latest time.Time
for _, item := range items {
if item.Updated.After(latest) {
latest = item.Updated
}
}
return time.Since(latest) < maxAge, nil
}
// ── AudioStore ────────────────────────────────────────────────────────────────
func (s *Store) AudioObjectKey(slug string, n int, voice string) string {
return AudioObjectKey(slug, n, voice)
}
func (s *Store) AudioExists(ctx context.Context, key string) bool {
return s.mc.objectExists(ctx, s.mc.bucketAudio, key)
}
func (s *Store) PutAudio(ctx context.Context, key string, data []byte) error {
return s.mc.putObject(ctx, s.mc.bucketAudio, key, "audio/mpeg", data)
}
// ── PresignStore ──────────────────────────────────────────────────────────────
func (s *Store) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) {
return s.mc.presignGet(ctx, s.mc.bucketChapters, ChapterObjectKey(slug, n), expires)
}
func (s *Store) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) {
return s.mc.presignGet(ctx, s.mc.bucketAudio, key, expires)
}
func (s *Store) PresignAvatarUpload(ctx context.Context, userID, ext string) (uploadURL, key string, err error) {
key = AvatarObjectKey(userID, ext)
uploadURL, err = s.mc.presignPut(ctx, s.mc.bucketAvatars, key, 15*time.Minute)
return
}
func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) {
for _, ext := range []string{"jpg", "png", "webp"} {
key := AvatarObjectKey(userID, ext)
if s.mc.objectExists(ctx, s.mc.bucketAvatars, key) {
u, err := s.mc.presignGet(ctx, s.mc.bucketAvatars, key, 1*time.Hour)
return u, true, err
}
}
return "", false, nil
}
func (s *Store) DeleteAvatar(ctx context.Context, userID string) error {
return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
}
// ── ProgressStore ─────────────────────────────────────────────────────────────
func (s *Store) GetProgress(ctx context.Context, sessionID, slug string) (domain.ReadingProgress, bool) {
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
items, err := s.pb.listAll(ctx, "progress", filter, "")
if err != nil || len(items) == 0 {
return domain.ReadingProgress{}, false
}
var rec struct {
Slug string `json:"slug"`
Chapter int `json:"chapter"`
UpdatedAt string `json:"updated"`
}
json.Unmarshal(items[0], &rec)
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
return domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t}, true
}
func (s *Store) SetProgress(ctx context.Context, sessionID string, p domain.ReadingProgress) error {
payload := map[string]any{
"session_id": sessionID,
"slug": p.Slug,
"chapter": p.Chapter,
}
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, p.Slug)
items, err := s.pb.listAll(ctx, "progress", filter, "")
if err != nil && err != ErrNotFound {
return err
}
if len(items) == 0 {
return s.pb.post(ctx, "/api/collections/progress/records", payload, nil)
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID), payload)
}
func (s *Store) AllProgress(ctx context.Context, sessionID string) ([]domain.ReadingProgress, error) {
filter := fmt.Sprintf(`session_id=%q`, sessionID)
items, err := s.pb.listAll(ctx, "progress", filter, "-updated")
if err != nil {
return nil, err
}
result := make([]domain.ReadingProgress, 0, len(items))
for _, raw := range items {
var rec struct {
Slug string `json:"slug"`
Chapter int `json:"chapter"`
UpdatedAt string `json:"updated"`
}
json.Unmarshal(raw, &rec)
t, _ := time.Parse(time.RFC3339, rec.UpdatedAt)
result = append(result, domain.ReadingProgress{Slug: rec.Slug, Chapter: rec.Chapter, UpdatedAt: t})
}
return result, nil
}
func (s *Store) DeleteProgress(ctx context.Context, sessionID, slug string) error {
filter := fmt.Sprintf(`session_id=%q&&slug=%q`, sessionID, slug)
items, err := s.pb.listAll(ctx, "progress", filter, "")
if err != nil || len(items) == 0 {
return nil
}
var rec struct {
ID string `json:"id"`
}
json.Unmarshal(items[0], &rec)
return s.pb.delete(ctx, fmt.Sprintf("/api/collections/progress/records/%s", rec.ID))
}
// ── taskqueue.Producer ────────────────────────────────────────────────────────
func (s *Store) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) {
payload := map[string]any{
"kind": kind,
"target_url": targetURL,
"from_chapter": fromChapter,
"to_chapter": toChapter,
"status": string(domain.TaskStatusPending),
"started": time.Now().UTC().Format(time.RFC3339),
}
var rec struct {
ID string `json:"id"`
}
if err := s.pb.post(ctx, "/api/collections/scraping_tasks/records", payload, &rec); err != nil {
return "", err
}
return rec.ID, nil
}
func (s *Store) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) {
cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, voice)
payload := map[string]any{
"cache_key": cacheKey,
"slug": slug,
"chapter": chapter,
"voice": voice,
"status": string(domain.TaskStatusPending),
"started": time.Now().UTC().Format(time.RFC3339),
}
var rec struct {
ID string `json:"id"`
}
if err := s.pb.post(ctx, "/api/collections/audio_jobs/records", payload, &rec); err != nil {
return "", err
}
return rec.ID, nil
}
func (s *Store) CancelTask(ctx context.Context, id string) error {
// Try scraping_tasks first, then audio_jobs.
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),
map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id),
map[string]string{"status": string(domain.TaskStatusCancelled)})
}
// ── taskqueue.Consumer ────────────────────────────────────────────────────────
func (s *Store) ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error) {
raw, err := s.pb.claimRecord(ctx, "scraping_tasks", workerID, nil)
if err != nil {
return domain.ScrapeTask{}, false, err
}
if raw == nil {
return domain.ScrapeTask{}, false, nil
}
task, err := parseScrapeTask(raw)
return task, err == nil, err
}
func (s *Store) ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error) {
raw, err := s.pb.claimRecord(ctx, "audio_jobs", workerID, nil)
if err != nil {
return domain.AudioTask{}, false, err
}
if raw == nil {
return domain.AudioTask{}, false, nil
}
task, err := parseAudioTask(raw)
return task, err == nil, err
}
func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
status := string(domain.TaskStatusDone)
if result.ErrorMessage != "" {
status = string(domain.TaskStatusFailed)
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), map[string]any{
"status": status,
"books_found": result.BooksFound,
"chapters_scraped": result.ChaptersScraped,
"chapters_skipped": result.ChaptersSkipped,
"errors": result.Errors,
"error_message": result.ErrorMessage,
"finished": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *Store) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error {
status := string(domain.TaskStatusDone)
if result.ErrorMessage != "" {
status = string(domain.TaskStatusFailed)
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), map[string]any{
"status": status,
"error_message": result.ErrorMessage,
"finished": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
payload := map[string]any{
"status": string(domain.TaskStatusFailed),
"error_message": errMsg,
"finished": time.Now().UTC().Format(time.RFC3339),
}
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
}
// HeartbeatTask updates the heartbeat_at field on a running task.
// Tries scraping_tasks first, then audio_jobs (same pattern as FailTask).
func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
payload := map[string]any{
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
}
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
}
// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing
// or older than staleAfter, and resets them to pending so they can be
// re-claimed. Returns the number of tasks reaped.
func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) {
threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339)
// Match tasks that are running AND (heartbeat_at is empty OR heartbeat_at < threshold).
filter := fmt.Sprintf(`status="running"&&(heartbeat_at=""||heartbeat_at<"%s")`, threshold)
resetPayload := map[string]any{
"status": string(domain.TaskStatusPending),
"worker_id": "",
"heartbeat_at": "",
}
total := 0
for _, collection := range []string{"scraping_tasks", "audio_jobs"} {
items, err := s.pb.listAll(ctx, collection, filter, "")
if err != nil {
return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err)
}
for _, raw := range items {
var rec struct {
ID string `json:"id"`
}
if err := json.Unmarshal(raw, &rec); err != nil || rec.ID == "" {
continue
}
path := fmt.Sprintf("/api/collections/%s/records/%s", collection, rec.ID)
if err := s.pb.patch(ctx, path, resetPayload); err != nil {
s.log.Warn("ReapStaleTasks: patch failed", "collection", collection, "id", rec.ID, "err", err)
continue
}
total++
}
}
return total, nil
}
// ── taskqueue.Reader ──────────────────────────────────────────────────────────
func (s *Store) ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error) {
items, err := s.pb.listAll(ctx, "scraping_tasks", "", "-started")
if err != nil {
return nil, err
}
tasks := make([]domain.ScrapeTask, 0, len(items))
for _, raw := range items {
t, err := parseScrapeTask(raw)
if err == nil {
tasks = append(tasks, t)
}
}
return tasks, nil
}
func (s *Store) GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error) {
var raw json.RawMessage
if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), &raw); err != nil {
if err == ErrNotFound {
return domain.ScrapeTask{}, false, nil
}
return domain.ScrapeTask{}, false, err
}
t, err := parseScrapeTask(raw)
return t, err == nil, err
}
func (s *Store) ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error) {
items, err := s.pb.listAll(ctx, "audio_jobs", "", "-started")
if err != nil {
return nil, err
}
tasks := make([]domain.AudioTask, 0, len(items))
for _, raw := range items {
t, err := parseAudioTask(raw)
if err == nil {
tasks = append(tasks, t)
}
}
return tasks, nil
}
func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error) {
filter := fmt.Sprintf(`cache_key=%q`, cacheKey)
items, err := s.pb.listAll(ctx, "audio_jobs", filter, "-started")
if err != nil || len(items) == 0 {
return domain.AudioTask{}, false, err
}
t, err := parseAudioTask(items[0])
return t, err == nil, err
}
// ── Parsers ───────────────────────────────────────────────────────────────────
func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) {
var rec struct {
ID string `json:"id"`
Kind string `json:"kind"`
TargetURL string `json:"target_url"`
FromChapter int `json:"from_chapter"`
ToChapter int `json:"to_chapter"`
WorkerID string `json:"worker_id"`
Status string `json:"status"`
BooksFound int `json:"books_found"`
ChaptersScraped int `json:"chapters_scraped"`
ChaptersSkipped int `json:"chapters_skipped"`
Errors int `json:"errors"`
Started string `json:"started"`
Finished string `json:"finished"`
ErrorMessage string `json:"error_message"`
}
if err := json.Unmarshal(raw, &rec); err != nil {
return domain.ScrapeTask{}, err
}
started, _ := time.Parse(time.RFC3339, rec.Started)
finished, _ := time.Parse(time.RFC3339, rec.Finished)
return domain.ScrapeTask{
ID: rec.ID,
Kind: rec.Kind,
TargetURL: rec.TargetURL,
FromChapter: rec.FromChapter,
ToChapter: rec.ToChapter,
WorkerID: rec.WorkerID,
Status: domain.TaskStatus(rec.Status),
BooksFound: rec.BooksFound,
ChaptersScraped: rec.ChaptersScraped,
ChaptersSkipped: rec.ChaptersSkipped,
Errors: rec.Errors,
Started: started,
Finished: finished,
ErrorMessage: rec.ErrorMessage,
}, nil
}
func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
var rec struct {
ID string `json:"id"`
CacheKey string `json:"cache_key"`
Slug string `json:"slug"`
Chapter int `json:"chapter"`
Voice string `json:"voice"`
WorkerID string `json:"worker_id"`
Status string `json:"status"`
ErrorMessage string `json:"error_message"`
Started string `json:"started"`
Finished string `json:"finished"`
}
if err := json.Unmarshal(raw, &rec); err != nil {
return domain.AudioTask{}, err
}
started, _ := time.Parse(time.RFC3339, rec.Started)
finished, _ := time.Parse(time.RFC3339, rec.Finished)
return domain.AudioTask{
ID: rec.ID,
CacheKey: rec.CacheKey,
Slug: rec.Slug,
Chapter: rec.Chapter,
Voice: rec.Voice,
WorkerID: rec.WorkerID,
Status: domain.TaskStatus(rec.Status),
ErrorMessage: rec.ErrorMessage,
Started: started,
Finished: finished,
}, nil
}