feat: add v2 stack (backend, runner, ui-v2) with release workflow
All checks were successful
Release / Scraper / Test (push) Successful in 10s
Release / UI / Build (push) Successful in 26s
Release / v2 / Build ui-v2 (push) Successful in 17s
Release / Scraper / Docker (push) Successful in 47s
Release / UI / Docker (push) Successful in 56s
CI / Scraper / Lint (pull_request) Successful in 7s
CI / Scraper / Test (pull_request) Successful in 8s
CI / UI / Build (pull_request) Successful in 16s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 56s
Release / v2 / Test backend (push) Successful in 4m35s
iOS CI / Build (pull_request) Successful in 4m28s
Release / v2 / Docker / backend (push) Successful in 1m29s
Release / v2 / Docker / runner (push) Successful in 1m39s
iOS CI / Test (pull_request) Successful in 9m51s

- backend/: Go API server and runner binaries with PocketBase + MinIO storage
- ui-v2/: SvelteKit frontend rewrite
- docker-compose-new.yml: compose file for the v2 stack
- .gitea/workflows/release-v2.yaml: CI/CD for backend, runner, and ui-v2 Docker Hub images
- scripts/pb-init.sh: migrate from wget to curl, add superuser bootstrap for fresh installs
- .env.example: document DOCKER_BUILDKIT=1 for Colima users
This commit is contained in:
Admin
2026-03-15 19:32:40 +05:00
parent 1642434a79
commit 5825b859b7
142 changed files with 22768 additions and 26 deletions

View File

@@ -0,0 +1,84 @@
// Package taskqueue defines the interfaces for creating and consuming
// scrape/audio tasks stored in PocketBase.
//
// Interface segregation:
// - Producer is used only by the backend (creates tasks, cancels tasks).
// - Consumer is used only by the runner (claims tasks, reports results).
// - Reader is used by the backend for status/history endpoints.
//
// Concrete implementations live in internal/storage.
package taskqueue
import (
"context"
"time"
"github.com/libnovel/backend/internal/domain"
)
// Producer is the write side of the task queue used by the backend service.
// It creates new tasks in PocketBase for the runner to pick up.
type Producer interface {
// CreateScrapeTask inserts a new scrape task with status=pending and
// returns the assigned PocketBase record ID.
// kind is one of "catalogue", "book", or "book_range".
// targetURL is the book URL (empty for catalogue-wide tasks).
CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error)
// CreateAudioTask inserts a new audio task with status=pending and
// returns the assigned PocketBase record ID.
CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error)
// CancelTask transitions a pending task to status=cancelled.
// Returns ErrNotFound if the task does not exist.
CancelTask(ctx context.Context, id string) error
}
// Consumer is the read/claim side of the task queue used by the runner.
type Consumer interface {
// ClaimNextScrapeTask atomically finds the oldest pending scrape task,
// sets its status=running and worker_id=workerID, and returns it.
// Returns (zero, false, nil) when the queue is empty.
ClaimNextScrapeTask(ctx context.Context, workerID string) (domain.ScrapeTask, bool, error)
// ClaimNextAudioTask atomically finds the oldest pending audio task,
// sets its status=running and worker_id=workerID, and returns it.
// Returns (zero, false, nil) when the queue is empty.
ClaimNextAudioTask(ctx context.Context, workerID string) (domain.AudioTask, bool, error)
// FinishScrapeTask marks a running scrape task as done and records the result.
FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error
// FinishAudioTask marks a running audio task as done and records the result.
FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error
// FailTask marks a task (scrape or audio) as failed with an error message.
FailTask(ctx context.Context, id, errMsg string) error
// HeartbeatTask updates the heartbeat_at timestamp on a running task.
// Should be called periodically by the runner while the task is active so
// the reaper knows the task is still alive.
HeartbeatTask(ctx context.Context, id string) error
// ReapStaleTasks finds all running tasks whose heartbeat_at is older than
// staleAfter (or was never set) and resets them to pending so they can be
// re-claimed by a healthy runner. Returns the number of tasks reaped.
ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error)
}
// Reader is the read-only side used by the backend for status pages.
type Reader interface {
// ListScrapeTasks returns all scrape tasks sorted by started descending.
ListScrapeTasks(ctx context.Context) ([]domain.ScrapeTask, error)
// GetScrapeTask returns a single scrape task by ID.
// Returns (zero, false, nil) if not found.
GetScrapeTask(ctx context.Context, id string) (domain.ScrapeTask, bool, error)
// ListAudioTasks returns all audio tasks sorted by started descending.
ListAudioTasks(ctx context.Context) ([]domain.AudioTask, error)
// GetAudioTask returns the most recent audio task for cacheKey.
// Returns (zero, false, nil) if not found.
GetAudioTask(ctx context.Context, cacheKey string) (domain.AudioTask, bool, error)
}

View File

@@ -0,0 +1,138 @@
package taskqueue_test
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/taskqueue"
)
// ── Compile-time interface satisfaction ───────────────────────────────────────
// stubStore satisfies all three taskqueue interfaces.
// Any method that is called but not expected panics — making accidental
// calls immediately visible in tests.
type stubStore struct{}
func (s *stubStore) CreateScrapeTask(_ context.Context, _, _ string, _, _ int) (string, error) {
return "task-1", nil
}
func (s *stubStore) CreateAudioTask(_ context.Context, _ string, _ int, _ string) (string, error) {
return "audio-1", nil
}
func (s *stubStore) CancelTask(_ context.Context, _ string) error { return nil }
func (s *stubStore) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
return domain.ScrapeTask{ID: "task-1", Status: domain.TaskStatusRunning}, true, nil
}
func (s *stubStore) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
return domain.AudioTask{ID: "audio-1", Status: domain.TaskStatusRunning}, true, nil
}
func (s *stubStore) FinishScrapeTask(_ context.Context, _ string, _ domain.ScrapeResult) error {
return nil
}
func (s *stubStore) FinishAudioTask(_ context.Context, _ string, _ domain.AudioResult) error {
return nil
}
func (s *stubStore) FailTask(_ context.Context, _, _ string) error { return nil }
func (s *stubStore) HeartbeatTask(_ context.Context, _ string) error { return nil }
func (s *stubStore) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) {
return 0, nil
}
func (s *stubStore) ListScrapeTasks(_ context.Context) ([]domain.ScrapeTask, error) { return nil, nil }
func (s *stubStore) GetScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
return domain.ScrapeTask{}, false, nil
}
func (s *stubStore) ListAudioTasks(_ context.Context) ([]domain.AudioTask, error) { return nil, nil }
func (s *stubStore) GetAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
return domain.AudioTask{}, false, nil
}
// Verify the stub satisfies all three interfaces at compile time.
var _ taskqueue.Producer = (*stubStore)(nil)
var _ taskqueue.Consumer = (*stubStore)(nil)
var _ taskqueue.Reader = (*stubStore)(nil)
// ── Behavioural tests (using stub) ────────────────────────────────────────────
func TestProducer_CreateScrapeTask(t *testing.T) {
var p taskqueue.Producer = &stubStore{}
id, err := p.CreateScrapeTask(context.Background(), "book", "https://example.com/book/slug", 0, 0)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if id == "" {
t.Error("expected non-empty task ID")
}
}
func TestConsumer_ClaimNextScrapeTask(t *testing.T) {
var c taskqueue.Consumer = &stubStore{}
task, ok, err := c.ClaimNextScrapeTask(context.Background(), "worker-1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !ok {
t.Fatal("expected a task to be claimed")
}
if task.Status != domain.TaskStatusRunning {
t.Errorf("want running, got %q", task.Status)
}
}
func TestConsumer_ClaimNextAudioTask(t *testing.T) {
var c taskqueue.Consumer = &stubStore{}
task, ok, err := c.ClaimNextAudioTask(context.Background(), "worker-1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !ok {
t.Fatal("expected an audio task to be claimed")
}
if task.ID == "" {
t.Error("expected non-empty task ID")
}
}
// ── domain.ScrapeResult / domain.AudioResult JSON shape ──────────────────────
func TestScrapeResult_JSONRoundtrip(t *testing.T) {
cases := []domain.ScrapeResult{
{BooksFound: 5, ChaptersScraped: 100, ChaptersSkipped: 2, Errors: 0},
{BooksFound: 0, ChaptersScraped: 0, Errors: 1, ErrorMessage: "timeout"},
}
for _, orig := range cases {
b, err := json.Marshal(orig)
if err != nil {
t.Fatalf("marshal: %v", err)
}
var got domain.ScrapeResult
if err := json.Unmarshal(b, &got); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if got != orig {
t.Errorf("want %+v, got %+v", orig, got)
}
}
}
func TestAudioResult_JSONRoundtrip(t *testing.T) {
cases := []domain.AudioResult{
{ObjectKey: "audio/slug/1/af_bella.mp3"},
{ErrorMessage: "kokoro unavailable"},
}
for _, orig := range cases {
b, _ := json.Marshal(orig)
var got domain.AudioResult
json.Unmarshal(b, &got)
if got != orig {
t.Errorf("want %+v, got %+v", orig, got)
}
}
}