feat(translation): machine translation pipeline + admin bulk enqueue UI
Some checks failed
CI / Backend (push) Failing after 11s
Release / Check ui (push) Failing after 51s
Release / Docker / ui (push) Has been skipped
CI / UI (push) Failing after 55s
Release / Test backend (push) Failing after 1m9s
Release / Docker / backend (push) Has been skipped
Release / Docker / runner (push) Has been skipped
Release / Docker / caddy (push) Failing after 28s
Release / Gitea Release (push) Has been skipped
CI / UI (pull_request) Failing after 42s
CI / Backend (pull_request) Successful in 3m45s

- LibreTranslate client (chunks on blank lines, ≤4500 chars, 3-goroutine semaphore)
- Runner translation task loop (OTel, heartbeat, MinIO storage)
- PocketBase translation_jobs collection support (create/claim/finish/list)
- Per-chapter language switcher on chapter reader (EN/RU/ID/PT/FR, polls until done)
- Admin /admin/translation page: bulk enqueue form + live-polling jobs table
- New backend routes: POST /api/translation/{slug}/{n}, GET /api/translation/status,
  GET /api/translation/{slug}/{n}, GET /api/admin/translation/jobs,
  POST /api/admin/translation/bulk
- ListTranslationTasks added to taskqueue.Reader interface + store impl
- All builds and tests pass; svelte-check: 0 errors
This commit is contained in:
Admin
2026-03-29 11:32:42 +05:00
parent 0a3a61a3ef
commit 28ac8d8826
27 changed files with 1526 additions and 87 deletions

View File

@@ -17,12 +17,13 @@ import (
// minioClient wraps the official minio-go client with bucket names.
type minioClient struct {
client *minio.Client // internal — all read/write operations
pubClient *minio.Client // presign-only — initialised against the public endpoint
bucketChapters string
bucketAudio string
bucketAvatars string
bucketBrowse string
client *minio.Client // internal — all read/write operations
pubClient *minio.Client // presign-only — initialised against the public endpoint
bucketChapters string
bucketAudio string
bucketAvatars string
bucketBrowse string
bucketTranslations string
}
func newMinioClient(cfg config.MinIO) (*minioClient, error) {
@@ -74,18 +75,19 @@ func newMinioClient(cfg config.MinIO) (*minioClient, error) {
}
return &minioClient{
client: internal,
pubClient: pub,
bucketChapters: cfg.BucketChapters,
bucketAudio: cfg.BucketAudio,
bucketAvatars: cfg.BucketAvatars,
bucketBrowse: cfg.BucketBrowse,
client: internal,
pubClient: pub,
bucketChapters: cfg.BucketChapters,
bucketAudio: cfg.BucketAudio,
bucketAvatars: cfg.BucketAvatars,
bucketBrowse: cfg.BucketBrowse,
bucketTranslations: cfg.BucketTranslations,
}, nil
}
// ensureBuckets creates all required buckets if they don't already exist.
func (m *minioClient) ensureBuckets(ctx context.Context) error {
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse} {
for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse, m.bucketTranslations} {
exists, err := m.client.BucketExists(ctx, bucket)
if err != nil {
return fmt.Errorf("minio: check bucket %q: %w", bucket, err)
@@ -125,6 +127,12 @@ func CoverObjectKey(slug string) string {
return fmt.Sprintf("covers/%s.jpg", slug)
}
// TranslationObjectKey returns the MinIO object key for a translated chapter.
// Format: {lang}/{slug}/{n:06d}.md
func TranslationObjectKey(lang, slug string, n int) string {
return fmt.Sprintf("%s/%s/%06d.md", lang, slug, n)
}
// chapterNumberFromKey extracts the chapter number from a MinIO object key.
// e.g. "my-book/chapter-000042.md" → 42
func chapterNumberFromKey(key string) int {

View File

@@ -51,6 +51,7 @@ var _ bookstore.AudioStore = (*Store)(nil)
var _ bookstore.PresignStore = (*Store)(nil)
var _ bookstore.ProgressStore = (*Store)(nil)
var _ bookstore.CoverStore = (*Store)(nil)
var _ bookstore.TranslationStore = (*Store)(nil)
var _ taskqueue.Producer = (*Store)(nil)
var _ taskqueue.Consumer = (*Store)(nil)
var _ taskqueue.Reader = (*Store)(nil)
@@ -535,13 +536,36 @@ func (s *Store) CreateAudioTask(ctx context.Context, slug string, chapter int, v
return rec.ID, nil
}
func (s *Store) CreateTranslationTask(ctx context.Context, slug string, chapter int, lang string) (string, error) {
cacheKey := fmt.Sprintf("%s/%d/%s", slug, chapter, lang)
payload := map[string]any{
"cache_key": cacheKey,
"slug": slug,
"chapter": chapter,
"lang": lang,
"status": string(domain.TaskStatusPending),
"started": time.Now().UTC().Format(time.RFC3339),
}
var rec struct {
ID string `json:"id"`
}
if err := s.pb.post(ctx, "/api/collections/translation_jobs/records", payload, &rec); err != nil {
return "", err
}
return rec.ID, nil
}
func (s *Store) CancelTask(ctx context.Context, id string) error {
// Try scraping_tasks first, then audio_jobs.
// Try scraping_tasks first, then audio_jobs, then translation_jobs.
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),
map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id),
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id),
map[string]string{"status": string(domain.TaskStatusCancelled)}); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id),
map[string]string{"status": string(domain.TaskStatusCancelled)})
}
@@ -571,6 +595,18 @@ func (s *Store) ClaimNextAudioTask(ctx context.Context, workerID string) (domain
return task, err == nil, err
}
func (s *Store) ClaimNextTranslationTask(ctx context.Context, workerID string) (domain.TranslationTask, bool, error) {
raw, err := s.pb.claimRecord(ctx, "translation_jobs", workerID, nil)
if err != nil {
return domain.TranslationTask{}, false, err
}
if raw == nil {
return domain.TranslationTask{}, false, nil
}
task, err := parseTranslationTask(raw)
return task, err == nil, err
}
func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
status := string(domain.TaskStatusDone)
if result.ErrorMessage != "" {
@@ -599,6 +635,18 @@ func (s *Store) FinishAudioTask(ctx context.Context, id string, result domain.Au
})
}
func (s *Store) FinishTranslationTask(ctx context.Context, id string, result domain.TranslationResult) error {
status := string(domain.TaskStatusDone)
if result.ErrorMessage != "" {
status = string(domain.TaskStatusFailed)
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), map[string]any{
"status": status,
"error_message": result.ErrorMessage,
"finished": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
payload := map[string]any{
"status": string(domain.TaskStatusFailed),
@@ -608,11 +656,14 @@ func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), payload)
}
// HeartbeatTask updates the heartbeat_at field on a running task.
// Tries scraping_tasks first, then audio_jobs (same pattern as FailTask).
// Tries scraping_tasks first, then audio_jobs, then translation_jobs.
func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
payload := map[string]any{
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
@@ -620,7 +671,10 @@ func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload)
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), payload)
}
// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing
@@ -638,7 +692,7 @@ func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (i
}
total := 0
for _, collection := range []string{"scraping_tasks", "audio_jobs"} {
for _, collection := range []string{"scraping_tasks", "audio_jobs", "translation_jobs"} {
items, err := s.pb.listAll(ctx, collection, filter, "")
if err != nil {
return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err)
@@ -715,6 +769,31 @@ func (s *Store) GetAudioTask(ctx context.Context, cacheKey string) (domain.Audio
return t, err == nil, err
}
func (s *Store) ListTranslationTasks(ctx context.Context) ([]domain.TranslationTask, error) {
items, err := s.pb.listAll(ctx, "translation_jobs", "", "-started")
if err != nil {
return nil, err
}
tasks := make([]domain.TranslationTask, 0, len(items))
for _, raw := range items {
t, err := parseTranslationTask(raw)
if err == nil {
tasks = append(tasks, t)
}
}
return tasks, nil
}
func (s *Store) GetTranslationTask(ctx context.Context, cacheKey string) (domain.TranslationTask, bool, error) {
filter := fmt.Sprintf(`cache_key='%s'`, cacheKey)
items, err := s.pb.listAll(ctx, "translation_jobs", filter, "-started")
if err != nil || len(items) == 0 {
return domain.TranslationTask{}, false, err
}
t, err := parseTranslationTask(items[0])
return t, err == nil, err
}
// ── Parsers ───────────────────────────────────────────────────────────────────
func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) {
@@ -789,6 +868,38 @@ func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
}, nil
}
func parseTranslationTask(raw json.RawMessage) (domain.TranslationTask, error) {
var rec struct {
ID string `json:"id"`
CacheKey string `json:"cache_key"`
Slug string `json:"slug"`
Chapter int `json:"chapter"`
Lang string `json:"lang"`
WorkerID string `json:"worker_id"`
Status string `json:"status"`
ErrorMessage string `json:"error_message"`
Started string `json:"started"`
Finished string `json:"finished"`
}
if err := json.Unmarshal(raw, &rec); err != nil {
return domain.TranslationTask{}, err
}
started, _ := time.Parse(time.RFC3339, rec.Started)
finished, _ := time.Parse(time.RFC3339, rec.Finished)
return domain.TranslationTask{
ID: rec.ID,
CacheKey: rec.CacheKey,
Slug: rec.Slug,
Chapter: rec.Chapter,
Lang: rec.Lang,
WorkerID: rec.WorkerID,
Status: domain.TaskStatus(rec.Status),
ErrorMessage: rec.ErrorMessage,
Started: started,
Finished: finished,
}, nil
}
// ── CoverStore ─────────────────────────────────────────────────────────────────
func (s *Store) PutCover(ctx context.Context, slug string, data []byte, contentType string) error {
@@ -818,3 +929,25 @@ func (s *Store) GetCover(ctx context.Context, slug string) ([]byte, string, bool
func (s *Store) CoverExists(ctx context.Context, slug string) bool {
return s.mc.coverExists(ctx, CoverObjectKey(slug))
}
// ── TranslationStore ───────────────────────────────────────────────────────────
func (s *Store) TranslationObjectKey(lang, slug string, n int) string {
return TranslationObjectKey(lang, slug, n)
}
func (s *Store) TranslationExists(ctx context.Context, key string) bool {
return s.mc.objectExists(ctx, s.mc.bucketTranslations, key)
}
func (s *Store) PutTranslation(ctx context.Context, key string, data []byte) error {
return s.mc.putObject(ctx, s.mc.bucketTranslations, key, "text/markdown; charset=utf-8", data)
}
func (s *Store) GetTranslation(ctx context.Context, key string) (string, error) {
data, err := s.mc.getObject(ctx, s.mc.bucketTranslations, key)
if err != nil {
return "", fmt.Errorf("GetTranslation: %w", err)
}
return string(data), nil
}