feat(translation): machine translation pipeline + admin bulk enqueue UI
Some checks failed
CI / Backend (push) Failing after 11s
Release / Check ui (push) Failing after 51s
Release / Docker / ui (push) Has been skipped
CI / UI (push) Failing after 55s
Release / Test backend (push) Failing after 1m9s
Release / Docker / backend (push) Has been skipped
Release / Docker / runner (push) Has been skipped
Release / Docker / caddy (push) Failing after 28s
Release / Gitea Release (push) Has been skipped
CI / UI (pull_request) Failing after 42s
CI / Backend (pull_request) Successful in 3m45s
Some checks failed
CI / Backend (push) Failing after 11s
Release / Check ui (push) Failing after 51s
Release / Docker / ui (push) Has been skipped
CI / UI (push) Failing after 55s
Release / Test backend (push) Failing after 1m9s
Release / Docker / backend (push) Has been skipped
Release / Docker / runner (push) Has been skipped
Release / Docker / caddy (push) Failing after 28s
Release / Gitea Release (push) Has been skipped
CI / UI (pull_request) Failing after 42s
CI / Backend (pull_request) Successful in 3m45s
- LibreTranslate client (chunks on blank lines, ≤4500 chars, 3-goroutine semaphore)
- Runner translation task loop (OTel, heartbeat, MinIO storage)
- PocketBase translation_jobs collection support (create/claim/finish/list)
- Per-chapter language switcher on chapter reader (EN/RU/ID/PT/FR, polls until done)
- Admin /admin/translation page: bulk enqueue form + live-polling jobs table
- New backend routes: POST /api/translation/{slug}/{n}, GET /api/translation/status,
GET /api/translation/{slug}/{n}, GET /api/admin/translation/jobs,
POST /api/admin/translation/bulk
- ListTranslationTasks added to taskqueue.Reader interface + store impl
- All builds and tests pass; svelte-check: 0 errors
This commit is contained in:
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/libnovel/backend/internal/bookstore"
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
"github.com/libnovel/backend/internal/libretranslate"
|
||||
"github.com/libnovel/backend/internal/meili"
|
||||
"github.com/libnovel/backend/internal/orchestrator"
|
||||
"github.com/libnovel/backend/internal/pockettts"
|
||||
@@ -48,6 +49,8 @@ type Config struct {
|
||||
MaxConcurrentScrape int
|
||||
// MaxConcurrentAudio limits simultaneous audio-generation goroutines.
|
||||
MaxConcurrentAudio int
|
||||
// MaxConcurrentTranslation limits simultaneous translation goroutines.
|
||||
MaxConcurrentTranslation int
|
||||
// OrchestratorWorkers is the chapter-scraping parallelism inside each book run.
|
||||
OrchestratorWorkers int
|
||||
// HeartbeatInterval is how often active tasks PATCH their heartbeat_at
|
||||
@@ -95,6 +98,8 @@ type Dependencies struct {
|
||||
BookReader bookstore.BookReader
|
||||
// AudioStore persists generated audio and checks key existence.
|
||||
AudioStore bookstore.AudioStore
|
||||
// TranslationStore persists translated markdown and checks key existence.
|
||||
TranslationStore bookstore.TranslationStore
|
||||
// CoverStore stores book cover images in MinIO.
|
||||
CoverStore bookstore.CoverStore
|
||||
// SearchIndex indexes books in Meilisearch after scraping.
|
||||
@@ -107,6 +112,9 @@ type Dependencies struct {
|
||||
// PocketTTS is the pocket-tts client (CPU, kyutai voices: alba, marius, etc.).
|
||||
// If nil, pocket-tts voice tasks will fail with a clear error.
|
||||
PocketTTS pockettts.Client
|
||||
// LibreTranslate is the machine translation client.
|
||||
// If nil, translation tasks will fail with a clear error.
|
||||
LibreTranslate libretranslate.Client
|
||||
// Log is the structured logger.
|
||||
Log *slog.Logger
|
||||
}
|
||||
@@ -137,6 +145,9 @@ func New(cfg Config, deps Dependencies) *Runner {
|
||||
if cfg.MaxConcurrentAudio <= 0 {
|
||||
cfg.MaxConcurrentAudio = 1
|
||||
}
|
||||
if cfg.MaxConcurrentTranslation <= 0 {
|
||||
cfg.MaxConcurrentTranslation = 1
|
||||
}
|
||||
if cfg.WorkerID == "" {
|
||||
cfg.WorkerID = "runner"
|
||||
}
|
||||
@@ -175,6 +186,7 @@ func (r *Runner) Run(ctx context.Context) error {
|
||||
"mode", r.mode(),
|
||||
"max_scrape", r.cfg.MaxConcurrentScrape,
|
||||
"max_audio", r.cfg.MaxConcurrentAudio,
|
||||
"max_translation", r.cfg.MaxConcurrentTranslation,
|
||||
"catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval,
|
||||
"metrics_addr", r.cfg.MetricsAddr,
|
||||
)
|
||||
@@ -208,6 +220,7 @@ func (r *Runner) mode() string {
|
||||
func (r *Runner) runPoll(ctx context.Context) error {
|
||||
scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
|
||||
audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio)
|
||||
translationSem := make(chan struct{}, r.cfg.MaxConcurrentTranslation)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
tick := time.NewTicker(r.cfg.PollInterval)
|
||||
@@ -227,7 +240,7 @@ func (r *Runner) runPoll(ctx context.Context) error {
|
||||
|
||||
// Run one poll immediately on startup, then on each tick.
|
||||
for {
|
||||
r.poll(ctx, scrapeSem, audioSem, &wg)
|
||||
r.poll(ctx, scrapeSem, audioSem, translationSem, &wg)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -252,7 +265,7 @@ func (r *Runner) runPoll(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// poll claims all available pending tasks and dispatches them to goroutines.
|
||||
func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) {
|
||||
func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem, translationSem chan struct{}, wg *sync.WaitGroup) {
|
||||
// ── Heartbeat file ────────────────────────────────────────────────────
|
||||
// Touch /tmp/runner.alive so the Docker health check can confirm the
|
||||
// runner is actively polling. Failure is non-fatal — just log it.
|
||||
@@ -335,6 +348,39 @@ audioLoop:
|
||||
r.runAudioTask(ctx, t)
|
||||
}(task)
|
||||
}
|
||||
|
||||
// ── Translation tasks ─────────────────────────────────────────────────
|
||||
translationLoop:
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case translationSem <- struct{}{}:
|
||||
// Slot acquired — proceed to claim a task.
|
||||
default:
|
||||
// All slots busy; leave remaining pending tasks for next tick.
|
||||
break translationLoop
|
||||
}
|
||||
task, ok, err := r.deps.Consumer.ClaimNextTranslationTask(ctx, r.cfg.WorkerID)
|
||||
if err != nil {
|
||||
<-translationSem
|
||||
r.deps.Log.Error("runner: ClaimNextTranslationTask failed", "err", err)
|
||||
break
|
||||
}
|
||||
if !ok {
|
||||
<-translationSem
|
||||
break
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
wg.Add(1)
|
||||
go func(t domain.TranslationTask) {
|
||||
defer wg.Done()
|
||||
defer func() { <-translationSem }()
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runTranslationTask(ctx, t)
|
||||
}(task)
|
||||
}
|
||||
}
|
||||
|
||||
// newOrchestrator builds an orchestrator with the Meilisearch post-hook wired in.
|
||||
|
||||
@@ -48,6 +48,10 @@ func (s *stubConsumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.A
|
||||
return t, true, nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) ClaimNextTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) {
|
||||
return domain.TranslationTask{}, false, nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) FinishScrapeTask(_ context.Context, id string, _ domain.ScrapeResult) error {
|
||||
s.finished = append(s.finished, id)
|
||||
return nil
|
||||
@@ -58,6 +62,11 @@ func (s *stubConsumer) FinishAudioTask(_ context.Context, id string, _ domain.Au
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) FinishTranslationTask(_ context.Context, id string, _ domain.TranslationResult) error {
|
||||
s.finished = append(s.finished, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) FailTask(_ context.Context, id, _ string) error {
|
||||
s.failCalled = append(s.failCalled, id)
|
||||
return nil
|
||||
|
||||
97
backend/internal/runner/translation.go
Normal file
97
backend/internal/runner/translation.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
)
|
||||
|
||||
// runTranslationTask executes one machine-translation task end-to-end and
|
||||
// reports the result back to PocketBase.
|
||||
func (r *Runner) runTranslationTask(ctx context.Context, task domain.TranslationTask) {
|
||||
ctx, span := otel.Tracer("runner").Start(ctx, "runner.translation_task")
|
||||
defer span.End()
|
||||
span.SetAttributes(
|
||||
attribute.String("task.id", task.ID),
|
||||
attribute.String("book.slug", task.Slug),
|
||||
attribute.Int("chapter.number", task.Chapter),
|
||||
attribute.String("translation.lang", task.Lang),
|
||||
)
|
||||
|
||||
log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "lang", task.Lang)
|
||||
log.Info("runner: translation task starting")
|
||||
|
||||
// Heartbeat goroutine — keeps the task alive while translation runs.
|
||||
hbCtx, hbCancel := context.WithCancel(ctx)
|
||||
defer hbCancel()
|
||||
go func() {
|
||||
tick := time.NewTicker(r.cfg.HeartbeatInterval)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-hbCtx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil {
|
||||
log.Warn("runner: heartbeat failed", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
fail := func(msg string) {
|
||||
log.Error("runner: translation task failed", "reason", msg)
|
||||
r.tasksFailed.Add(1)
|
||||
span.SetStatus(codes.Error, msg)
|
||||
result := domain.TranslationResult{ErrorMessage: msg}
|
||||
if err := r.deps.Consumer.FinishTranslationTask(ctx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishTranslationTask failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Guard: LibreTranslate must be configured.
|
||||
if r.deps.LibreTranslate == nil {
|
||||
fail("libretranslate client not configured (LIBRETRANSLATE_URL is empty)")
|
||||
return
|
||||
}
|
||||
|
||||
// 1. Read raw markdown chapter.
|
||||
raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter)
|
||||
if err != nil {
|
||||
fail(fmt.Sprintf("read chapter: %v", err))
|
||||
return
|
||||
}
|
||||
if raw == "" {
|
||||
fail("chapter text is empty")
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Translate (chunked, concurrent).
|
||||
translated, err := r.deps.LibreTranslate.Translate(ctx, raw, "en", task.Lang)
|
||||
if err != nil {
|
||||
fail(fmt.Sprintf("translate: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Store translated markdown in MinIO.
|
||||
key := r.deps.TranslationStore.TranslationObjectKey(task.Lang, task.Slug, task.Chapter)
|
||||
if err := r.deps.TranslationStore.PutTranslation(ctx, key, []byte(translated)); err != nil {
|
||||
fail(fmt.Sprintf("put translation: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// 4. Report success.
|
||||
r.tasksCompleted.Add(1)
|
||||
span.SetStatus(codes.Ok, "")
|
||||
result := domain.TranslationResult{ObjectKey: key}
|
||||
if err := r.deps.Consumer.FinishTranslationTask(ctx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishTranslationTask failed", "err", err)
|
||||
}
|
||||
log.Info("runner: translation task finished", "key", key)
|
||||
}
|
||||
Reference in New Issue
Block a user