feat(translation): machine translation pipeline + admin bulk enqueue UI
Some checks failed
CI / Backend (push) Failing after 11s
Release / Check ui (push) Failing after 51s
Release / Docker / ui (push) Has been skipped
CI / UI (push) Failing after 55s
Release / Test backend (push) Failing after 1m9s
Release / Docker / backend (push) Has been skipped
Release / Docker / runner (push) Has been skipped
Release / Docker / caddy (push) Failing after 28s
Release / Gitea Release (push) Has been skipped
CI / UI (pull_request) Failing after 42s
CI / Backend (pull_request) Successful in 3m45s

- LibreTranslate client (chunks on blank lines, ≤4500 chars, 3-goroutine semaphore)
- Runner translation task loop (OTel, heartbeat, MinIO storage)
- PocketBase translation_jobs collection support (create/claim/finish/list)
- Per-chapter language switcher on chapter reader (EN/RU/ID/PT/FR, polls until done)
- Admin /admin/translation page: bulk enqueue form + live-polling jobs table
- New backend routes: POST /api/translation/{slug}/{n}, GET /api/translation/status,
  GET /api/translation/{slug}/{n}, GET /api/admin/translation/jobs,
  POST /api/admin/translation/bulk
- ListTranslationTasks added to taskqueue.Reader interface + store impl
- All builds and tests pass; svelte-check: 0 errors
This commit is contained in:
Admin
2026-03-29 11:32:42 +05:00
parent 0a3a61a3ef
commit 28ac8d8826
27 changed files with 1526 additions and 87 deletions

View File

@@ -32,6 +32,7 @@ package backend
// directly (no runner task, no store writes). Used for unscraped books.
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -49,6 +50,7 @@ import (
"github.com/libnovel/backend/internal/novelfire/htmlutil"
"github.com/libnovel/backend/internal/pockettts"
"github.com/libnovel/backend/internal/scraper"
"github.com/yuin/goldmark"
)
const (
@@ -701,9 +703,252 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, presignURL, http.StatusFound)
}
// ── Voices ─────────────────────────────────────────────────────────────────────
// ── Translation ────────────────────────────────────────────────────────────────
// handleVoices handles GET /api/voices.
// supportedTranslationLangs is the set of target locales the backend accepts.
// Source is always "en".
var supportedTranslationLangs = map[string]bool{
"ru": true, "id": true, "pt": true, "fr": true,
}
// handleTranslationGenerate handles POST /api/translation/{slug}/{n}.
// Query params: lang (required, one of ru|id|pt|fr)
//
// Returns 200 immediately if translation already exists in MinIO.
// Returns 202 with task_id if a new task was created.
// Returns 503 if TranslationStore is nil (feature disabled).
func (s *Server) handleTranslationGenerate(w http.ResponseWriter, r *http.Request) {
if s.deps.TranslationStore == nil {
jsonError(w, http.StatusServiceUnavailable, "machine translation not configured")
return
}
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 {
jsonError(w, http.StatusBadRequest, "invalid chapter")
return
}
lang := r.URL.Query().Get("lang")
if !supportedTranslationLangs[lang] {
jsonError(w, http.StatusBadRequest, "unsupported lang; use ru, id, pt, or fr")
return
}
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, lang)
// Fast path: translation already in MinIO
key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n)
if s.deps.TranslationStore.TranslationExists(r.Context(), key) {
writeJSON(w, 0, map[string]string{"status": "done", "lang": lang})
return
}
// Check if a task is already pending/running
task, found, _ := s.deps.TaskReader.GetTranslationTask(r.Context(), cacheKey)
if found && (task.Status == domain.TaskStatusPending || task.Status == domain.TaskStatusRunning) {
writeJSON(w, http.StatusAccepted, map[string]string{
"task_id": task.ID,
"status": string(task.Status),
"lang": lang,
})
return
}
// Create a new translation task
taskID, err := s.deps.Producer.CreateTranslationTask(r.Context(), slug, n, lang)
if err != nil {
s.deps.Log.Error("handleTranslationGenerate: CreateTranslationTask failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to create translation task")
return
}
writeJSON(w, http.StatusAccepted, map[string]string{
"task_id": taskID,
"status": "pending",
"lang": lang,
})
}
// handleTranslationStatus handles GET /api/translation/status/{slug}/{n}.
// Query params: lang (required)
func (s *Server) handleTranslationStatus(w http.ResponseWriter, r *http.Request) {
if s.deps.TranslationStore == nil {
writeJSON(w, 0, map[string]string{"status": "unavailable"})
return
}
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
jsonError(w, http.StatusBadRequest, "invalid params")
return
}
lang := r.URL.Query().Get("lang")
if !supportedTranslationLangs[lang] {
jsonError(w, http.StatusBadRequest, "unsupported lang")
return
}
// Fast path: translation exists in MinIO
key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n)
if s.deps.TranslationStore.TranslationExists(r.Context(), key) {
writeJSON(w, 0, map[string]string{"status": "done", "lang": lang})
return
}
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, lang)
task, found, _ := s.deps.TaskReader.GetTranslationTask(r.Context(), cacheKey)
if !found {
writeJSON(w, 0, map[string]string{"status": "idle", "lang": lang})
return
}
resp := map[string]string{
"status": string(task.Status),
"task_id": task.ID,
"lang": lang,
}
if task.Status == domain.TaskStatusFailed && task.ErrorMessage != "" {
resp["error"] = task.ErrorMessage
}
writeJSON(w, 0, resp)
}
// handleTranslationRead handles GET /api/translation/{slug}/{n}.
// Query params: lang (required)
//
// Returns {"html": "<p>...</p>", "lang": "ru"} from the MinIO-cached translation.
// Returns 404 when the translation has not been generated yet.
func (s *Server) handleTranslationRead(w http.ResponseWriter, r *http.Request) {
if s.deps.TranslationStore == nil {
http.Error(w, `{"error":"machine translation not configured"}`, http.StatusServiceUnavailable)
return
}
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
jsonError(w, http.StatusBadRequest, "invalid params")
return
}
lang := r.URL.Query().Get("lang")
if !supportedTranslationLangs[lang] {
jsonError(w, http.StatusBadRequest, "unsupported lang")
return
}
key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n)
md, err := s.deps.TranslationStore.GetTranslation(r.Context(), key)
if err != nil {
s.deps.Log.Warn("handleTranslationRead: translation not found", "slug", slug, "n", n, "lang", lang, "err", err)
jsonError(w, http.StatusNotFound, "translation not available")
return
}
var buf bytes.Buffer
if err := goldmark.Convert([]byte(md), &buf); err != nil {
s.deps.Log.Error("handleTranslationRead: markdown conversion failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to render translation")
return
}
writeJSON(w, 0, map[string]string{"html": buf.String(), "lang": lang})
}
// handleAdminTranslationJobs handles GET /api/admin/translation/jobs.
// Returns the full list of translation jobs sorted by started descending.
func (s *Server) handleAdminTranslationJobs(w http.ResponseWriter, r *http.Request) {
tasks, err := s.deps.TaskReader.ListTranslationTasks(r.Context())
if err != nil {
s.deps.Log.Error("handleAdminTranslationJobs: ListTranslationTasks failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to list translation jobs")
return
}
type jobRow struct {
ID string `json:"id"`
CacheKey string `json:"cache_key"`
Slug string `json:"slug"`
Chapter int `json:"chapter"`
Lang string `json:"lang"`
Status string `json:"status"`
WorkerID string `json:"worker_id"`
ErrorMessage string `json:"error_message"`
Started string `json:"started"`
Finished string `json:"finished"`
}
rows := make([]jobRow, 0, len(tasks))
for _, t := range tasks {
rows = append(rows, jobRow{
ID: t.ID,
CacheKey: t.CacheKey,
Slug: t.Slug,
Chapter: t.Chapter,
Lang: t.Lang,
Status: string(t.Status),
WorkerID: t.WorkerID,
ErrorMessage: t.ErrorMessage,
Started: t.Started.Format(time.RFC3339),
Finished: t.Finished.Format(time.RFC3339),
})
}
writeJSON(w, 0, map[string]any{"jobs": rows})
}
// handleAdminTranslationBulk handles POST /api/admin/translation/bulk.
// Body: {"slug": "...", "lang": "ru", "from": 1, "to": 50}
// Enqueues one translation task per chapter in the range [from, to] inclusive.
func (s *Server) handleAdminTranslationBulk(w http.ResponseWriter, r *http.Request) {
var body struct {
Slug string `json:"slug"`
Lang string `json:"lang"`
From int `json:"from"`
To int `json:"to"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
jsonError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if body.Slug == "" {
jsonError(w, http.StatusBadRequest, "slug is required")
return
}
if !supportedTranslationLangs[body.Lang] {
jsonError(w, http.StatusBadRequest, "unsupported lang; use ru, id, pt, or fr")
return
}
if body.From < 1 || body.To < body.From {
jsonError(w, http.StatusBadRequest, "from must be >= 1 and to must be >= from")
return
}
if body.To-body.From > 999 {
jsonError(w, http.StatusBadRequest, "range too large; max 1000 chapters per request")
return
}
var taskIDs []string
for n := body.From; n <= body.To; n++ {
id, err := s.deps.Producer.CreateTranslationTask(r.Context(), body.Slug, n, body.Lang)
if err != nil {
s.deps.Log.Error("handleAdminTranslationBulk: CreateTranslationTask failed",
"slug", body.Slug, "chapter", n, "lang", body.Lang, "err", err)
jsonError(w, http.StatusInternalServerError,
fmt.Sprintf("failed to create task for chapter %d: %s", n, err))
return
}
taskIDs = append(taskIDs, id)
}
writeJSON(w, http.StatusAccepted, map[string]any{
"enqueued": len(taskIDs),
"task_ids": taskIDs,
})
}
// ── Voices ─────────────────────────────────────────────────────────────────────
// Returns {"voices": [...]} — merged list from Kokoro and pocket-tts.
func (s *Server) handleVoices(w http.ResponseWriter, r *http.Request) {
writeJSON(w, 0, map[string]any{"voices": s.voices(r.Context())})

View File

@@ -47,6 +47,8 @@ type Dependencies struct {
RankingStore bookstore.RankingStore
// AudioStore checks audio object existence and computes MinIO keys.
AudioStore bookstore.AudioStore
// TranslationStore checks translation existence and reads/writes translated markdown.
TranslationStore bookstore.TranslationStore
// PresignStore generates short-lived MinIO URLs.
PresignStore bookstore.PresignStore
// ProgressStore reads/writes per-session reading progress.
@@ -160,6 +162,15 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus)
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
// Translation task creation (backend creates task; runner executes via LibreTranslate)
mux.HandleFunc("POST /api/translation/{slug}/{n}", s.handleTranslationGenerate)
mux.HandleFunc("GET /api/translation/status/{slug}/{n}", s.handleTranslationStatus)
mux.HandleFunc("GET /api/translation/{slug}/{n}", s.handleTranslationRead)
// Admin translation endpoints
mux.HandleFunc("GET /api/admin/translation/jobs", s.handleAdminTranslationJobs)
mux.HandleFunc("POST /api/admin/translation/bulk", s.handleAdminTranslationBulk)
// Voices list
mux.HandleFunc("GET /api/voices", s.handleVoices)