feat(translation): machine translation pipeline + admin bulk enqueue UI
Some checks failed
CI / Backend (push) Failing after 11s
Release / Check ui (push) Failing after 51s
Release / Docker / ui (push) Has been skipped
CI / UI (push) Failing after 55s
Release / Test backend (push) Failing after 1m9s
Release / Docker / backend (push) Has been skipped
Release / Docker / runner (push) Has been skipped
Release / Docker / caddy (push) Failing after 28s
Release / Gitea Release (push) Has been skipped
CI / UI (pull_request) Failing after 42s
CI / Backend (pull_request) Successful in 3m45s
Some checks failed
CI / Backend (push) Failing after 11s
Release / Check ui (push) Failing after 51s
Release / Docker / ui (push) Has been skipped
CI / UI (push) Failing after 55s
Release / Test backend (push) Failing after 1m9s
Release / Docker / backend (push) Has been skipped
Release / Docker / runner (push) Has been skipped
Release / Docker / caddy (push) Failing after 28s
Release / Gitea Release (push) Has been skipped
CI / UI (pull_request) Failing after 42s
CI / Backend (pull_request) Successful in 3m45s
- LibreTranslate client (chunks on blank lines, ≤4500 chars, 3-goroutine semaphore)
- Runner translation task loop (OTel, heartbeat, MinIO storage)
- PocketBase translation_jobs collection support (create/claim/finish/list)
- Per-chapter language switcher on chapter reader (EN/RU/ID/PT/FR, polls until done)
- Admin /admin/translation page: bulk enqueue form + live-polling jobs table
- New backend routes: POST /api/translation/{slug}/{n}, GET /api/translation/status,
GET /api/translation/{slug}/{n}, GET /api/admin/translation/jobs,
POST /api/admin/translation/bulk
- ListTranslationTasks added to taskqueue.Reader interface + store impl
- All builds and tests pass; svelte-check: 0 errors
This commit is contained in:
@@ -32,6 +32,7 @@ package backend
|
||||
// directly (no runner task, no store writes). Used for unscraped books.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -49,6 +50,7 @@ import (
|
||||
"github.com/libnovel/backend/internal/novelfire/htmlutil"
|
||||
"github.com/libnovel/backend/internal/pockettts"
|
||||
"github.com/libnovel/backend/internal/scraper"
|
||||
"github.com/yuin/goldmark"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -701,9 +703,252 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, presignURL, http.StatusFound)
|
||||
}
|
||||
|
||||
// ── Voices ─────────────────────────────────────────────────────────────────────
|
||||
// ── Translation ────────────────────────────────────────────────────────────────
|
||||
|
||||
// handleVoices handles GET /api/voices.
|
||||
// supportedTranslationLangs is the set of target locales the backend accepts.
|
||||
// Source is always "en".
|
||||
var supportedTranslationLangs = map[string]bool{
|
||||
"ru": true, "id": true, "pt": true, "fr": true,
|
||||
}
|
||||
|
||||
// handleTranslationGenerate handles POST /api/translation/{slug}/{n}.
|
||||
// Query params: lang (required, one of ru|id|pt|fr)
|
||||
//
|
||||
// Returns 200 immediately if translation already exists in MinIO.
|
||||
// Returns 202 with task_id if a new task was created.
|
||||
// Returns 503 if TranslationStore is nil (feature disabled).
|
||||
func (s *Server) handleTranslationGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
if s.deps.TranslationStore == nil {
|
||||
jsonError(w, http.StatusServiceUnavailable, "machine translation not configured")
|
||||
return
|
||||
}
|
||||
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 {
|
||||
jsonError(w, http.StatusBadRequest, "invalid chapter")
|
||||
return
|
||||
}
|
||||
|
||||
lang := r.URL.Query().Get("lang")
|
||||
if !supportedTranslationLangs[lang] {
|
||||
jsonError(w, http.StatusBadRequest, "unsupported lang; use ru, id, pt, or fr")
|
||||
return
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, lang)
|
||||
|
||||
// Fast path: translation already in MinIO
|
||||
key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n)
|
||||
if s.deps.TranslationStore.TranslationExists(r.Context(), key) {
|
||||
writeJSON(w, 0, map[string]string{"status": "done", "lang": lang})
|
||||
return
|
||||
}
|
||||
|
||||
// Check if a task is already pending/running
|
||||
task, found, _ := s.deps.TaskReader.GetTranslationTask(r.Context(), cacheKey)
|
||||
if found && (task.Status == domain.TaskStatusPending || task.Status == domain.TaskStatusRunning) {
|
||||
writeJSON(w, http.StatusAccepted, map[string]string{
|
||||
"task_id": task.ID,
|
||||
"status": string(task.Status),
|
||||
"lang": lang,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Create a new translation task
|
||||
taskID, err := s.deps.Producer.CreateTranslationTask(r.Context(), slug, n, lang)
|
||||
if err != nil {
|
||||
s.deps.Log.Error("handleTranslationGenerate: CreateTranslationTask failed", "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "failed to create translation task")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusAccepted, map[string]string{
|
||||
"task_id": taskID,
|
||||
"status": "pending",
|
||||
"lang": lang,
|
||||
})
|
||||
}
|
||||
|
||||
// handleTranslationStatus handles GET /api/translation/status/{slug}/{n}.
|
||||
// Query params: lang (required)
|
||||
func (s *Server) handleTranslationStatus(w http.ResponseWriter, r *http.Request) {
|
||||
if s.deps.TranslationStore == nil {
|
||||
writeJSON(w, 0, map[string]string{"status": "unavailable"})
|
||||
return
|
||||
}
|
||||
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
jsonError(w, http.StatusBadRequest, "invalid params")
|
||||
return
|
||||
}
|
||||
|
||||
lang := r.URL.Query().Get("lang")
|
||||
if !supportedTranslationLangs[lang] {
|
||||
jsonError(w, http.StatusBadRequest, "unsupported lang")
|
||||
return
|
||||
}
|
||||
|
||||
// Fast path: translation exists in MinIO
|
||||
key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n)
|
||||
if s.deps.TranslationStore.TranslationExists(r.Context(), key) {
|
||||
writeJSON(w, 0, map[string]string{"status": "done", "lang": lang})
|
||||
return
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("%s/%d/%s", slug, n, lang)
|
||||
task, found, _ := s.deps.TaskReader.GetTranslationTask(r.Context(), cacheKey)
|
||||
if !found {
|
||||
writeJSON(w, 0, map[string]string{"status": "idle", "lang": lang})
|
||||
return
|
||||
}
|
||||
|
||||
resp := map[string]string{
|
||||
"status": string(task.Status),
|
||||
"task_id": task.ID,
|
||||
"lang": lang,
|
||||
}
|
||||
if task.Status == domain.TaskStatusFailed && task.ErrorMessage != "" {
|
||||
resp["error"] = task.ErrorMessage
|
||||
}
|
||||
writeJSON(w, 0, resp)
|
||||
}
|
||||
|
||||
// handleTranslationRead handles GET /api/translation/{slug}/{n}.
|
||||
// Query params: lang (required)
|
||||
//
|
||||
// Returns {"html": "<p>...</p>", "lang": "ru"} from the MinIO-cached translation.
|
||||
// Returns 404 when the translation has not been generated yet.
|
||||
func (s *Server) handleTranslationRead(w http.ResponseWriter, r *http.Request) {
|
||||
if s.deps.TranslationStore == nil {
|
||||
http.Error(w, `{"error":"machine translation not configured"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
slug := r.PathValue("slug")
|
||||
n, err := strconv.Atoi(r.PathValue("n"))
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
jsonError(w, http.StatusBadRequest, "invalid params")
|
||||
return
|
||||
}
|
||||
|
||||
lang := r.URL.Query().Get("lang")
|
||||
if !supportedTranslationLangs[lang] {
|
||||
jsonError(w, http.StatusBadRequest, "unsupported lang")
|
||||
return
|
||||
}
|
||||
|
||||
key := s.deps.TranslationStore.TranslationObjectKey(lang, slug, n)
|
||||
md, err := s.deps.TranslationStore.GetTranslation(r.Context(), key)
|
||||
if err != nil {
|
||||
s.deps.Log.Warn("handleTranslationRead: translation not found", "slug", slug, "n", n, "lang", lang, "err", err)
|
||||
jsonError(w, http.StatusNotFound, "translation not available")
|
||||
return
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := goldmark.Convert([]byte(md), &buf); err != nil {
|
||||
s.deps.Log.Error("handleTranslationRead: markdown conversion failed", "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "failed to render translation")
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, 0, map[string]string{"html": buf.String(), "lang": lang})
|
||||
}
|
||||
|
||||
// handleAdminTranslationJobs handles GET /api/admin/translation/jobs.
|
||||
// Returns the full list of translation jobs sorted by started descending.
|
||||
func (s *Server) handleAdminTranslationJobs(w http.ResponseWriter, r *http.Request) {
|
||||
tasks, err := s.deps.TaskReader.ListTranslationTasks(r.Context())
|
||||
if err != nil {
|
||||
s.deps.Log.Error("handleAdminTranslationJobs: ListTranslationTasks failed", "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "failed to list translation jobs")
|
||||
return
|
||||
}
|
||||
type jobRow struct {
|
||||
ID string `json:"id"`
|
||||
CacheKey string `json:"cache_key"`
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
Lang string `json:"lang"`
|
||||
Status string `json:"status"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
ErrorMessage string `json:"error_message"`
|
||||
Started string `json:"started"`
|
||||
Finished string `json:"finished"`
|
||||
}
|
||||
rows := make([]jobRow, 0, len(tasks))
|
||||
for _, t := range tasks {
|
||||
rows = append(rows, jobRow{
|
||||
ID: t.ID,
|
||||
CacheKey: t.CacheKey,
|
||||
Slug: t.Slug,
|
||||
Chapter: t.Chapter,
|
||||
Lang: t.Lang,
|
||||
Status: string(t.Status),
|
||||
WorkerID: t.WorkerID,
|
||||
ErrorMessage: t.ErrorMessage,
|
||||
Started: t.Started.Format(time.RFC3339),
|
||||
Finished: t.Finished.Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
writeJSON(w, 0, map[string]any{"jobs": rows})
|
||||
}
|
||||
|
||||
// handleAdminTranslationBulk handles POST /api/admin/translation/bulk.
|
||||
// Body: {"slug": "...", "lang": "ru", "from": 1, "to": 50}
|
||||
// Enqueues one translation task per chapter in the range [from, to] inclusive.
|
||||
func (s *Server) handleAdminTranslationBulk(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Slug string `json:"slug"`
|
||||
Lang string `json:"lang"`
|
||||
From int `json:"from"`
|
||||
To int `json:"to"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
jsonError(w, http.StatusBadRequest, "invalid JSON body")
|
||||
return
|
||||
}
|
||||
if body.Slug == "" {
|
||||
jsonError(w, http.StatusBadRequest, "slug is required")
|
||||
return
|
||||
}
|
||||
if !supportedTranslationLangs[body.Lang] {
|
||||
jsonError(w, http.StatusBadRequest, "unsupported lang; use ru, id, pt, or fr")
|
||||
return
|
||||
}
|
||||
if body.From < 1 || body.To < body.From {
|
||||
jsonError(w, http.StatusBadRequest, "from must be >= 1 and to must be >= from")
|
||||
return
|
||||
}
|
||||
if body.To-body.From > 999 {
|
||||
jsonError(w, http.StatusBadRequest, "range too large; max 1000 chapters per request")
|
||||
return
|
||||
}
|
||||
|
||||
var taskIDs []string
|
||||
for n := body.From; n <= body.To; n++ {
|
||||
id, err := s.deps.Producer.CreateTranslationTask(r.Context(), body.Slug, n, body.Lang)
|
||||
if err != nil {
|
||||
s.deps.Log.Error("handleAdminTranslationBulk: CreateTranslationTask failed",
|
||||
"slug", body.Slug, "chapter", n, "lang", body.Lang, "err", err)
|
||||
jsonError(w, http.StatusInternalServerError,
|
||||
fmt.Sprintf("failed to create task for chapter %d: %s", n, err))
|
||||
return
|
||||
}
|
||||
taskIDs = append(taskIDs, id)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"enqueued": len(taskIDs),
|
||||
"task_ids": taskIDs,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Voices ─────────────────────────────────────────────────────────────────────
|
||||
// Returns {"voices": [...]} — merged list from Kokoro and pocket-tts.
|
||||
func (s *Server) handleVoices(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 0, map[string]any{"voices": s.voices(r.Context())})
|
||||
|
||||
@@ -47,6 +47,8 @@ type Dependencies struct {
|
||||
RankingStore bookstore.RankingStore
|
||||
// AudioStore checks audio object existence and computes MinIO keys.
|
||||
AudioStore bookstore.AudioStore
|
||||
// TranslationStore checks translation existence and reads/writes translated markdown.
|
||||
TranslationStore bookstore.TranslationStore
|
||||
// PresignStore generates short-lived MinIO URLs.
|
||||
PresignStore bookstore.PresignStore
|
||||
// ProgressStore reads/writes per-session reading progress.
|
||||
@@ -160,6 +162,15 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus)
|
||||
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
||||
|
||||
// Translation task creation (backend creates task; runner executes via LibreTranslate)
|
||||
mux.HandleFunc("POST /api/translation/{slug}/{n}", s.handleTranslationGenerate)
|
||||
mux.HandleFunc("GET /api/translation/status/{slug}/{n}", s.handleTranslationStatus)
|
||||
mux.HandleFunc("GET /api/translation/{slug}/{n}", s.handleTranslationRead)
|
||||
|
||||
// Admin translation endpoints
|
||||
mux.HandleFunc("GET /api/admin/translation/jobs", s.handleAdminTranslationJobs)
|
||||
mux.HandleFunc("POST /api/admin/translation/bulk", s.handleAdminTranslationBulk)
|
||||
|
||||
// Voices list
|
||||
mux.HandleFunc("GET /api/voices", s.handleVoices)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user