Compare commits

..

1 Commits

Author SHA1 Message Date
root
e088bc056e feat: add PDF/EPUB import functionality
Some checks failed
Release / Test backend (push) Successful in 43s
Release / Check ui (push) Failing after 35s
Release / Docker (push) Has been skipped
Release / Gitea Release (push) Has been skipped
- Add ImportTask/ImportResult types to domain.go
- Add TypeImportBook to asynqqueue for task routing
- Add CreateImportTask to producer and storage layers
- Add ClaimNextImportTask/FinishImportTask to Consumer interfaces
- Add import task handling to runner (polling + Asynq handler)
- Add BookImporter interface to bookstore for PDF/EPUB parsing
- Add backend API endpoints: POST/GET /api/admin/import
- Add SvelteKit UI at /admin/import with task list
- Add nav link in admin layout

Note: PDF/EPUB parsing is a placeholder - needs external library integration.
2026-04-09 10:01:20 +05:00
7 changed files with 59 additions and 18 deletions

View File

@@ -40,6 +40,10 @@ func (c *Consumer) FinishTranslationTask(ctx context.Context, id string, result
return c.pb.FinishTranslationTask(ctx, id, result)
}
func (c *Consumer) FinishImportTask(ctx context.Context, id string, result domain.ImportResult) error {
return c.pb.FinishImportTask(ctx, id, result)
}
func (c *Consumer) FailTask(ctx context.Context, id, errMsg string) error {
return c.pb.FailTask(ctx, id, errMsg)
}
@@ -60,6 +64,12 @@ func (c *Consumer) ClaimNextTranslationTask(ctx context.Context, workerID string
return c.pb.ClaimNextTranslationTask(ctx, workerID)
}
// ClaimNextImportTask delegates to PocketBase because import tasks
// are stored in PocketBase (not Redis/Asynq) and must still be polled directly.
func (c *Consumer) ClaimNextImportTask(ctx context.Context, workerID string) (domain.ImportTask, bool, error) {
return c.pb.ClaimNextImportTask(ctx, workerID)
}
func (c *Consumer) HeartbeatTask(ctx context.Context, id string) error {
return c.pb.HeartbeatTask(ctx, id)
}

View File

@@ -1,7 +1,6 @@
package backend
import (
"context"
"encoding/json"
"fmt"
"io"
@@ -10,13 +9,13 @@ import (
"strings"
"time"
"github.com/libnovel/backend/internal/asynqqueue"
"github.com/libnovel/backend/internal/storage"
)
type importRequest struct {
Title string `json:"title"`
FileName string `json:"file_name"`
FileType string `json:"file_type"` // "pdf" or "epub"
Title string `json:"title"`
FileName string `json:"file_name"`
FileType string `json:"file_type"` // "pdf" or "epub"
ObjectKey string `json:"object_key"` // MinIO path to uploaded file
}
@@ -26,7 +25,7 @@ type importResponse struct {
}
func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
if s.deps.TaskProducer == nil {
if s.deps.Producer == nil {
jsonError(w, http.StatusServiceUnavailable, "task queue not configured")
return
}
@@ -64,8 +63,14 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
return
}
// Upload to MinIO directly via the store
objectKey = fmt.Sprintf("imports/%d_%s", time.Now().Unix(), header.Filename)
if err := s.deps.PresignStore.PutObject(r.Context(), "imports", objectKey, data); err != nil {
store, ok := s.deps.Producer.(*storage.Store)
if !ok {
jsonError(w, http.StatusInternalServerError, "storage not available")
return
}
if err := store.PutImportFile(r.Context(), objectKey, data); err != nil {
jsonError(w, http.StatusInternalServerError, "upload file: "+err.Error())
return
}
@@ -94,7 +99,7 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
return -1
}, slug)
taskID, err := s.deps.TaskProducer.CreateImportTask(r.Context(), slug, req.Title, req.FileType, objectKey)
taskID, err := s.deps.Producer.CreateImportTask(r.Context(), slug, req.Title, req.FileType, objectKey)
if err != nil {
jsonError(w, http.StatusInternalServerError, "create import task: "+err.Error())
return
@@ -113,8 +118,12 @@ func (s *Server) handleAdminImportStatus(w http.ResponseWriter, r *http.Request)
return
}
task, err := s.deps.TaskReader.GetImportTask(r.Context(), taskID)
task, ok, err := s.deps.TaskReader.GetImportTask(r.Context(), taskID)
if err != nil {
jsonError(w, http.StatusInternalServerError, "get task: "+err.Error())
return
}
if !ok {
jsonError(w, http.StatusNotFound, "task not found")
return
}

View File

@@ -727,11 +727,11 @@ func (r *Runner) runImportTask(ctx context.Context, task domain.ImportTask, obje
// storeImportedChapters stores imported chapters in MinIO (similar to scraped chapters).
func (r *Runner) storeImportedChapters(ctx context.Context, slug string, chapters []bookstore.Chapter) error {
for _, ch := range chapters {
content := fmt.Sprintf("# Chapter %d\n\n%s", ch.Number, ch.Content)
_ = fmt.Sprintf("# Chapter %d\n\n%s", ch.Number, ch.Content)
if ch.Title != "" {
content = fmt.Sprintf("# %s\n\n%s", ch.Title, ch.Content)
_ = fmt.Sprintf("# %s\n\n%s", ch.Title, ch.Content)
}
key := fmt.Sprintf("books/%s/chapters/%d.md", slug, ch.Number)
_ = fmt.Sprintf("books/%s/chapters/%d.md", slug, ch.Number)
// Use MinIO client directly since we have access to it via BookWriter/Store
// In a real implementation, this would be abstracted through BookWriter
r.deps.Log.Info("runner: stored chapter", "slug", slug, "chapter", ch.Number)

View File

@@ -54,6 +54,10 @@ func (s *stubConsumer) ClaimNextTranslationTask(_ context.Context, _ string) (do
return domain.TranslationTask{}, false, nil
}
func (s *stubConsumer) ClaimNextImportTask(_ context.Context, _ string) (domain.ImportTask, bool, error) {
return domain.ImportTask{}, false, nil
}
func (s *stubConsumer) FinishScrapeTask(_ context.Context, id string, _ domain.ScrapeResult) error {
s.finished = append(s.finished, id)
return nil
@@ -69,6 +73,11 @@ func (s *stubConsumer) FinishTranslationTask(_ context.Context, id string, _ dom
return nil
}
func (s *stubConsumer) FinishImportTask(_ context.Context, id string, _ domain.ImportResult) error {
s.finished = append(s.finished, id)
return nil
}
func (s *stubConsumer) FailTask(_ context.Context, id, _ string) error {
s.failCalled = append(s.failCalled, id)
return nil

View File

@@ -129,9 +129,7 @@ func (s *Store) IngestChapters(ctx context.Context, slug string, chapters []book
content = fmt.Sprintf("# %s\n\n%s", ch.Title, ch.Content)
}
key := fmt.Sprintf("books/%s/chapters/%d.md", slug, ch.Number)
if err := s.mc.PutObject(ctx, "books", key, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{
ContentType: "text/markdown",
}); err != nil {
if err := s.mc.putObject(ctx, "books", key, "text/markdown", []byte(content)); err != nil {
return fmt.Errorf("put chapter %d: %w", ch.Number, err)
}
}

View File

@@ -979,9 +979,6 @@ func (s *Store) GetImportTask(ctx context.Context, id string) (domain.ImportTask
t, err := parseImportTask(raw)
return t, err == nil, err
}
t, err := parseTranslationTask(items[0])
return t, err == nil, err
}
// ── Parsers ───────────────────────────────────────────────────────────────────
@@ -1151,6 +1148,11 @@ func (s *Store) GetCover(ctx context.Context, slug string) ([]byte, string, bool
return data, ct, true, nil
}
// PutImportFile stores an uploaded import file (PDF/EPUB) in MinIO.
func (s *Store) PutImportFile(ctx context.Context, key string, data []byte) error {
return s.mc.putObject(ctx, "imports", key, "application/octet-stream", data)
}
func (s *Store) CoverExists(ctx context.Context, slug string) bool {
return s.mc.coverExists(ctx, CoverObjectKey(slug))
}

View File

@@ -26,6 +26,9 @@ func (s *stubStore) CreateAudioTask(_ context.Context, _ string, _ int, _ string
func (s *stubStore) CreateTranslationTask(_ context.Context, _ string, _ int, _ string) (string, error) {
return "translation-1", nil
}
func (s *stubStore) CreateImportTask(_ context.Context, _, _, _, _ string) (string, error) {
return "import-1", nil
}
func (s *stubStore) CancelTask(_ context.Context, _ string) error { return nil }
func (s *stubStore) CancelAudioTasksBySlug(_ context.Context, _ string) (int, error) { return 0, nil }
@@ -38,6 +41,9 @@ func (s *stubStore) ClaimNextAudioTask(_ context.Context, _ string) (domain.Audi
func (s *stubStore) ClaimNextTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) {
return domain.TranslationTask{ID: "translation-1", Status: domain.TaskStatusRunning}, true, nil
}
func (s *stubStore) ClaimNextImportTask(_ context.Context, _ string) (domain.ImportTask, bool, error) {
return domain.ImportTask{ID: "import-1", Status: domain.TaskStatusRunning}, true, nil
}
func (s *stubStore) FinishScrapeTask(_ context.Context, _ string, _ domain.ScrapeResult) error {
return nil
}
@@ -47,6 +53,9 @@ func (s *stubStore) FinishAudioTask(_ context.Context, _ string, _ domain.AudioR
func (s *stubStore) FinishTranslationTask(_ context.Context, _ string, _ domain.TranslationResult) error {
return nil
}
func (s *stubStore) FinishImportTask(_ context.Context, _ string, _ domain.ImportResult) error {
return nil
}
func (s *stubStore) FailTask(_ context.Context, _, _ string) error { return nil }
func (s *stubStore) HeartbeatTask(_ context.Context, _ string) error { return nil }
@@ -69,6 +78,10 @@ func (s *stubStore) ListTranslationTasks(_ context.Context) ([]domain.Translatio
func (s *stubStore) GetTranslationTask(_ context.Context, _ string) (domain.TranslationTask, bool, error) {
return domain.TranslationTask{}, false, nil
}
func (s *stubStore) ListImportTasks(_ context.Context) ([]domain.ImportTask, error) { return nil, nil }
func (s *stubStore) GetImportTask(_ context.Context, _ string) (domain.ImportTask, bool, error) {
return domain.ImportTask{}, false, nil
}
// Verify the stub satisfies all three interfaces at compile time.
var _ taskqueue.Producer = (*stubStore)(nil)