feat: add PDF/EPUB import functionality
- Add ImportTask/ImportResult types to domain.go - Add TypeImportBook to asynqqueue for task routing - Add CreateImportTask to producer and storage layers - Add ClaimNextImportTask/FinishImportTask to Consumer - Add import task handling to runner (polling + Asynq handler) - Add BookImporter interface to bookstore for PDF/EPUB parsing - Add backend API endpoints: POST/GET /api/admin/import - Add SvelteKit UI at /admin/import with task list - Add nav link in admin layout Note: PDF/EPUB parsing is a placeholder - needs external library integration.
This commit is contained in:
root
166
backend/internal/storage/import.go
Normal file
166
backend/internal/storage/import.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/libnovel/backend/internal/bookstore"
|
||||
"github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
var (
|
||||
chapterPattern = regexp.MustCompile(`(?i)chapter\s+(\d+)|The\s+Eminence\s+in\s+Shadow\s+(\d+)\s*-\s*(\d+)`)
|
||||
)
|
||||
|
||||
type importer struct {
|
||||
mc *minio.Client
|
||||
}
|
||||
|
||||
// NewBookImporter creates a BookImporter that reads files from MinIO.
|
||||
func NewBookImporter(mc *minio.Client) bookstore.BookImporter {
|
||||
return &importer{mc: mc}
|
||||
}
|
||||
|
||||
func (i *importer) Import(ctx context.Context, objectKey, fileType string) ([]bookstore.Chapter, error) {
|
||||
if fileType != "pdf" && fileType != "epub" {
|
||||
return nil, fmt.Errorf("unsupported file type: %s", fileType)
|
||||
}
|
||||
|
||||
obj, err := i.mc.GetObject(ctx, "imports", objectKey, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get object from minio: %w", err)
|
||||
}
|
||||
defer obj.Close()
|
||||
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read object: %w", err)
|
||||
}
|
||||
|
||||
if fileType == "pdf" {
|
||||
return i.parsePDF(data)
|
||||
}
|
||||
return i.parseEPUB(data)
|
||||
}
|
||||
|
||||
func (i *importer) parsePDF(data []byte) ([]bookstore.Chapter, error) {
|
||||
return nil, errors.New("PDF parsing not yet implemented - requires external library")
|
||||
}
|
||||
|
||||
func (i *importer) parseEPUB(data []byte) ([]bookstore.Chapter, error) {
|
||||
return nil, errors.New("EPUB parsing not yet implemented - requires external library")
|
||||
}
|
||||
|
||||
// extractChaptersFromText is a helper that splits raw text into chapters.
|
||||
// Used as a fallback when the PDF parser library returns raw text.
|
||||
func extractChaptersFromText(text string) []bookstore.Chapter {
|
||||
var chapters []bookstore.Chapter
|
||||
var currentChapter *bookstore.Chapter
|
||||
|
||||
lines := strings.Split(text, "\n")
|
||||
chapterNum := 0
|
||||
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if len(line) < 3 {
|
||||
continue
|
||||
}
|
||||
|
||||
matches := chapterPattern.FindStringSubmatch(line)
|
||||
if matches != nil {
|
||||
if currentChapter != nil && currentChapter.Content != "" {
|
||||
chapters = append(chapters, *currentChapter)
|
||||
}
|
||||
chapterNum++
|
||||
if matches[1] != "" {
|
||||
chapterNum, _ = fmt.Sscanf(matches[1], "%d", &chapterNum)
|
||||
}
|
||||
currentChapter = &bookstore.Chapter{
|
||||
Number: chapterNum,
|
||||
Title: line,
|
||||
Content: "",
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if currentChapter != nil {
|
||||
if currentChapter.Content != "" {
|
||||
currentChapter.Content += " "
|
||||
}
|
||||
currentChapter.Content += line
|
||||
}
|
||||
}
|
||||
|
||||
if currentChapter != nil && currentChapter.Content != "" {
|
||||
chapters = append(chapters, *currentChapter)
|
||||
}
|
||||
|
||||
// If no chapters found via regex, try splitting by double newlines
|
||||
if len(chapters) == 0 {
|
||||
paragraphs := strings.Split(text, "\n\n")
|
||||
for i, para := range paragraphs {
|
||||
para = strings.TrimSpace(para)
|
||||
if len(para) > 50 {
|
||||
chapters = append(chapters, bookstore.Chapter{
|
||||
Number: i + 1,
|
||||
Title: fmt.Sprintf("Chapter %d", i+1),
|
||||
Content: para,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return chapters
|
||||
}
|
||||
|
||||
// IngestChapters stores extracted chapters for a book via BookWriter.
|
||||
// This is called by the runner after extracting chapters from PDF/EPUB.
|
||||
func (s *Store) IngestChapters(ctx context.Context, slug string, chapters []bookstore.Chapter) error {
|
||||
// For now, store each chapter as plain text in MinIO (similar to scraped chapters)
|
||||
// The BookWriter interface expects markdown, so we'll store the content as-is
|
||||
for _, ch := range chapters {
|
||||
content := fmt.Sprintf("# Chapter %d\n\n%s", ch.Number, ch.Content)
|
||||
if ch.Title != "" {
|
||||
content = fmt.Sprintf("# %s\n\n%s", ch.Title, ch.Content)
|
||||
}
|
||||
key := fmt.Sprintf("books/%s/chapters/%d.md", slug, ch.Number)
|
||||
if err := s.mc.PutObject(ctx, "books", key, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{
|
||||
ContentType: "text/markdown",
|
||||
}); err != nil {
|
||||
return fmt.Errorf("put chapter %d: %w", ch.Number, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Also create a simple metadata entry in the books collection
|
||||
// (in a real implementation, we'd update the existing book or create a placeholder)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetImportObjectKey returns the MinIO object key for an uploaded import file.
|
||||
func GetImportObjectKey(filename string) string {
|
||||
return fmt.Sprintf("imports/%s", filename)
|
||||
}
|
||||
|
||||
func parsePDFWithPython(data []byte) ([]bookstore.Chapter, error) {
|
||||
// This would require calling an external Python script or service
|
||||
// For now, return placeholder - in production, this would integrate with
|
||||
// the Python pypdf library via subprocess or API call
|
||||
return nil, errors.New("PDF parsing requires Python integration")
|
||||
}
|
||||
|
||||
// Debug helper - decode a base64-encoded PDF from bytes and extract text
|
||||
func extractTextFromPDFBytes(data []byte) (string, error) {
|
||||
// This is a placeholder - in production we'd use a proper Go PDF library
|
||||
// like github.com/ledongthuc/pdf or the Python approach
|
||||
var buf bytes.Buffer
|
||||
_, err := buf.Write(data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "", errors.New("PDF text extraction not implemented in Go")
|
||||
}
|
||||
@@ -647,6 +647,26 @@ func (s *Store) CreateTranslationTask(ctx context.Context, slug string, chapter
|
||||
return rec.ID, nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateImportTask(ctx context.Context, slug, title, fileType, objectKey string) (string, error) {
|
||||
payload := map[string]any{
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"file_name": slug + "." + fileType,
|
||||
"file_type": fileType,
|
||||
"status": string(domain.TaskStatusPending),
|
||||
"chapters_done": 0,
|
||||
"chapters_total": 0,
|
||||
"started": time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := s.pb.post(ctx, "/api/collections/import_tasks/records", payload, &rec); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return rec.ID, nil
|
||||
}
|
||||
|
||||
func (s *Store) CancelTask(ctx context.Context, id string) error {
|
||||
// Try scraping_tasks first, then audio_jobs, then translation_jobs.
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),
|
||||
@@ -721,6 +741,18 @@ func (s *Store) ClaimNextTranslationTask(ctx context.Context, workerID string) (
|
||||
return task, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) ClaimNextImportTask(ctx context.Context, workerID string) (domain.ImportTask, bool, error) {
|
||||
raw, err := s.pb.claimRecord(ctx, "import_tasks", workerID, nil)
|
||||
if err != nil {
|
||||
return domain.ImportTask{}, false, err
|
||||
}
|
||||
if raw == nil {
|
||||
return domain.ImportTask{}, false, nil
|
||||
}
|
||||
task, err := parseImportTask(raw)
|
||||
return task, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
|
||||
status := string(domain.TaskStatusDone)
|
||||
if result.ErrorMessage != "" {
|
||||
@@ -761,6 +793,20 @@ func (s *Store) FinishTranslationTask(ctx context.Context, id string, result dom
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) FinishImportTask(ctx context.Context, id string, result domain.ImportResult) error {
|
||||
status := string(domain.TaskStatusDone)
|
||||
if result.ErrorMessage != "" {
|
||||
status = string(domain.TaskStatusFailed)
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/import_tasks/records/%s", id), map[string]any{
|
||||
"status": status,
|
||||
"chapters_done": result.ChaptersImported,
|
||||
"chapters_total": result.ChaptersImported,
|
||||
"error_message": result.ErrorMessage,
|
||||
"finished": time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
|
||||
payload := map[string]any{
|
||||
"status": string(domain.TaskStatusFailed),
|
||||
@@ -899,8 +945,7 @@ func (s *Store) ListTranslationTasks(ctx context.Context) ([]domain.TranslationT
|
||||
}
|
||||
|
||||
func (s *Store) GetTranslationTask(ctx context.Context, cacheKey string) (domain.TranslationTask, bool, error) {
|
||||
filter := fmt.Sprintf(`cache_key='%s'`, cacheKey)
|
||||
items, err := s.pb.listAll(ctx, "translation_jobs", filter, "-started")
|
||||
items, err := s.pb.listAll(ctx, "translation_jobs", fmt.Sprintf("cache_key=%q", cacheKey), "-started")
|
||||
if err != nil || len(items) == 0 {
|
||||
return domain.TranslationTask{}, false, err
|
||||
}
|
||||
@@ -908,6 +953,36 @@ func (s *Store) GetTranslationTask(ctx context.Context, cacheKey string) (domain
|
||||
return t, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) ListImportTasks(ctx context.Context) ([]domain.ImportTask, error) {
|
||||
items, err := s.pb.listAll(ctx, "import_tasks", "", "-started")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks := make([]domain.ImportTask, 0, len(items))
|
||||
for _, raw := range items {
|
||||
t, err := parseImportTask(raw)
|
||||
if err == nil {
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetImportTask(ctx context.Context, id string) (domain.ImportTask, bool, error) {
|
||||
var raw json.RawMessage
|
||||
if err := s.pb.get(ctx, fmt.Sprintf("/api/collections/import_tasks/records/%s", id), &raw); err != nil {
|
||||
if err == ErrNotFound {
|
||||
return domain.ImportTask{}, false, nil
|
||||
}
|
||||
return domain.ImportTask{}, false, err
|
||||
}
|
||||
t, err := parseImportTask(raw)
|
||||
return t, err == nil, err
|
||||
}
|
||||
t, err := parseTranslationTask(items[0])
|
||||
return t, err == nil, err
|
||||
}
|
||||
|
||||
// ── Parsers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
func parseScrapeTask(raw json.RawMessage) (domain.ScrapeTask, error) {
|
||||
@@ -1014,6 +1089,42 @@ func parseTranslationTask(raw json.RawMessage) (domain.TranslationTask, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseImportTask(raw json.RawMessage) (domain.ImportTask, error) {
|
||||
var rec struct {
|
||||
ID string `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
FileName string `json:"file_name"`
|
||||
FileType string `json:"file_type"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
Status string `json:"status"`
|
||||
ChaptersDone int `json:"chapters_done"`
|
||||
ChaptersTotal int `json:"chapters_total"`
|
||||
ErrorMessage string `json:"error_message"`
|
||||
Started string `json:"started"`
|
||||
Finished string `json:"finished"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &rec); err != nil {
|
||||
return domain.ImportTask{}, err
|
||||
}
|
||||
started, _ := time.Parse(time.RFC3339, rec.Started)
|
||||
finished, _ := time.Parse(time.RFC3339, rec.Finished)
|
||||
return domain.ImportTask{
|
||||
ID: rec.ID,
|
||||
Slug: rec.Slug,
|
||||
Title: rec.Title,
|
||||
FileName: rec.FileName,
|
||||
FileType: rec.FileType,
|
||||
WorkerID: rec.WorkerID,
|
||||
Status: domain.TaskStatus(rec.Status),
|
||||
ChaptersDone: rec.ChaptersDone,
|
||||
ChaptersTotal: rec.ChaptersTotal,
|
||||
ErrorMessage: rec.ErrorMessage,
|
||||
Started: started,
|
||||
Finished: finished,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ── CoverStore ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) PutCover(ctx context.Context, slug string, data []byte, contentType string) error {
|
||||
|
||||
Reference in New Issue
Block a user