feat(import): move PDF parsing to backend; fix heartbeat/reap for import_tasks
- parsePDF function restored in import.go (body was orphaned outside function) - ParseImportFile() called at upload time with 3-min timeout; chapters stored as JSON in MinIO - runner.go: prefer ChaptersKey path (read pre-parsed JSON) over BookImport.Import() - ImportChapterStore interface added; store wired in runner/main.go - HeartbeatTask and ReapStaleTasks now include import_tasks collection - parseImportTask now returns ChaptersKey in domain.ImportTask - asynq_runner.go handleImportTask passes ChaptersKey - pb-init-v3.sh: chapters_key field added to import_tasks schema
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package backend
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -45,6 +46,8 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
|
||||
ct := r.Header.Get("Content-Type")
|
||||
var req importRequest
|
||||
var objectKey string
|
||||
var chaptersKey string
|
||||
var chapterCount int
|
||||
|
||||
if strings.HasPrefix(ct, "multipart/form-data") {
|
||||
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||
@@ -96,7 +99,20 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Upload to MinIO for actual import
|
||||
// Parse PDF/EPUB on the backend (with timeout) and store chapters as JSON.
|
||||
// The runner only needs to ingest pre-parsed chapters — no PDF parsing on runner.
|
||||
parseCtx, parseCancel := context.WithTimeout(r.Context(), 3*time.Minute)
|
||||
defer parseCancel()
|
||||
chapters, parseErr := storage.ParseImportFile(parseCtx, data, req.FileType)
|
||||
if parseErr != nil || len(chapters) == 0 {
|
||||
jsonError(w, http.StatusUnprocessableEntity, "could not parse file: "+func() string {
|
||||
if parseErr != nil { return parseErr.Error() }
|
||||
return "no chapters found"
|
||||
}())
|
||||
return
|
||||
}
|
||||
|
||||
// Store raw file in MinIO (for reference/re-import).
|
||||
objectKey = fmt.Sprintf("imports/%d_%s", time.Now().Unix(), header.Filename)
|
||||
if s.deps.ImportFileStore == nil {
|
||||
jsonError(w, http.StatusInternalServerError, "storage not available")
|
||||
@@ -106,6 +122,15 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
|
||||
jsonError(w, http.StatusInternalServerError, "upload file: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Store pre-parsed chapters JSON in MinIO so runner can ingest without re-parsing.
|
||||
chaptersJSON, _ := json.Marshal(chapters)
|
||||
chaptersKey = fmt.Sprintf("imports/%d_%s_chapters.json", time.Now().Unix(), strings.TrimSuffix(header.Filename, filepath.Ext(header.Filename)))
|
||||
if err := s.deps.ImportFileStore.PutImportChapters(r.Context(), chaptersKey, chaptersJSON); err != nil {
|
||||
jsonError(w, http.StatusInternalServerError, "store chapters: "+err.Error())
|
||||
return
|
||||
}
|
||||
chapterCount = len(chapters)
|
||||
} else {
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
jsonError(w, http.StatusBadRequest, "parse body: "+err.Error())
|
||||
@@ -141,6 +166,8 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
|
||||
BookStatus: req.BookStatus,
|
||||
FileType: req.FileType,
|
||||
ObjectKey: objectKey,
|
||||
ChaptersKey: chaptersKey,
|
||||
ChaptersTotal: chapterCount,
|
||||
InitiatorUserID: "",
|
||||
})
|
||||
if err != nil {
|
||||
@@ -149,8 +176,9 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
writeJSON(w, 0, importResponse{
|
||||
TaskID: taskID,
|
||||
Slug: slug,
|
||||
TaskID: taskID,
|
||||
Slug: slug,
|
||||
Preview: &importPreview{Chapters: chapterCount},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user