feat(import): move PDF parsing to backend; fix heartbeat/reap for import_tasks
All checks were successful
Release / Test backend (push) Successful in 51s
Release / Check ui (push) Successful in 2m2s
Release / Docker (push) Successful in 7m32s
Release / Gitea Release (push) Successful in 1m0s

- parsePDF function restored in import.go (body was orphaned outside function)
- ParseImportFile() called at upload time with 3-min timeout; chapters stored as JSON in MinIO
- runner.go: prefer ChaptersKey path (read pre-parsed JSON) over BookImport.Import()
- ImportChapterStore interface added; store wired in runner/main.go
- HeartbeatTask and ReapStaleTasks now include import_tasks collection
- parseImportTask now returns ChaptersKey in domain.ImportTask
- asynq_runner.go handleImportTask passes ChaptersKey
- pb-init-v3.sh: chapters_key field added to import_tasks schema
This commit is contained in:
root
2026-04-09 21:19:43 +05:00
parent d82aa9d4b4
commit 899c504d1f
11 changed files with 174 additions and 59 deletions

View File

@@ -1,6 +1,7 @@
package backend
import (
"context"
"encoding/json"
"fmt"
"io"
@@ -45,6 +46,8 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
ct := r.Header.Get("Content-Type")
var req importRequest
var objectKey string
var chaptersKey string
var chapterCount int
if strings.HasPrefix(ct, "multipart/form-data") {
if err := r.ParseMultipartForm(32 << 20); err != nil {
@@ -96,7 +99,20 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
return
}
// Upload to MinIO for actual import
// Parse PDF/EPUB on the backend (with timeout) and store chapters as JSON.
// The runner only needs to ingest pre-parsed chapters — no PDF parsing on runner.
parseCtx, parseCancel := context.WithTimeout(r.Context(), 3*time.Minute)
defer parseCancel()
chapters, parseErr := storage.ParseImportFile(parseCtx, data, req.FileType)
if parseErr != nil || len(chapters) == 0 {
jsonError(w, http.StatusUnprocessableEntity, "could not parse file: "+func() string {
if parseErr != nil { return parseErr.Error() }
return "no chapters found"
}())
return
}
// Store raw file in MinIO (for reference/re-import).
objectKey = fmt.Sprintf("imports/%d_%s", time.Now().Unix(), header.Filename)
if s.deps.ImportFileStore == nil {
jsonError(w, http.StatusInternalServerError, "storage not available")
@@ -106,6 +122,15 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
jsonError(w, http.StatusInternalServerError, "upload file: "+err.Error())
return
}
// Store pre-parsed chapters JSON in MinIO so runner can ingest without re-parsing.
chaptersJSON, _ := json.Marshal(chapters)
chaptersKey = fmt.Sprintf("imports/%d_%s_chapters.json", time.Now().Unix(), strings.TrimSuffix(header.Filename, filepath.Ext(header.Filename)))
if err := s.deps.ImportFileStore.PutImportChapters(r.Context(), chaptersKey, chaptersJSON); err != nil {
jsonError(w, http.StatusInternalServerError, "store chapters: "+err.Error())
return
}
chapterCount = len(chapters)
} else {
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
jsonError(w, http.StatusBadRequest, "parse body: "+err.Error())
@@ -141,6 +166,8 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
BookStatus: req.BookStatus,
FileType: req.FileType,
ObjectKey: objectKey,
ChaptersKey: chaptersKey,
ChaptersTotal: chapterCount,
InitiatorUserID: "",
})
if err != nil {
@@ -149,8 +176,9 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
}
writeJSON(w, 0, importResponse{
TaskID: taskID,
Slug: slug,
TaskID: taskID,
Slug: slug,
Preview: &importPreview{Chapters: chapterCount},
})
}