feat(import): move PDF parsing to backend; fix heartbeat/reap for import_tasks
All checks were successful
Release / Test backend (push) Successful in 51s
Release / Check ui (push) Successful in 2m2s
Release / Docker (push) Successful in 7m32s
Release / Gitea Release (push) Successful in 1m0s

- parsePDF function restored in import.go (body was orphaned outside function)
- ParseImportFile() called at upload time with 3-min timeout; chapters stored as JSON in MinIO
- runner.go: prefer ChaptersKey path (read pre-parsed JSON) over BookImport.Import()
- ImportChapterStore interface added; store wired in runner/main.go
- HeartbeatTask and ReapStaleTasks now include import_tasks collection
- parseImportTask now returns ChaptersKey in domain.ImportTask
- asynq_runner.go handleImportTask passes ChaptersKey
- pb-init-v3.sh: chapters_key field added to import_tasks schema
This commit is contained in:
root
2026-04-09 21:19:43 +05:00
parent d82aa9d4b4
commit 899c504d1f
11 changed files with 174 additions and 59 deletions

View File

@@ -109,6 +109,40 @@ func decryptPDF(data []byte) ([]byte, error) {
return out.Bytes(), nil
}
// ParseImportFile parses a PDF or EPUB and returns chapters.
// Unlike AnalyzeFile it respects ctx cancellation so callers can apply a timeout.
// For PDFs it first attempts to strip encryption with an empty password.
func ParseImportFile(ctx context.Context, data []byte, fileType string) ([]bookstore.Chapter, error) {
type result struct {
chapters []bookstore.Chapter
err error
}
ch := make(chan result, 1)
go func() {
var chapters []bookstore.Chapter
var err error
switch fileType {
case "pdf":
chapters, err = parsePDF(data)
case "epub":
chapters, err = parseEPUB(data)
default:
err = fmt.Errorf("unsupported file type: %s", fileType)
}
ch <- result{chapters, err}
}()
select {
case <-ctx.Done():
return nil, fmt.Errorf("parse timed out: %w", ctx.Err())
case r := <-ch:
return r.chapters, r.err
}
}
// parsePDF extracts chapters from PDF bytes using dslipak/pdf.
// It first attempts to decrypt the PDF with an empty password in case the file
// uses owner-only encryption (copy/print restrictions), which is common for
// publisher PDFs that open normally in readers but confuse raw parsers.
func parsePDF(data []byte) ([]bookstore.Chapter, error) {
// If the PDF is encrypted, try to decrypt it with an empty password.
// Many publisher PDFs use owner-only encryption (copy/print restrictions)

View File

@@ -654,13 +654,14 @@ func (s *Store) CreateImportTask(ctx context.Context, task domain.ImportTask) (s
"file_name": task.Slug + "." + task.FileType,
"file_type": task.FileType,
"object_key": task.ObjectKey,
"chapters_key": task.ChaptersKey,
"author": task.Author,
"cover_url": task.CoverURL,
"summary": task.Summary,
"book_status": task.BookStatus,
"status": string(domain.TaskStatusPending),
"chapters_done": 0,
"chapters_total": 0,
"chapters_total": task.ChaptersTotal,
"started": time.Now().UTC().Format(time.RFC3339),
"initiator_user_id": task.InitiatorUserID,
}
@@ -914,7 +915,7 @@ func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
}
// HeartbeatTask updates the heartbeat_at field on a running task.
// Tries scraping_tasks first, then audio_jobs, then translation_jobs.
// Tries scraping_tasks, audio_jobs, translation_jobs, then import_tasks.
func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
payload := map[string]any{
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
@@ -925,7 +926,10 @@ func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), payload)
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), payload); err == nil {
return nil
}
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/import_tasks/records/%s", id), payload)
}
// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing
@@ -943,7 +947,7 @@ func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (i
}
total := 0
for _, collection := range []string{"scraping_tasks", "audio_jobs", "translation_jobs"} {
for _, collection := range []string{"scraping_tasks", "audio_jobs", "translation_jobs", "import_tasks"} {
items, err := s.pb.listAll(ctx, collection, filter, "")
if err != nil {
return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err)
@@ -1185,6 +1189,7 @@ func parseImportTask(raw json.RawMessage) (domain.ImportTask, error) {
FileName string `json:"file_name"`
FileType string `json:"file_type"`
ObjectKey string `json:"object_key"`
ChaptersKey string `json:"chapters_key"`
Author string `json:"author"`
CoverURL string `json:"cover_url"`
Genres string `json:"genres"` // stored as comma-separated
@@ -1219,6 +1224,7 @@ func parseImportTask(raw json.RawMessage) (domain.ImportTask, error) {
FileName: rec.FileName,
FileType: rec.FileType,
ObjectKey: rec.ObjectKey,
ChaptersKey: rec.ChaptersKey,
Author: rec.Author,
CoverURL: rec.CoverURL,
Genres: genres,
@@ -1266,6 +1272,20 @@ func (s *Store) PutImportFile(ctx context.Context, key string, data []byte) erro
return s.mc.putObject(ctx, "imports", key, "application/octet-stream", data)
}
// PutImportChapters stores a pre-parsed chapters JSON blob in MinIO.
func (s *Store) PutImportChapters(ctx context.Context, key string, data []byte) error {
return s.mc.putObject(ctx, "imports", key, "application/json", data)
}
// GetImportChapters retrieves the pre-parsed chapters JSON from MinIO.
func (s *Store) GetImportChapters(ctx context.Context, key string) ([]byte, error) {
data, err := s.mc.getObject(ctx, "imports", key)
if err != nil {
return nil, fmt.Errorf("get chapters object: %w", err)
}
return data, nil
}
func (s *Store) CoverExists(ctx context.Context, slug string) bool {
return s.mc.coverExists(ctx, CoverObjectKey(slug))
}