feat(import): move PDF parsing to backend; fix heartbeat/reap for import_tasks
- parsePDF function restored in import.go (body was orphaned outside function) - ParseImportFile() called at upload time with 3-min timeout; chapters stored as JSON in MinIO - runner.go: prefer ChaptersKey path (read pre-parsed JSON) over BookImport.Import() - ImportChapterStore interface added; store wired in runner/main.go - HeartbeatTask and ReapStaleTasks now include import_tasks collection - parseImportTask now returns ChaptersKey in domain.ImportTask - asynq_runner.go handleImportTask passes ChaptersKey - pb-init-v3.sh: chapters_key field added to import_tasks schema
This commit is contained in:
@@ -654,13 +654,14 @@ func (s *Store) CreateImportTask(ctx context.Context, task domain.ImportTask) (s
|
||||
"file_name": task.Slug + "." + task.FileType,
|
||||
"file_type": task.FileType,
|
||||
"object_key": task.ObjectKey,
|
||||
"chapters_key": task.ChaptersKey,
|
||||
"author": task.Author,
|
||||
"cover_url": task.CoverURL,
|
||||
"summary": task.Summary,
|
||||
"book_status": task.BookStatus,
|
||||
"status": string(domain.TaskStatusPending),
|
||||
"chapters_done": 0,
|
||||
"chapters_total": 0,
|
||||
"chapters_total": task.ChaptersTotal,
|
||||
"started": time.Now().UTC().Format(time.RFC3339),
|
||||
"initiator_user_id": task.InitiatorUserID,
|
||||
}
|
||||
@@ -914,7 +915,7 @@ func (s *Store) FailTask(ctx context.Context, id, errMsg string) error {
|
||||
}
|
||||
|
||||
// HeartbeatTask updates the heartbeat_at field on a running task.
|
||||
// Tries scraping_tasks first, then audio_jobs, then translation_jobs.
|
||||
// Tries scraping_tasks, audio_jobs, translation_jobs, then import_tasks.
|
||||
func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
|
||||
payload := map[string]any{
|
||||
"heartbeat_at": time.Now().UTC().Format(time.RFC3339),
|
||||
@@ -925,7 +926,10 @@ func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/audio_jobs/records/%s", id), payload); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), payload)
|
||||
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/translation_jobs/records/%s", id), payload); err == nil {
|
||||
return nil
|
||||
}
|
||||
return s.pb.patch(ctx, fmt.Sprintf("/api/collections/import_tasks/records/%s", id), payload)
|
||||
}
|
||||
|
||||
// ReapStaleTasks finds all running tasks whose heartbeat_at is either missing
|
||||
@@ -943,7 +947,7 @@ func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (i
|
||||
}
|
||||
|
||||
total := 0
|
||||
for _, collection := range []string{"scraping_tasks", "audio_jobs", "translation_jobs"} {
|
||||
for _, collection := range []string{"scraping_tasks", "audio_jobs", "translation_jobs", "import_tasks"} {
|
||||
items, err := s.pb.listAll(ctx, collection, filter, "")
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("ReapStaleTasks list %s: %w", collection, err)
|
||||
@@ -1185,6 +1189,7 @@ func parseImportTask(raw json.RawMessage) (domain.ImportTask, error) {
|
||||
FileName string `json:"file_name"`
|
||||
FileType string `json:"file_type"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
ChaptersKey string `json:"chapters_key"`
|
||||
Author string `json:"author"`
|
||||
CoverURL string `json:"cover_url"`
|
||||
Genres string `json:"genres"` // stored as comma-separated
|
||||
@@ -1219,6 +1224,7 @@ func parseImportTask(raw json.RawMessage) (domain.ImportTask, error) {
|
||||
FileName: rec.FileName,
|
||||
FileType: rec.FileType,
|
||||
ObjectKey: rec.ObjectKey,
|
||||
ChaptersKey: rec.ChaptersKey,
|
||||
Author: rec.Author,
|
||||
CoverURL: rec.CoverURL,
|
||||
Genres: genres,
|
||||
@@ -1266,6 +1272,20 @@ func (s *Store) PutImportFile(ctx context.Context, key string, data []byte) erro
|
||||
return s.mc.putObject(ctx, "imports", key, "application/octet-stream", data)
|
||||
}
|
||||
|
||||
// PutImportChapters stores a pre-parsed chapters JSON blob in MinIO.
|
||||
func (s *Store) PutImportChapters(ctx context.Context, key string, data []byte) error {
|
||||
return s.mc.putObject(ctx, "imports", key, "application/json", data)
|
||||
}
|
||||
|
||||
// GetImportChapters retrieves the pre-parsed chapters JSON from MinIO.
|
||||
func (s *Store) GetImportChapters(ctx context.Context, key string) ([]byte, error) {
|
||||
data, err := s.mc.getObject(ctx, "imports", key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get chapters object: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (s *Store) CoverExists(ctx context.Context, slug string) bool {
|
||||
return s.mc.coverExists(ctx, CoverObjectKey(slug))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user