Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3a9f3b773e | ||
|
|
6776d9106f |
@@ -241,7 +241,7 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo in
|
||||
}
|
||||
|
||||
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
|
||||
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
|
||||
s.log.Debug("scraping chapter list", "page", page, "url", pageURL)
|
||||
|
||||
raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second)
|
||||
if err != nil {
|
||||
|
||||
@@ -68,7 +68,7 @@ func New(cfg Config, novel scraper.NovelScraper, store bookstore.BookWriter, log
|
||||
// Returns a ScrapeResult with counters. The result's ErrorMessage is non-empty
|
||||
// if the run failed at the metadata or chapter-list level.
|
||||
func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) domain.ScrapeResult {
|
||||
o.log.Info("orchestrator: RunBook starting",
|
||||
o.log.Debug("orchestrator: RunBook starting",
|
||||
"task_id", task.ID,
|
||||
"kind", task.Kind,
|
||||
"url", task.TargetURL,
|
||||
@@ -103,7 +103,7 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma
|
||||
}
|
||||
}
|
||||
|
||||
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
|
||||
o.log.Debug("metadata saved", "slug", meta.Slug, "title", meta.Title)
|
||||
|
||||
// ── Step 2: Chapter list ──────────────────────────────────────────────────
|
||||
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL, task.ToChapter)
|
||||
@@ -114,7 +114,7 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma
|
||||
return result
|
||||
}
|
||||
|
||||
o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs))
|
||||
o.log.Debug("chapter list fetched", "slug", meta.Slug, "chapters", len(refs))
|
||||
|
||||
// Persist chapter refs (without text) so the index exists early.
|
||||
if wErr := o.store.WriteChapterRefs(ctx, meta.Slug, refs); wErr != nil {
|
||||
|
||||
@@ -505,7 +505,11 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
|
||||
log.Warn("runner: unknown task kind")
|
||||
}
|
||||
|
||||
if err := r.deps.Consumer.FinishScrapeTask(ctx, task.ID, result); err != nil {
|
||||
// Use a fresh context for the final write so a cancelled task context doesn't
|
||||
// prevent the result counters from being persisted to PocketBase.
|
||||
finishCtx, finishCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer finishCancel()
|
||||
if err := r.deps.Consumer.FinishScrapeTask(finishCtx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishScrapeTask failed", "err", err)
|
||||
}
|
||||
|
||||
@@ -551,7 +555,7 @@ func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o
|
||||
TargetURL: entry.URL,
|
||||
}
|
||||
bookResult := o.RunBook(ctx, bookTask)
|
||||
result.BooksFound += bookResult.BooksFound + 1
|
||||
result.BooksFound += bookResult.BooksFound
|
||||
result.ChaptersScraped += bookResult.ChaptersScraped
|
||||
result.ChaptersSkipped += bookResult.ChaptersSkipped
|
||||
result.Errors += bookResult.Errors
|
||||
|
||||
Reference in New Issue
Block a user