Compare commits

...

2 Commits

Author SHA1 Message Date
root
3a9f3b773e fix: reduce log noise during catalogue/book scrapes
All checks were successful
Release / Test backend (push) Successful in 55s
Release / Check ui (push) Successful in 2m0s
Release / Docker (push) Successful in 5m57s
Release / Gitea Release (push) Successful in 32s
Demote per-book and per-chapter-list-page Info logs to Debug — these
fire hundreds of times per catalogue run and drown out meaningful signals:
- orchestrator: RunBook starting (per book)
- metadata saved (per book)
- chapter list fetched (per book)
- scraping chapter list page N (per pagination page per book)

The 'book scrape finished' summary log (with scraped/skipped/errors
counters) remains at Info — it is the useful signal per book.
2026-04-11 12:39:41 +05:00
root
6776d9106f fix: catalogue job always shows 0 counters after cancel/finish
All checks were successful
Release / Test backend (push) Successful in 48s
Release / Check ui (push) Successful in 1m55s
Release / Docker (push) Successful in 6m5s
Release / Gitea Release (push) Successful in 32s
Two bugs fixed in runScrapeTask / runCatalogueTask:

1. FinishScrapeTask was called with the task's own context, which is
   already cancelled when the task is stopped. The PATCH to PocketBase
   failed silently, leaving all counters at their initial zero values.
   Fix: use a fresh context.WithTimeout(Background, 15s) for the write.

2. BooksFound was double-counted: RunBook already sets BooksFound=1 on
   success, but the accumulation loop added an extra +1 unconditionally,
   reporting 2 books per successful scrape.
   Fix: result.BooksFound += bookResult.BooksFound  (drop the + 1).
2026-04-11 12:33:30 +05:00
3 changed files with 10 additions and 6 deletions

View File

@@ -241,7 +241,7 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo in
}
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
s.log.Debug("scraping chapter list", "page", page, "url", pageURL)
raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second)
if err != nil {

View File

@@ -68,7 +68,7 @@ func New(cfg Config, novel scraper.NovelScraper, store bookstore.BookWriter, log
// Returns a ScrapeResult with counters. The result's ErrorMessage is non-empty
// if the run failed at the metadata or chapter-list level.
func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) domain.ScrapeResult {
o.log.Info("orchestrator: RunBook starting",
o.log.Debug("orchestrator: RunBook starting",
"task_id", task.ID,
"kind", task.Kind,
"url", task.TargetURL,
@@ -103,7 +103,7 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma
}
}
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
o.log.Debug("metadata saved", "slug", meta.Slug, "title", meta.Title)
// ── Step 2: Chapter list ──────────────────────────────────────────────────
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL, task.ToChapter)
@@ -114,7 +114,7 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma
return result
}
o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs))
o.log.Debug("chapter list fetched", "slug", meta.Slug, "chapters", len(refs))
// Persist chapter refs (without text) so the index exists early.
if wErr := o.store.WriteChapterRefs(ctx, meta.Slug, refs); wErr != nil {

View File

@@ -505,7 +505,11 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
log.Warn("runner: unknown task kind")
}
if err := r.deps.Consumer.FinishScrapeTask(ctx, task.ID, result); err != nil {
// Use a fresh context for the final write so a cancelled task context doesn't
// prevent the result counters from being persisted to PocketBase.
finishCtx, finishCancel := context.WithTimeout(context.Background(), 15*time.Second)
defer finishCancel()
if err := r.deps.Consumer.FinishScrapeTask(finishCtx, task.ID, result); err != nil {
log.Error("runner: FinishScrapeTask failed", "err", err)
}
@@ -551,7 +555,7 @@ func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o
TargetURL: entry.URL,
}
bookResult := o.RunBook(ctx, bookTask)
result.BooksFound += bookResult.BooksFound + 1
result.BooksFound += bookResult.BooksFound
result.ChaptersScraped += bookResult.ChaptersScraped
result.ChaptersSkipped += bookResult.ChaptersSkipped
result.Errors += bookResult.Errors