kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

fix: reduce log noise during catalogue/book scrapes
All checks were successful
Release / Test backend (push) Successful in 55s
Details
Release / Check ui (push) Successful in 2m0s
Details
Release / Docker (push) Successful in 5m57s
Details
Release / Gitea Release (push) Successful in 32s
Details

Browse Source 
Demote per-book and per-chapter-list-page Info logs to Debug — these
fire hundreds of times per catalogue run and drown out meaningful signals:
- orchestrator: RunBook starting (per book)
- metadata saved (per book)
- chapter list fetched (per book)
- scraping chapter list page N (per pagination page per book)

The 'book scrape finished' summary log (with scraped/skipped/errors
counters) remains at Info — it is the useful signal per book.
This commit is contained in:
root
2026-04-11 12:39:41 +05:00
parent 6776d9106f
commit 3a9f3b773e
2 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/internal/novelfire/scraper.go
View File

@@ -241,7 +241,7 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo in
} }
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page) pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
s.log.Info("scraping chapter list", "page", page, "url", pageURL) s.log.Debug("scraping chapter list", "page", page, "url", pageURL)
raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second) raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second)
if err != nil { if err != nil {

6
backend/internal/orchestrator/orchestrator.go
View File

@@ -68,7 +68,7 @@ func New(cfg Config, novel scraper.NovelScraper, store bookstore.BookWriter, log
// Returns a ScrapeResult with counters. The result's ErrorMessage is non-empty // Returns a ScrapeResult with counters. The result's ErrorMessage is non-empty
// if the run failed at the metadata or chapter-list level. // if the run failed at the metadata or chapter-list level.
func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) domain.ScrapeResult { func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) domain.ScrapeResult {
o.log.Info("orchestrator: RunBook starting", o.log.Debug("orchestrator: RunBook starting",
"task_id", task.ID, "task_id", task.ID,
"kind", task.Kind, "kind", task.Kind,
"url", task.TargetURL, "url", task.TargetURL,
@@ -103,7 +103,7 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma
} }
} }
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title) o.log.Debug("metadata saved", "slug", meta.Slug, "title", meta.Title)
// ── Step 2: Chapter list ────────────────────────────────────────────────── // ── Step 2: Chapter list ──────────────────────────────────────────────────
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL, task.ToChapter) refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL, task.ToChapter)
@@ -114,7 +114,7 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma
return result return result
} }
o.log.Info("chapter list fetched", "slug", meta.Slug, "chapters", len(refs)) o.log.Debug("chapter list fetched", "slug", meta.Slug, "chapters", len(refs))
// Persist chapter refs (without text) so the index exists early. // Persist chapter refs (without text) so the index exists early.
if wErr := o.store.WriteChapterRefs(ctx, meta.Slug, refs); wErr != nil { if wErr := o.store.WriteChapterRefs(ctx, meta.Slug, refs); wErr != nil {