fix(scraper): fix ScrapeCatalogue selectors, zero-byte cache skip, and wire ScrapeRanking into runAsync

- ScrapeCatalogue: use li.novel-item (was div), extract href from outer <a> and title from h4.novel-title (was h3), detect next page via rel=next (was class=next)
- handleBrowse/triggerBrowseSnapshot: treat zero-byte MinIO cache entries as misses, skip storing empty SingleFile output
- runAsync: after a successful full-catalogue run, call ScrapeRanking and upsert each result into PocketBase ranking collection
This commit is contained in:
Admin
2026-03-04 16:54:00 +05:00
parent 8de374cd35
commit d14644238f
2 changed files with 70 additions and 12 deletions

View File

@@ -829,6 +829,36 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
o := orchestrator.New(cfg, s.novel, s.log, s.store)
runErr := o.Run(ctx)
// After a successful full-catalogue run, refresh the ranking list.
if runErr == nil && cfg.SingleBookURL == "" {
s.log.Info("runAsync: starting ScrapeRanking after catalogue run")
rankCtx, rankCancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer rankCancel()
rankEntries, rankErrs := s.novel.ScrapeRanking(rankCtx, 0)
rank := 1
for meta := range rankEntries {
item := storage.RankingItem{
Rank: rank,
Slug: meta.Slug,
Title: meta.Title,
Author: meta.Author,
Cover: meta.Cover,
Status: meta.Status,
Genres: meta.Genres,
SourceURL: meta.SourceURL,
}
if werr := s.store.WriteRankingItem(rankCtx, item); werr != nil {
s.log.Warn("runAsync: WriteRankingItem failed", "slug", meta.Slug, "err", werr)
}
rank++
}
if rerr := <-rankErrs; rerr != nil {
s.log.Warn("runAsync: ScrapeRanking finished with error", "err", rerr)
} else {
s.log.Info("runAsync: ScrapeRanking complete", "count", rank-1)
}
}
// Determine final status.
finalStatus := "done"
errMsg := ""
@@ -938,7 +968,7 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 {
novels, hasNext := parseBrowsePage(strings.NewReader(html))
s.log.Debug("browse: served from cache", "key", cacheKey)
w.Header().Set("Content-Type", "application/json")
@@ -1062,6 +1092,11 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
"key", cacheKey, "err", readErr)
return
}
if len(htmlBytes) == 0 {
s.log.Warn("triggerBrowseSnapshot: SingleFile produced empty output, skipping cache",
"key", cacheKey)
return
}
// Store the HTML snapshot.
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {