fix(scraper): fix ScrapeCatalogue selectors, zero-byte cache skip, and wire ScrapeRanking into runAsync
- ScrapeCatalogue: use li.novel-item (was div), extract href from outer <a> and title from h4.novel-title (was h3), detect next page via rel=next (was class=next) - handleBrowse/triggerBrowseSnapshot: treat zero-byte MinIO cache entries as misses, skip storing empty SingleFile output - runAsync: after a successful full-catalogue run, call ScrapeRanking and upsert each result into PocketBase ranking collection
This commit is contained in:
@@ -829,6 +829,36 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
||||
o := orchestrator.New(cfg, s.novel, s.log, s.store)
|
||||
runErr := o.Run(ctx)
|
||||
|
||||
// After a successful full-catalogue run, refresh the ranking list.
|
||||
if runErr == nil && cfg.SingleBookURL == "" {
|
||||
s.log.Info("runAsync: starting ScrapeRanking after catalogue run")
|
||||
rankCtx, rankCancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer rankCancel()
|
||||
rankEntries, rankErrs := s.novel.ScrapeRanking(rankCtx, 0)
|
||||
rank := 1
|
||||
for meta := range rankEntries {
|
||||
item := storage.RankingItem{
|
||||
Rank: rank,
|
||||
Slug: meta.Slug,
|
||||
Title: meta.Title,
|
||||
Author: meta.Author,
|
||||
Cover: meta.Cover,
|
||||
Status: meta.Status,
|
||||
Genres: meta.Genres,
|
||||
SourceURL: meta.SourceURL,
|
||||
}
|
||||
if werr := s.store.WriteRankingItem(rankCtx, item); werr != nil {
|
||||
s.log.Warn("runAsync: WriteRankingItem failed", "slug", meta.Slug, "err", werr)
|
||||
}
|
||||
rank++
|
||||
}
|
||||
if rerr := <-rankErrs; rerr != nil {
|
||||
s.log.Warn("runAsync: ScrapeRanking finished with error", "err", rerr)
|
||||
} else {
|
||||
s.log.Info("runAsync: ScrapeRanking complete", "count", rank-1)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine final status.
|
||||
finalStatus := "done"
|
||||
errMsg := ""
|
||||
@@ -938,7 +968,7 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
|
||||
cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
|
||||
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
|
||||
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 {
|
||||
novels, hasNext := parseBrowsePage(strings.NewReader(html))
|
||||
s.log.Debug("browse: served from cache", "key", cacheKey)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@@ -1062,6 +1092,11 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
|
||||
"key", cacheKey, "err", readErr)
|
||||
return
|
||||
}
|
||||
if len(htmlBytes) == 0 {
|
||||
s.log.Warn("triggerBrowseSnapshot: SingleFile produced empty output, skipping cache",
|
||||
"key", cacheKey)
|
||||
return
|
||||
}
|
||||
|
||||
// Store the HTML snapshot.
|
||||
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
|
||||
|
||||
Reference in New Issue
Block a user