fix(scraper): fix ScrapeCatalogue selectors, zero-byte cache skip, and wire ScrapeRanking into runAsync

- ScrapeCatalogue: use li.novel-item (was div), extract href from outer <a> and title from h4.novel-title (was h3), detect next page via rel=next (was class=next)
- handleBrowse/triggerBrowseSnapshot: treat zero-byte MinIO cache entries as misses, skip storing empty SingleFile output
- runAsync: after a successful full-catalogue run, call ScrapeRanking and upsert each result into PocketBase ranking collection
This commit is contained in:
Admin
2026-03-04 16:54:00 +05:00
parent 8de374cd35
commit d14644238f
2 changed files with 70 additions and 12 deletions

View File

@@ -142,24 +142,28 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
return
}
// Extract novel cards: <div class="novel-item">
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true})
// Extract novel cards: <li class="novel-item">
// <a href="/book/slug" title="Title">
// <figure class="novel-cover"><img data-src="..."></figure>
// <h4 class="novel-title text2row">Title</h4>
// </a>
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
if len(cards) == 0 {
s.log.Warn("no novel cards found, stopping pagination", "page", page)
return
}
for _, card := range cards {
// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"})
// The outer <a> carries the href; <h4 class="novel-title"> has the title text.
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"})
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})
var title, href string
if linkNode != nil {
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
}
if titleNode != nil {
linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
if linkNode != nil {
title = htmlutil.ExtractText(linkNode, scraper.Selector{})
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
}
title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{}))
}
if href == "" || title == "" {
continue
@@ -173,8 +177,27 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
}
}
// Find next page link: <a class="next" href="...">
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
// Find next page link: <a rel="next" href="..."> (same structure as ranking pages)
if !hasNextPageLink(root) {
break
}
nextHref := ""
for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) {
isNext := false
for _, attr := range a.Attr {
if attr.Key == "rel" && attr.Val == "next" {
isNext = true
}
}
if isNext {
for _, attr := range a.Attr {
if attr.Key == "href" {
nextHref = attr.Val
}
}
break
}
}
if nextHref == "" {
break
}