fix(scraper): fix ScrapeCatalogue selectors, zero-byte cache skip, and wire ScrapeRanking into runAsync
- ScrapeCatalogue: use li.novel-item (was div), extract href from outer <a> and title from h4.novel-title (was h3), detect next page via rel=next (was class=next) - handleBrowse/triggerBrowseSnapshot: treat zero-byte MinIO cache entries as misses, skip storing empty SingleFile output - runAsync: after a successful full-catalogue run, call ScrapeRanking and upsert each result into PocketBase ranking collection
This commit is contained in:
@@ -142,24 +142,28 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
|
||||
return
|
||||
}
|
||||
|
||||
// Extract novel cards: <div class="novel-item">
|
||||
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true})
|
||||
// Extract novel cards: <li class="novel-item">
|
||||
// <a href="/book/slug" title="Title">
|
||||
// <figure class="novel-cover"><img data-src="..."></figure>
|
||||
// <h4 class="novel-title text2row">Title</h4>
|
||||
// </a>
|
||||
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
|
||||
if len(cards) == 0 {
|
||||
s.log.Warn("no novel cards found, stopping pagination", "page", page)
|
||||
return
|
||||
}
|
||||
|
||||
for _, card := range cards {
|
||||
// Title: <h3 class="novel-title"><a href="/book/slug">Title</a>
|
||||
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"})
|
||||
// The outer <a> carries the href; <h4 class="novel-title"> has the title text.
|
||||
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})
|
||||
|
||||
var title, href string
|
||||
if linkNode != nil {
|
||||
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
}
|
||||
if titleNode != nil {
|
||||
linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
if linkNode != nil {
|
||||
title = htmlutil.ExtractText(linkNode, scraper.Selector{})
|
||||
href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
|
||||
}
|
||||
title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{}))
|
||||
}
|
||||
if href == "" || title == "" {
|
||||
continue
|
||||
@@ -173,8 +177,27 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
|
||||
}
|
||||
}
|
||||
|
||||
// Find next page link: <a class="next" href="...">
|
||||
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
|
||||
// Find next page link: <a rel="next" href="..."> (same structure as ranking pages)
|
||||
if !hasNextPageLink(root) {
|
||||
break
|
||||
}
|
||||
nextHref := ""
|
||||
for _, a := range htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true}) {
|
||||
isNext := false
|
||||
for _, attr := range a.Attr {
|
||||
if attr.Key == "rel" && attr.Val == "next" {
|
||||
isNext = true
|
||||
}
|
||||
}
|
||||
if isNext {
|
||||
for _, attr := range a.Attr {
|
||||
if attr.Key == "href" {
|
||||
nextHref = attr.Val
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if nextHref == "" {
|
||||
break
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user