diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go index 10b55f1..720752d 100644 --- a/scraper/internal/novelfire/scraper.go +++ b/scraper/internal/novelfire/scraper.go @@ -142,24 +142,28 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue return } - // Extract novel cards:
- cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true}) + // Extract novel cards:
  • + // + //
    + //

    Title

    + //
    + cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true}) if len(cards) == 0 { s.log.Warn("no novel cards found, stopping pagination", "page", page) return } for _, card := range cards { - // Title:

    Title - titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"}) + // The outer carries the href;

    has the title text. + linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a", Attr: "href"}) + titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}) var title, href string + if linkNode != nil { + href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) + } if titleNode != nil { - linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"}) - if linkNode != nil { - title = htmlutil.ExtractText(linkNode, scraper.Selector{}) - href = htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) - } + title = strings.TrimSpace(htmlutil.ExtractText(titleNode, scraper.Selector{})) } if href == "" || title == "" { continue @@ -173,8 +177,27 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue } } - // Find next page link: