From ca33f8c3cf45045cf3aa798d4c9c90c45a9e081e Mon Sep 17 00:00:00 2001 From: Admin Date: Sun, 1 Mar 2026 21:54:11 +0500 Subject: [PATCH] fix: correct ScrapeRanking DOM selectors for real novelfire.net HTML The previous selectors were based on a hypothetical structure that does not match the actual site. The real novelfire.net popular listing uses:
  • (not
    )

    Title
    • And pagination uses nodes for rel="next" - Import golang.org/x/net/html for Node.Attr access - Test fixtures rewritten to match real structure (li/h4/rel=next) - Status and genres removed from ranking items (not present on listing page) Verified end-to-end: ranking.json written with 24 items on first fetch --- scraper/internal/novelfire/ranking_test.go | 67 ++++++++-------- scraper/internal/novelfire/scraper.go | 91 +++++++++++++--------- 2 files changed, 92 insertions(+), 66 deletions(-) diff --git a/scraper/internal/novelfire/ranking_test.go b/scraper/internal/novelfire/ranking_test.go index df1a2a0..15611a8 100644 --- a/scraper/internal/novelfire/ranking_test.go +++ b/scraper/internal/novelfire/ranking_test.go @@ -14,46 +14,51 @@ import ( // rankingPage1HTML is a realistic mock of the popular genre listing page // (novelfire.net/genre-all/sort-popular/status-all/all-novel?page=1). -// It contains two novel-item cards and a "next" link for pagination tests. +// It uses the real novelfire.net DOM:
  • cards with +//

    and a rel="next" pagination link. func rankingPage1HTML() string { return ` -
    -
    -
    -
    -

    The Iron Throne

    - Ongoing -
    FantasyAction
    -
    -
    -
    -
    -
    -

    Shadow Mage

    - Completed -
    Magic
    -
    -
    -
    - + + ` } func rankingPage2HTML() string { return ` -
    -
    -
    -
    -

    Void Hunter

    - Ongoing -
    Sci-Fi
    -
    -
    -
    - + + + ` } diff --git a/scraper/internal/novelfire/scraper.go b/scraper/internal/novelfire/scraper.go index 0f74266..2fc1445 100644 --- a/scraper/internal/novelfire/scraper.go +++ b/scraper/internal/novelfire/scraper.go @@ -21,6 +21,7 @@ import ( "github.com/libnovel/scraper/internal/browser" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/scraper/htmlutil" + "golang.org/x/net/html" ) const ( @@ -367,6 +368,22 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra // ─── RankingProvider ─────────────────────────────────────────────────────────── +// hasNextPageLink returns true if the HTML document contains a pagination link +// with rel="next". novelfire.net uses: +// +//
  • + //
    + //

    Title

    + //     + //
    • + cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true}) if len(cards) == 0 { s.log.Debug("no novel cards found, stopping pagination", "page", page) break } for _, card := range cards { - // Cover:
    + // The outer carries the href and title attribute. + linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"}) + if linkNode == nil { + continue + } + href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) + bookURL := resolveURL(baseURL, href) + if bookURL == "" { + continue + } + + // Title: prefer

    text; fall back to + title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"})) + if title == "" { + title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"})) + } + if title == "" { + continue + } + + // Cover:
    var cover string - if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil { - cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"}) + if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil { + cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"}) if cover == "" { - cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"}) + cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"}) + } + // Filter out base64 placeholder images. + if strings.HasPrefix(cover, "data:") { + cover = "" } if cover != "" && !strings.HasPrefix(cover, "http") { cover = baseURL + cover } } - // Title and URL:

    Title

    - titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"}) - var title, bookURL string - if titleNode != nil { - linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"}) - if linkNode != nil { - title = htmlutil.ExtractText(linkNode, scraper.Selector{}) - href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"}) - bookURL = resolveURL(baseURL, href) - } - } - if title == "" || bookURL == "" { - continue - } - - // Status: Ongoing - status := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "span", Class: "status"})) - - // Genres:
    Genre... - var genres []string - if genresNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "div", Class: "genres"}); genresNode != nil { - genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true}) - } - slug := slugFromURL(bookURL) meta := scraper.BookMeta{ Slug: slug, Title: title, Cover: cover, - Status: status, - Genres: genres, SourceURL: bookURL, Ranking: rank, } @@ -494,9 +515,9 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap } } - // Stop if no next-page link exists (natural end of listing). - nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"}) - if nextHref == "" { + // Stop if no next-page link exists. + // The real pagination uses