kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

fix: correct ScrapeRanking DOM selectors for real novelfire.net HTML
Some checks failed
CI / Lint (push) Has been cancelled
Details
CI / Test (push) Has been cancelled
Details
CI / Build (push) Has been cancelled
Details

Browse Source 
The previous selectors were based on a hypothetical structure that does not
match the actual site. The real novelfire.net popular listing uses:

  <li class="novel-item">          (not <div>)
    <a href="/book/slug" title="Title">
      <figure class="novel-cover"><img data-src="/path.jpg"></figure>
      <h4 class="novel-title text2row">Title</h4>
    </a>
  </li>

And pagination uses <a rel="next"> (not <a class="next">).

Changes:
- ScrapeRanking: use li.novel-item, h4.novel-title, figure.novel-cover,
  img[data-src]; strip base64 placeholder covers
- hasNextPageLink(): new helper walking all <a> nodes for rel="next"
- Import golang.org/x/net/html for Node.Attr access
- Test fixtures rewritten to match real structure (li/h4/rel=next)
- Status and genres removed from ranking items (not present on listing page)

Verified end-to-end: ranking.json written with 24 items on first fetch
This commit is contained in:
Admin
2026-03-01 21:54:11 +05:00
parent 9837c89075
commit ca33f8c3cf
2 changed files with 92 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
scraper/internal/novelfire/ranking_test.go
View File

@@ -14,46 +14,51 @@ import (
// rankingPage1HTML is a realistic mock of the popular genre listing page // rankingPage1HTML is a realistic mock of the popular genre listing page
// (novelfire.net/genre-all/sort-popular/status-all/all-novel?page=1). // (novelfire.net/genre-all/sort-popular/status-all/all-novel?page=1).
// It contains two novel-item cards and a "next" link for pagination tests. // It uses the real novelfire.net DOM: <li class="novel-item"> cards with
// <h4 class="novel-title"> and a rel="next" pagination link.
func rankingPage1HTML() string { func rankingPage1HTML() string {
return `<!DOCTYPE html> return `<!DOCTYPE html>
<html><body> <html><body>
<div class="list-novel"> <ul class="list-novel">
<div class="novel-item"> <li class="novel-item">
<figure class="cover"><img src="/covers/iron-throne.jpg"></figure> <a title="The Iron Throne" href="/book/the-iron-throne">
<div class="item-body"> <figure class="novel-cover"><img class="lazy" src="data:image/gif;base64,R0lG" data-src="/covers/iron-throne.jpg" alt="The Iron Throne"></figure>
<h3 class="novel-title"><a href="/book/the-iron-throne">The Iron Throne</a></h3> <h4 class="novel-title text2row">The Iron Throne</h4>
<span class="status">Ongoing</span> </a>
<div class="genres"><a>Fantasy</a><a>Action</a></div> <div class="novel-stats"><i class="icon-book-open"></i> 500 Chapters</div>
</div> </li>
</div> <li class="novel-item">
<div class="novel-item"> <a title="Shadow Mage" href="/book/shadow-mage">
<figure class="cover"><img src="/covers/shadow-mage.jpg"></figure> <figure class="novel-cover"><img class="lazy" src="data:image/gif;base64,R0lG" data-src="/covers/shadow-mage.jpg" alt="Shadow Mage"></figure>
<div class="item-body"> <h4 class="novel-title text2row">Shadow Mage</h4>
<h3 class="novel-title"><a href="/book/shadow-mage">Shadow Mage</a></h3> </a>
<span class="status">Completed</span> <div class="novel-stats"><i class="icon-book-open"></i> 200 Chapters</div>
<div class="genres"><a>Magic</a></div> </li>
</div> </ul>
</div> <ul class="pagination">
</div> <li class="page-item active"><span class="page-link">1</span></li>
<a class="next" href="/genre-all/sort-popular/status-all/all-novel?page=2">Next</a> <li class="page-item"><a class="page-link" href="/genre-all/sort-popular/status-all/all-novel?page=2" rel="next" aria-label="Next">›</a></li>
</ul>
</body></html>` </body></html>`
} }
func rankingPage2HTML() string { func rankingPage2HTML() string {
return `<!DOCTYPE html> return `<!DOCTYPE html>
<html><body> <html><body>
<div class="list-novel"> <ul class="list-novel">
<div class="novel-item"> <li class="novel-item">
<figure class="cover"><img src="/covers/void-hunter.jpg"></figure> <a title="Void Hunter" href="/book/void-hunter">
<div class="item-body"> <figure class="novel-cover"><img class="lazy" src="data:image/gif;base64,R0lG" data-src="/covers/void-hunter.jpg" alt="Void Hunter"></figure>
<h3 class="novel-title"><a href="/book/void-hunter">Void Hunter</a></h3> <h4 class="novel-title text2row">Void Hunter</h4>
<span class="status">Ongoing</span> </a>
<div class="genres"><a>Sci-Fi</a></div> <div class="novel-stats"><i class="icon-book-open"></i> 100 Chapters</div>
</div> </li>
</div> </ul>
</div> <!-- no rel="next" link → last page -->
<!-- no .next link → last page --> <ul class="pagination">
<li class="page-item"><a class="page-link" href="?page=1" rel="prev">‹</a></li>
<li class="page-item active"><span class="page-link">2</span></li>
</ul>
</body></html>` </body></html>`
} }

91
scraper/internal/novelfire/scraper.go
View File

@@ -21,6 +21,7 @@ import (
"github.com/libnovel/scraper/internal/browser" "github.com/libnovel/scraper/internal/browser"
"github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/scraper/htmlutil" "github.com/libnovel/scraper/internal/scraper/htmlutil"
"golang.org/x/net/html"
) )
const ( const (
@@ -367,6 +368,22 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
// ─── RankingProvider ─────────────────────────────────────────────────────────── // ─── RankingProvider ───────────────────────────────────────────────────────────
// hasNextPageLink returns true if the HTML document contains a pagination link
// with rel="next". novelfire.net uses:
//
// <a class="page-link" href="...?page=N" rel="next" ...>
func hasNextPageLink(root *html.Node) bool {
links := htmlutil.FindAll(root, scraper.Selector{Tag: "a", Multiple: true})
for _, a := range links {
for _, attr := range a.Attr {
if attr.Key == "rel" && attr.Val == "next" {
return true
}
}
}
return false
}
// ScrapeRanking pages through up to maxPages pages of the popular-novels genre // ScrapeRanking pages through up to maxPages pages of the popular-novels genre
// listing on novelfire.net (/genre-all/sort-popular/status-all/all-novel). // listing on novelfire.net (/genre-all/sort-popular/status-all/all-novel).
// Pages are fetched one at a time, strictly sequentially. // Pages are fetched one at a time, strictly sequentially.
@@ -430,58 +447,62 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
return return
} }
// Genre listing uses div.novel-item cards (same structure as catalogue). // Real novelfire.net popular listing structure:
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "div", Class: "novel-item", Multiple: true}) // <li class="novel-item">
// <a href="/book/slug" title="Title">
// <figure class="novel-cover"><img data-src="..."></figure>
// <h4 class="novel-title text2row">Title</h4>
// </a>
// </li>
cards := htmlutil.FindAll(root, scraper.Selector{Tag: "li", Class: "novel-item", Multiple: true})
if len(cards) == 0 { if len(cards) == 0 {
s.log.Debug("no novel cards found, stopping pagination", "page", page) s.log.Debug("no novel cards found, stopping pagination", "page", page)
break break
} }
for _, card := range cards { for _, card := range cards {
// Cover: <figure class="cover"><img src="..." or data-src="..."> // The outer <a> carries the href and title attribute.
linkNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "a"})
if linkNode == nil {
continue
}
href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
bookURL := resolveURL(baseURL, href)
if bookURL == "" {
continue
}
// Title: prefer <h4 class="novel-title"> text; fall back to <a title="...">
title := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "h4", Class: "novel-title"}))
if title == "" {
title = strings.TrimSpace(htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "title"}))
}
if title == "" {
continue
}
// Cover: <figure class="novel-cover"><img data-src="...">
var cover string var cover string
if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil { if fig := htmlutil.FindFirst(card, scraper.Selector{Tag: "figure", Class: "novel-cover"}); fig != nil {
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"}) cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
if cover == "" { if cover == "" {
cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"}) cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "src"})
}
// Filter out base64 placeholder images.
if strings.HasPrefix(cover, "data:") {
cover = ""
} }
if cover != "" && !strings.HasPrefix(cover, "http") { if cover != "" && !strings.HasPrefix(cover, "http") {
cover = baseURL + cover cover = baseURL + cover
} }
} }
// Title and URL: <h3 class="novel-title"><a href="/book/slug">Title</a></h3>
titleNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "h3", Class: "novel-title"})
var title, bookURL string
if titleNode != nil {
linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a", Attr: "href"})
if linkNode != nil {
title = htmlutil.ExtractText(linkNode, scraper.Selector{})
href := htmlutil.ExtractText(linkNode, scraper.Selector{Tag: "a", Attr: "href"})
bookURL = resolveURL(baseURL, href)
}
}
if title == "" || bookURL == "" {
continue
}
// Status: <span class="status">Ongoing</span>
status := strings.TrimSpace(htmlutil.ExtractFirst(card, scraper.Selector{Tag: "span", Class: "status"}))
// Genres: <div class="genres"><a>Genre</a>...
var genres []string
if genresNode := htmlutil.FindFirst(card, scraper.Selector{Tag: "div", Class: "genres"}); genresNode != nil {
genres = htmlutil.ExtractAll(genresNode, scraper.Selector{Tag: "a", Multiple: true})
}
slug := slugFromURL(bookURL) slug := slugFromURL(bookURL)
meta := scraper.BookMeta{ meta := scraper.BookMeta{
Slug: slug, Slug: slug,
Title: title, Title: title,
Cover: cover, Cover: cover,
Status: status,
Genres: genres,
SourceURL: bookURL, SourceURL: bookURL,
Ranking: rank, Ranking: rank,
} }
@@ -494,9 +515,9 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
} }
} }
// Stop if no next-page link exists (natural end of listing). // Stop if no next-page link exists.
nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"}) // The real pagination uses <a rel="next" ...> inside .pagination.
if nextHref == "" { if !hasNextPageLink(root) {
s.log.Debug("no next-page link found, stopping pagination", "page", page) s.log.Debug("no next-page link found, stopping pagination", "page", page)
break break
} }