ca33f8c3cf
The previous selectors were based on a hypothetical structure that does not
match the actual site. The real novelfire.net popular listing uses:
<li class="novel-item"> (not <div>)
<a href="/book/slug" title="Title">
<figure class="novel-cover"><img data-src="/path.jpg"></figure>
<h4 class="novel-title text2row">Title</h4>
</a>
</li>
And pagination uses <a rel="next"> (not <a class="next">).
Changes:
- ScrapeRanking: use li.novel-item, h4.novel-title, figure.novel-cover,
img[data-src]; strip base64 placeholder covers
- hasNextPageLink(): new helper walking all <a> nodes for rel="next"
- Import golang.org/x/net/html for Node.Attr access
- Test fixtures rewritten to match real structure (li/h4/rel=next)
- Status and genres removed from ranking items (not present on listing page)
Verified end-to-end: ranking.json written with 24 items on first fetch
297 lines
11 KiB
Go
297 lines
11 KiB
Go
package novelfire
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"testing"
|
||
|
||
"github.com/libnovel/scraper/internal/browser"
|
||
"github.com/libnovel/scraper/internal/scraper"
|
||
"github.com/libnovel/scraper/internal/writer"
|
||
)
|
||
|
||
// rankingPage1HTML is a realistic mock of the popular genre listing page
|
||
// (novelfire.net/genre-all/sort-popular/status-all/all-novel?page=1).
|
||
// It uses the real novelfire.net DOM: <li class="novel-item"> cards with
|
||
// <h4 class="novel-title"> and a rel="next" pagination link.
|
||
func rankingPage1HTML() string {
|
||
return `<!DOCTYPE html>
|
||
<html><body>
|
||
<ul class="list-novel">
|
||
<li class="novel-item">
|
||
<a title="The Iron Throne" href="/book/the-iron-throne">
|
||
<figure class="novel-cover"><img class="lazy" src="data:image/gif;base64,R0lG" data-src="/covers/iron-throne.jpg" alt="The Iron Throne"></figure>
|
||
<h4 class="novel-title text2row">The Iron Throne</h4>
|
||
</a>
|
||
<div class="novel-stats"><i class="icon-book-open"></i> 500 Chapters</div>
|
||
</li>
|
||
<li class="novel-item">
|
||
<a title="Shadow Mage" href="/book/shadow-mage">
|
||
<figure class="novel-cover"><img class="lazy" src="data:image/gif;base64,R0lG" data-src="/covers/shadow-mage.jpg" alt="Shadow Mage"></figure>
|
||
<h4 class="novel-title text2row">Shadow Mage</h4>
|
||
</a>
|
||
<div class="novel-stats"><i class="icon-book-open"></i> 200 Chapters</div>
|
||
</li>
|
||
</ul>
|
||
<ul class="pagination">
|
||
<li class="page-item active"><span class="page-link">1</span></li>
|
||
<li class="page-item"><a class="page-link" href="/genre-all/sort-popular/status-all/all-novel?page=2" rel="next" aria-label="Next">›</a></li>
|
||
</ul>
|
||
</body></html>`
|
||
}
|
||
|
||
func rankingPage2HTML() string {
|
||
return `<!DOCTYPE html>
|
||
<html><body>
|
||
<ul class="list-novel">
|
||
<li class="novel-item">
|
||
<a title="Void Hunter" href="/book/void-hunter">
|
||
<figure class="novel-cover"><img class="lazy" src="data:image/gif;base64,R0lG" data-src="/covers/void-hunter.jpg" alt="Void Hunter"></figure>
|
||
<h4 class="novel-title text2row">Void Hunter</h4>
|
||
</a>
|
||
<div class="novel-stats"><i class="icon-book-open"></i> 100 Chapters</div>
|
||
</li>
|
||
</ul>
|
||
<!-- no rel="next" link → last page -->
|
||
<ul class="pagination">
|
||
<li class="page-item"><a class="page-link" href="?page=1" rel="prev">‹</a></li>
|
||
<li class="page-item active"><span class="page-link">2</span></li>
|
||
</ul>
|
||
</body></html>`
|
||
}
|
||
|
||
// drainRanking collects all entries from a ScrapeRanking call without
|
||
// deadlocking. It uses "for A != nil || B != nil" — nil channels are never
|
||
// selected, so setting one to nil effectively removes it from the select.
|
||
func drainRanking(t *testing.T, entryCh <-chan scraper.BookMeta, errCh <-chan error) []scraper.BookMeta {
|
||
t.Helper()
|
||
var entries []scraper.BookMeta
|
||
for entryCh != nil || errCh != nil {
|
||
select {
|
||
case meta, ok := <-entryCh:
|
||
if !ok {
|
||
entryCh = nil
|
||
} else {
|
||
entries = append(entries, meta)
|
||
}
|
||
case err, ok := <-errCh:
|
||
if !ok {
|
||
errCh = nil
|
||
} else if err != nil {
|
||
t.Fatalf("unexpected scrape error: %v", err)
|
||
}
|
||
}
|
||
}
|
||
return entries
|
||
}
|
||
|
||
// TestScrapeRanking_SinglePage verifies a single page is parsed into entries
|
||
// with sequential Ranking numbers using a stub client.
|
||
// ScrapeRanking uses s.client (the main client, not urlClient) because the
|
||
// ranking page is fully server-rendered.
|
||
func TestScrapeRanking_SinglePage(t *testing.T) {
|
||
// newScraper passes the stub as s.client — exactly what ScrapeRanking uses.
|
||
s := newScraper(rankingPage1HTML())
|
||
entryCh, errCh := s.ScrapeRanking(context.Background(), 1)
|
||
entries := drainRanking(t, entryCh, errCh)
|
||
|
||
if len(entries) != 2 {
|
||
t.Fatalf("expected 2 entries, got %d", len(entries))
|
||
}
|
||
if entries[0].Ranking != 1 || entries[0].Title != "The Iron Throne" {
|
||
t.Errorf("entry[0]: got rank=%d title=%q, want rank=1 title=%q",
|
||
entries[0].Ranking, entries[0].Title, "The Iron Throne")
|
||
}
|
||
if entries[1].Ranking != 2 || entries[1].Title != "Shadow Mage" {
|
||
t.Errorf("entry[1]: got rank=%d title=%q, want rank=2 title=%q",
|
||
entries[1].Ranking, entries[1].Title, "Shadow Mage")
|
||
}
|
||
}
|
||
|
||
// TestScrapeRanking_MultiPage verifies pagination across two pages yields
|
||
// contiguous rank numbers (1, 2, 3).
|
||
func TestScrapeRanking_MultiPage(t *testing.T) {
|
||
// Use pagedStubClient for s.client so each GetContent call returns the
|
||
// next page. ScrapeRanking now calls s.client directly.
|
||
urlClient := &pagedStubClient{pages: []string{rankingPage1HTML(), rankingPage2HTML()}}
|
||
s := New(urlClient, nil, nil, nil) // nil cache — no disk I/O in tests
|
||
|
||
entryCh, errCh := s.ScrapeRanking(context.Background(), 0) // 0 = all pages
|
||
entries := drainRanking(t, entryCh, errCh)
|
||
|
||
if len(entries) != 3 {
|
||
t.Fatalf("expected 3 entries across 2 pages, got %d", len(entries))
|
||
}
|
||
want := []struct {
|
||
rank int
|
||
title string
|
||
}{
|
||
{1, "The Iron Throne"},
|
||
{2, "Shadow Mage"},
|
||
{3, "Void Hunter"},
|
||
}
|
||
for i, w := range want {
|
||
if entries[i].Ranking != w.rank || entries[i].Title != w.title {
|
||
t.Errorf("entry[%d]: got rank=%d title=%q, want rank=%d title=%q",
|
||
i, entries[i].Ranking, entries[i].Title, w.rank, w.title)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestScrapeRanking_EmptyPage verifies that a page with no .novel-item
|
||
// cards produces zero entries and closes channels cleanly (no deadlock).
|
||
func TestScrapeRanking_EmptyPage(t *testing.T) {
|
||
s := newScraper(`<!DOCTYPE html><html><body><div class="no-rankings"></div></body></html>`)
|
||
entryCh, errCh := s.ScrapeRanking(context.Background(), 1)
|
||
entries := drainRanking(t, entryCh, errCh)
|
||
|
||
if len(entries) != 0 {
|
||
t.Errorf("expected 0 entries for empty page, got %d", len(entries))
|
||
}
|
||
}
|
||
|
||
// TestWriteRanking_RoundTrip verifies WriteRanking → ReadRankingItems
|
||
// faithfully reconstructs the original slice.
|
||
func TestWriteRanking_RoundTrip(t *testing.T) {
|
||
dir := t.TempDir()
|
||
w := writer.New(dir)
|
||
|
||
items := []writer.RankingItem{
|
||
{Rank: 1, Slug: "the-iron-throne", Title: "The Iron Throne", Status: "Ongoing",
|
||
Genres: []string{"Fantasy", "Action"}, SourceURL: "https://novelfire.net/book/the-iron-throne"},
|
||
{Rank: 2, Slug: "shadow-mage", Title: "Shadow Mage", Status: "Completed",
|
||
Genres: []string{"Magic"}, SourceURL: "https://novelfire.net/book/shadow-mage"},
|
||
}
|
||
|
||
if err := w.WriteRanking(items); err != nil {
|
||
t.Fatalf("WriteRanking failed: %v", err)
|
||
}
|
||
|
||
rankingFile := filepath.Join(dir, "ranking.json")
|
||
if _, err := os.Stat(rankingFile); err != nil {
|
||
t.Fatalf("ranking.json not created: %v", err)
|
||
}
|
||
|
||
got, err := w.ReadRankingItems()
|
||
if err != nil {
|
||
t.Fatalf("ReadRankingItems failed: %v", err)
|
||
}
|
||
if len(got) != len(items) {
|
||
t.Fatalf("expected %d items, got %d", len(items), len(got))
|
||
}
|
||
for i, want := range items {
|
||
if got[i].Rank != want.Rank {
|
||
t.Errorf("item[%d].Rank = %d, want %d", i, got[i].Rank, want.Rank)
|
||
}
|
||
if got[i].Slug != want.Slug {
|
||
t.Errorf("item[%d].Slug = %q, want %q", i, got[i].Slug, want.Slug)
|
||
}
|
||
if got[i].Title != want.Title {
|
||
t.Errorf("item[%d].Title = %q, want %q", i, got[i].Title, want.Title)
|
||
}
|
||
if got[i].Status != want.Status {
|
||
t.Errorf("item[%d].Status = %q, want %q", i, got[i].Status, want.Status)
|
||
}
|
||
if len(got[i].Genres) != len(want.Genres) {
|
||
t.Errorf("item[%d].Genres len = %d, want %d", i, len(got[i].Genres), len(want.Genres))
|
||
} else {
|
||
for j, g := range want.Genres {
|
||
if got[i].Genres[j] != g {
|
||
t.Errorf("item[%d].Genres[%d] = %q, want %q", i, j, got[i].Genres[j], g)
|
||
}
|
||
}
|
||
}
|
||
if got[i].SourceURL != want.SourceURL {
|
||
t.Errorf("item[%d].SourceURL = %q, want %q", i, got[i].SourceURL, want.SourceURL)
|
||
}
|
||
}
|
||
}
|
||
|
||
// ── in-memory page cacher ─────────────────────────────────────────────────────
|
||
|
||
// memPageCacher is a RankingPageCacher backed by an in-memory map.
|
||
// It records how many times each page was written and exposes the stored HTML.
|
||
type memPageCacher struct {
|
||
pages map[int]string
|
||
writes map[int]int
|
||
}
|
||
|
||
func newMemPageCacher() *memPageCacher {
|
||
return &memPageCacher{pages: make(map[int]string), writes: make(map[int]int)}
|
||
}
|
||
|
||
func (c *memPageCacher) WriteRankingPageCache(page int, html string) error {
|
||
c.pages[page] = html
|
||
c.writes[page]++
|
||
return nil
|
||
}
|
||
|
||
func (c *memPageCacher) ReadRankingPageCache(page int) (string, error) {
|
||
return c.pages[page], nil // returns "" on miss, satisfying the interface contract
|
||
}
|
||
|
||
var _ scraper.RankingPageCacher = (*memPageCacher)(nil) // compile-time check
|
||
|
||
// TestScrapeRanking_CacheHit verifies that when a page is already in the cache
|
||
// ScrapeRanking serves from cache and does NOT call the browser client.
|
||
func TestScrapeRanking_CacheHit(t *testing.T) {
|
||
cache := newMemPageCacher()
|
||
// Pre-populate the cache with page 1 HTML.
|
||
if err := cache.WriteRankingPageCache(1, rankingPage1HTML()); err != nil {
|
||
t.Fatalf("cache write: %v", err)
|
||
}
|
||
cache.writes[1] = 0 // reset write counter — we only care about fetches
|
||
|
||
// The stub client panics on any GetContent call so we can prove it is not used.
|
||
panicClient := &panicOnGetContent{}
|
||
s := New(panicClient, nil, panicClient, cache)
|
||
|
||
entryCh, errCh := s.ScrapeRanking(context.Background(), 1)
|
||
entries := drainRanking(t, entryCh, errCh)
|
||
|
||
if len(entries) != 2 {
|
||
t.Fatalf("expected 2 entries from cache, got %d", len(entries))
|
||
}
|
||
// Cache should not have been written again (we served from cache).
|
||
if cache.writes[1] != 0 {
|
||
t.Errorf("expected 0 cache writes on a hit, got %d", cache.writes[1])
|
||
}
|
||
}
|
||
|
||
// TestScrapeRanking_CacheMiss verifies that on a cache miss the page is fetched
|
||
// from the network and the result is written to the cache.
|
||
func TestScrapeRanking_CacheMiss(t *testing.T) {
|
||
cache := newMemPageCacher() // empty cache
|
||
s := New(&stubClient{html: rankingPage1HTML()}, nil, nil, cache)
|
||
|
||
entryCh, errCh := s.ScrapeRanking(context.Background(), 1)
|
||
entries := drainRanking(t, entryCh, errCh)
|
||
|
||
if len(entries) != 2 {
|
||
t.Fatalf("expected 2 entries, got %d", len(entries))
|
||
}
|
||
if cache.writes[1] != 1 {
|
||
t.Errorf("expected 1 cache write on a miss, got %d", cache.writes[1])
|
||
}
|
||
if cache.pages[1] == "" {
|
||
t.Error("expected page 1 to be stored in cache after miss")
|
||
}
|
||
}
|
||
|
||
// panicOnGetContent is a BrowserClient whose GetContent panics, letting tests
|
||
// assert that it is never called (i.e. the cache was used instead).
|
||
type panicOnGetContent struct{}
|
||
|
||
func (p *panicOnGetContent) Strategy() browser.Strategy { return browser.StrategyContent }
|
||
func (p *panicOnGetContent) GetContent(_ context.Context, req browser.ContentRequest) (string, error) {
|
||
panic(fmt.Sprintf("unexpected GetContent call for URL %s — should have been served from cache", req.URL))
|
||
}
|
||
func (p *panicOnGetContent) ScrapePage(_ context.Context, _ browser.ScrapeRequest) (browser.ScrapeResponse, error) {
|
||
return browser.ScrapeResponse{}, nil
|
||
}
|
||
func (p *panicOnGetContent) CDPSession(_ context.Context, _ string, _ browser.CDPSessionFunc) error {
|
||
return nil
|
||
}
|