package novelfire import ( "context" "fmt" "os" "path/filepath" "testing" "github.com/libnovel/scraper/internal/browser" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/writer" ) // rankingPage1HTML is a realistic mock of the popular genre listing page // (novelfire.net/genre-all/sort-popular/status-all/all-novel?page=1). // It uses the real novelfire.net DOM:
  • cards with //

    and a rel="next" pagination link. func rankingPage1HTML() string { return ` ` } func rankingPage2HTML() string { return ` ` } // drainRanking collects all entries from a ScrapeRanking call without // deadlocking. It uses "for A != nil || B != nil" — nil channels are never // selected, so setting one to nil effectively removes it from the select. func drainRanking(t *testing.T, entryCh <-chan scraper.BookMeta, errCh <-chan error) []scraper.BookMeta { t.Helper() var entries []scraper.BookMeta for entryCh != nil || errCh != nil { select { case meta, ok := <-entryCh: if !ok { entryCh = nil } else { entries = append(entries, meta) } case err, ok := <-errCh: if !ok { errCh = nil } else if err != nil { t.Fatalf("unexpected scrape error: %v", err) } } } return entries } // TestScrapeRanking_SinglePage verifies a single page is parsed into entries // with sequential Ranking numbers using a stub client. // ScrapeRanking uses s.client (the main client, not urlClient) because the // ranking page is fully server-rendered. func TestScrapeRanking_SinglePage(t *testing.T) { // newScraper passes the stub as s.client — exactly what ScrapeRanking uses. s := newScraper(rankingPage1HTML()) entryCh, errCh := s.ScrapeRanking(context.Background(), 1) entries := drainRanking(t, entryCh, errCh) if len(entries) != 2 { t.Fatalf("expected 2 entries, got %d", len(entries)) } if entries[0].Ranking != 1 || entries[0].Title != "The Iron Throne" { t.Errorf("entry[0]: got rank=%d title=%q, want rank=1 title=%q", entries[0].Ranking, entries[0].Title, "The Iron Throne") } if entries[1].Ranking != 2 || entries[1].Title != "Shadow Mage" { t.Errorf("entry[1]: got rank=%d title=%q, want rank=2 title=%q", entries[1].Ranking, entries[1].Title, "Shadow Mage") } } // TestScrapeRanking_MultiPage verifies pagination across two pages yields // contiguous rank numbers (1, 2, 3). func TestScrapeRanking_MultiPage(t *testing.T) { // Use pagedStubClient for s.client so each GetContent call returns the // next page. ScrapeRanking now calls s.client directly. urlClient := &pagedStubClient{pages: []string{rankingPage1HTML(), rankingPage2HTML()}} s := New(urlClient, nil, nil, nil) // nil cache — no disk I/O in tests entryCh, errCh := s.ScrapeRanking(context.Background(), 0) // 0 = all pages entries := drainRanking(t, entryCh, errCh) if len(entries) != 3 { t.Fatalf("expected 3 entries across 2 pages, got %d", len(entries)) } want := []struct { rank int title string }{ {1, "The Iron Throne"}, {2, "Shadow Mage"}, {3, "Void Hunter"}, } for i, w := range want { if entries[i].Ranking != w.rank || entries[i].Title != w.title { t.Errorf("entry[%d]: got rank=%d title=%q, want rank=%d title=%q", i, entries[i].Ranking, entries[i].Title, w.rank, w.title) } } } // TestScrapeRanking_EmptyPage verifies that a page with no .novel-item // cards produces zero entries and closes channels cleanly (no deadlock). func TestScrapeRanking_EmptyPage(t *testing.T) { s := newScraper(`
    `) entryCh, errCh := s.ScrapeRanking(context.Background(), 1) entries := drainRanking(t, entryCh, errCh) if len(entries) != 0 { t.Errorf("expected 0 entries for empty page, got %d", len(entries)) } } // TestWriteRanking_RoundTrip verifies WriteRanking → ReadRankingItems // faithfully reconstructs the original slice. func TestWriteRanking_RoundTrip(t *testing.T) { dir := t.TempDir() w := writer.New(dir) items := []writer.RankingItem{ {Rank: 1, Slug: "the-iron-throne", Title: "The Iron Throne", Status: "Ongoing", Genres: []string{"Fantasy", "Action"}, SourceURL: "https://novelfire.net/book/the-iron-throne"}, {Rank: 2, Slug: "shadow-mage", Title: "Shadow Mage", Status: "Completed", Genres: []string{"Magic"}, SourceURL: "https://novelfire.net/book/shadow-mage"}, } if err := w.WriteRanking(items); err != nil { t.Fatalf("WriteRanking failed: %v", err) } rankingFile := filepath.Join(dir, "ranking.json") if _, err := os.Stat(rankingFile); err != nil { t.Fatalf("ranking.json not created: %v", err) } got, err := w.ReadRankingItems() if err != nil { t.Fatalf("ReadRankingItems failed: %v", err) } if len(got) != len(items) { t.Fatalf("expected %d items, got %d", len(items), len(got)) } for i, want := range items { if got[i].Rank != want.Rank { t.Errorf("item[%d].Rank = %d, want %d", i, got[i].Rank, want.Rank) } if got[i].Slug != want.Slug { t.Errorf("item[%d].Slug = %q, want %q", i, got[i].Slug, want.Slug) } if got[i].Title != want.Title { t.Errorf("item[%d].Title = %q, want %q", i, got[i].Title, want.Title) } if got[i].Status != want.Status { t.Errorf("item[%d].Status = %q, want %q", i, got[i].Status, want.Status) } if len(got[i].Genres) != len(want.Genres) { t.Errorf("item[%d].Genres len = %d, want %d", i, len(got[i].Genres), len(want.Genres)) } else { for j, g := range want.Genres { if got[i].Genres[j] != g { t.Errorf("item[%d].Genres[%d] = %q, want %q", i, j, got[i].Genres[j], g) } } } if got[i].SourceURL != want.SourceURL { t.Errorf("item[%d].SourceURL = %q, want %q", i, got[i].SourceURL, want.SourceURL) } } } // ── in-memory page cacher ───────────────────────────────────────────────────── // memPageCacher is a RankingPageCacher backed by an in-memory map. // It records how many times each page was written and exposes the stored HTML. type memPageCacher struct { pages map[int]string writes map[int]int } func newMemPageCacher() *memPageCacher { return &memPageCacher{pages: make(map[int]string), writes: make(map[int]int)} } func (c *memPageCacher) WriteRankingPageCache(page int, html string) error { c.pages[page] = html c.writes[page]++ return nil } func (c *memPageCacher) ReadRankingPageCache(page int) (string, error) { return c.pages[page], nil // returns "" on miss, satisfying the interface contract } var _ scraper.RankingPageCacher = (*memPageCacher)(nil) // compile-time check // TestScrapeRanking_CacheHit verifies that when a page is already in the cache // ScrapeRanking serves from cache and does NOT call the browser client. func TestScrapeRanking_CacheHit(t *testing.T) { cache := newMemPageCacher() // Pre-populate the cache with page 1 HTML. if err := cache.WriteRankingPageCache(1, rankingPage1HTML()); err != nil { t.Fatalf("cache write: %v", err) } cache.writes[1] = 0 // reset write counter — we only care about fetches // The stub client panics on any GetContent call so we can prove it is not used. panicClient := &panicOnGetContent{} s := New(panicClient, nil, panicClient, cache) entryCh, errCh := s.ScrapeRanking(context.Background(), 1) entries := drainRanking(t, entryCh, errCh) if len(entries) != 2 { t.Fatalf("expected 2 entries from cache, got %d", len(entries)) } // Cache should not have been written again (we served from cache). if cache.writes[1] != 0 { t.Errorf("expected 0 cache writes on a hit, got %d", cache.writes[1]) } } // TestScrapeRanking_CacheMiss verifies that on a cache miss the page is fetched // from the network and the result is written to the cache. func TestScrapeRanking_CacheMiss(t *testing.T) { cache := newMemPageCacher() // empty cache s := New(&stubClient{html: rankingPage1HTML()}, nil, nil, cache) entryCh, errCh := s.ScrapeRanking(context.Background(), 1) entries := drainRanking(t, entryCh, errCh) if len(entries) != 2 { t.Fatalf("expected 2 entries, got %d", len(entries)) } if cache.writes[1] != 1 { t.Errorf("expected 1 cache write on a miss, got %d", cache.writes[1]) } if cache.pages[1] == "" { t.Error("expected page 1 to be stored in cache after miss") } } // panicOnGetContent is a BrowserClient whose GetContent panics, letting tests // assert that it is never called (i.e. the cache was used instead). type panicOnGetContent struct{} func (p *panicOnGetContent) Strategy() browser.Strategy { return browser.StrategyContent } func (p *panicOnGetContent) GetContent(_ context.Context, req browser.ContentRequest) (string, error) { panic(fmt.Sprintf("unexpected GetContent call for URL %s — should have been served from cache", req.URL)) } func (p *panicOnGetContent) ScrapePage(_ context.Context, _ browser.ScrapeRequest) (browser.ScrapeResponse, error) { return browser.ScrapeResponse{}, nil } func (p *panicOnGetContent) CDPSession(_ context.Context, _ string, _ browser.CDPSessionFunc) error { return nil }