kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

refactor(ranking): replace blob cache with per-item PocketBase storage

Browse Source 
- Replace SetRanking/GetRanking/SetRankingPageHTML/GetRankingPageHTML blob methods
  with WriteRankingItem/ReadRankingItems/RankingFreshEnough per-item operations
- Add 24h staleness gate in ScrapeRanking to skip re-scraping fresh data
- Add GET /api/ranking endpoint returning []RankingItem sorted by rank
- Remove RankingPageCacher interface and rankingCacheAdapter adapter
- Update integration tests to use new per-item upsert semantics
- Include e2e test suite (scraper/internal/e2e/)
This commit is contained in:
Admin
2026-03-03 19:37:49 +05:00
parent 56bf4dde22
commit b8d4d94b18
10 changed files with 1173 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
scraper/internal/novelfire/ranking_test.go
View File

@@ -2,12 +2,10 @@ package novelfire
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"github.com/libnovel/scraper/internal/browser"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/writer"
)
@@ -208,89 +206,3 @@ func TestWriteRanking_RoundTrip(t *testing.T) {
}
}
}
// ── in-memory page cacher ─────────────────────────────────────────────────────
// memPageCacher is a RankingPageCacher backed by an in-memory map.
// It records how many times each page was written and exposes the stored HTML.
type memPageCacher struct {
pages map[int]string
writes map[int]int
}
func newMemPageCacher() *memPageCacher {
return &memPageCacher{pages: make(map[int]string), writes: make(map[int]int)}
}
func (c *memPageCacher) WriteRankingPageCache(page int, html string) error {
c.pages[page] = html
c.writes[page]++
return nil
}
func (c *memPageCacher) ReadRankingPageCache(page int) (string, error) {
return c.pages[page], nil // returns "" on miss, satisfying the interface contract
}
var _ scraper.RankingPageCacher = (*memPageCacher)(nil) // compile-time check
// TestScrapeRanking_CacheHit verifies that when a page is already in the cache
// ScrapeRanking serves from cache and does NOT call the browser client.
func TestScrapeRanking_CacheHit(t *testing.T) {
cache := newMemPageCacher()
// Pre-populate the cache with page 1 HTML.
if err := cache.WriteRankingPageCache(1, rankingPage1HTML()); err != nil {
t.Fatalf("cache write: %v", err)
}
cache.writes[1] = 0 // reset write counter — we only care about fetches
// The stub client panics on any GetContent call so we can prove it is not used.
panicClient := &panicOnGetContent{}
s := New(panicClient, nil, panicClient, cache)
entryCh, errCh := s.ScrapeRanking(context.Background(), 1)
entries := drainRanking(t, entryCh, errCh)
if len(entries) != 2 {
t.Fatalf("expected 2 entries from cache, got %d", len(entries))
}
// Cache should not have been written again (we served from cache).
if cache.writes[1] != 0 {
t.Errorf("expected 0 cache writes on a hit, got %d", cache.writes[1])
}
}
// TestScrapeRanking_CacheMiss verifies that on a cache miss the page is fetched
// from the network and the result is written to the cache.
func TestScrapeRanking_CacheMiss(t *testing.T) {
cache := newMemPageCacher() // empty cache
s := New(&stubClient{html: rankingPage1HTML()}, nil, nil, cache)
entryCh, errCh := s.ScrapeRanking(context.Background(), 1)
entries := drainRanking(t, entryCh, errCh)
if len(entries) != 2 {
t.Fatalf("expected 2 entries, got %d", len(entries))
}
if cache.writes[1] != 1 {
t.Errorf("expected 1 cache write on a miss, got %d", cache.writes[1])
}
if cache.pages[1] == "" {
t.Error("expected page 1 to be stored in cache after miss")
}
}
// panicOnGetContent is a BrowserClient whose GetContent panics, letting tests
// assert that it is never called (i.e. the cache was used instead).
type panicOnGetContent struct{}
func (p *panicOnGetContent) Strategy() browser.Strategy { return browser.StrategyContent }
func (p *panicOnGetContent) GetContent(_ context.Context, req browser.ContentRequest) (string, error) {
panic(fmt.Sprintf("unexpected GetContent call for URL %s — should have been served from cache", req.URL))
}
func (p *panicOnGetContent) ScrapePage(_ context.Context, _ browser.ScrapeRequest) (browser.ScrapeResponse, error) {
return browser.ScrapeResponse{}, nil
}
func (p *panicOnGetContent) CDPSession(_ context.Context, _ string, _ browser.CDPSessionFunc) error {
return nil
}

151
scraper/internal/novelfire/scraper.go
View File

@@ -21,6 +21,7 @@ import (
"github.com/libnovel/scraper/internal/browser"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/scraper/htmlutil"
"github.com/libnovel/scraper/internal/storage"
"golang.org/x/net/html"
)
@@ -49,27 +50,33 @@ var rejectResourceTypes = []string{
"websocket",
}
// RankingStore is the subset of storage.Store consumed by ScrapeRanking.
type RankingStore interface {
WriteRankingItem(ctx context.Context, item storage.RankingItem) error
RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error)
}
// Scraper is the novelfire.net implementation of scraper.NovelScraper.
// It uses the /content strategy by default (rendered HTML via Browserless).
type Scraper struct {
client browser.BrowserClient
urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
pageCache scraper.RankingPageCacher
log *slog.Logger
client browser.BrowserClient
urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
rankingStore RankingStore
log *slog.Logger
}
// New returns a new novelfire Scraper.
// client is used for content fetching, urlClient is used for URL retrieval (chapter list).
// If urlClient is nil, client will be used for both.
// pageCache is optional; pass nil to disable ranking page caching.
func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, pageCache scraper.RankingPageCacher) *Scraper {
// rankingStore is optional; pass nil to disable freshness checks and per-item persistence.
func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient, rankingStore RankingStore) *Scraper {
if log == nil {
log = slog.Default()
}
if urlClient == nil {
urlClient = client
}
return &Scraper{client: client, urlClient: urlClient, pageCache: pageCache, log: log}
return &Scraper{client: client, urlClient: urlClient, rankingStore: rankingStore, log: log}
}
// SourceName implements NovelScraper.
@@ -366,6 +373,61 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
return refs, nil
}
// ScrapeChapterListPage fetches and parses a single chapter-list page URL and
// returns all chapter refs found on that page without following pagination.
// pageURL should be the full URL including query params, e.g.:
//
// https://novelfire.net/book/shadow-slave/chapters?page=1
func (s *Scraper) ScrapeChapterListPage(ctx context.Context, pageURL string) ([]scraper.ChapterRef, error) {
s.log.Info("scraping chapter list page (single)", "url", pageURL)
raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
WaitForTimeout: 2000,
RejectResourceTypes: rejectResourceTypes,
GotoOptions: &browser.GotoOptions{Timeout: 60000},
})
if err != nil {
return nil, fmt.Errorf("chapter list page fetch: %w", err)
}
root, err := htmlutil.ParseHTML(raw)
if err != nil {
return nil, fmt.Errorf("chapter list page parse: %w", err)
}
chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
if chapterList == nil {
return nil, fmt.Errorf("chapter list container not found in %s", pageURL)
}
items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
var refs []scraper.ChapterRef
for _, item := range items {
linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
if linkNode == nil {
continue
}
href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
chTitle := htmlutil.ExtractText(linkNode, scraper.Selector{})
if href == "" {
continue
}
chURL := resolveURL(baseURL, href)
num := chapterNumberFromURL(chURL)
if num <= 0 {
num = len(refs) + 1
}
refs = append(refs, scraper.ChapterRef{
Number: num,
Title: strings.TrimSpace(chTitle),
URL: chURL,
})
}
return refs, nil
}
// ─── RankingProvider ───────────────────────────────────────────────────────────
// hasNextPageLink returns true if the HTML document contains a pagination link
@@ -388,6 +450,9 @@ func hasNextPageLink(root *html.Node) bool {
// listing on novelfire.net (/genre-all/sort-popular/status-all/all-novel).
// Pages are fetched one at a time, strictly sequentially.
// maxPages <= 0 means "fetch all pages until no more are found".
//
// If a RankingStore was provided and the stored ranking is fresh (< 24 hours old),
// both channels are closed immediately without any network traffic.
func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scraper.BookMeta, <-chan error) {
entries := make(chan scraper.BookMeta, 32)
errs := make(chan error, 16)
@@ -396,6 +461,17 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
defer close(entries)
defer close(errs)
// Freshness check: skip scraping if data is recent enough.
if s.rankingStore != nil {
fresh, err := s.rankingStore.RankingFreshEnough(ctx, 24*time.Hour)
if err != nil {
s.log.Warn("ranking freshness check failed, proceeding with scrape", "err", err)
} else if fresh {
s.log.Info("ranking data is fresh, skipping scrape")
return
}
}
rank := 1
for page := 1; maxPages <= 0 || page <= maxPages; page++ {
@@ -407,38 +483,17 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
pageURL := fmt.Sprintf("%s%s?page=%d", baseURL, rankingPath, page)
// Try to serve from disk cache before hitting the network.
var raw string
if s.pageCache != nil {
if cached, err := s.pageCache.ReadRankingPageCache(page); err != nil {
s.log.Warn("ranking page cache read error", "page", page, "err", err)
} else if cached != "" {
s.log.Info("serving ranking page from cache", "page", page)
raw = cached
}
}
if raw == "" {
s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
fetched, err := s.client.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
RejectResourceTypes: rejectResourceTypes,
GotoOptions: &browser.GotoOptions{Timeout: 60000},
})
if err != nil {
s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
errs <- fmt.Errorf("ranking page %d: %w", page, err)
return
}
raw = fetched
// Persist to cache for future runs.
if s.pageCache != nil {
if werr := s.pageCache.WriteRankingPageCache(page, raw); werr != nil {
s.log.Warn("ranking page cache write error", "page", page, "err", werr)
}
}
s.log.Info("scraping popular ranking page", "page", page, "url", pageURL)
raw, err := s.client.GetContent(ctx, browser.ContentRequest{
URL: pageURL,
WaitFor: &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
RejectResourceTypes: rejectResourceTypes,
GotoOptions: &browser.GotoOptions{Timeout: 60000},
})
if err != nil {
s.log.Debug("ranking page fetch failed", "page", page, "url", pageURL, "err", err)
errs <- fmt.Errorf("ranking page %d: %w", page, err)
return
}
root, err := htmlutil.ParseHTML(raw)
@@ -497,10 +552,10 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
}
}
slug := slugFromURL(bookURL)
bookSlug := slugFromURL(bookURL)
meta := scraper.BookMeta{
Slug: slug,
Slug: bookSlug,
Title: title,
Cover: cover,
SourceURL: bookURL,
@@ -508,6 +563,20 @@ func (s *Scraper) ScrapeRanking(ctx context.Context, maxPages int) (<-chan scrap
}
rank++
// Persist item to store immediately.
if s.rankingStore != nil {
item := storage.RankingItem{
Rank: meta.Ranking,
Slug: meta.Slug,
Title: meta.Title,
Cover: meta.Cover,
SourceURL: meta.SourceURL,
}
if werr := s.rankingStore.WriteRankingItem(ctx, item); werr != nil {
s.log.Warn("ranking item write failed", "slug", meta.Slug, "err", werr)
}
}
select {
case <-ctx.Done():
return