feat: ranking page pagination, popular URL, and per-page HTML disk cache
Some checks failed
CI / Lint (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Build (push) Has been cancelled

- Switch ScrapeRanking to novelfire.net/genre-all/sort-popular URL and updated DOM selectors (div.novel-item, h3.novel-title, div.genres)
- Replace 5 hardcoded refresh buttons with dynamic 100-page paginator (smart ellipsis via rankingPageNums)
- Add RankingPageCacher interface and writer methods to cache raw HTML per page under static/books/_ranking_cache/page-N.html
- ScrapeRanking serves from disk cache on hit and writes to cache on miss, skipping Browserless round-trip
- Thread writer as PageCacher through novelfire.New and main.go
- Add TestScrapeRanking_CacheHit and TestScrapeRanking_CacheMiss tests
This commit is contained in:
Admin
2026-03-01 21:32:50 +05:00
parent 73869f01fa
commit 9cf94576d8
7 changed files with 334 additions and 103 deletions

View File

@@ -403,6 +403,57 @@ func (w *Writer) rankingPath() string {
return filepath.Join(w.root, "ranking.md")
}
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
// rankingCacheDir returns the directory that stores per-page HTML caches.
func (w *Writer) rankingCacheDir() string {
return filepath.Join(w.root, "_ranking_cache")
}
// rankingPageCachePath returns the path for a cached ranking page HTML file.
func (w *Writer) rankingPageCachePath(page int) string {
return filepath.Join(w.rankingCacheDir(), fmt.Sprintf("page-%d.html", page))
}
// WriteRankingPageCache persists raw HTML for the given ranking page number.
func (w *Writer) WriteRankingPageCache(page int, html string) error {
dir := w.rankingCacheDir()
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("writer: mkdir ranking cache %s: %w", dir, err)
}
path := w.rankingPageCachePath(page)
if err := os.WriteFile(path, []byte(html), 0o644); err != nil {
return fmt.Errorf("writer: write ranking page cache %s: %w", path, err)
}
return nil
}
// ReadRankingPageCache reads the cached HTML for the given ranking page.
// Returns ("", nil) when no cache file exists yet.
func (w *Writer) ReadRankingPageCache(page int) (string, error) {
data, err := os.ReadFile(w.rankingPageCachePath(page))
if err != nil {
if os.IsNotExist(err) {
return "", nil
}
return "", fmt.Errorf("writer: read ranking page cache page %d: %w", page, err)
}
return string(data), nil
}
// RankingPageCacheInfo returns os.FileInfo for a cached ranking page file.
// Returns (nil, nil) when the file does not exist.
func (w *Writer) RankingPageCacheInfo(page int) (os.FileInfo, error) {
info, err := os.Stat(w.rankingPageCachePath(page))
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
return info, nil
}
// bookDir returns the root directory for a book slug.
func (w *Writer) bookDir(slug string) string {
return filepath.Join(w.root, slug)