feat: ranking page pagination, popular URL, and per-page HTML disk cache
Some checks failed
CI / Lint (push) Has been cancelled
CI / Test (push) Has been cancelled
CI / Build (push) Has been cancelled

- Switch ScrapeRanking to novelfire.net/genre-all/sort-popular URL and updated DOM selectors (div.novel-item, h3.novel-title, div.genres)
- Replace 5 hardcoded refresh buttons with dynamic 100-page paginator (smart ellipsis via rankingPageNums)
- Add RankingPageCacher interface and writer methods to cache raw HTML per page under static/books/_ranking_cache/page-N.html
- ScrapeRanking serves from disk cache on hit and writes to cache on miss, skipping Browserless round-trip
- Thread writer as PageCacher through novelfire.New and main.go
- Add TestScrapeRanking_CacheHit and TestScrapeRanking_CacheMiss tests
This commit is contained in:
Admin
2026-03-01 21:32:50 +05:00
parent 73869f01fa
commit 9cf94576d8
7 changed files with 334 additions and 103 deletions

View File

@@ -88,7 +88,9 @@ func run(log *slog.Logger) error {
bc := newBrowserClient(strategy, browserCfg)
urlClient := newBrowserClient(urlStrategy, browserCfg)
nf := novelfire.New(bc, log, urlClient)
staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books")
w := writer.New(staticRoot)
nf := novelfire.New(bc, log, urlClient, w)
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -103,7 +105,7 @@ func run(log *slog.Logger) error {
oCfg := orchestrator.Config{
Workers: workers,
StaticRoot: envOr("SCRAPER_STATIC_ROOT", "./static/books"),
StaticRoot: staticRoot,
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)