feat: ranking page pagination, popular URL, and per-page HTML disk cache
- Switch ScrapeRanking to novelfire.net/genre-all/sort-popular URL and updated DOM selectors (div.novel-item, h3.novel-title, div.genres) - Replace 5 hardcoded refresh buttons with dynamic 100-page paginator (smart ellipsis via rankingPageNums) - Add RankingPageCacher interface and writer methods to cache raw HTML per page under static/books/_ranking_cache/page-N.html - ScrapeRanking serves from disk cache on hit and writes to cache on miss, skipping Browserless round-trip - Thread writer as PageCacher through novelfire.New and main.go - Add TestScrapeRanking_CacheHit and TestScrapeRanking_CacheMiss tests
This commit is contained in:
@@ -88,7 +88,9 @@ func run(log *slog.Logger) error {
|
||||
bc := newBrowserClient(strategy, browserCfg)
|
||||
urlClient := newBrowserClient(urlStrategy, browserCfg)
|
||||
|
||||
nf := novelfire.New(bc, log, urlClient)
|
||||
staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books")
|
||||
w := writer.New(staticRoot)
|
||||
nf := novelfire.New(bc, log, urlClient, w)
|
||||
|
||||
workers := 0
|
||||
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
|
||||
@@ -103,7 +105,7 @@ func run(log *slog.Logger) error {
|
||||
|
||||
oCfg := orchestrator.Config{
|
||||
Workers: workers,
|
||||
StaticRoot: envOr("SCRAPER_STATIC_ROOT", "./static/books"),
|
||||
StaticRoot: staticRoot,
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
Reference in New Issue
Block a user