refactor(ranking): replace blob cache with per-item PocketBase storage

- Replace SetRanking/GetRanking/SetRankingPageHTML/GetRankingPageHTML blob methods
  with WriteRankingItem/ReadRankingItems/RankingFreshEnough per-item operations
- Add 24h staleness gate in ScrapeRanking to skip re-scraping fresh data
- Add GET /api/ranking endpoint returning []RankingItem sorted by rank
- Remove RankingPageCacher interface and rankingCacheAdapter adapter
- Update integration tests to use new per-item upsert semantics
- Include e2e test suite (scraper/internal/e2e/)
This commit is contained in:
Admin
2026-03-03 19:37:49 +05:00
parent 56bf4dde22
commit b8d4d94b18
10 changed files with 1173 additions and 351 deletions

View File

@@ -119,7 +119,7 @@ func run(log *slog.Logger) error {
return fmt.Errorf("storage init failed: %w", err)
}
nf := novelfire.New(bc, log, urlClient, &rankingCacheAdapter{store: store})
nf := novelfire.New(bc, log, urlClient, store)
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -215,20 +215,6 @@ func envOr(key, fallback string) string {
return fallback
}
// rankingCacheAdapter bridges storage.HybridStore (context-aware) to the
// context-free scraper.RankingPageCacher interface expected by novelfire.New.
type rankingCacheAdapter struct {
store *storage.HybridStore
}
func (a *rankingCacheAdapter) WriteRankingPageCache(page int, html string) error {
return a.store.WriteRankingPageCache(context.Background(), page, html)
}
func (a *rankingCacheAdapter) ReadRankingPageCache(page int) (string, error) {
return a.store.ReadRankingPageCache(context.Background(), page)
}
func printUsage() {
fmt.Fprintf(os.Stderr, `libnovel scraper