refactor(ranking): replace blob cache with per-item PocketBase storage

- Replace SetRanking/GetRanking/SetRankingPageHTML/GetRankingPageHTML blob methods
  with WriteRankingItem/ReadRankingItems/RankingFreshEnough per-item operations
- Add 24h staleness gate in ScrapeRanking to skip re-scraping fresh data
- Add GET /api/ranking endpoint returning []RankingItem sorted by rank
- Remove RankingPageCacher interface and rankingCacheAdapter adapter
- Update integration tests to use new per-item upsert semantics
- Include e2e test suite (scraper/internal/e2e/)
This commit is contained in:
Admin
2026-03-03 19:37:49 +05:00
parent 56bf4dde22
commit b8d4d94b18
10 changed files with 1173 additions and 351 deletions

View File

@@ -38,16 +38,15 @@ import (
// Server wraps an HTTP mux with the scraping endpoints.
type Server struct {
addr string
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
store storage.Store
mu sync.Mutex
running bool
rankingRunning bool
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
kokoroVoice string // default voice, e.g. af_bella
addr string
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
store storage.Store
mu sync.Mutex
running bool
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
kokoroVoice string // default voice, e.g. af_bella
// voiceMu guards cachedVoices.
voiceMu sync.RWMutex
@@ -122,6 +121,8 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
// Browse API — fetches and parses novelfire catalogue page
mux.HandleFunc("GET /api/browse", s.handleBrowse)
// Ranking API
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
// Scrape status
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
// Progress API
@@ -172,6 +173,21 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}
// handleGetRanking returns all ranking items sorted by rank ascending.
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
items, err := s.store.ReadRankingItems(r.Context())
if err != nil {
s.log.Error("ranking read failed", "err", err)
http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
if items == nil {
items = []storage.RankingItem{}
}
_ = json.NewEncoder(w).Encode(items)
}
// ─── Session cookie helpers ───────────────────────────────────────────────────
const sessionCookieName = "libnovel_session"
@@ -410,6 +426,24 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
_ = s.store.SetAudioCache(r.Context(), cacheKey, filename)
// Download generated audio from Kokoro and persist to MinIO so that
// presigned URLs for the audio object are accessible.
go func() {
audioData, dlErr := s.downloadFromKokoro(context.Background(), filename)
if dlErr != nil {
s.log.Warn("audio MinIO upload skipped: kokoro download failed",
"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
return
}
minioKey := s.store.AudioObjectKey(slug, n, voice, speed)
if putErr := s.store.PutAudio(context.Background(), minioKey, audioData); putErr != nil {
s.log.Warn("audio MinIO upload failed",
"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
} else {
s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
}
}()
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
s.writeAudioResponse(w, slug, n, voice, speed, filename)
}
@@ -463,6 +497,29 @@ func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed f
return filename, nil
}
// downloadFromKokoro downloads a generated audio file from Kokoro's temp storage
// using GET /v1/download/{filename} and returns the raw bytes.
func (s *Server) downloadFromKokoro(ctx context.Context, filename string) ([]byte, error) {
url := s.kokoroURL + "/v1/download/" + filename
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("build download request: %w", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("kokoro download request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("kokoro download status %d", resp.StatusCode)
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("read kokoro download body: %w", err)
}
return data, nil
}
// writeAudioResponse writes the JSON response for a generated audio chapter.
// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) {