refactor(ranking): replace blob cache with per-item PocketBase storage
- Replace SetRanking/GetRanking/SetRankingPageHTML/GetRankingPageHTML blob methods with WriteRankingItem/ReadRankingItems/RankingFreshEnough per-item operations - Add 24h staleness gate in ScrapeRanking to skip re-scraping fresh data - Add GET /api/ranking endpoint returning []RankingItem sorted by rank - Remove RankingPageCacher interface and rankingCacheAdapter adapter - Update integration tests to use new per-item upsert semantics - Include e2e test suite (scraper/internal/e2e/)
This commit is contained in:
@@ -38,16 +38,15 @@ import (
|
||||
|
||||
// Server wraps an HTTP mux with the scraping endpoints.
|
||||
type Server struct {
|
||||
addr string
|
||||
oCfg orchestrator.Config
|
||||
novel scraper.NovelScraper
|
||||
log *slog.Logger
|
||||
store storage.Store
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
rankingRunning bool
|
||||
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
||||
kokoroVoice string // default voice, e.g. af_bella
|
||||
addr string
|
||||
oCfg orchestrator.Config
|
||||
novel scraper.NovelScraper
|
||||
log *slog.Logger
|
||||
store storage.Store
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
||||
kokoroVoice string // default voice, e.g. af_bella
|
||||
|
||||
// voiceMu guards cachedVoices.
|
||||
voiceMu sync.RWMutex
|
||||
@@ -122,6 +121,8 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
||||
// Browse API — fetches and parses novelfire catalogue page
|
||||
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
||||
// Ranking API
|
||||
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
|
||||
// Scrape status
|
||||
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
|
||||
// Progress API
|
||||
@@ -172,6 +173,21 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// handleGetRanking returns all ranking items sorted by rank ascending.
|
||||
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
|
||||
items, err := s.store.ReadRankingItems(r.Context())
|
||||
if err != nil {
|
||||
s.log.Error("ranking read failed", "err", err)
|
||||
http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if items == nil {
|
||||
items = []storage.RankingItem{}
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(items)
|
||||
}
|
||||
|
||||
// ─── Session cookie helpers ───────────────────────────────────────────────────
|
||||
|
||||
const sessionCookieName = "libnovel_session"
|
||||
@@ -410,6 +426,24 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
_ = s.store.SetAudioCache(r.Context(), cacheKey, filename)
|
||||
|
||||
// Download generated audio from Kokoro and persist to MinIO so that
|
||||
// presigned URLs for the audio object are accessible.
|
||||
go func() {
|
||||
audioData, dlErr := s.downloadFromKokoro(context.Background(), filename)
|
||||
if dlErr != nil {
|
||||
s.log.Warn("audio MinIO upload skipped: kokoro download failed",
|
||||
"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
|
||||
return
|
||||
}
|
||||
minioKey := s.store.AudioObjectKey(slug, n, voice, speed)
|
||||
if putErr := s.store.PutAudio(context.Background(), minioKey, audioData); putErr != nil {
|
||||
s.log.Warn("audio MinIO upload failed",
|
||||
"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
|
||||
} else {
|
||||
s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
|
||||
}
|
||||
}()
|
||||
|
||||
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
|
||||
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
||||
}
|
||||
@@ -463,6 +497,29 @@ func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed f
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
// downloadFromKokoro downloads a generated audio file from Kokoro's temp storage
|
||||
// using GET /v1/download/{filename} and returns the raw bytes.
|
||||
func (s *Server) downloadFromKokoro(ctx context.Context, filename string) ([]byte, error) {
|
||||
url := s.kokoroURL + "/v1/download/" + filename
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build download request: %w", err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kokoro download request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("kokoro download status %d", resp.StatusCode)
|
||||
}
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read kokoro download body: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// writeAudioResponse writes the JSON response for a generated audio chapter.
|
||||
// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
|
||||
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) {
|
||||
|
||||
Reference in New Issue
Block a user