fix: serve browse pages from MinIO cache; fix ReapStaleTasks PocketBase filter
All checks were successful
Release / Scraper / Test (push) Successful in 20s
Release / UI / Build (push) Successful in 25s
CI / Scraper / Lint (pull_request) Successful in 10s
Release / v2 / Build ui-v2 (push) Successful in 28s
CI / Scraper / Test (pull_request) Successful in 15s
CI / UI / Build (pull_request) Successful in 25s
Release / Scraper / Docker (push) Successful in 55s
Release / UI / Docker (push) Successful in 43s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 36s
Release / v2 / Test backend (push) Successful in 3m51s
Release / v2 / Docker / runner (push) Successful in 36s
Release / v2 / Docker / backend (push) Successful in 1m34s
iOS CI / Build (pull_request) Successful in 5m33s
iOS CI / Test (pull_request) Successful in 11m25s

- Runner fetches 9 browse combos (genre×sort×status) every 6h and stores
  JSON snapshots in MinIO libnovel-browse bucket (browse_refresh.go)
- Backend handleBrowse reads page-1 results from MinIO first; falls back
  to live novelfire.net fetch; returns empty+cached:false on total failure
  instead of 502
- Add BrowseStore interface (bookstore.go), MinIO put/get helpers (minio.go),
  Store methods + compile-time assertion (store.go), BucketBrowse config,
  wiring in cmd/backend and cmd/runner, docker-compose-new bucket init
- Fix ReapStaleTasks: PocketBase datetime fields require heartbeat_at=null
  (not heartbeat_at="") in filter expressions, and nil (not "") in patch
  payload — was causing 400 errors on every reap cycle
This commit is contained in:
Admin
2026-03-15 21:19:28 +05:00
parent 5825b859b7
commit 3918bc8dc3
12 changed files with 302 additions and 16 deletions

View File

@@ -204,16 +204,36 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
pageNum = 1
}
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
novelFireBase, genre, sortBy, status, novelType, pageNum)
// ── Try MinIO cache first ─────────────────────────────────────────────
// Only page 1 is cached; higher pages fall through to live fetch.
if pageNum == 1 && s.deps.BrowseStore != nil {
if data, ok, err := s.deps.BrowseStore.GetBrowsePage(r.Context(), genre, sortBy, status, novelType, 1); err == nil && ok {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "public, max-age=300")
_, _ = w.Write(data)
return
}
}
// ── Fall back to live novelfire.net fetch ──────────────────────────────
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
defer cancel()
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
novelFireBase, genre, sortBy, status, novelType, pageNum)
novels, hasNext, err := s.fetchBrowsePage(ctx, targetURL)
if err != nil {
s.deps.Log.Error("handleBrowse: fetch failed", "url", targetURL, "err", err)
jsonError(w, http.StatusBadGateway, err.Error())
// Live fetch also failed — return empty list with cached=false flag so
// the UI can show a "not ready yet" state instead of a hard error.
s.deps.Log.Error("handleBrowse: fetch failed (no cache)", "url", targetURL, "err", err)
w.Header().Set("Cache-Control", "no-store")
writeJSON(w, 0, map[string]any{
"novels": []any{},
"page": pageNum,
"hasNext": false,
"cached": false,
})
return
}
@@ -222,6 +242,7 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
"novels": novels,
"page": pageNum,
"hasNext": hasNext,
"cached": false,
})
}

View File

@@ -46,6 +46,8 @@ type Dependencies struct {
PresignStore bookstore.PresignStore
// ProgressStore reads/writes per-session reading progress.
ProgressStore bookstore.ProgressStore
// BrowseStore reads cached browse page snapshots from MinIO.
BrowseStore bookstore.BrowseStore
// Producer creates scrape/audio tasks in PocketBase.
Producer taskqueue.Producer
// TaskReader reads scrape/audio task records from PocketBase.