fix: serve browse pages from MinIO cache; fix ReapStaleTasks PocketBase filter
All checks were successful
Release / Scraper / Test (push) Successful in 20s
Release / UI / Build (push) Successful in 25s
CI / Scraper / Lint (pull_request) Successful in 10s
Release / v2 / Build ui-v2 (push) Successful in 28s
CI / Scraper / Test (pull_request) Successful in 15s
CI / UI / Build (pull_request) Successful in 25s
Release / Scraper / Docker (push) Successful in 55s
Release / UI / Docker (push) Successful in 43s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 36s
Release / v2 / Test backend (push) Successful in 3m51s
Release / v2 / Docker / runner (push) Successful in 36s
Release / v2 / Docker / backend (push) Successful in 1m34s
iOS CI / Build (pull_request) Successful in 5m33s
iOS CI / Test (pull_request) Successful in 11m25s

- Runner fetches 9 browse combos (genre×sort×status) every 6h and stores
  JSON snapshots in MinIO libnovel-browse bucket (browse_refresh.go)
- Backend handleBrowse reads page-1 results from MinIO first; falls back
  to live novelfire.net fetch; returns empty+cached:false on total failure
  instead of 502
- Add BrowseStore interface (bookstore.go), MinIO put/get helpers (minio.go),
  Store methods + compile-time assertion (store.go), BucketBrowse config,
  wiring in cmd/backend and cmd/runner, docker-compose-new bucket init
- Fix ReapStaleTasks: PocketBase datetime fields require heartbeat_at=null
  (not heartbeat_at="") in filter expressions, and nil (not "") in patch
  payload — was causing 400 errors on every reap cycle
This commit is contained in:
Admin
2026-03-15 21:19:28 +05:00
parent 5825b859b7
commit 3918bc8dc3
12 changed files with 302 additions and 16 deletions

View File

@@ -44,6 +44,9 @@ type Config struct {
// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
// task is considered orphaned and reset to pending. Defaults to 2m when 0.
StaleTaskThreshold time.Duration
// BrowseRefreshInterval is how often the runner pre-fetches browse page
// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
BrowseRefreshInterval time.Duration
}
// Dependencies are the external services the runner depends on.
@@ -56,6 +59,8 @@ type Dependencies struct {
BookReader bookstore.BookReader
// AudioStore persists generated audio and checks key existence.
AudioStore bookstore.AudioStore
// BrowseStore stores browse page snapshots in MinIO.
BrowseStore bookstore.BrowseStore
// Novel is the scraper implementation.
Novel scraper.NovelScraper
// Kokoro is the TTS client.
@@ -91,6 +96,9 @@ func New(cfg Config, deps Dependencies) *Runner {
if cfg.StaleTaskThreshold <= 0 {
cfg.StaleTaskThreshold = 2 * time.Minute
}
if cfg.BrowseRefreshInterval <= 0 {
cfg.BrowseRefreshInterval = 6 * time.Hour
}
if deps.Log == nil {
deps.Log = slog.Default()
}
@@ -121,6 +129,7 @@ func (r *Runner) Run(ctx context.Context) error {
"poll_interval", r.cfg.PollInterval,
"max_scrape", r.cfg.MaxConcurrentScrape,
"max_audio", r.cfg.MaxConcurrentAudio,
"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
)
scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
@@ -134,6 +143,12 @@ func (r *Runner) Run(ctx context.Context) error {
tick := time.NewTicker(r.cfg.PollInterval)
defer tick.Stop()
browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
defer browseTick.Stop()
// Run one browse refresh and one poll immediately on startup.
go r.runBrowseRefresh(ctx)
// Run one poll immediately on startup, then on each tick.
for {
r.poll(ctx, scrapeSem, audioSem, &wg)
@@ -154,6 +169,8 @@ func (r *Runner) Run(ctx context.Context) error {
r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
}
return nil
case <-browseTick.C:
go r.runBrowseRefresh(ctx)
case <-tick.C:
}
}