fix: serve browse pages from MinIO cache; fix ReapStaleTasks PocketBase filter

- Runner fetches 9 browse combos (genre×sort×status) every 6h and stores
  JSON snapshots in MinIO libnovel-browse bucket (browse_refresh.go)
- Backend handleBrowse reads page-1 results from MinIO first; falls back
  to live novelfire.net fetch; returns empty+cached:false on total failure
  instead of 502
- Add BrowseStore interface (bookstore.go), MinIO put/get helpers (minio.go),
  Store methods + compile-time assertion (store.go), BucketBrowse config,
  wiring in cmd/backend and cmd/runner, docker-compose-new bucket init
- Fix ReapStaleTasks: PocketBase datetime fields require heartbeat_at=null
  (not heartbeat_at="") in filter expressions, and nil (not "") in patch
  payload — was causing 400 errors on every reap cycle

.env.example

@@ -79,6 +79,7 @@ MINIO_ROOT_USER=admin
 MINIO_ROOT_PASSWORD=changeme123
 MINIO_BUCKET_CHAPTERS=libnovel-chapters
 MINIO_BUCKET_AUDIO=libnovel-audio
+MINIO_BUCKET_BROWSE=libnovel-browse
 
 # ── PocketBase ────────────────────────────────────────────────────────────────
 # Admin credentials (used by scraper + UI server-side)

backend/cmd/backend/main.go

@@ -84,6 +84,7 @@ func run() error {
 			AudioStore:    store,
 			PresignStore:  store,
 			ProgressStore: store,
+			BrowseStore:   store,
 			Producer:      store,
 			TaskReader:    store,
 			Kokoro:        kokoroClient,

backend/cmd/runner/main.go

@@ -97,13 +97,14 @@ func run() error {
 		OrchestratorWorkers: workers,
 	}
 	deps := runner.Dependencies{
-		Consumer:   store,
-		BookWriter: store,
-		BookReader: store,
-		AudioStore: store,
-		Novel:      novel,
-		Kokoro:     kokoroClient,
-		Log:        log,
+		Consumer:    store,
+		BookWriter:  store,
+		BookReader:  store,
+		AudioStore:  store,
+		BrowseStore: store,
+		Novel:       novel,
+		Kokoro:      kokoroClient,
+		Log:         log,
 	}
 	r := runner.New(rCfg, deps)
 

backend/internal/backend/handlers.go

@@ -204,16 +204,36 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 		pageNum = 1
 	}
 
-	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
-		novelFireBase, genre, sortBy, status, novelType, pageNum)
+	// ── Try MinIO cache first ─────────────────────────────────────────────
+	// Only page 1 is cached; higher pages fall through to live fetch.
+	if pageNum == 1 && s.deps.BrowseStore != nil {
+		if data, ok, err := s.deps.BrowseStore.GetBrowsePage(r.Context(), genre, sortBy, status, novelType, 1); err == nil && ok {
+			w.Header().Set("Content-Type", "application/json")
+			w.Header().Set("Cache-Control", "public, max-age=300")
+			_, _ = w.Write(data)
+			return
+		}
+	}
 
+	// ── Fall back to live novelfire.net fetch ──────────────────────────────
 	ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
 	defer cancel()
 
+	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
+		novelFireBase, genre, sortBy, status, novelType, pageNum)
+
 	novels, hasNext, err := s.fetchBrowsePage(ctx, targetURL)
 	if err != nil {
-		s.deps.Log.Error("handleBrowse: fetch failed", "url", targetURL, "err", err)
-		jsonError(w, http.StatusBadGateway, err.Error())
+		// Live fetch also failed — return empty list with cached=false flag so
+		// the UI can show a "not ready yet" state instead of a hard error.
+		s.deps.Log.Error("handleBrowse: fetch failed (no cache)", "url", targetURL, "err", err)
+		w.Header().Set("Cache-Control", "no-store")
+		writeJSON(w, 0, map[string]any{
+			"novels":  []any{},
+			"page":    pageNum,
+			"hasNext": false,
+			"cached":  false,
+		})
 		return
 	}
 
@@ -222,6 +242,7 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 		"novels":  novels,
 		"page":    pageNum,
 		"hasNext": hasNext,
+		"cached":  false,
 	})
 }
 

backend/internal/backend/server.go

@@ -46,6 +46,8 @@ type Dependencies struct {
 	PresignStore bookstore.PresignStore
 	// ProgressStore reads/writes per-session reading progress.
 	ProgressStore bookstore.ProgressStore
+	// BrowseStore reads cached browse page snapshots from MinIO.
+	BrowseStore bookstore.BrowseStore
 	// Producer creates scrape/audio tasks in PocketBase.
 	Producer taskqueue.Producer
 	// TaskReader reads scrape/audio task records from PocketBase.

backend/internal/bookstore/bookstore.go

@@ -123,3 +123,15 @@ type ProgressStore interface {
 	// DeleteProgress removes progress for a specific slug.
 	DeleteProgress(ctx context.Context, sessionID, slug string) error
 }
+
+// BrowseStore covers browse page snapshot storage.
+// The runner writes snapshots; the backend reads them.
+type BrowseStore interface {
+	// PutBrowsePage stores a raw JSON snapshot for a browse page.
+	// genre, sort, status, novelType and page identify the page.
+	PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error
+
+	// GetBrowsePage retrieves a raw JSON snapshot. Returns (nil, false, nil)
+	// when no snapshot exists for the given parameters.
+	GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error)
+}

backend/internal/config/config.go

@@ -44,6 +44,8 @@ type MinIO struct {
 	BucketAudio string
 	// BucketAvatars is the bucket that holds user avatar images.
 	BucketAvatars string
+	// BucketBrowse is the bucket that holds cached browse page snapshots (JSON).
+	BucketBrowse string
 }
 
 // Kokoro holds connection settings for the Kokoro-FastAPI TTS service.
@@ -118,6 +120,7 @@ func Load() Config {
 			BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
 			BucketAudio:    envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
 			BucketAvatars:  envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"),
+			BucketBrowse:   envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
 		},
 
 		Kokoro: Kokoro{

backend/internal/runner/browse_refresh.go

@@ -0,0 +1,176 @@
+package runner
+
+// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
+// browse page snapshots and stores them in MinIO.
+//
+// Design:
+//   - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
+//   - Fetches page 1 for each combination of the standard genre/sort/status
+//     filter values and stores the parsed JSON blob in MinIO via BrowseStore.
+//   - The backend's handleBrowse then serves from MinIO instead of calling
+//     novelfire.net live, which avoids IP-based rate-limiting on the server.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
+type browseNovelListing struct {
+	Slug  string `json:"slug"`
+	Title string `json:"title"`
+	Cover string `json:"cover"`
+	URL   string `json:"url"`
+}
+
+// browseSnapshot is the JSON structure stored in MinIO.
+type browseSnapshot struct {
+	Novels  []browseNovelListing `json:"novels"`
+	Page    int                  `json:"page"`
+	HasNext bool                 `json:"hasNext"`
+	// CachedAt is the UTC time the snapshot was written (ISO 8601).
+	CachedAt string `json:"cachedAt"`
+}
+
+// browseCombos lists the filter combinations to pre-fetch.
+// Each entry is (genre, sort, status, novelType).
+var browseCombos = []struct{ genre, sort, status, novelType string }{
+	{"all", "popular", "all", "all-novel"},
+	{"all", "popular", "ongoing", "all-novel"},
+	{"all", "popular", "completed", "all-novel"},
+	{"all", "new", "all", "all-novel"},
+	{"all", "new", "ongoing", "all-novel"},
+	{"all", "new", "completed", "all-novel"},
+	{"all", "top-rated", "all", "all-novel"},
+	{"all", "top-rated", "ongoing", "all-novel"},
+	{"all", "top-rated", "completed", "all-novel"},
+}
+
+const novelFireBrowseBase = "https://novelfire.net"
+
+// runBrowseRefresh fetches all browse combos from novelfire.net and stores
+// the results in MinIO. Errors per-combo are logged but do not abort the
+// whole refresh cycle.
+func (r *Runner) runBrowseRefresh(ctx context.Context) {
+	if r.deps.BrowseStore == nil {
+		r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
+		return
+	}
+
+	log := r.deps.Log.With("op", "browse_refresh")
+	log.Info("runner: browse refresh starting", "combos", len(browseCombos))
+
+	ok, fail := 0, 0
+	for _, c := range browseCombos {
+		if ctx.Err() != nil {
+			break
+		}
+		novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
+		if err != nil {
+			log.Warn("runner: browse fetch failed",
+				"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
+			fail++
+			continue
+		}
+
+		snap := browseSnapshot{
+			Novels:   novels,
+			Page:     1,
+			HasNext:  hasNext,
+			CachedAt: time.Now().UTC().Format(time.RFC3339),
+		}
+		data, _ := json.Marshal(snap)
+		if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
+			log.Warn("runner: browse put failed",
+				"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
+			fail++
+			continue
+		}
+		ok++
+	}
+
+	log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
+}
+
+// fetchBrowsePage calls novelfire.net and returns a list of novel listings
+// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
+func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
+	pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
+		novelFireBrowseBase, genre, sort, status, novelType)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
+	if err != nil {
+		return nil, false, fmt.Errorf("build request: %w", err)
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+
+	httpClient := &http.Client{Timeout: 45 * time.Second}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
+	}
+
+	return parseBrowseHTML(resp.Body)
+}
+
+// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
+// in backend/handlers.go — kept separate to avoid coupling packages.
+func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return nil, false, err
+	}
+	body := string(data)
+
+	hasNext := strings.Contains(body, `rel="next"`) ||
+		strings.Contains(body, `aria-label="Next"`) ||
+		strings.Contains(body, `class="next"`)
+
+	slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
+	titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
+	coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
+
+	slugMatches := slugRe.FindAllStringSubmatch(body, -1)
+	titleMatches := titleRe.FindAllStringSubmatch(body, -1)
+	coverMatches := coverRe.FindAllStringSubmatch(body, -1)
+
+	var novels []browseNovelListing
+	seen := make(map[string]bool)
+	for i, sm := range slugMatches {
+		slug := sm[1]
+		if seen[slug] {
+			continue
+		}
+		seen[slug] = true
+
+		item := browseNovelListing{
+			Slug: slug,
+			URL:  novelFireBrowseBase + "/book/" + slug,
+		}
+		if i < len(titleMatches) {
+			item.Title = strings.TrimSpace(titleMatches[i][1])
+		}
+		if i < len(coverMatches) {
+			item.Cover = coverMatches[i][1]
+		}
+		if item.Title != "" {
+			novels = append(novels, item)
+		}
+	}
+
+	return novels, hasNext, nil
+}

backend/internal/runner/runner.go

@@ -44,6 +44,9 @@ type Config struct {
 	// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
 	// task is considered orphaned and reset to pending. Defaults to 2m when 0.
 	StaleTaskThreshold time.Duration
+	// BrowseRefreshInterval is how often the runner pre-fetches browse page
+	// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
+	BrowseRefreshInterval time.Duration
 }
 
 // Dependencies are the external services the runner depends on.
@@ -56,6 +59,8 @@ type Dependencies struct {
 	BookReader bookstore.BookReader
 	// AudioStore persists generated audio and checks key existence.
 	AudioStore bookstore.AudioStore
+	// BrowseStore stores browse page snapshots in MinIO.
+	BrowseStore bookstore.BrowseStore
 	// Novel is the scraper implementation.
 	Novel scraper.NovelScraper
 	// Kokoro is the TTS client.
@@ -91,6 +96,9 @@ func New(cfg Config, deps Dependencies) *Runner {
 	if cfg.StaleTaskThreshold <= 0 {
 		cfg.StaleTaskThreshold = 2 * time.Minute
 	}
+	if cfg.BrowseRefreshInterval <= 0 {
+		cfg.BrowseRefreshInterval = 6 * time.Hour
+	}
 	if deps.Log == nil {
 		deps.Log = slog.Default()
 	}
@@ -121,6 +129,7 @@ func (r *Runner) Run(ctx context.Context) error {
 		"poll_interval", r.cfg.PollInterval,
 		"max_scrape", r.cfg.MaxConcurrentScrape,
 		"max_audio", r.cfg.MaxConcurrentAudio,
+		"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
 	)
 
 	scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
@@ -134,6 +143,12 @@ func (r *Runner) Run(ctx context.Context) error {
 	tick := time.NewTicker(r.cfg.PollInterval)
 	defer tick.Stop()
 
+	browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
+	defer browseTick.Stop()
+
+	// Run one browse refresh and one poll immediately on startup.
+	go r.runBrowseRefresh(ctx)
+
 	// Run one poll immediately on startup, then on each tick.
 	for {
 		r.poll(ctx, scrapeSem, audioSem, &wg)
@@ -154,6 +169,8 @@ func (r *Runner) Run(ctx context.Context) error {
 				r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
 			}
 			return nil
+		case <-browseTick.C:
+			go r.runBrowseRefresh(ctx)
 		case <-tick.C:
 		}
 	}

backend/internal/storage/minio.go

@@ -22,6 +22,7 @@ type minioClient struct {
 	bucketChapters string
 	bucketAudio    string
 	bucketAvatars  string
+	bucketBrowse   string
 }
 
 func newMinioClient(cfg config.MinIO) (*minioClient, error) {
@@ -78,12 +79,13 @@ func newMinioClient(cfg config.MinIO) (*minioClient, error) {
 		bucketChapters: cfg.BucketChapters,
 		bucketAudio:    cfg.BucketAudio,
 		bucketAvatars:  cfg.BucketAvatars,
+		bucketBrowse:   cfg.BucketBrowse,
 	}, nil
 }
 
 // ensureBuckets creates all required buckets if they don't already exist.
 func (m *minioClient) ensureBuckets(ctx context.Context) error {
-	for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars} {
+	for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse} {
 		exists, err := m.client.BucketExists(ctx, bucket)
 		if err != nil {
 			return fmt.Errorf("minio: check bucket %q: %w", bucket, err)
@@ -117,6 +119,12 @@ func AvatarObjectKey(userID, ext string) string {
 	return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
 }
 
+// BrowseObjectKey returns the MinIO object key for a cached browse page snapshot.
+// Format: browse/{genre}/{sort}/{status}/{type}/page-{n}.json
+func BrowseObjectKey(genre, sort, status, novelType string, page int) string {
+	return fmt.Sprintf("browse/%s/%s/%s/%s/page-%d.json", genre, sort, status, novelType, page)
+}
+
 // chapterNumberFromKey extracts the chapter number from a MinIO object key.
 // e.g. "my-book/chapter-000042.md" → 42
 func chapterNumberFromKey(key string) int {
@@ -192,3 +200,23 @@ func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string)
 	}
 	return keys, nil
 }
+
+// ── Browse operations ─────────────────────────────────────────────────────────
+
+// putBrowse stores raw JSON bytes for a browse page snapshot.
+func (m *minioClient) putBrowse(ctx context.Context, key string, data []byte) error {
+	return m.putObject(ctx, m.bucketBrowse, key, "application/json", data)
+}
+
+// getBrowse retrieves a browse page snapshot. Returns (nil, false, nil) when
+// the object does not exist.
+func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool, error) {
+	if !m.objectExists(ctx, m.bucketBrowse, key) {
+		return nil, false, nil
+	}
+	data, err := m.getObject(ctx, m.bucketBrowse, key)
+	if err != nil {
+		return nil, false, err
+	}
+	return data, true, nil
+}

backend/internal/storage/store.go

@@ -50,6 +50,7 @@ var _ bookstore.RankingStore = (*Store)(nil)
 var _ bookstore.AudioStore = (*Store)(nil)
 var _ bookstore.PresignStore = (*Store)(nil)
 var _ bookstore.ProgressStore = (*Store)(nil)
+var _ bookstore.BrowseStore = (*Store)(nil)
 var _ taskqueue.Producer = (*Store)(nil)
 var _ taskqueue.Consumer = (*Store)(nil)
 var _ taskqueue.Reader = (*Store)(nil)
@@ -608,12 +609,13 @@ func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
 // re-claimed. Returns the number of tasks reaped.
 func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) {
 	threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339)
-	// Match tasks that are running AND (heartbeat_at is empty OR heartbeat_at < threshold).
-	filter := fmt.Sprintf(`status="running"&&(heartbeat_at=""||heartbeat_at<"%s")`, threshold)
+	// Match tasks that are running AND (heartbeat_at is null OR heartbeat_at < threshold).
+	// PocketBase datetime fields require `=null` not `=""` in filter expressions.
+	filter := fmt.Sprintf(`status="running"&&(heartbeat_at=null||heartbeat_at<"%s")`, threshold)
 	resetPayload := map[string]any{
 		"status":       string(domain.TaskStatusPending),
 		"worker_id":    "",
-		"heartbeat_at": "",
+		"heartbeat_at": nil,
 	}
 
 	total := 0
@@ -767,3 +769,22 @@ func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
 		Finished:     finished,
 	}, nil
 }
+
+// ── BrowseStore ────────────────────────────────────────────────────────────────
+
+func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error {
+	key := BrowseObjectKey(genre, sort, status, novelType, page)
+	if err := s.mc.putBrowse(ctx, key, data); err != nil {
+		return fmt.Errorf("PutBrowsePage: %w", err)
+	}
+	return nil
+}
+
+func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) {
+	key := BrowseObjectKey(genre, sort, status, novelType, page)
+	data, ok, err := s.mc.getBrowse(ctx, key)
+	if err != nil {
+		return nil, false, fmt.Errorf("GetBrowsePage: %w", err)
+	}
+	return data, ok, nil
+}

docker-compose-new.yml

@@ -30,6 +30,7 @@ services:
         mc mb --ignore-existing local/libnovel-chapters;
         mc mb --ignore-existing local/libnovel-audio;
         mc mb --ignore-existing local/libnovel-avatars;
+        mc mb --ignore-existing local/libnovel-browse;
         echo 'buckets ready';
       "
     environment:
@@ -88,7 +89,7 @@ services:
     environment:
       BACKEND_HTTP_ADDR: ":8080"
       LOG_LEVEL: "${LOG_LEVEL:-info}"
-      # MinIO
+       # MinIO
       MINIO_ENDPOINT: "minio:9000"
       MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
       MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
@@ -96,6 +97,7 @@ services:
       MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
       MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
       MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
+      MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
       MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
       MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}"
       # PocketBase
@@ -152,6 +154,7 @@ services:
       MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
       MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
       MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
+      MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
       MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
       MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}"
       # PocketBase