docs: add architecture diagrams (D2 + Mermaid)

Adds docs/architecture.d2 and docs/architecture.mermaid.md showing the
docker-compose-new.yml service topology — storage, application, init
containers, and external dependencies with annotated connections.

Also includes the rendered docs/architecture.svg (D2 output).

View live: d2 --watch docs/architecture.d2
View in Gitea: navigate to docs/architecture.mermaid.md in the web UI.

.env.example

@@ -79,6 +79,7 @@ MINIO_ROOT_USER=admin
 MINIO_ROOT_PASSWORD=changeme123
 MINIO_BUCKET_CHAPTERS=libnovel-chapters
 MINIO_BUCKET_AUDIO=libnovel-audio
+MINIO_BUCKET_BROWSE=libnovel-browse
 
 # ── PocketBase ────────────────────────────────────────────────────────────────
 # Admin credentials (used by scraper + UI server-side)

backend/cmd/backend/main.go

@@ -84,6 +84,7 @@ func run() error {
 			AudioStore:    store,
 			PresignStore:  store,
 			ProgressStore: store,
+			BrowseStore:   store,
 			Producer:      store,
 			TaskReader:    store,
 			Kokoro:        kokoroClient,

backend/cmd/runner/main.go

@@ -97,13 +97,14 @@ func run() error {
 		OrchestratorWorkers: workers,
 	}
 	deps := runner.Dependencies{
-		Consumer:   store,
+		Consumer:    store,
-		BookWriter: store,
+		BookWriter:  store,
-		BookReader: store,
+		BookReader:  store,
-		AudioStore: store,
+		AudioStore:  store,
-		Novel:      novel,
+		BrowseStore: store,
-		Kokoro:     kokoroClient,
+		Novel:       novel,
-		Log:        log,
+		Kokoro:      kokoroClient,
+		Log:         log,
 	}
 	r := runner.New(rCfg, deps)
 

backend/internal/backend/handlers.go

@@ -204,16 +204,36 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 		pageNum = 1
 	}
 
-	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
+	// ── Try MinIO cache first ─────────────────────────────────────────────
-		novelFireBase, genre, sortBy, status, novelType, pageNum)
+	// Only page 1 is cached; higher pages fall through to live fetch.
+	if pageNum == 1 && s.deps.BrowseStore != nil {
+		if data, ok, err := s.deps.BrowseStore.GetBrowsePage(r.Context(), genre, sortBy, status, novelType, 1); err == nil && ok {
+			w.Header().Set("Content-Type", "application/json")
+			w.Header().Set("Cache-Control", "public, max-age=300")
+			_, _ = w.Write(data)
+			return
+		}
+	}
 
+	// ── Fall back to live novelfire.net fetch ──────────────────────────────
 	ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
 	defer cancel()
 
+	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
+		novelFireBase, genre, sortBy, status, novelType, pageNum)
+
 	novels, hasNext, err := s.fetchBrowsePage(ctx, targetURL)
 	if err != nil {
-		s.deps.Log.Error("handleBrowse: fetch failed", "url", targetURL, "err", err)
+		// Live fetch also failed — return empty list with cached=false flag so
-		jsonError(w, http.StatusBadGateway, err.Error())
+		// the UI can show a "not ready yet" state instead of a hard error.
+		s.deps.Log.Error("handleBrowse: fetch failed (no cache)", "url", targetURL, "err", err)
+		w.Header().Set("Cache-Control", "no-store")
+		writeJSON(w, 0, map[string]any{
+			"novels":  []any{},
+			"page":    pageNum,
+			"hasNext": false,
+			"cached":  false,
+		})
 		return
 	}
 
@@ -222,6 +242,7 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 		"novels":  novels,
 		"page":    pageNum,
 		"hasNext": hasNext,
+		"cached":  false,
 	})
 }
 

backend/internal/backend/server.go

@@ -46,6 +46,8 @@ type Dependencies struct {
 	PresignStore bookstore.PresignStore
 	// ProgressStore reads/writes per-session reading progress.
 	ProgressStore bookstore.ProgressStore
+	// BrowseStore reads cached browse page snapshots from MinIO.
+	BrowseStore bookstore.BrowseStore
 	// Producer creates scrape/audio tasks in PocketBase.
 	Producer taskqueue.Producer
 	// TaskReader reads scrape/audio task records from PocketBase.

backend/internal/bookstore/bookstore.go

@@ -123,3 +123,15 @@ type ProgressStore interface {
 	// DeleteProgress removes progress for a specific slug.
 	DeleteProgress(ctx context.Context, sessionID, slug string) error
 }
+
+// BrowseStore covers browse page snapshot storage.
+// The runner writes snapshots; the backend reads them.
+type BrowseStore interface {
+	// PutBrowsePage stores a raw JSON snapshot for a browse page.
+	// genre, sort, status, novelType and page identify the page.
+	PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error
+
+	// GetBrowsePage retrieves a raw JSON snapshot. Returns (nil, false, nil)
+	// when no snapshot exists for the given parameters.
+	GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error)
+}

backend/internal/config/config.go

@@ -44,6 +44,8 @@ type MinIO struct {
 	BucketAudio string
 	// BucketAvatars is the bucket that holds user avatar images.
 	BucketAvatars string
+	// BucketBrowse is the bucket that holds cached browse page snapshots (JSON).
+	BucketBrowse string
 }
 
 // Kokoro holds connection settings for the Kokoro-FastAPI TTS service.
@@ -118,6 +120,7 @@ func Load() Config {
 			BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
 			BucketAudio:    envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
 			BucketAvatars:  envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"),
+			BucketBrowse:   envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
 		},
 
 		Kokoro: Kokoro{

backend/internal/runner/browse_refresh.go

@@ -0,0 +1,176 @@
+package runner
+
+// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
+// browse page snapshots and stores them in MinIO.
+//
+// Design:
+//   - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
+//   - Fetches page 1 for each combination of the standard genre/sort/status
+//     filter values and stores the parsed JSON blob in MinIO via BrowseStore.
+//   - The backend's handleBrowse then serves from MinIO instead of calling
+//     novelfire.net live, which avoids IP-based rate-limiting on the server.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
+type browseNovelListing struct {
+	Slug  string `json:"slug"`
+	Title string `json:"title"`
+	Cover string `json:"cover"`
+	URL   string `json:"url"`
+}
+
+// browseSnapshot is the JSON structure stored in MinIO.
+type browseSnapshot struct {
+	Novels  []browseNovelListing `json:"novels"`
+	Page    int                  `json:"page"`
+	HasNext bool                 `json:"hasNext"`
+	// CachedAt is the UTC time the snapshot was written (ISO 8601).
+	CachedAt string `json:"cachedAt"`
+}
+
+// browseCombos lists the filter combinations to pre-fetch.
+// Each entry is (genre, sort, status, novelType).
+var browseCombos = []struct{ genre, sort, status, novelType string }{
+	{"all", "popular", "all", "all-novel"},
+	{"all", "popular", "ongoing", "all-novel"},
+	{"all", "popular", "completed", "all-novel"},
+	{"all", "new", "all", "all-novel"},
+	{"all", "new", "ongoing", "all-novel"},
+	{"all", "new", "completed", "all-novel"},
+	{"all", "top-rated", "all", "all-novel"},
+	{"all", "top-rated", "ongoing", "all-novel"},
+	{"all", "top-rated", "completed", "all-novel"},
+}
+
+const novelFireBrowseBase = "https://novelfire.net"
+
+// runBrowseRefresh fetches all browse combos from novelfire.net and stores
+// the results in MinIO. Errors per-combo are logged but do not abort the
+// whole refresh cycle.
+func (r *Runner) runBrowseRefresh(ctx context.Context) {
+	if r.deps.BrowseStore == nil {
+		r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
+		return
+	}
+
+	log := r.deps.Log.With("op", "browse_refresh")
+	log.Info("runner: browse refresh starting", "combos", len(browseCombos))
+
+	ok, fail := 0, 0
+	for _, c := range browseCombos {
+		if ctx.Err() != nil {
+			break
+		}
+		novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
+		if err != nil {
+			log.Warn("runner: browse fetch failed",
+				"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
+			fail++
+			continue
+		}
+
+		snap := browseSnapshot{
+			Novels:   novels,
+			Page:     1,
+			HasNext:  hasNext,
+			CachedAt: time.Now().UTC().Format(time.RFC3339),
+		}
+		data, _ := json.Marshal(snap)
+		if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
+			log.Warn("runner: browse put failed",
+				"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
+			fail++
+			continue
+		}
+		ok++
+	}
+
+	log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
+}
+
+// fetchBrowsePage calls novelfire.net and returns a list of novel listings
+// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
+func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
+	pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
+		novelFireBrowseBase, genre, sort, status, novelType)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
+	if err != nil {
+		return nil, false, fmt.Errorf("build request: %w", err)
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+
+	httpClient := &http.Client{Timeout: 45 * time.Second}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
+	}
+
+	return parseBrowseHTML(resp.Body)
+}
+
+// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
+// in backend/handlers.go — kept separate to avoid coupling packages.
+func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return nil, false, err
+	}
+	body := string(data)
+
+	hasNext := strings.Contains(body, `rel="next"`) ||
+		strings.Contains(body, `aria-label="Next"`) ||
+		strings.Contains(body, `class="next"`)
+
+	slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
+	titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
+	coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
+
+	slugMatches := slugRe.FindAllStringSubmatch(body, -1)
+	titleMatches := titleRe.FindAllStringSubmatch(body, -1)
+	coverMatches := coverRe.FindAllStringSubmatch(body, -1)
+
+	var novels []browseNovelListing
+	seen := make(map[string]bool)
+	for i, sm := range slugMatches {
+		slug := sm[1]
+		if seen[slug] {
+			continue
+		}
+		seen[slug] = true
+
+		item := browseNovelListing{
+			Slug: slug,
+			URL:  novelFireBrowseBase + "/book/" + slug,
+		}
+		if i < len(titleMatches) {
+			item.Title = strings.TrimSpace(titleMatches[i][1])
+		}
+		if i < len(coverMatches) {
+			item.Cover = coverMatches[i][1]
+		}
+		if item.Title != "" {
+			novels = append(novels, item)
+		}
+	}
+
+	return novels, hasNext, nil
+}

backend/internal/runner/runner.go

@@ -44,6 +44,9 @@ type Config struct {
 	// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
 	// task is considered orphaned and reset to pending. Defaults to 2m when 0.
 	StaleTaskThreshold time.Duration
+	// BrowseRefreshInterval is how often the runner pre-fetches browse page
+	// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
+	BrowseRefreshInterval time.Duration
 }
 
 // Dependencies are the external services the runner depends on.
@@ -56,6 +59,8 @@ type Dependencies struct {
 	BookReader bookstore.BookReader
 	// AudioStore persists generated audio and checks key existence.
 	AudioStore bookstore.AudioStore
+	// BrowseStore stores browse page snapshots in MinIO.
+	BrowseStore bookstore.BrowseStore
 	// Novel is the scraper implementation.
 	Novel scraper.NovelScraper
 	// Kokoro is the TTS client.
@@ -91,6 +96,9 @@ func New(cfg Config, deps Dependencies) *Runner {
 	if cfg.StaleTaskThreshold <= 0 {
 		cfg.StaleTaskThreshold = 2 * time.Minute
 	}
+	if cfg.BrowseRefreshInterval <= 0 {
+		cfg.BrowseRefreshInterval = 6 * time.Hour
+	}
 	if deps.Log == nil {
 		deps.Log = slog.Default()
 	}
@@ -121,6 +129,7 @@ func (r *Runner) Run(ctx context.Context) error {
 		"poll_interval", r.cfg.PollInterval,
 		"max_scrape", r.cfg.MaxConcurrentScrape,
 		"max_audio", r.cfg.MaxConcurrentAudio,
+		"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
 	)
 
 	scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
@@ -134,6 +143,12 @@ func (r *Runner) Run(ctx context.Context) error {
 	tick := time.NewTicker(r.cfg.PollInterval)
 	defer tick.Stop()
 
+	browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
+	defer browseTick.Stop()
+
+	// Run one browse refresh and one poll immediately on startup.
+	go r.runBrowseRefresh(ctx)
+
 	// Run one poll immediately on startup, then on each tick.
 	for {
 		r.poll(ctx, scrapeSem, audioSem, &wg)
@@ -154,6 +169,8 @@ func (r *Runner) Run(ctx context.Context) error {
 				r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
 			}
 			return nil
+		case <-browseTick.C:
+			go r.runBrowseRefresh(ctx)
 		case <-tick.C:
 		}
 	}

backend/internal/storage/minio.go

@@ -22,6 +22,7 @@ type minioClient struct {
 	bucketChapters string
 	bucketAudio    string
 	bucketAvatars  string
+	bucketBrowse   string
 }
 
 func newMinioClient(cfg config.MinIO) (*minioClient, error) {
@@ -78,12 +79,13 @@ func newMinioClient(cfg config.MinIO) (*minioClient, error) {
 		bucketChapters: cfg.BucketChapters,
 		bucketAudio:    cfg.BucketAudio,
 		bucketAvatars:  cfg.BucketAvatars,
+		bucketBrowse:   cfg.BucketBrowse,
 	}, nil
 }
 
 // ensureBuckets creates all required buckets if they don't already exist.
 func (m *minioClient) ensureBuckets(ctx context.Context) error {
-	for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars} {
+	for _, bucket := range []string{m.bucketChapters, m.bucketAudio, m.bucketAvatars, m.bucketBrowse} {
 		exists, err := m.client.BucketExists(ctx, bucket)
 		if err != nil {
 			return fmt.Errorf("minio: check bucket %q: %w", bucket, err)
@@ -117,6 +119,12 @@ func AvatarObjectKey(userID, ext string) string {
 	return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
 }
 
+// BrowseObjectKey returns the MinIO object key for a cached browse page snapshot.
+// Format: browse/{genre}/{sort}/{status}/{type}/page-{n}.json
+func BrowseObjectKey(genre, sort, status, novelType string, page int) string {
+	return fmt.Sprintf("browse/%s/%s/%s/%s/page-%d.json", genre, sort, status, novelType, page)
+}
+
 // chapterNumberFromKey extracts the chapter number from a MinIO object key.
 // e.g. "my-book/chapter-000042.md" → 42
 func chapterNumberFromKey(key string) int {
@@ -192,3 +200,23 @@ func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string)
 	}
 	return keys, nil
 }
+
+// ── Browse operations ─────────────────────────────────────────────────────────
+
+// putBrowse stores raw JSON bytes for a browse page snapshot.
+func (m *minioClient) putBrowse(ctx context.Context, key string, data []byte) error {
+	return m.putObject(ctx, m.bucketBrowse, key, "application/json", data)
+}
+
+// getBrowse retrieves a browse page snapshot. Returns (nil, false, nil) when
+// the object does not exist.
+func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool, error) {
+	if !m.objectExists(ctx, m.bucketBrowse, key) {
+		return nil, false, nil
+	}
+	data, err := m.getObject(ctx, m.bucketBrowse, key)
+	if err != nil {
+		return nil, false, err
+	}
+	return data, true, nil
+}

backend/internal/storage/store.go

@@ -50,6 +50,7 @@ var _ bookstore.RankingStore = (*Store)(nil)
 var _ bookstore.AudioStore = (*Store)(nil)
 var _ bookstore.PresignStore = (*Store)(nil)
 var _ bookstore.ProgressStore = (*Store)(nil)
+var _ bookstore.BrowseStore = (*Store)(nil)
 var _ taskqueue.Producer = (*Store)(nil)
 var _ taskqueue.Consumer = (*Store)(nil)
 var _ taskqueue.Reader = (*Store)(nil)
@@ -608,12 +609,13 @@ func (s *Store) HeartbeatTask(ctx context.Context, id string) error {
 // re-claimed. Returns the number of tasks reaped.
 func (s *Store) ReapStaleTasks(ctx context.Context, staleAfter time.Duration) (int, error) {
 	threshold := time.Now().UTC().Add(-staleAfter).Format(time.RFC3339)
-	// Match tasks that are running AND (heartbeat_at is empty OR heartbeat_at < threshold).
+	// Match tasks that are running AND (heartbeat_at is null OR heartbeat_at < threshold).
-	filter := fmt.Sprintf(`status="running"&&(heartbeat_at=""||heartbeat_at<"%s")`, threshold)
+	// PocketBase datetime fields require `=null` not `=""` in filter expressions.
+	filter := fmt.Sprintf(`status="running"&&(heartbeat_at=null||heartbeat_at<"%s")`, threshold)
 	resetPayload := map[string]any{
 		"status":       string(domain.TaskStatusPending),
 		"worker_id":    "",
-		"heartbeat_at": "",
+		"heartbeat_at": nil,
 	}
 
 	total := 0
@@ -767,3 +769,22 @@ func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
 		Finished:     finished,
 	}, nil
 }
+
+// ── BrowseStore ────────────────────────────────────────────────────────────────
+
+func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error {
+	key := BrowseObjectKey(genre, sort, status, novelType, page)
+	if err := s.mc.putBrowse(ctx, key, data); err != nil {
+		return fmt.Errorf("PutBrowsePage: %w", err)
+	}
+	return nil
+}
+
+func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) {
+	key := BrowseObjectKey(genre, sort, status, novelType, page)
+	data, ok, err := s.mc.getBrowse(ctx, key)
+	if err != nil {
+		return nil, false, fmt.Errorf("GetBrowsePage: %w", err)
+	}
+	return data, ok, nil
+}

docker-compose-new.yml

@@ -30,6 +30,7 @@ services:
         mc mb --ignore-existing local/libnovel-chapters;
         mc mb --ignore-existing local/libnovel-audio;
         mc mb --ignore-existing local/libnovel-avatars;
+        mc mb --ignore-existing local/libnovel-browse;
         echo 'buckets ready';
       "
     environment:
@@ -64,7 +65,7 @@ services:
       POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
       POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
     volumes:
-      - ./scripts/pb-init.sh:/pb-init.sh:ro
+      - ./scripts/pb-init-v2.sh:/pb-init.sh:ro
     entrypoint: ["sh", "/pb-init.sh"]
 
   # ─── Backend API ──────────────────────────────────────────────────────────────
@@ -88,7 +89,7 @@ services:
     environment:
       BACKEND_HTTP_ADDR: ":8080"
       LOG_LEVEL: "${LOG_LEVEL:-info}"
-      # MinIO
+       # MinIO
       MINIO_ENDPOINT: "minio:9000"
       MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
       MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
@@ -96,6 +97,7 @@ services:
       MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
       MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
       MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
+      MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
       MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
       MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}"
       # PocketBase
@@ -152,6 +154,7 @@ services:
       MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
       MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
       MINIO_BUCKET_AVATARS: "${MINIO_BUCKET_AVATARS:-libnovel-avatars}"
+      MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
       MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}"
       MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}"
       # PocketBase

docs/architecture.d2

@@ -0,0 +1,99 @@
+direction: right
+
+# ─── External ─────────────────────────────────────────────────────────────────
+
+novelfire: novelfire.net {
+  shape: cloud
+  style.fill: "#f0f4ff"
+}
+
+kokoro: Kokoro-FastAPI TTS {
+  shape: cloud
+  style.fill: "#f0f4ff"
+}
+
+browser: Browser / iOS App {
+  shape: person
+  style.fill: "#fff9e6"
+}
+
+# ─── Init containers (one-shot) ───────────────────────────────────────────────
+
+init: Init containers {
+  style.fill: "#f5f5f5"
+  style.stroke-dash: 4
+
+  minio-init: minio-init {
+    shape: rectangle
+    label: "minio-init\n(mc: create buckets)"
+  }
+
+  pb-init: pb-init {
+    shape: rectangle
+    label: "pb-init\n(bootstrap collections)"
+  }
+}
+
+# ─── Storage ──────────────────────────────────────────────────────────────────
+
+storage: Storage {
+  style.fill: "#eaf7ea"
+
+  minio: MinIO {
+    shape: cylinder
+    label: "MinIO :9000\n\nbuckets:\n  libnovel-chapters\n  libnovel-audio\n  libnovel-avatars\n  libnovel-browse"
+  }
+
+  pocketbase: PocketBase {
+    shape: cylinder
+    label: "PocketBase :8090\n\ncollections:\n  books  chapters_idx\n  audio_cache  progress\n  scrape_jobs  app_users\n  ranking"
+  }
+}
+
+# ─── Application ──────────────────────────────────────────────────────────────
+
+app: Application {
+  style.fill: "#eef3ff"
+
+  backend: backend {
+    shape: rectangle
+    label: "Backend API :8080\n(Go — HTTP API server)"
+  }
+
+  runner: runner {
+    shape: rectangle
+    label: "Runner\n(Go — background worker\nscraping + TTS jobs)"
+  }
+
+  ui: ui {
+    shape: rectangle
+    label: "SvelteKit UI :5252\n(adapter-node)"
+  }
+}
+
+# ─── Init → Storage deps ──────────────────────────────────────────────────────
+
+init.minio-init -> storage.minio: create buckets {style.stroke-dash: 4}
+init.pb-init -> storage.pocketbase: bootstrap schema {style.stroke-dash: 4}
+
+# ─── App → Storage ────────────────────────────────────────────────────────────
+
+app.backend -> storage.minio: blobs (chapters, audio,\navatars, browse)
+app.backend -> storage.pocketbase: structured records\n(books, progress, jobs…)
+
+app.runner -> storage.minio: write chapter markdown\n& audio MP3s
+app.runner -> storage.pocketbase: read/update scrape jobs\nwrite book records
+
+# ─── App internal ─────────────────────────────────────────────────────────────
+
+app.ui -> app.backend: REST API calls\n(server-side)
+
+# ─── External → App ───────────────────────────────────────────────────────────
+
+app.runner -> novelfire: scrape\n(HTTP GET)
+app.runner -> kokoro: TTS generation\n(HTTP POST)
+
+# ─── Browser ──────────────────────────────────────────────────────────────────
+
+browser -> app.ui: HTTPS :5252
+browser -> storage.minio: presigned URLs\n(audio / chapter downloads)

docs/architecture.mermaid.md

@@ -0,0 +1,47 @@
+```mermaid
+graph LR
+    %% ── External ──────────────────────────────────────────────────────────
+    NF([novelfire.net])
+    KK([Kokoro-FastAPI TTS])
+    CL([Browser / iOS App])
+
+    %% ── Init containers ───────────────────────────────────────────────────
+    subgraph INIT["Init containers (one-shot)"]
+        MI[minio-init\nmc: create buckets]
+        PI[pb-init\nbootstrap collections]
+    end
+
+    %% ── Storage ───────────────────────────────────────────────────────────
+    subgraph STORAGE["Storage"]
+        MN[(MinIO :9000\nchapters · audio\navatars · browse)]
+        PB[(PocketBase :8090\nbooks · chapters_idx\naudio_cache · progress\nscrape_jobs · app_users · ranking)]
+    end
+
+    %% ── Application ───────────────────────────────────────────────────────
+    subgraph APP["Application"]
+        BE[Backend API :8080\nGo HTTP server]
+        RN[Runner\nGo background worker]
+        UI[SvelteKit UI :5252]
+    end
+
+    %% ── Init → Storage ────────────────────────────────────────────────────
+    MI -.->|create buckets| MN
+    PI -.->|bootstrap schema| PB
+
+    %% ── App → Storage ─────────────────────────────────────────────────────
+    BE -->|blobs| MN
+    BE -->|structured records| PB
+    RN -->|chapter markdown & audio| MN
+    RN -->|read/update jobs & books| PB
+
+    %% ── App internal ──────────────────────────────────────────────────────
+    UI -->|REST API| BE
+
+    %% ── Runner → External ─────────────────────────────────────────────────
+    RN -->|scrape HTTP GET| NF
+    RN -->|TTS HTTP POST| KK
+
+    %% ── Client ────────────────────────────────────────────────────────────
+    CL -->|HTTPS :5252| UI
+    CL -->|presigned URLs| MN
+```

scraper/go.mod

@@ -10,6 +10,7 @@ require (
 
 require (
 	github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect

scraper/go.sum

@@ -1,5 +1,7 @@
 github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
 github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=

scraper/internal/browser/http.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"strings"
 	"time"
+
+	"github.com/andybalholm/brotli"
 )
 
 type httpClient struct {
@@ -106,16 +108,17 @@ func (c *httpClient) GetContent(ctx context.Context, req ContentRequest) (string
 	// net/http decompresses gzip automatically only when it sets the header
 	// itself; since we set Accept-Encoding explicitly we must do it ourselves.
 	body := resp.Body
-	if strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
+	switch strings.ToLower(resp.Header.Get("Content-Encoding")) {
+	case "gzip":
 		gr, gzErr := gzip.NewReader(resp.Body)
 		if gzErr != nil {
 			return "", fmt.Errorf("http: gzip reader: %w", gzErr)
 		}
 		defer gr.Close()
 		body = gr
+	case "br":
+		body = io.NopCloser(brotli.NewReader(resp.Body))
 	}
-	// br (Brotli) decompression requires an external package; skip for now —
-	// the server will fall back to gzip or plain text for unknown encodings.
 
 	raw, err := io.ReadAll(body)
 	if err != nil {

scripts/pb-init-v2.sh

@@ -0,0 +1,257 @@
+#!/bin/sh
+# pb-init-v2.sh — idempotent PocketBase collection bootstrap for the v2 stack
+#
+# Creates all collections required by libnovel v2 (backend + runner + ui-v2).
+# Safe to re-run: POST returns 400/422 when a collection already exists; both
+# are treated as success.  The ensure_field helper adds fields to existing
+# instances without touching fields that are already present.
+#
+# Collections created:
+#   books           — book metadata
+#   chapters_idx    — per-chapter index (title, number)
+#   ranking         — novelfire ranking snapshots
+#   progress        — per-session reading progress
+#   scraping_tasks  — scrape job queue (runner ↔ backend)
+#   audio_jobs      — TTS job queue (runner ↔ backend)
+#
+# Required env vars (with defaults matching docker-compose-new.yml):
+#   POCKETBASE_URL            http://pocketbase:8090
+#   POCKETBASE_ADMIN_EMAIL    admin@libnovel.local
+#   POCKETBASE_ADMIN_PASSWORD changeme123
+
+set -e
+
+PB_URL="${POCKETBASE_URL:-http://pocketbase:8090}"
+PB_EMAIL="${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
+PB_PASSWORD="${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
+
+log() { echo "[pb-init-v2] $*"; }
+
+# ─── 0. Ensure curl and python3 are available ────────────────────────────────
+if ! command -v curl > /dev/null 2>&1; then
+  apk add --no-cache curl > /dev/null 2>&1
+fi
+if ! command -v python3 > /dev/null 2>&1; then
+  apk add --no-cache python3 > /dev/null 2>&1
+fi
+
+# ─── 1. Wait for PocketBase to be ready ──────────────────────────────────────
+log "waiting for PocketBase at $PB_URL ..."
+until curl -sf "$PB_URL/api/health" > /dev/null 2>&1; do
+  sleep 2
+done
+log "PocketBase is up"
+
+# ─── 2. Ensure the superuser exists ──────────────────────────────────────────
+#
+# On a fresh install PocketBase v0.23+ exposes a one-time install token in the
+# /_/ redirect Location header.  Use it to create the superuser if needed; on
+# subsequent runs the token is gone and we fall through to normal auth.
+
+log "ensuring superuser $PB_EMAIL exists ..."
+
+LOCATION=$(curl -sf -o /dev/null -w "%{redirect_url}" "$PB_URL/_/" 2>/dev/null || true)
+if echo "$LOCATION" | grep -q "pbinstal/"; then
+  INSTALL_TOKEN=$(echo "$LOCATION" | sed 's|.*pbinstal/||' | tr -d ' \r\n')
+  log "install token found — creating superuser via install endpoint"
+  curl -sf -X POST "$PB_URL/api/collections/_superusers/records" \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer $INSTALL_TOKEN" \
+    -d "{\"email\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\",\"passwordConfirm\":\"$PB_PASSWORD\"}" \
+    > /dev/null 2>&1 || true
+  log "superuser create attempted (may already exist)"
+fi
+
+# ─── 3. Authenticate and obtain a superuser token ────────────────────────────
+log "authenticating as $PB_EMAIL ..."
+AUTH_RESPONSE=$(curl -sf -X POST "$PB_URL/api/collections/_superusers/auth-with-password" \
+  -H "Content-Type: application/json" \
+  -d "{\"identity\":\"$PB_EMAIL\",\"password\":\"$PB_PASSWORD\"}")
+
+TOKEN=$(echo "$AUTH_RESPONSE" | sed 's/.*"token":"\([^"]*\)".*/\1/')
+if [ -z "$TOKEN" ] || [ "$TOKEN" = "$AUTH_RESPONSE" ]; then
+  log "ERROR: failed to obtain auth token. Response: $AUTH_RESPONSE"
+  exit 1
+fi
+log "auth token obtained"
+
+# ─── 4. Helpers ──────────────────────────────────────────────────────────────
+
+# create_collection NAME JSON_BODY
+# POSTs to /api/collections.  400/422 = already exists → treated as success.
+create_collection() {
+  NAME="$1"
+  BODY="$2"
+  STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
+    -X POST "$PB_URL/api/collections" \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer $TOKEN" \
+    -d "$BODY")
+  case "$STATUS" in
+    200|201) log "created collection: $NAME" ;;
+    400|422) log "collection already exists (skipped): $NAME" ;;
+    *)       log "WARNING: unexpected status $STATUS for collection: $NAME" ;;
+  esac
+}
+
+# ensure_field COLLECTION FIELD_NAME FIELD_TYPE
+#
+# Uses python3 to parse the collection schema, then PATCHes the full fields
+# array with the new field appended — only if it is not already present.
+# python3 is required to correctly extract the top-level collection id from
+# the JSON response (sed-based extraction is unreliable on multi-field schemas
+# because the greedy pattern picks up a field id instead of the collection id).
+ensure_field() {
+  COLL="$1"
+  FIELD_NAME="$2"
+  FIELD_TYPE="$3"
+
+  SCHEMA=$(curl -sf \
+    -H "Authorization: Bearer $TOKEN" \
+    "$PB_URL/api/collections/$COLL" 2>/dev/null)
+
+  PARSED=$(echo "$SCHEMA" | python3 -c "
+import sys, json
+try:
+    d = json.load(sys.stdin)
+    fields = d.get('fields', [])
+    exists = any(f.get('name') == '$FIELD_NAME' for f in fields)
+    print('exists=' + str(exists))
+    print('id=' + d.get('id', ''))
+    if not exists:
+        fields.append({'name': '$FIELD_NAME', 'type': '$FIELD_TYPE'})
+        print('fields=' + json.dumps(fields))
+except Exception as e:
+    print('error=' + str(e))
+" 2>/dev/null)
+
+  if echo "$PARSED" | grep -q "^exists=True"; then
+    log "field $COLL.$FIELD_NAME already exists — skipping"
+    return
+  fi
+
+  COLLECTION_ID=$(echo "$PARSED" | grep "^id=" | sed 's/^id=//')
+  if [ -z "$COLLECTION_ID" ]; then
+    log "WARNING: could not get id for collection $COLL — skipping ensure_field"
+    return
+  fi
+
+  NEW_FIELDS=$(echo "$PARSED" | grep "^fields=" | sed 's/^fields=//')
+  STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
+    -X PATCH "$PB_URL/api/collections/$COLLECTION_ID" \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer $TOKEN" \
+    -d "{\"fields\":${NEW_FIELDS}}")
+  case "$STATUS" in
+    200|201) log "patched $COLL — added field: $FIELD_NAME ($FIELD_TYPE)" ;;
+    *)       log "WARNING: patch returned $STATUS when adding $FIELD_NAME to $COLL" ;;
+  esac
+}
+
+# ─── 5. Collections ───────────────────────────────────────────────────────────
+
+# books — one record per scraped novel
+create_collection "books" '{
+  "name": "books",
+  "type": "base",
+  "fields": [
+    {"name": "slug",           "type": "text",   "required": true},
+    {"name": "title",          "type": "text",   "required": true},
+    {"name": "author",         "type": "text"},
+    {"name": "cover",          "type": "text"},
+    {"name": "status",         "type": "text"},
+    {"name": "genres",         "type": "json"},
+    {"name": "summary",        "type": "text"},
+    {"name": "total_chapters", "type": "number"},
+    {"name": "source_url",     "type": "text"},
+    {"name": "ranking",        "type": "number"}
+  ]
+}'
+
+# chapters_idx — lightweight chapter list (no content; content lives in MinIO)
+create_collection "chapters_idx" '{
+  "name": "chapters_idx",
+  "type": "base",
+  "fields": [
+    {"name": "slug",   "type": "text",   "required": true},
+    {"name": "number", "type": "number", "required": true},
+    {"name": "title",  "type": "text"}
+  ]
+}'
+
+# ranking — periodic novelfire ranking snapshots
+create_collection "ranking" '{
+  "name": "ranking",
+  "type": "base",
+  "fields": [
+    {"name": "rank",       "type": "number", "required": true},
+    {"name": "slug",       "type": "text",   "required": true},
+    {"name": "title",      "type": "text"},
+    {"name": "author",     "type": "text"},
+    {"name": "cover",      "type": "text"},
+    {"name": "status",     "type": "text"},
+    {"name": "genres",     "type": "json"},
+    {"name": "source_url", "type": "text"}
+  ]
+}'
+
+# progress — per-session reading progress (no user accounts required)
+create_collection "progress" '{
+  "name": "progress",
+  "type": "base",
+  "fields": [
+    {"name": "session_id", "type": "text",   "required": true},
+    {"name": "slug",       "type": "text",   "required": true},
+    {"name": "chapter",    "type": "number"}
+  ]
+}'
+
+# scraping_tasks — scrape job queue consumed by the runner
+create_collection "scraping_tasks" '{
+  "name": "scraping_tasks",
+  "type": "base",
+  "fields": [
+    {"name": "kind",             "type": "text"},
+    {"name": "target_url",       "type": "text"},
+    {"name": "from_chapter",     "type": "number"},
+    {"name": "to_chapter",       "type": "number"},
+    {"name": "worker_id",        "type": "text"},
+    {"name": "status",           "type": "text",   "required": true},
+    {"name": "books_found",      "type": "number"},
+    {"name": "chapters_scraped", "type": "number"},
+    {"name": "chapters_skipped", "type": "number"},
+    {"name": "errors",           "type": "number"},
+    {"name": "error_message",    "type": "text"},
+    {"name": "started",          "type": "date"},
+    {"name": "finished",         "type": "date"},
+    {"name": "heartbeat_at",     "type": "date"}
+  ]
+}'
+
+# audio_jobs — TTS generation queue consumed by the runner
+create_collection "audio_jobs" '{
+  "name": "audio_jobs",
+  "type": "base",
+  "fields": [
+    {"name": "cache_key",     "type": "text",   "required": true},
+    {"name": "slug",          "type": "text",   "required": true},
+    {"name": "chapter",       "type": "number", "required": true},
+    {"name": "voice",         "type": "text"},
+    {"name": "worker_id",     "type": "text"},
+    {"name": "status",        "type": "text",   "required": true},
+    {"name": "error_message", "type": "text"},
+    {"name": "started",       "type": "date"},
+    {"name": "finished",      "type": "date"},
+    {"name": "heartbeat_at",  "type": "date"}
+  ]
+}'
+
+# ─── 6. Schema migrations (idempotent — safe to re-run on existing instances) ─
+#
+# heartbeat_at was added after the initial v2 deploy.  ensure_field is a no-op
+# if the field already exists (e.g. fresh installs that ran this script from
+# the start already have it from the create_collection call above).
+ensure_field "scraping_tasks" "heartbeat_at" "date"
+ensure_field "audio_jobs"     "heartbeat_at" "date"
+
+log "all collections ready"

scripts/pb-init.sh

@@ -98,22 +98,34 @@ ensure_field() {
     -H "Authorization: Bearer $TOKEN" \
     "$PB_URL/api/collections/$COLL" 2>/dev/null)
 
-  # Check if the field already exists (look for "name":"<FIELD_NAME>" in the fields array)
+  # Use python3 to reliably parse the JSON schema.
-  if echo "$SCHEMA" | grep -q "\"name\":\"$FIELD_NAME\""; then
+  PARSED=$(echo "$SCHEMA" | python3 -c "
+import sys, json
+try:
+    d = json.load(sys.stdin)
+    fields = d.get('fields', [])
+    exists = any(f.get('name') == '$FIELD_NAME' for f in fields)
+    print('exists=' + str(exists))
+    print('id=' + d.get('id', ''))
+    if not exists:
+        fields.append({'name': '$FIELD_NAME', 'type': '$FIELD_TYPE'})
+        print('fields=' + json.dumps(fields))
+except Exception as e:
+    print('error=' + str(e))
+" 2>/dev/null)
+
+  if echo "$PARSED" | grep -q "^exists=True"; then
     log "field $COLL.$FIELD_NAME already exists — skipping"
     return
   fi
 
-  COLLECTION_ID=$(echo "$SCHEMA" | sed 's/.*"id":"\([^"]*\)".*/\1/')
+  COLLECTION_ID=$(echo "$PARSED" | grep "^id=" | sed 's/^id=//')
-  if [ -z "$COLLECTION_ID" ] || [ "$COLLECTION_ID" = "$SCHEMA" ]; then
+  if [ -z "$COLLECTION_ID" ]; then
     log "WARNING: could not get id for collection $COLL — skipping ensure_field"
     return
   fi
 
-  # Extract current fields array and append the new field before the closing bracket.
+  NEW_FIELDS=$(echo "$PARSED" | grep "^fields=" | sed 's/^fields=//')
-  CURRENT_FIELDS=$(echo "$SCHEMA" | sed 's/.*"fields":\(\[.*\]\).*/\1/')
-  TRIMMED=$(echo "$CURRENT_FIELDS" | sed 's/]$//')
-  NEW_FIELDS="${TRIMMED},{\"name\":\"${FIELD_NAME}\",\"type\":\"${FIELD_TYPE}\"}]"
   PATCH_BODY="{\"fields\":${NEW_FIELDS}}"
 
   STATUS=$(curl -s -o /dev/null -w "%{http_code}" \