fix(browse): replace SingleFile with direct Go HTTP fetch; fix ranking schema and cache-hit ranking gap

- Remove SingleFile CLI and Node.js from Dockerfile; runtime base reverted to alpine:3.21
- Replace triggerBrowseSnapshot with triggerDirectScrape: plain Go HTTP GET to novelfire.net
  (page is server-rendered, no browser needed)
- Fix Accept-Encoding bug: stop setting header manually so Go transport auto-decompresses gzip
- Add in-memory browse cache fallback (browseMemCache) for when MinIO and upstream both fail
- Add warmBrowseCache() startup goroutine
- Call triggerDirectScrape on MinIO cache hits too, so PocketBase ranking records are
  repopulated after a schema reset or fresh deploy without waiting for a cache miss
- Fix grid view cards: wrap in <a> instead of <div> so clicking navigates to book detail
- Remove SINGLEFILE_PATH and BROWSERLESS_URL env vars from Dockerfile

scraper/Dockerfile

@@ -13,20 +13,12 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
     go build -ldflags="-s -w" -o /scraper ./cmd/scraper
 
 # ── Runtime stage ──────────────────────────────────────────────────────────────
-# Use node:22-alpine so single-file-cli (npm package) runs natively without
+FROM alpine:3.21
-# any glibc shims.  The pre-compiled binary release requires glibc symbols
-# (e.g. __res_init) that Alpine's gcompat shim does not provide.
-FROM node:22-alpine
 
 # ca-certificates: HTTPS to novelfire.net
 # tzdata: timezone data
 RUN apk add --no-cache ca-certificates tzdata
 
-# Install single-file-cli as a global npm package.
-# It runs via Node.js (no Deno/glibc needed) and connects to an external
-# Chromium via the CDP --browser-server flag.
-RUN npm install -g single-file-cli
-
 WORKDIR /app
 
 COPY --from=builder /scraper /app/scraper
@@ -40,12 +32,9 @@ RUN chown -R scraper:scraper /app
 USER scraper
 
 # ── Configuration ─────────────────────────────────────────────────────────────
-ENV BROWSERLESS_URL=http://browserless:3030
-ENV BROWSERLESS_STRATEGY=content
 ENV SCRAPER_WORKERS=0
 ENV SCRAPER_STATIC_ROOT=/app/static/books
 ENV SCRAPER_HTTP_ADDR=:8080
-ENV SINGLEFILE_PATH=single-file
 
 EXPOSE 8080
 

scraper/internal/server/server.go

@@ -25,8 +25,6 @@ import (
 	"io"
 	"log/slog"
 	"net/http"
-	"os"
-	"os/exec"
 	"strconv"
 	"strings"
 	"sync"
@@ -50,10 +48,6 @@ type Server struct {
 	kokoroURL   string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
 	kokoroVoice string // default voice, e.g. af_bella
 
-	// SingleFile CLI settings for browse-page snapshots.
-	singleFilePath string // path to single-file binary, e.g. /usr/local/bin/single-file
-	browserlessURL string // Browserless base URL, e.g. http://browserless:3000
-
 	// voiceMu guards cachedVoices.
 	voiceMu      sync.RWMutex
 	cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
@@ -64,10 +58,23 @@ type Server struct {
 	audioMu       sync.Mutex
 	audioInFlight map[string]chan struct{} // cacheKey → closed when done
 
-	// browseMu guards browseInFlight — keys of MinIO objects currently being
+	// browseMu guards browseInFlight — keys currently being refreshed
-	// captured by a background SingleFile goroutine.
+	// in the background.
 	browseMu       sync.Mutex
 	browseInFlight map[string]struct{}
+
+	// browseMemCache is a short-lived in-process cache for browse results.
+	// It is populated whenever a live upstream fetch succeeds and used as a
+	// last-resort fallback when both MinIO and the upstream are unavailable.
+	// Key: the MinIO cache key (same as used for BrowseHTMLKey).
+	browseMemCacheMu sync.RWMutex
+	browseMemCache   map[string]browseCacheEntry
+}
+
+type browseCacheEntry struct {
+	novels   []NovelListing
+	hasNext  bool
+	cachedAt time.Time
 }
 
 // New creates a new Server.
@@ -80,10 +87,9 @@ func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log
 		store:          store,
 		kokoroURL:      kokoroURL,
 		kokoroVoice:    kokoroVoice,
-		singleFilePath: os.Getenv("SINGLEFILE_PATH"),
-		browserlessURL: os.Getenv("BROWSERLESS_URL"),
 		audioInFlight:  make(map[string]chan struct{}),
 		browseInFlight: make(map[string]struct{}),
+		browseMemCache: make(map[string]browseCacheEntry),
 	}
 }
 
@@ -192,6 +198,11 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
 	// has playable previews without requiring a manual trigger.
 	go s.warmVoiceSamples(ctx)
 
+	// Warm the browse cache on startup: if page 1 is not cached in MinIO yet,
+	// trigger a background SingleFile snapshot immediately so the first user
+	// request is served from cache rather than hitting novelfire.net live.
+	go s.warmBrowseCache()
+
 	select {
 	case <-ctx.Done():
 		shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
@@ -222,7 +233,7 @@ func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
 		items = []storage.RankingItem{}
 	}
 	// Rewrite cover keys to proxy URLs.
-	// Keys stored by triggerBrowseSnapshot look like:
+	// Keys stored by triggerDirectScrape look like:
 	//   "novelfire.net/assets/book-covers/shadow-slave.jpg"
 	// We expose them as:
 	//   "/api/cover/novelfire.net/shadow-slave"
@@ -968,6 +979,11 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 	if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 {
 		novels, hasNext := parseBrowsePage(strings.NewReader(html))
 		s.log.Debug("browse: served from cache", "key", cacheKey)
+		// Still fire background ranking population in case PocketBase ranking
+		// records are missing (e.g. after a schema reset / fresh deploy).
+		targetURLForRanking := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
+			novelFireBase, genre, sortBy, status, novelType, page)
+		s.triggerDirectScrape(cacheKey, targetURLForRanking)
 		w.Header().Set("Content-Type", "application/json")
 		w.Header().Set("Cache-Control", "public, max-age=300")
 		_ = json.NewEncoder(w).Encode(map[string]interface{}{
@@ -1005,7 +1021,10 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
 		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
 		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
-		req.Header.Set("Accept-Encoding", "gzip, deflate, br")
+		// Do NOT set Accept-Encoding manually: Go's http.Transport handles
+		// transparent gzip decompression only when it adds the header itself.
+		// If we set it explicitly, Transport disables auto-decompression and
+		// parseBrowsePage receives raw gzip bytes instead of HTML.
 		req.Header.Set("Cache-Control", "no-cache")
 		req.Header.Set("Pragma", "no-cache")
 
@@ -1031,14 +1050,41 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 	}
 	if fetchErr != nil {
 		s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr)
+		// ── In-memory fallback: use cached result from a prior successful fetch ──
+		s.browseMemCacheMu.RLock()
+		entry, memHit := s.browseMemCache[cacheKey]
+		s.browseMemCacheMu.RUnlock()
+		if memHit {
+			s.log.Warn("browse: upstream unavailable, serving stale in-memory cache",
+				"key", cacheKey, "age", time.Since(entry.cachedAt).Round(time.Second))
+			w.Header().Set("Content-Type", "application/json")
+			w.Header().Set("Cache-Control", "public, max-age=60")
+			_ = json.NewEncoder(w).Encode(map[string]interface{}{
+				"novels":  entry.novels,
+				"page":    pageNum,
+				"hasNext": entry.hasNext,
+			})
+			return
+		}
 		http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway)
 		return
 	}
 
-	// ── Background: populate MinIO cache via SingleFile ───────────────────
+	// ── Populate in-memory cache with the fresh upstream result ──────────
-	// Fire-and-forget: capture the JS-rendered page with SingleFile, store
+	if len(novels) > 0 {
-	// it in MinIO, then parse it to populate the ranking collection.
+		s.browseMemCacheMu.Lock()
-	s.triggerBrowseSnapshot(cacheKey, targetURL)
+		s.browseMemCache[cacheKey] = browseCacheEntry{
+			novels:   novels,
+			hasNext:  hasNext,
+			cachedAt: time.Now(),
+		}
+		s.browseMemCacheMu.Unlock()
+	}
+
+	// ── Background: fetch and cache page directly from novelfire.net ─────
+	// Fire-and-forget: stores raw HTML in MinIO and populates the ranking
+	// collection in PocketBase (no browser/SingleFile needed).
+	s.triggerDirectScrape(cacheKey, targetURL)
 
 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Cache-Control", "public, max-age=300")
@@ -1049,25 +1095,20 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
-// triggerBrowseSnapshot fires a background goroutine that:
+// triggerDirectScrape fires a background goroutine that:
-//  1. Runs SingleFile CLI to capture the fully-rendered novelfire browse page
+//  1. Fetches pageURL directly from novelfire.net using Go's HTTP client
-//     and stores the self-contained HTML at {domain}/html/page-N.html in MinIO.
+//     (no browser/SingleFile needed — the page is server-rendered HTML).
-//  2. Parses the stored HTML to extract novel listings.
+//  2. Stores the raw HTML in MinIO at cacheKey so future requests are served
-//  3. For each listing, upserts a ranking record in PocketBase (rank, slug,
+//     from cache without hitting the origin.
+//  3. Parses the HTML to extract novel listings.
+//  4. For each listing, upserts a ranking record in PocketBase (rank, slug,
 //     title, cover key, source_url).
-//  4. Fires a separate goroutine per cover image to download and store it at
+//  5. Fires a separate goroutine per cover image to download and store it at
 //     {domain}/assets/book-covers/{slug}.jpg in MinIO.
 //
-// It is a no-op when:
+// It is a no-op when a refresh for this cache key is already in progress.
-//   - SINGLEFILE_PATH is not set (SingleFile not installed)
-//   - a capture for this cache key is already in progress
-//
 // The goroutine uses a fresh context so it outlives the HTTP request.
-func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
+func (s *Server) triggerDirectScrape(cacheKey, pageURL string) {
-	if s.singleFilePath == "" {
-		return
-	}
-
 	s.browseMu.Lock()
 	if _, inflight := s.browseInFlight[cacheKey]; inflight {
 		s.browseMu.Unlock()
@@ -1083,64 +1124,56 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
 			s.browseMu.Unlock()
 		}()
 
-		ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 		defer cancel()
 
-		// Convert http(s) → ws(s) for the SingleFile --browser-server flag.
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
-		wsEndpoint := s.browserlessURL
-		wsEndpoint = strings.Replace(wsEndpoint, "http://", "ws://", 1)
-		wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)
-
-		tmpFile, err := os.CreateTemp("", "libnovel-browse-*.html")
 		if err != nil {
-			s.log.Warn("triggerBrowseSnapshot: create temp file failed", "key", cacheKey, "err", err)
+			s.log.Warn("triggerDirectScrape: build request failed", "key", cacheKey, "err", err)
 			return
 		}
-		tmpPath := tmpFile.Name()
+		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
-		tmpFile.Close()
+		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
-		defer os.Remove(tmpPath)
+		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
 
-		//nolint:gosec
+		resp, err := http.DefaultClient.Do(req)
-		cmd := exec.CommandContext(ctx, s.singleFilePath,
+		if err != nil {
-			pageURL,
+			s.log.Warn("triggerDirectScrape: fetch failed", "key", cacheKey, "err", err)
-			"--browser-server="+wsEndpoint,
+			return
-			"--output="+tmpPath,
+		}
-		)
+		defer resp.Body.Close()
-		if out, runErr := cmd.CombinedOutput(); runErr != nil {
+
-			s.log.Warn("triggerBrowseSnapshot: SingleFile failed",
+		if resp.StatusCode != http.StatusOK {
-				"key", cacheKey, "err", runErr, "output", string(out))
+			s.log.Warn("triggerDirectScrape: non-200 response", "key", cacheKey, "status", resp.StatusCode)
 			return
 		}
 
-		htmlBytes, readErr := os.ReadFile(tmpPath)
+		htmlBytes, readErr := io.ReadAll(resp.Body)
 		if readErr != nil {
-			s.log.Warn("triggerBrowseSnapshot: read output failed",
+			s.log.Warn("triggerDirectScrape: read body failed", "key", cacheKey, "err", readErr)
-				"key", cacheKey, "err", readErr)
 			return
 		}
 		if len(htmlBytes) == 0 {
-			s.log.Warn("triggerBrowseSnapshot: SingleFile produced empty output, skipping cache",
+			s.log.Warn("triggerDirectScrape: empty response body", "key", cacheKey)
-				"key", cacheKey)
 			return
 		}
 
-		// Store the HTML snapshot.
+		// Store the HTML in MinIO so subsequent requests are cache-hits.
 		if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
-			s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
+			s.log.Warn("triggerDirectScrape: SaveBrowsePage failed", "key", cacheKey, "err", putErr)
-				"key", cacheKey, "err", putErr)
+			// Non-fatal: continue to populate PocketBase/covers even if MinIO write fails.
-			return
+		} else {
+			s.log.Info("triggerDirectScrape: cached browse page", "key", cacheKey, "bytes", len(htmlBytes))
 		}
-		s.log.Info("triggerBrowseSnapshot: cached browse page",
-			"key", cacheKey, "bytes", len(htmlBytes))
 
-		// Parse the stored HTML to extract novel listings.
+		// Parse to extract novel listings.
 		novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
 		if len(novels) == 0 {
+			s.log.Warn("triggerDirectScrape: no novels parsed", "key", cacheKey)
 			return
 		}
 
-		// Upsert each novel into the ranking PocketBase collection and
+		// Upsert each novel into PocketBase ranking and kick off cover downloads.
-		// kick off a background cover download.
 		for i, novel := range novels {
 			rank := i + 1
 			coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
@@ -1153,21 +1186,37 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
 				SourceURL: novel.URL,
 			}
 			if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
-				s.log.Warn("triggerBrowseSnapshot: WriteRankingItem failed",
+				s.log.Warn("triggerDirectScrape: WriteRankingItem failed",
 					"slug", novel.Slug, "err", werr)
 			}
 
-			// Download and store the cover image in a separate goroutine.
+			if novel.Cover != "" {
-			coverURL := novel.Cover
+				go s.downloadAndStoreCover(coverKey, novel.Cover)
-			if coverURL != "" {
-				go s.downloadAndStoreCover(coverKey, coverURL)
 			}
 		}
 
-		s.log.Info("triggerBrowseSnapshot: ranking populated", "count", len(novels), "key", cacheKey)
+		s.log.Info("triggerDirectScrape: ranking populated", "count", len(novels), "key", cacheKey)
 	}()
 }
 
+// warmBrowseCache checks whether the browse cache for page 1 is populated in
+// MinIO and, if not, triggers a background direct scrape.  This is called
+// once on server startup so the first user request is likely served from cache.
+func (s *Server) warmBrowseCache() {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	cacheKey := s.store.BrowseHTMLKey(novelFireDomain, 1)
+	if _, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
+		s.log.Debug("warmBrowseCache: page 1 already cached, skipping")
+		return
+	}
+
+	targetURL := fmt.Sprintf("%s/genre-all/sort-popular/status-all/all-novel?page=1", novelFireBase)
+	s.log.Info("warmBrowseCache: page 1 not cached, triggering background scrape")
+	s.triggerDirectScrape(cacheKey, targetURL)
+}
+
 // downloadAndStoreCover fetches a cover image URL and stores it in MinIO under
 // the given key. Errors are logged but not propagated — this is best-effort.
 func (s *Server) downloadAndStoreCover(key, imageURL string) {

ui/src/routes/browse/+page.svelte

@@ -250,7 +250,10 @@
 	<!-- ── Grid view ─────────────────────────────────────────────────────── -->
 	<div class="grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-5 xl:grid-cols-6 gap-4">
 		{#each data.novels as novel}
-			<div class="group flex flex-col rounded-lg overflow-hidden bg-zinc-800 border border-zinc-700 hover:border-zinc-500 transition-colors relative">
+			<a
+				href="/books/{novel.slug}"
+				class="group flex flex-col rounded-lg overflow-hidden bg-zinc-800 border border-zinc-700 hover:border-zinc-500 transition-colors relative"
+			>
 				<!-- Cover -->
 				<div class="aspect-[2/3] bg-zinc-900 overflow-hidden relative">
 					{#if novel.cover}
@@ -302,7 +305,7 @@
 								<span class="text-xs text-red-400 font-medium">Error</span>
 							{:else}
 								<button
-									onclick={() => scrapeNovel(novel)}
+									onclick={(e) => { e.preventDefault(); scrapeNovel(novel); }}
 									disabled={scraping[novel.slug]}
 									class="w-full text-xs px-2 py-1 rounded bg-amber-500/20 text-amber-300 hover:bg-amber-500/40 transition-colors disabled:opacity-50 disabled:cursor-not-allowed border border-amber-500/30"
 								>
@@ -312,7 +315,7 @@
 						</div>
 					{/if}
 				</div>
-			</div>
+			</a>
 		{/each}
 	</div>