feat(browse): add SingleFile browse-page snapshot cache via MinIO

- New MinIO bucket 'libnovel-browse' (MINIO_BUCKET_BROWSE env) for storing self-contained HTML snapshots of novelfire browse pages - Store interface gains SaveBrowsePage / GetBrowsePage / BrowsePageKey methods - handleBrowse is now cache-first: serves from MinIO snapshot when available, then fires a background triggerBrowseSnapshot goroutine to populate cache on live-fetch (de-duplicated, 90s timeout) - New 'save-browse' CLI subcommand to bulk-capture pages via SingleFile CLI - Dockerfile: downloads pinned single-file-x86_64-linux binary (v2.0.83), adds gcompat + libstdc++ to Alpine runtime for glibc compatibility - docker-compose: adds libnovel-browse bucket init and SINGLEFILE_PATH env - .gitignore: exclude scraper/scraper build artifact
2026-03-04 10:42:26 +05:00
parent c2d6ce1c5b
commit 555973c053
9 changed files with 365 additions and 16 deletions
--- a/scraper/internal/server/server.go
+++ b/scraper/internal/server/server.go
@@ -25,6 +25,8 @@ import (
 	"io"
 	"log/slog"
 	"net/http"
+	"os"
+	"os/exec"
 	"strconv"
 	"strings"
 	"sync"
@@ -48,6 +50,10 @@ type Server struct {
 	kokoroURL   string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
 	kokoroVoice string // default voice, e.g. af_bella

+	// SingleFile CLI settings for browse-page snapshots.
+	singleFilePath string // path to single-file binary, e.g. /usr/local/bin/single-file
+	browserlessURL string // Browserless base URL, e.g. http://browserless:3000
+
 	// voiceMu guards cachedVoices.
 	voiceMu      sync.RWMutex
 	cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
@@ -57,19 +63,27 @@ type Server struct {
 	// audioInFlight deduplicates concurrent generation requests for the same key.
 	audioMu       sync.Mutex
 	audioInFlight map[string]chan struct{} // cacheKey → closed when done
+
+	// browseMu guards browseInFlight — keys of MinIO objects currently being
+	// captured by a background SingleFile goroutine.
+	browseMu       sync.Mutex
+	browseInFlight map[string]struct{}
 }

 // New creates a new Server.
 func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
 	return &Server{
-		addr:          addr,
-		oCfg:          oCfg,
-		novel:         novel,
-		log:           log,
-		store:         store,
-		kokoroURL:     kokoroURL,
-		kokoroVoice:   kokoroVoice,
-		audioInFlight: make(map[string]chan struct{}),
+		addr:           addr,
+		oCfg:           oCfg,
+		novel:          novel,
+		log:            log,
+		store:          store,
+		kokoroURL:      kokoroURL,
+		kokoroVoice:    kokoroVoice,
+		singleFilePath: os.Getenv("SINGLEFILE_PATH"),
+		browserlessURL: os.Getenv("BROWSERLESS_URL"),
+		audioInFlight:  make(map[string]chan struct{}),
+		browseInFlight: make(map[string]struct{}),
 	}
 }

@@ -815,6 +829,9 @@ const novelFireBase = "https://novelfire.net"
 //	type    (default "all-novel")
 //
 // Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
+//
+// Cache strategy: check MinIO browse bucket first; if a snapshot exists,
+// parse and return it. Otherwise fetch live from novelfire.net.
 func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 	q := r.URL.Query()
 	page := q.Get("page")
@@ -838,13 +855,34 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 		novelType = "all-novel"
 	}

-	// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
-	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
-		novelFireBase, genre, sortBy, status, novelType, page)
+	pageNum, _ := strconv.Atoi(page)
+	if pageNum <= 0 {
+		pageNum = 1
+	}

 	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
 	defer cancel()

+	// ── Cache-first: try MinIO snapshot ──────────────────────────────────
+	cacheKey := s.store.BrowsePageKey(genre, sortBy, status, novelType, pageNum)
+	if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
+		novels, hasNext := parseBrowsePage(strings.NewReader(html))
+		s.log.Debug("browse: served from cache", "key", cacheKey)
+		w.Header().Set("Content-Type", "application/json")
+		w.Header().Set("Cache-Control", "public, max-age=300")
+		_ = json.NewEncoder(w).Encode(map[string]interface{}{
+			"novels":  novels,
+			"page":    pageNum,
+			"hasNext": hasNext,
+		})
+		return
+	}
+
+	// ── Live fallback: fetch from novelfire.net ───────────────────────────
+	// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
+	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
+		novelFireBase, genre, sortBy, status, novelType, page)
+
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
 	if err != nil {
 		http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
@@ -867,7 +905,11 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 	}

 	novels, hasNext := parseBrowsePage(resp.Body)
-	pageNum, _ := strconv.Atoi(page)
+
+	// ── Background: populate MinIO cache via SingleFile ───────────────────
+	// Fire-and-forget: capture the JS-rendered page with SingleFile and store
+	// it in MinIO so the next request is served from cache.
+	s.triggerBrowseSnapshot(cacheKey, targetURL)

 	w.Header().Set("Content-Type", "application/json")
 	w.Header().Set("Cache-Control", "public, max-age=300")
@@ -878,6 +920,80 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
 	})
 }

+// triggerBrowseSnapshot fires a background goroutine that uses SingleFile CLI
+// to capture the fully-rendered novelfire browse page and store it in MinIO.
+// It is a no-op when:
+//   - SINGLEFILE_PATH is not set (SingleFile not installed)
+//   - a capture for this cache key is already in progress
+//
+// The goroutine uses a fresh context so it outlives the HTTP request.
+func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
+	if s.singleFilePath == "" {
+		return
+	}
+
+	s.browseMu.Lock()
+	if _, inflight := s.browseInFlight[cacheKey]; inflight {
+		s.browseMu.Unlock()
+		return
+	}
+	s.browseInFlight[cacheKey] = struct{}{}
+	s.browseMu.Unlock()
+
+	go func() {
+		defer func() {
+			s.browseMu.Lock()
+			delete(s.browseInFlight, cacheKey)
+			s.browseMu.Unlock()
+		}()
+
+		ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+		defer cancel()
+
+		// Convert http(s) → ws(s) for the SingleFile --browser-server flag.
+		wsEndpoint := s.browserlessURL
+		wsEndpoint = strings.Replace(wsEndpoint, "http://", "ws://", 1)
+		wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)
+
+		tmpFile, err := os.CreateTemp("", "libnovel-browse-*.html")
+		if err != nil {
+			s.log.Warn("triggerBrowseSnapshot: create temp file failed", "key", cacheKey, "err", err)
+			return
+		}
+		tmpPath := tmpFile.Name()
+		tmpFile.Close()
+		defer os.Remove(tmpPath)
+
+		//nolint:gosec
+		cmd := exec.CommandContext(ctx, s.singleFilePath,
+			pageURL,
+			"--browser-server="+wsEndpoint,
+			"--output="+tmpPath,
+		)
+		if out, runErr := cmd.CombinedOutput(); runErr != nil {
+			s.log.Warn("triggerBrowseSnapshot: SingleFile failed",
+				"key", cacheKey, "err", runErr, "output", string(out))
+			return
+		}
+
+		htmlBytes, readErr := os.ReadFile(tmpPath)
+		if readErr != nil {
+			s.log.Warn("triggerBrowseSnapshot: read output failed",
+				"key", cacheKey, "err", readErr)
+			return
+		}
+
+		if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
+			s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
+				"key", cacheKey, "err", putErr)
+			return
+		}
+
+		s.log.Info("triggerBrowseSnapshot: cached browse page",
+			"key", cacheKey, "bytes", len(htmlBytes))
+	}()
+}
+
 // parseBrowsePage parses the novelfire HTML and extracts novel listings.
 // Returns novels and whether a "next page" link was found.
 func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {