feat(browse): add SingleFile browse-page snapshot cache via MinIO
- New MinIO bucket 'libnovel-browse' (MINIO_BUCKET_BROWSE env) for storing self-contained HTML snapshots of novelfire browse pages - Store interface gains SaveBrowsePage / GetBrowsePage / BrowsePageKey methods - handleBrowse is now cache-first: serves from MinIO snapshot when available, then fires a background triggerBrowseSnapshot goroutine to populate cache on live-fetch (de-duplicated, 90s timeout) - New 'save-browse' CLI subcommand to bulk-capture pages via SingleFile CLI - Dockerfile: downloads pinned single-file-x86_64-linux binary (v2.0.83), adds gcompat + libstdc++ to Alpine runtime for glibc compatibility - docker-compose: adds libnovel-browse bucket init and SINGLEFILE_PATH env - .gitignore: exclude scraper/scraper build artifact
This commit is contained in:
@@ -25,6 +25,8 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -48,6 +50,10 @@ type Server struct {
|
||||
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
||||
kokoroVoice string // default voice, e.g. af_bella
|
||||
|
||||
// SingleFile CLI settings for browse-page snapshots.
|
||||
singleFilePath string // path to single-file binary, e.g. /usr/local/bin/single-file
|
||||
browserlessURL string // Browserless base URL, e.g. http://browserless:3000
|
||||
|
||||
// voiceMu guards cachedVoices.
|
||||
voiceMu sync.RWMutex
|
||||
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
|
||||
@@ -57,19 +63,27 @@ type Server struct {
|
||||
// audioInFlight deduplicates concurrent generation requests for the same key.
|
||||
audioMu sync.Mutex
|
||||
audioInFlight map[string]chan struct{} // cacheKey → closed when done
|
||||
|
||||
// browseMu guards browseInFlight — keys of MinIO objects currently being
|
||||
// captured by a background SingleFile goroutine.
|
||||
browseMu sync.Mutex
|
||||
browseInFlight map[string]struct{}
|
||||
}
|
||||
|
||||
// New creates a new Server.
|
||||
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
|
||||
return &Server{
|
||||
addr: addr,
|
||||
oCfg: oCfg,
|
||||
novel: novel,
|
||||
log: log,
|
||||
store: store,
|
||||
kokoroURL: kokoroURL,
|
||||
kokoroVoice: kokoroVoice,
|
||||
audioInFlight: make(map[string]chan struct{}),
|
||||
addr: addr,
|
||||
oCfg: oCfg,
|
||||
novel: novel,
|
||||
log: log,
|
||||
store: store,
|
||||
kokoroURL: kokoroURL,
|
||||
kokoroVoice: kokoroVoice,
|
||||
singleFilePath: os.Getenv("SINGLEFILE_PATH"),
|
||||
browserlessURL: os.Getenv("BROWSERLESS_URL"),
|
||||
audioInFlight: make(map[string]chan struct{}),
|
||||
browseInFlight: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -815,6 +829,9 @@ const novelFireBase = "https://novelfire.net"
|
||||
// type (default "all-novel")
|
||||
//
|
||||
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
|
||||
//
|
||||
// Cache strategy: check MinIO browse bucket first; if a snapshot exists,
|
||||
// parse and return it. Otherwise fetch live from novelfire.net.
|
||||
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page := q.Get("page")
|
||||
@@ -838,13 +855,34 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
novelType = "all-novel"
|
||||
}
|
||||
|
||||
// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
|
||||
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
pageNum, _ := strconv.Atoi(page)
|
||||
if pageNum <= 0 {
|
||||
pageNum = 1
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// ── Cache-first: try MinIO snapshot ──────────────────────────────────
|
||||
cacheKey := s.store.BrowsePageKey(genre, sortBy, status, novelType, pageNum)
|
||||
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
|
||||
novels, hasNext := parseBrowsePage(strings.NewReader(html))
|
||||
s.log.Debug("browse: served from cache", "key", cacheKey)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"novels": novels,
|
||||
"page": pageNum,
|
||||
"hasNext": hasNext,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// ── Live fallback: fetch from novelfire.net ───────────────────────────
|
||||
// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
|
||||
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
|
||||
@@ -867,7 +905,11 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
novels, hasNext := parseBrowsePage(resp.Body)
|
||||
pageNum, _ := strconv.Atoi(page)
|
||||
|
||||
// ── Background: populate MinIO cache via SingleFile ───────────────────
|
||||
// Fire-and-forget: capture the JS-rendered page with SingleFile and store
|
||||
// it in MinIO so the next request is served from cache.
|
||||
s.triggerBrowseSnapshot(cacheKey, targetURL)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
@@ -878,6 +920,80 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// triggerBrowseSnapshot fires a background goroutine that uses SingleFile CLI
|
||||
// to capture the fully-rendered novelfire browse page and store it in MinIO.
|
||||
// It is a no-op when:
|
||||
// - SINGLEFILE_PATH is not set (SingleFile not installed)
|
||||
// - a capture for this cache key is already in progress
|
||||
//
|
||||
// The goroutine uses a fresh context so it outlives the HTTP request.
|
||||
func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
|
||||
if s.singleFilePath == "" {
|
||||
return
|
||||
}
|
||||
|
||||
s.browseMu.Lock()
|
||||
if _, inflight := s.browseInFlight[cacheKey]; inflight {
|
||||
s.browseMu.Unlock()
|
||||
return
|
||||
}
|
||||
s.browseInFlight[cacheKey] = struct{}{}
|
||||
s.browseMu.Unlock()
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
s.browseMu.Lock()
|
||||
delete(s.browseInFlight, cacheKey)
|
||||
s.browseMu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Convert http(s) → ws(s) for the SingleFile --browser-server flag.
|
||||
wsEndpoint := s.browserlessURL
|
||||
wsEndpoint = strings.Replace(wsEndpoint, "http://", "ws://", 1)
|
||||
wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)
|
||||
|
||||
tmpFile, err := os.CreateTemp("", "libnovel-browse-*.html")
|
||||
if err != nil {
|
||||
s.log.Warn("triggerBrowseSnapshot: create temp file failed", "key", cacheKey, "err", err)
|
||||
return
|
||||
}
|
||||
tmpPath := tmpFile.Name()
|
||||
tmpFile.Close()
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
//nolint:gosec
|
||||
cmd := exec.CommandContext(ctx, s.singleFilePath,
|
||||
pageURL,
|
||||
"--browser-server="+wsEndpoint,
|
||||
"--output="+tmpPath,
|
||||
)
|
||||
if out, runErr := cmd.CombinedOutput(); runErr != nil {
|
||||
s.log.Warn("triggerBrowseSnapshot: SingleFile failed",
|
||||
"key", cacheKey, "err", runErr, "output", string(out))
|
||||
return
|
||||
}
|
||||
|
||||
htmlBytes, readErr := os.ReadFile(tmpPath)
|
||||
if readErr != nil {
|
||||
s.log.Warn("triggerBrowseSnapshot: read output failed",
|
||||
"key", cacheKey, "err", readErr)
|
||||
return
|
||||
}
|
||||
|
||||
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
|
||||
s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
|
||||
"key", cacheKey, "err", putErr)
|
||||
return
|
||||
}
|
||||
|
||||
s.log.Info("triggerBrowseSnapshot: cached browse page",
|
||||
"key", cacheKey, "bytes", len(htmlBytes))
|
||||
}()
|
||||
}
|
||||
|
||||
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
|
||||
// Returns novels and whether a "next page" link was found.
|
||||
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
|
||||
|
||||
@@ -304,6 +304,20 @@ func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time
|
||||
return h.minio.PresignAudio(ctx, key, expires)
|
||||
}
|
||||
|
||||
// ─── Browse page snapshots ────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) SaveBrowsePage(ctx context.Context, key, html string) error {
|
||||
return h.minio.PutBrowsePage(ctx, key, html)
|
||||
}
|
||||
|
||||
func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bool, error) {
|
||||
return h.minio.GetBrowsePage(ctx, key)
|
||||
}
|
||||
|
||||
func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
|
||||
return BrowsePageKey(genre, sortBy, status, novelType, page)
|
||||
}
|
||||
|
||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) {
|
||||
|
||||
@@ -22,6 +22,7 @@ type MinioConfig struct {
|
||||
PublicUseSSL bool // TLS for the public endpoint (usually true in prod)
|
||||
BucketChapters string // e.g. "libnovel-chapters"
|
||||
BucketAudio string // e.g. "libnovel-audio"
|
||||
BucketBrowse string // e.g. "libnovel-browse"
|
||||
}
|
||||
|
||||
// MinioClient wraps a minio.Client and exposes object operations for
|
||||
@@ -58,7 +59,10 @@ func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error)
|
||||
}
|
||||
|
||||
mc := &MinioClient{c: c, pub: pub, cfg: cfg}
|
||||
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio} {
|
||||
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio, cfg.BucketBrowse} {
|
||||
if bucket == "" {
|
||||
continue
|
||||
}
|
||||
if err := mc.ensureBucket(ctx, bucket); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -206,6 +210,51 @@ func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
// ─── Browse page snapshots ────────────────────────────────────────────────────
|
||||
|
||||
// BrowsePageKey returns the MinIO object key for a cached browse-page snapshot.
|
||||
// Layout: {genre}/{sort}/{status}/{novelType}/page-{n}.html
|
||||
func BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
|
||||
return fmt.Sprintf("%s/%s/%s/%s/page-%d.html", genre, sortBy, status, novelType, page)
|
||||
}
|
||||
|
||||
// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket.
|
||||
func (m *MinioClient) PutBrowsePage(ctx context.Context, key, html string) error {
|
||||
data := []byte(html)
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: "text/html; charset=utf-8"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put browse page %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBrowsePage retrieves a SingleFile HTML snapshot from the browse bucket.
|
||||
// Returns ("", false, nil) when the object does not exist.
|
||||
func (m *MinioClient) GetBrowsePage(ctx context.Context, key string) (string, bool, error) {
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("minio: get browse page %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
// Check whether the object actually exists by inspecting the Stat.
|
||||
if _, statErr := obj.Stat(); statErr != nil {
|
||||
return "", false, nil // not found
|
||||
}
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("minio: read browse page %s: %w", key, err)
|
||||
}
|
||||
return string(data), true, nil
|
||||
}
|
||||
|
||||
// BrowsePageExists returns true if a snapshot object is present in the browse bucket.
|
||||
func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool {
|
||||
_, err := m.c.StatObject(ctx, m.cfg.BucketBrowse, key, minio.StatObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// sanitiseVoice converts a voice name to a filename-safe string.
|
||||
|
||||
@@ -139,6 +139,16 @@ type Store interface {
|
||||
// PresignAudio returns a presigned GET URL for an audio object.
|
||||
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
|
||||
|
||||
// ── Browse page snapshots (MinIO) ──────────────────────────────────────
|
||||
|
||||
// SaveBrowsePage stores a SingleFile HTML snapshot for the given cache key.
|
||||
SaveBrowsePage(ctx context.Context, key, html string) error
|
||||
// GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil)
|
||||
// when no snapshot exists for the key.
|
||||
GetBrowsePage(ctx context.Context, key string) (string, bool, error)
|
||||
// BrowsePageKey returns the MinIO object key for the given browse params.
|
||||
BrowsePageKey(genre, sortBy, status, novelType string, page int) string
|
||||
|
||||
// ── Scraping tasks ─────────────────────────────────────────────────────
|
||||
|
||||
// CreateScrapeTask inserts a new scraping_tasks record with status="running"
|
||||
|
||||
Reference in New Issue
Block a user