feat(browse): add SingleFile browse-page snapshot cache via MinIO

- New MinIO bucket 'libnovel-browse' (MINIO_BUCKET_BROWSE env) for storing
  self-contained HTML snapshots of novelfire browse pages
- Store interface gains SaveBrowsePage / GetBrowsePage / BrowsePageKey methods
- handleBrowse is now cache-first: serves from MinIO snapshot when available,
  then fires a background triggerBrowseSnapshot goroutine to populate cache
  on live-fetch (de-duplicated, 90s timeout)
- New 'save-browse' CLI subcommand to bulk-capture pages via SingleFile CLI
- Dockerfile: downloads pinned single-file-x86_64-linux binary (v2.0.83),
  adds gcompat + libstdc++ to Alpine runtime for glibc compatibility
- docker-compose: adds libnovel-browse bucket init and SINGLEFILE_PATH env
- .gitignore: exclude scraper/scraper build artifact
This commit is contained in:
Admin
2026-03-04 10:42:26 +05:00
parent c2d6ce1c5b
commit 555973c053
9 changed files with 365 additions and 16 deletions

View File

@@ -304,6 +304,20 @@ func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time
return h.minio.PresignAudio(ctx, key, expires)
}
// ─── Browse page snapshots ────────────────────────────────────────────────────
func (h *HybridStore) SaveBrowsePage(ctx context.Context, key, html string) error {
return h.minio.PutBrowsePage(ctx, key, html)
}
func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bool, error) {
return h.minio.GetBrowsePage(ctx, key)
}
func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
return BrowsePageKey(genre, sortBy, status, novelType, page)
}
// ─── Scraping tasks ───────────────────────────────────────────────────────────
func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) {

View File

@@ -22,6 +22,7 @@ type MinioConfig struct {
PublicUseSSL bool // TLS for the public endpoint (usually true in prod)
BucketChapters string // e.g. "libnovel-chapters"
BucketAudio string // e.g. "libnovel-audio"
BucketBrowse string // e.g. "libnovel-browse"
}
// MinioClient wraps a minio.Client and exposes object operations for
@@ -58,7 +59,10 @@ func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error)
}
mc := &MinioClient{c: c, pub: pub, cfg: cfg}
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio} {
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio, cfg.BucketBrowse} {
if bucket == "" {
continue
}
if err := mc.ensureBucket(ctx, bucket); err != nil {
return nil, err
}
@@ -206,6 +210,51 @@ func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time
return u.String(), nil
}
// ─── Browse page snapshots ────────────────────────────────────────────────────
// BrowsePageKey returns the MinIO object key for a cached browse-page snapshot.
// Layout: {genre}/{sort}/{status}/{novelType}/page-{n}.html
func BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
return fmt.Sprintf("%s/%s/%s/%s/page-%d.html", genre, sortBy, status, novelType, page)
}
// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket.
func (m *MinioClient) PutBrowsePage(ctx context.Context, key, html string) error {
data := []byte(html)
_, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: "text/html; charset=utf-8"})
if err != nil {
return fmt.Errorf("minio: put browse page %s: %w", key, err)
}
return nil
}
// GetBrowsePage retrieves a SingleFile HTML snapshot from the browse bucket.
// Returns ("", false, nil) when the object does not exist.
func (m *MinioClient) GetBrowsePage(ctx context.Context, key string) (string, bool, error) {
obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
if err != nil {
return "", false, fmt.Errorf("minio: get browse page %s: %w", key, err)
}
defer obj.Close()
// Check whether the object actually exists by inspecting the Stat.
if _, statErr := obj.Stat(); statErr != nil {
return "", false, nil // not found
}
data, err := io.ReadAll(obj)
if err != nil {
return "", false, fmt.Errorf("minio: read browse page %s: %w", key, err)
}
return string(data), true, nil
}
// BrowsePageExists returns true if a snapshot object is present in the browse bucket.
func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool {
_, err := m.c.StatObject(ctx, m.cfg.BucketBrowse, key, minio.StatObjectOptions{})
return err == nil
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// sanitiseVoice converts a voice name to a filename-safe string.

View File

@@ -139,6 +139,16 @@ type Store interface {
// PresignAudio returns a presigned GET URL for an audio object.
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
// ── Browse page snapshots (MinIO) ──────────────────────────────────────
// SaveBrowsePage stores a SingleFile HTML snapshot for the given cache key.
SaveBrowsePage(ctx context.Context, key, html string) error
// GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil)
// when no snapshot exists for the key.
GetBrowsePage(ctx context.Context, key string) (string, bool, error)
// BrowsePageKey returns the MinIO object key for the given browse params.
BrowsePageKey(genre, sortBy, status, novelType string, page int) string
// ── Scraping tasks ─────────────────────────────────────────────────────
// CreateScrapeTask inserts a new scraping_tasks record with status="running"