feat(browse): add SingleFile browse-page snapshot cache via MinIO

- New MinIO bucket 'libnovel-browse' (MINIO_BUCKET_BROWSE env) for storing
  self-contained HTML snapshots of novelfire browse pages
- Store interface gains SaveBrowsePage / GetBrowsePage / BrowsePageKey methods
- handleBrowse is now cache-first: serves from MinIO snapshot when available,
  then fires a background triggerBrowseSnapshot goroutine to populate cache
  on live-fetch (de-duplicated, 90s timeout)
- New 'save-browse' CLI subcommand to bulk-capture pages via SingleFile CLI
- Dockerfile: downloads pinned single-file-x86_64-linux binary (v2.0.83),
  adds gcompat + libstdc++ to Alpine runtime for glibc compatibility
- docker-compose: adds libnovel-browse bucket init and SINGLEFILE_PATH env
- .gitignore: exclude scraper/scraper build artifact
This commit is contained in:
Admin
2026-03-04 10:42:26 +05:00
parent c2d6ce1c5b
commit 555973c053
9 changed files with 365 additions and 16 deletions

View File

@@ -35,6 +35,7 @@ import (
"fmt"
"log/slog"
"os"
"os/exec"
"os/signal"
"runtime"
"strconv"
@@ -106,6 +107,7 @@ func run(log *slog.Logger) error {
PublicUseSSL: strings.ToLower(os.Getenv("MINIO_PUBLIC_USE_SSL")) != "false",
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
}
pbCfg := storage.PocketBaseConfig{
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
@@ -198,11 +200,140 @@ func run(log *slog.Logger) error {
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
return srv.ListenAndServe(ctx)
case "save-browse":
return runSaveBrowse(ctx, args[1:], store, log)
default:
return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd)
return fmt.Errorf("unknown command %q; use 'run', 'refresh', 'serve', or 'save-browse'", cmd)
}
}
// runSaveBrowse implements the `save-browse` subcommand.
// It iterates over browse pages on novelfire.net, captures each using
// SingleFile CLI (connected to the existing Browserless instance), and
// stores the resulting self-contained HTML in the MinIO browse bucket.
//
// Flags (all optional):
//
// --genre <value> genre slug (default: all)
// --sort <value> sort order (default: popular)
// --status <value> status (default: all)
// --type <value> novel type (default: all-novel)
// --max-pages <n> max pages (default: 5)
func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log *slog.Logger) error {
// Parse flags manually to avoid importing flag package.
genre := "all"
sortBy := "popular"
status := "all"
novelType := "all-novel"
maxPages := 5
for i := 0; i < len(args); i++ {
switch args[i] {
case "--genre":
if i+1 < len(args) {
genre = args[i+1]
i++
}
case "--sort":
if i+1 < len(args) {
sortBy = args[i+1]
i++
}
case "--status":
if i+1 < len(args) {
status = args[i+1]
i++
}
case "--type":
if i+1 < len(args) {
novelType = args[i+1]
i++
}
case "--max-pages":
if i+1 < len(args) {
if n, err := strconv.Atoi(args[i+1]); err == nil && n > 0 {
maxPages = n
}
i++
}
}
}
singleFilePath := envOr("SINGLEFILE_PATH", "single-file")
browserlessURL := envOr("BROWSERLESS_URL", "http://localhost:3030")
// SingleFile expects a WebSocket CDP endpoint.
// Browserless exposes /chromium at the WS root.
wsEndpoint := strings.Replace(browserlessURL, "http://", "ws://", 1)
wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)
log.Info("save-browse: starting",
"genre", genre, "sort", sortBy, "status", status,
"type", novelType, "max_pages", maxPages,
"singlefile", singleFilePath,
"browserless_ws", wsEndpoint,
)
tmpDir, err := os.MkdirTemp("", "libnovel-browse-*")
if err != nil {
return fmt.Errorf("save-browse: create temp dir: %w", err)
}
defer os.RemoveAll(tmpDir)
const novelFireBase = "https://novelfire.net"
for page := 1; page <= maxPages; page++ {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
novelFireBase, genre, sortBy, status, novelType, page)
key := store.BrowsePageKey(genre, sortBy, status, novelType, page)
outFile := fmt.Sprintf("%s/page-%d.html", tmpDir, page)
log.Info("save-browse: capturing page", "page", page, "url", pageURL)
//nolint:gosec // singleFilePath and pageURL are config/URL values, not user input.
cmd := exec.CommandContext(ctx, singleFilePath,
pageURL,
"--browser-server="+wsEndpoint,
"--output="+outFile,
)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if runErr := cmd.Run(); runErr != nil {
log.Warn("save-browse: SingleFile failed, skipping page",
"page", page, "err", runErr)
continue
}
htmlBytes, readErr := os.ReadFile(outFile)
if readErr != nil {
log.Warn("save-browse: failed to read output file",
"page", page, "file", outFile, "err", readErr)
continue
}
if putErr := store.SaveBrowsePage(ctx, key, string(htmlBytes)); putErr != nil {
log.Warn("save-browse: failed to store snapshot in MinIO",
"page", page, "key", key, "err", putErr)
continue
}
log.Info("save-browse: snapshot stored", "page", page, "key", key,
"bytes", len(htmlBytes))
}
log.Info("save-browse: done")
return nil
}
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
switch strategy {
case browser.StrategyScrape:
@@ -230,6 +361,12 @@ Commands:
run [--url <book-url>] One-shot: scrape full catalogue, or a single book
refresh <slug> Re-scrape a book from its saved source_url
serve Start HTTP server (POST /scrape, POST /scrape/book)
save-browse Capture browse pages via SingleFile → MinIO
--genre <slug> genre filter (default: all)
--sort <value> sort order (default: popular)
--status <value> status filter (default: all)
--type <value> novel type (default: all-novel)
--max-pages <n> pages to capture (default: 5)
Environment variables:
BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
@@ -242,6 +379,8 @@ Environment variables:
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled)
KOKORO_VOICE Default TTS voice (default: af_bella)
MINIO_BUCKET_BROWSE Browse snapshots bucket (default: libnovel-browse)
SINGLEFILE_PATH Path to single-file CLI (default: single-file)
LOG_LEVEL debug|info|warn|error (default: info)
`, runtime.NumCPU())
}