feat(browse): add SingleFile browse-page snapshot cache via MinIO
- New MinIO bucket 'libnovel-browse' (MINIO_BUCKET_BROWSE env) for storing self-contained HTML snapshots of novelfire browse pages - Store interface gains SaveBrowsePage / GetBrowsePage / BrowsePageKey methods - handleBrowse is now cache-first: serves from MinIO snapshot when available, then fires a background triggerBrowseSnapshot goroutine to populate cache on live-fetch (de-duplicated, 90s timeout) - New 'save-browse' CLI subcommand to bulk-capture pages via SingleFile CLI - Dockerfile: downloads pinned single-file-x86_64-linux binary (v2.0.83), adds gcompat + libstdc++ to Alpine runtime for glibc compatibility - docker-compose: adds libnovel-browse bucket init and SINGLEFILE_PATH env - .gitignore: exclude scraper/scraper build artifact
This commit is contained in:
@@ -35,6 +35,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"strconv"
|
||||
@@ -106,6 +107,7 @@ func run(log *slog.Logger) error {
|
||||
PublicUseSSL: strings.ToLower(os.Getenv("MINIO_PUBLIC_USE_SSL")) != "false",
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
|
||||
}
|
||||
pbCfg := storage.PocketBaseConfig{
|
||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||
@@ -198,11 +200,140 @@ func run(log *slog.Logger) error {
|
||||
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
|
||||
return srv.ListenAndServe(ctx)
|
||||
|
||||
case "save-browse":
|
||||
return runSaveBrowse(ctx, args[1:], store, log)
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd)
|
||||
return fmt.Errorf("unknown command %q; use 'run', 'refresh', 'serve', or 'save-browse'", cmd)
|
||||
}
|
||||
}
|
||||
|
||||
// runSaveBrowse implements the `save-browse` subcommand.
|
||||
// It iterates over browse pages on novelfire.net, captures each using
|
||||
// SingleFile CLI (connected to the existing Browserless instance), and
|
||||
// stores the resulting self-contained HTML in the MinIO browse bucket.
|
||||
//
|
||||
// Flags (all optional):
|
||||
//
|
||||
// --genre <value> genre slug (default: all)
|
||||
// --sort <value> sort order (default: popular)
|
||||
// --status <value> status (default: all)
|
||||
// --type <value> novel type (default: all-novel)
|
||||
// --max-pages <n> max pages (default: 5)
|
||||
func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log *slog.Logger) error {
|
||||
// Parse flags manually to avoid importing flag package.
|
||||
genre := "all"
|
||||
sortBy := "popular"
|
||||
status := "all"
|
||||
novelType := "all-novel"
|
||||
maxPages := 5
|
||||
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "--genre":
|
||||
if i+1 < len(args) {
|
||||
genre = args[i+1]
|
||||
i++
|
||||
}
|
||||
case "--sort":
|
||||
if i+1 < len(args) {
|
||||
sortBy = args[i+1]
|
||||
i++
|
||||
}
|
||||
case "--status":
|
||||
if i+1 < len(args) {
|
||||
status = args[i+1]
|
||||
i++
|
||||
}
|
||||
case "--type":
|
||||
if i+1 < len(args) {
|
||||
novelType = args[i+1]
|
||||
i++
|
||||
}
|
||||
case "--max-pages":
|
||||
if i+1 < len(args) {
|
||||
if n, err := strconv.Atoi(args[i+1]); err == nil && n > 0 {
|
||||
maxPages = n
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
singleFilePath := envOr("SINGLEFILE_PATH", "single-file")
|
||||
browserlessURL := envOr("BROWSERLESS_URL", "http://localhost:3030")
|
||||
// SingleFile expects a WebSocket CDP endpoint.
|
||||
// Browserless exposes /chromium at the WS root.
|
||||
wsEndpoint := strings.Replace(browserlessURL, "http://", "ws://", 1)
|
||||
wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)
|
||||
|
||||
log.Info("save-browse: starting",
|
||||
"genre", genre, "sort", sortBy, "status", status,
|
||||
"type", novelType, "max_pages", maxPages,
|
||||
"singlefile", singleFilePath,
|
||||
"browserless_ws", wsEndpoint,
|
||||
)
|
||||
|
||||
tmpDir, err := os.MkdirTemp("", "libnovel-browse-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("save-browse: create temp dir: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
const novelFireBase = "https://novelfire.net"
|
||||
|
||||
for page := 1; page <= maxPages; page++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
|
||||
key := store.BrowsePageKey(genre, sortBy, status, novelType, page)
|
||||
|
||||
outFile := fmt.Sprintf("%s/page-%d.html", tmpDir, page)
|
||||
|
||||
log.Info("save-browse: capturing page", "page", page, "url", pageURL)
|
||||
|
||||
//nolint:gosec // singleFilePath and pageURL are config/URL values, not user input.
|
||||
cmd := exec.CommandContext(ctx, singleFilePath,
|
||||
pageURL,
|
||||
"--browser-server="+wsEndpoint,
|
||||
"--output="+outFile,
|
||||
)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if runErr := cmd.Run(); runErr != nil {
|
||||
log.Warn("save-browse: SingleFile failed, skipping page",
|
||||
"page", page, "err", runErr)
|
||||
continue
|
||||
}
|
||||
|
||||
htmlBytes, readErr := os.ReadFile(outFile)
|
||||
if readErr != nil {
|
||||
log.Warn("save-browse: failed to read output file",
|
||||
"page", page, "file", outFile, "err", readErr)
|
||||
continue
|
||||
}
|
||||
|
||||
if putErr := store.SaveBrowsePage(ctx, key, string(htmlBytes)); putErr != nil {
|
||||
log.Warn("save-browse: failed to store snapshot in MinIO",
|
||||
"page", page, "key", key, "err", putErr)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Info("save-browse: snapshot stored", "page", page, "key", key,
|
||||
"bytes", len(htmlBytes))
|
||||
}
|
||||
|
||||
log.Info("save-browse: done")
|
||||
return nil
|
||||
}
|
||||
|
||||
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
|
||||
switch strategy {
|
||||
case browser.StrategyScrape:
|
||||
@@ -230,6 +361,12 @@ Commands:
|
||||
run [--url <book-url>] One-shot: scrape full catalogue, or a single book
|
||||
refresh <slug> Re-scrape a book from its saved source_url
|
||||
serve Start HTTP server (POST /scrape, POST /scrape/book)
|
||||
save-browse Capture browse pages via SingleFile → MinIO
|
||||
--genre <slug> genre filter (default: all)
|
||||
--sort <value> sort order (default: popular)
|
||||
--status <value> status filter (default: all)
|
||||
--type <value> novel type (default: all-novel)
|
||||
--max-pages <n> pages to capture (default: 5)
|
||||
|
||||
Environment variables:
|
||||
BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||
@@ -242,6 +379,8 @@ Environment variables:
|
||||
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||
KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled)
|
||||
KOKORO_VOICE Default TTS voice (default: af_bella)
|
||||
MINIO_BUCKET_BROWSE Browse snapshots bucket (default: libnovel-browse)
|
||||
SINGLEFILE_PATH Path to single-file CLI (default: single-file)
|
||||
LOG_LEVEL debug|info|warn|error (default: info)
|
||||
`, runtime.NumCPU())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user