feat(browse): add SingleFile browse-page snapshot cache via MinIO
- New MinIO bucket 'libnovel-browse' (MINIO_BUCKET_BROWSE env) for storing self-contained HTML snapshots of novelfire browse pages - Store interface gains SaveBrowsePage / GetBrowsePage / BrowsePageKey methods - handleBrowse is now cache-first: serves from MinIO snapshot when available, then fires a background triggerBrowseSnapshot goroutine to populate cache on live-fetch (de-duplicated, 90s timeout) - New 'save-browse' CLI subcommand to bulk-capture pages via SingleFile CLI - Dockerfile: downloads pinned single-file-x86_64-linux binary (v2.0.83), adds gcompat + libstdc++ to Alpine runtime for glibc compatibility - docker-compose: adds libnovel-browse bucket init and SINGLEFILE_PATH env - .gitignore: exclude scraper/scraper build artifact
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -6,6 +6,7 @@
|
|||||||
|
|
||||||
# ── Compiled binaries ──────────────────────────────────────────────────────────
|
# ── Compiled binaries ──────────────────────────────────────────────────────────
|
||||||
scraper/bin/
|
scraper/bin/
|
||||||
|
scraper/scraper
|
||||||
|
|
||||||
# ── Scraped output (large, machine-generated) ──────────────────────────────────
|
# ── Scraped output (large, machine-generated) ──────────────────────────────────
|
||||||
|
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ services:
|
|||||||
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
|
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
|
||||||
mc mb --ignore-existing local/libnovel-chapters;
|
mc mb --ignore-existing local/libnovel-chapters;
|
||||||
mc mb --ignore-existing local/libnovel-audio;
|
mc mb --ignore-existing local/libnovel-audio;
|
||||||
|
mc mb --ignore-existing local/libnovel-browse;
|
||||||
echo 'buckets ready';
|
echo 'buckets ready';
|
||||||
"
|
"
|
||||||
environment:
|
environment:
|
||||||
@@ -136,10 +137,13 @@ services:
|
|||||||
MINIO_USE_SSL: "false"
|
MINIO_USE_SSL: "false"
|
||||||
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
|
MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}"
|
||||||
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
|
MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}"
|
||||||
|
MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}"
|
||||||
# Public endpoint used to sign presigned audio URLs so browsers can reach them.
|
# Public endpoint used to sign presigned audio URLs so browsers can reach them.
|
||||||
# Leave empty to use MINIO_ENDPOINT (fine for local dev).
|
# Leave empty to use MINIO_ENDPOINT (fine for local dev).
|
||||||
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-}"
|
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-}"
|
||||||
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-true}"
|
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-true}"
|
||||||
|
# SingleFile CLI path for save-browse subcommand
|
||||||
|
SINGLEFILE_PATH: "${SINGLEFILE_PATH:-single-file}"
|
||||||
# PocketBase
|
# PocketBase
|
||||||
POCKETBASE_URL: "http://pocketbase:8090"
|
POCKETBASE_URL: "http://pocketbase:8090"
|
||||||
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
|
||||||
|
|||||||
@@ -12,15 +12,30 @@ COPY . .
|
|||||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
||||||
go build -ldflags="-s -w" -o /scraper ./cmd/scraper
|
go build -ldflags="-s -w" -o /scraper ./cmd/scraper
|
||||||
|
|
||||||
|
# ── single-file binary stage ───────────────────────────────────────────────────
|
||||||
|
# Download the official pre-compiled single-file-cli binary (Deno-compiled,
|
||||||
|
# statically linked for Linux x86_64 glibc). We pin to a specific version so
|
||||||
|
# image builds are reproducible; bump this ARG to upgrade.
|
||||||
|
FROM alpine:3.20 AS singlefile-downloader
|
||||||
|
ARG SINGLEFILE_VERSION=2.0.83
|
||||||
|
RUN apk add --no-cache wget ca-certificates && \
|
||||||
|
wget -q -O /single-file \
|
||||||
|
"https://github.com/gildas-lormeau/single-file-cli/releases/download/v${SINGLEFILE_VERSION}/single-file-x86_64-linux" && \
|
||||||
|
chmod +x /single-file
|
||||||
|
|
||||||
# ── Runtime stage ──────────────────────────────────────────────────────────────
|
# ── Runtime stage ──────────────────────────────────────────────────────────────
|
||||||
FROM alpine:3.20
|
FROM alpine:3.20
|
||||||
|
|
||||||
# ca-certificates is required for HTTPS requests to novelfire.net.
|
# ca-certificates: HTTPS to novelfire.net
|
||||||
RUN apk add --no-cache ca-certificates tzdata
|
# gcompat: glibc compatibility shim so the glibc-linked single-file binary
|
||||||
|
# runs on musl/Alpine without needing a full glibc image.
|
||||||
|
# libstdc++: required by the Deno runtime embedded in single-file
|
||||||
|
RUN apk add --no-cache ca-certificates tzdata gcompat libstdc++
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY --from=builder /scraper /app/scraper
|
COPY --from=builder /scraper /app/scraper
|
||||||
|
COPY --from=singlefile-downloader /single-file /usr/local/bin/single-file
|
||||||
|
|
||||||
# Create the default static output directory.
|
# Create the default static output directory.
|
||||||
RUN mkdir -p /app/static/books
|
RUN mkdir -p /app/static/books
|
||||||
@@ -36,6 +51,7 @@ ENV BROWSERLESS_STRATEGY=content
|
|||||||
ENV SCRAPER_WORKERS=0
|
ENV SCRAPER_WORKERS=0
|
||||||
ENV SCRAPER_STATIC_ROOT=/app/static/books
|
ENV SCRAPER_STATIC_ROOT=/app/static/books
|
||||||
ENV SCRAPER_HTTP_ADDR=:8080
|
ENV SCRAPER_HTTP_ADDR=:8080
|
||||||
|
ENV SINGLEFILE_PATH=/usr/local/bin/single-file
|
||||||
|
|
||||||
EXPOSE 8080
|
EXPOSE 8080
|
||||||
|
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
|
"os/exec"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -106,6 +107,7 @@ func run(log *slog.Logger) error {
|
|||||||
PublicUseSSL: strings.ToLower(os.Getenv("MINIO_PUBLIC_USE_SSL")) != "false",
|
PublicUseSSL: strings.ToLower(os.Getenv("MINIO_PUBLIC_USE_SSL")) != "false",
|
||||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||||
|
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
|
||||||
}
|
}
|
||||||
pbCfg := storage.PocketBaseConfig{
|
pbCfg := storage.PocketBaseConfig{
|
||||||
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
|
||||||
@@ -198,11 +200,140 @@ func run(log *slog.Logger) error {
|
|||||||
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
|
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
|
||||||
return srv.ListenAndServe(ctx)
|
return srv.ListenAndServe(ctx)
|
||||||
|
|
||||||
|
case "save-browse":
|
||||||
|
return runSaveBrowse(ctx, args[1:], store, log)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd)
|
return fmt.Errorf("unknown command %q; use 'run', 'refresh', 'serve', or 'save-browse'", cmd)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runSaveBrowse implements the `save-browse` subcommand.
|
||||||
|
// It iterates over browse pages on novelfire.net, captures each using
|
||||||
|
// SingleFile CLI (connected to the existing Browserless instance), and
|
||||||
|
// stores the resulting self-contained HTML in the MinIO browse bucket.
|
||||||
|
//
|
||||||
|
// Flags (all optional):
|
||||||
|
//
|
||||||
|
// --genre <value> genre slug (default: all)
|
||||||
|
// --sort <value> sort order (default: popular)
|
||||||
|
// --status <value> status (default: all)
|
||||||
|
// --type <value> novel type (default: all-novel)
|
||||||
|
// --max-pages <n> max pages (default: 5)
|
||||||
|
func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log *slog.Logger) error {
|
||||||
|
// Parse flags manually to avoid importing flag package.
|
||||||
|
genre := "all"
|
||||||
|
sortBy := "popular"
|
||||||
|
status := "all"
|
||||||
|
novelType := "all-novel"
|
||||||
|
maxPages := 5
|
||||||
|
|
||||||
|
for i := 0; i < len(args); i++ {
|
||||||
|
switch args[i] {
|
||||||
|
case "--genre":
|
||||||
|
if i+1 < len(args) {
|
||||||
|
genre = args[i+1]
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case "--sort":
|
||||||
|
if i+1 < len(args) {
|
||||||
|
sortBy = args[i+1]
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case "--status":
|
||||||
|
if i+1 < len(args) {
|
||||||
|
status = args[i+1]
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case "--type":
|
||||||
|
if i+1 < len(args) {
|
||||||
|
novelType = args[i+1]
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case "--max-pages":
|
||||||
|
if i+1 < len(args) {
|
||||||
|
if n, err := strconv.Atoi(args[i+1]); err == nil && n > 0 {
|
||||||
|
maxPages = n
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
singleFilePath := envOr("SINGLEFILE_PATH", "single-file")
|
||||||
|
browserlessURL := envOr("BROWSERLESS_URL", "http://localhost:3030")
|
||||||
|
// SingleFile expects a WebSocket CDP endpoint.
|
||||||
|
// Browserless exposes /chromium at the WS root.
|
||||||
|
wsEndpoint := strings.Replace(browserlessURL, "http://", "ws://", 1)
|
||||||
|
wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)
|
||||||
|
|
||||||
|
log.Info("save-browse: starting",
|
||||||
|
"genre", genre, "sort", sortBy, "status", status,
|
||||||
|
"type", novelType, "max_pages", maxPages,
|
||||||
|
"singlefile", singleFilePath,
|
||||||
|
"browserless_ws", wsEndpoint,
|
||||||
|
)
|
||||||
|
|
||||||
|
tmpDir, err := os.MkdirTemp("", "libnovel-browse-*")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("save-browse: create temp dir: %w", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(tmpDir)
|
||||||
|
|
||||||
|
const novelFireBase = "https://novelfire.net"
|
||||||
|
|
||||||
|
for page := 1; page <= maxPages; page++ {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
|
||||||
|
novelFireBase, genre, sortBy, status, novelType, page)
|
||||||
|
|
||||||
|
key := store.BrowsePageKey(genre, sortBy, status, novelType, page)
|
||||||
|
|
||||||
|
outFile := fmt.Sprintf("%s/page-%d.html", tmpDir, page)
|
||||||
|
|
||||||
|
log.Info("save-browse: capturing page", "page", page, "url", pageURL)
|
||||||
|
|
||||||
|
//nolint:gosec // singleFilePath and pageURL are config/URL values, not user input.
|
||||||
|
cmd := exec.CommandContext(ctx, singleFilePath,
|
||||||
|
pageURL,
|
||||||
|
"--browser-server="+wsEndpoint,
|
||||||
|
"--output="+outFile,
|
||||||
|
)
|
||||||
|
cmd.Stdout = os.Stdout
|
||||||
|
cmd.Stderr = os.Stderr
|
||||||
|
|
||||||
|
if runErr := cmd.Run(); runErr != nil {
|
||||||
|
log.Warn("save-browse: SingleFile failed, skipping page",
|
||||||
|
"page", page, "err", runErr)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
htmlBytes, readErr := os.ReadFile(outFile)
|
||||||
|
if readErr != nil {
|
||||||
|
log.Warn("save-browse: failed to read output file",
|
||||||
|
"page", page, "file", outFile, "err", readErr)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if putErr := store.SaveBrowsePage(ctx, key, string(htmlBytes)); putErr != nil {
|
||||||
|
log.Warn("save-browse: failed to store snapshot in MinIO",
|
||||||
|
"page", page, "key", key, "err", putErr)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("save-browse: snapshot stored", "page", page, "key", key,
|
||||||
|
"bytes", len(htmlBytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("save-browse: done")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
|
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
|
||||||
switch strategy {
|
switch strategy {
|
||||||
case browser.StrategyScrape:
|
case browser.StrategyScrape:
|
||||||
@@ -230,6 +361,12 @@ Commands:
|
|||||||
run [--url <book-url>] One-shot: scrape full catalogue, or a single book
|
run [--url <book-url>] One-shot: scrape full catalogue, or a single book
|
||||||
refresh <slug> Re-scrape a book from its saved source_url
|
refresh <slug> Re-scrape a book from its saved source_url
|
||||||
serve Start HTTP server (POST /scrape, POST /scrape/book)
|
serve Start HTTP server (POST /scrape, POST /scrape/book)
|
||||||
|
save-browse Capture browse pages via SingleFile → MinIO
|
||||||
|
--genre <slug> genre filter (default: all)
|
||||||
|
--sort <value> sort order (default: popular)
|
||||||
|
--status <value> status filter (default: all)
|
||||||
|
--type <value> novel type (default: all-novel)
|
||||||
|
--max-pages <n> pages to capture (default: 5)
|
||||||
|
|
||||||
Environment variables:
|
Environment variables:
|
||||||
BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
|
||||||
@@ -242,6 +379,8 @@ Environment variables:
|
|||||||
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
|
||||||
KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled)
|
KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled)
|
||||||
KOKORO_VOICE Default TTS voice (default: af_bella)
|
KOKORO_VOICE Default TTS voice (default: af_bella)
|
||||||
|
MINIO_BUCKET_BROWSE Browse snapshots bucket (default: libnovel-browse)
|
||||||
|
SINGLEFILE_PATH Path to single-file CLI (default: single-file)
|
||||||
LOG_LEVEL debug|info|warn|error (default: info)
|
LOG_LEVEL debug|info|warn|error (default: info)
|
||||||
`, runtime.NumCPU())
|
`, runtime.NumCPU())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -48,6 +50,10 @@ type Server struct {
|
|||||||
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
||||||
kokoroVoice string // default voice, e.g. af_bella
|
kokoroVoice string // default voice, e.g. af_bella
|
||||||
|
|
||||||
|
// SingleFile CLI settings for browse-page snapshots.
|
||||||
|
singleFilePath string // path to single-file binary, e.g. /usr/local/bin/single-file
|
||||||
|
browserlessURL string // Browserless base URL, e.g. http://browserless:3000
|
||||||
|
|
||||||
// voiceMu guards cachedVoices.
|
// voiceMu guards cachedVoices.
|
||||||
voiceMu sync.RWMutex
|
voiceMu sync.RWMutex
|
||||||
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
|
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
|
||||||
@@ -57,19 +63,27 @@ type Server struct {
|
|||||||
// audioInFlight deduplicates concurrent generation requests for the same key.
|
// audioInFlight deduplicates concurrent generation requests for the same key.
|
||||||
audioMu sync.Mutex
|
audioMu sync.Mutex
|
||||||
audioInFlight map[string]chan struct{} // cacheKey → closed when done
|
audioInFlight map[string]chan struct{} // cacheKey → closed when done
|
||||||
|
|
||||||
|
// browseMu guards browseInFlight — keys of MinIO objects currently being
|
||||||
|
// captured by a background SingleFile goroutine.
|
||||||
|
browseMu sync.Mutex
|
||||||
|
browseInFlight map[string]struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new Server.
|
// New creates a new Server.
|
||||||
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
|
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
|
||||||
return &Server{
|
return &Server{
|
||||||
addr: addr,
|
addr: addr,
|
||||||
oCfg: oCfg,
|
oCfg: oCfg,
|
||||||
novel: novel,
|
novel: novel,
|
||||||
log: log,
|
log: log,
|
||||||
store: store,
|
store: store,
|
||||||
kokoroURL: kokoroURL,
|
kokoroURL: kokoroURL,
|
||||||
kokoroVoice: kokoroVoice,
|
kokoroVoice: kokoroVoice,
|
||||||
audioInFlight: make(map[string]chan struct{}),
|
singleFilePath: os.Getenv("SINGLEFILE_PATH"),
|
||||||
|
browserlessURL: os.Getenv("BROWSERLESS_URL"),
|
||||||
|
audioInFlight: make(map[string]chan struct{}),
|
||||||
|
browseInFlight: make(map[string]struct{}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -815,6 +829,9 @@ const novelFireBase = "https://novelfire.net"
|
|||||||
// type (default "all-novel")
|
// type (default "all-novel")
|
||||||
//
|
//
|
||||||
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
|
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
|
||||||
|
//
|
||||||
|
// Cache strategy: check MinIO browse bucket first; if a snapshot exists,
|
||||||
|
// parse and return it. Otherwise fetch live from novelfire.net.
|
||||||
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||||
q := r.URL.Query()
|
q := r.URL.Query()
|
||||||
page := q.Get("page")
|
page := q.Get("page")
|
||||||
@@ -838,13 +855,34 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
|||||||
novelType = "all-novel"
|
novelType = "all-novel"
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
|
pageNum, _ := strconv.Atoi(page)
|
||||||
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
|
if pageNum <= 0 {
|
||||||
novelFireBase, genre, sortBy, status, novelType, page)
|
pageNum = 1
|
||||||
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
|
// ── Cache-first: try MinIO snapshot ──────────────────────────────────
|
||||||
|
cacheKey := s.store.BrowsePageKey(genre, sortBy, status, novelType, pageNum)
|
||||||
|
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
|
||||||
|
novels, hasNext := parseBrowsePage(strings.NewReader(html))
|
||||||
|
s.log.Debug("browse: served from cache", "key", cacheKey)
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||||
|
"novels": novels,
|
||||||
|
"page": pageNum,
|
||||||
|
"hasNext": hasNext,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Live fallback: fetch from novelfire.net ───────────────────────────
|
||||||
|
// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
|
||||||
|
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
|
||||||
|
novelFireBase, genre, sortBy, status, novelType, page)
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
|
http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
|
||||||
@@ -867,7 +905,11 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
novels, hasNext := parseBrowsePage(resp.Body)
|
novels, hasNext := parseBrowsePage(resp.Body)
|
||||||
pageNum, _ := strconv.Atoi(page)
|
|
||||||
|
// ── Background: populate MinIO cache via SingleFile ───────────────────
|
||||||
|
// Fire-and-forget: capture the JS-rendered page with SingleFile and store
|
||||||
|
// it in MinIO so the next request is served from cache.
|
||||||
|
s.triggerBrowseSnapshot(cacheKey, targetURL)
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||||
@@ -878,6 +920,80 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// triggerBrowseSnapshot fires a background goroutine that uses SingleFile CLI
|
||||||
|
// to capture the fully-rendered novelfire browse page and store it in MinIO.
|
||||||
|
// It is a no-op when:
|
||||||
|
// - SINGLEFILE_PATH is not set (SingleFile not installed)
|
||||||
|
// - a capture for this cache key is already in progress
|
||||||
|
//
|
||||||
|
// The goroutine uses a fresh context so it outlives the HTTP request.
|
||||||
|
func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
|
||||||
|
if s.singleFilePath == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.browseMu.Lock()
|
||||||
|
if _, inflight := s.browseInFlight[cacheKey]; inflight {
|
||||||
|
s.browseMu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.browseInFlight[cacheKey] = struct{}{}
|
||||||
|
s.browseMu.Unlock()
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer func() {
|
||||||
|
s.browseMu.Lock()
|
||||||
|
delete(s.browseInFlight, cacheKey)
|
||||||
|
s.browseMu.Unlock()
|
||||||
|
}()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// Convert http(s) → ws(s) for the SingleFile --browser-server flag.
|
||||||
|
wsEndpoint := s.browserlessURL
|
||||||
|
wsEndpoint = strings.Replace(wsEndpoint, "http://", "ws://", 1)
|
||||||
|
wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)
|
||||||
|
|
||||||
|
tmpFile, err := os.CreateTemp("", "libnovel-browse-*.html")
|
||||||
|
if err != nil {
|
||||||
|
s.log.Warn("triggerBrowseSnapshot: create temp file failed", "key", cacheKey, "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
tmpPath := tmpFile.Name()
|
||||||
|
tmpFile.Close()
|
||||||
|
defer os.Remove(tmpPath)
|
||||||
|
|
||||||
|
//nolint:gosec
|
||||||
|
cmd := exec.CommandContext(ctx, s.singleFilePath,
|
||||||
|
pageURL,
|
||||||
|
"--browser-server="+wsEndpoint,
|
||||||
|
"--output="+tmpPath,
|
||||||
|
)
|
||||||
|
if out, runErr := cmd.CombinedOutput(); runErr != nil {
|
||||||
|
s.log.Warn("triggerBrowseSnapshot: SingleFile failed",
|
||||||
|
"key", cacheKey, "err", runErr, "output", string(out))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
htmlBytes, readErr := os.ReadFile(tmpPath)
|
||||||
|
if readErr != nil {
|
||||||
|
s.log.Warn("triggerBrowseSnapshot: read output failed",
|
||||||
|
"key", cacheKey, "err", readErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
|
||||||
|
s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
|
||||||
|
"key", cacheKey, "err", putErr)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.log.Info("triggerBrowseSnapshot: cached browse page",
|
||||||
|
"key", cacheKey, "bytes", len(htmlBytes))
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
|
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
|
||||||
// Returns novels and whether a "next page" link was found.
|
// Returns novels and whether a "next page" link was found.
|
||||||
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
|
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
|
||||||
|
|||||||
@@ -304,6 +304,20 @@ func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time
|
|||||||
return h.minio.PresignAudio(ctx, key, expires)
|
return h.minio.PresignAudio(ctx, key, expires)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── Browse page snapshots ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
func (h *HybridStore) SaveBrowsePage(ctx context.Context, key, html string) error {
|
||||||
|
return h.minio.PutBrowsePage(ctx, key, html)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bool, error) {
|
||||||
|
return h.minio.GetBrowsePage(ctx, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
|
||||||
|
return BrowsePageKey(genre, sortBy, status, novelType, page)
|
||||||
|
}
|
||||||
|
|
||||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) {
|
func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) {
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ type MinioConfig struct {
|
|||||||
PublicUseSSL bool // TLS for the public endpoint (usually true in prod)
|
PublicUseSSL bool // TLS for the public endpoint (usually true in prod)
|
||||||
BucketChapters string // e.g. "libnovel-chapters"
|
BucketChapters string // e.g. "libnovel-chapters"
|
||||||
BucketAudio string // e.g. "libnovel-audio"
|
BucketAudio string // e.g. "libnovel-audio"
|
||||||
|
BucketBrowse string // e.g. "libnovel-browse"
|
||||||
}
|
}
|
||||||
|
|
||||||
// MinioClient wraps a minio.Client and exposes object operations for
|
// MinioClient wraps a minio.Client and exposes object operations for
|
||||||
@@ -58,7 +59,10 @@ func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
mc := &MinioClient{c: c, pub: pub, cfg: cfg}
|
mc := &MinioClient{c: c, pub: pub, cfg: cfg}
|
||||||
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio} {
|
for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio, cfg.BucketBrowse} {
|
||||||
|
if bucket == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
if err := mc.ensureBucket(ctx, bucket); err != nil {
|
if err := mc.ensureBucket(ctx, bucket); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -206,6 +210,51 @@ func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time
|
|||||||
return u.String(), nil
|
return u.String(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── Browse page snapshots ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// BrowsePageKey returns the MinIO object key for a cached browse-page snapshot.
|
||||||
|
// Layout: {genre}/{sort}/{status}/{novelType}/page-{n}.html
|
||||||
|
func BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
|
||||||
|
return fmt.Sprintf("%s/%s/%s/%s/page-%d.html", genre, sortBy, status, novelType, page)
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket.
|
||||||
|
func (m *MinioClient) PutBrowsePage(ctx context.Context, key, html string) error {
|
||||||
|
data := []byte(html)
|
||||||
|
_, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
|
||||||
|
bytes.NewReader(data), int64(len(data)),
|
||||||
|
minio.PutObjectOptions{ContentType: "text/html; charset=utf-8"})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("minio: put browse page %s: %w", key, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBrowsePage retrieves a SingleFile HTML snapshot from the browse bucket.
|
||||||
|
// Returns ("", false, nil) when the object does not exist.
|
||||||
|
func (m *MinioClient) GetBrowsePage(ctx context.Context, key string) (string, bool, error) {
|
||||||
|
obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return "", false, fmt.Errorf("minio: get browse page %s: %w", key, err)
|
||||||
|
}
|
||||||
|
defer obj.Close()
|
||||||
|
// Check whether the object actually exists by inspecting the Stat.
|
||||||
|
if _, statErr := obj.Stat(); statErr != nil {
|
||||||
|
return "", false, nil // not found
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(obj)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, fmt.Errorf("minio: read browse page %s: %w", key, err)
|
||||||
|
}
|
||||||
|
return string(data), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BrowsePageExists returns true if a snapshot object is present in the browse bucket.
|
||||||
|
func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool {
|
||||||
|
_, err := m.c.StatObject(ctx, m.cfg.BucketBrowse, key, minio.StatObjectOptions{})
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
// sanitiseVoice converts a voice name to a filename-safe string.
|
// sanitiseVoice converts a voice name to a filename-safe string.
|
||||||
|
|||||||
@@ -139,6 +139,16 @@ type Store interface {
|
|||||||
// PresignAudio returns a presigned GET URL for an audio object.
|
// PresignAudio returns a presigned GET URL for an audio object.
|
||||||
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
|
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
|
||||||
|
|
||||||
|
// ── Browse page snapshots (MinIO) ──────────────────────────────────────
|
||||||
|
|
||||||
|
// SaveBrowsePage stores a SingleFile HTML snapshot for the given cache key.
|
||||||
|
SaveBrowsePage(ctx context.Context, key, html string) error
|
||||||
|
// GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil)
|
||||||
|
// when no snapshot exists for the key.
|
||||||
|
GetBrowsePage(ctx context.Context, key string) (string, bool, error)
|
||||||
|
// BrowsePageKey returns the MinIO object key for the given browse params.
|
||||||
|
BrowsePageKey(genre, sortBy, status, novelType string, page int) string
|
||||||
|
|
||||||
// ── Scraping tasks ─────────────────────────────────────────────────────
|
// ── Scraping tasks ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
// CreateScrapeTask inserts a new scraping_tasks record with status="running"
|
// CreateScrapeTask inserts a new scraping_tasks record with status="running"
|
||||||
|
|||||||
BIN
scraper/scraper
BIN
scraper/scraper
Binary file not shown.
Reference in New Issue
Block a user