diff --git a/.gitignore b/.gitignore index 0e45f0f..02102a6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ # ── Compiled binaries ────────────────────────────────────────────────────────── scraper/bin/ +scraper/scraper # ── Scraped output (large, machine-generated) ────────────────────────────────── diff --git a/docker-compose.yml b/docker-compose.yml index 8339f04..7251ab2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,7 @@ services: mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123}; mc mb --ignore-existing local/libnovel-chapters; mc mb --ignore-existing local/libnovel-audio; + mc mb --ignore-existing local/libnovel-browse; echo 'buckets ready'; " environment: @@ -136,10 +137,13 @@ services: MINIO_USE_SSL: "false" MINIO_BUCKET_CHAPTERS: "${MINIO_BUCKET_CHAPTERS:-libnovel-chapters}" MINIO_BUCKET_AUDIO: "${MINIO_BUCKET_AUDIO:-libnovel-audio}" + MINIO_BUCKET_BROWSE: "${MINIO_BUCKET_BROWSE:-libnovel-browse}" # Public endpoint used to sign presigned audio URLs so browsers can reach them. # Leave empty to use MINIO_ENDPOINT (fine for local dev). MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-}" MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-true}" + # SingleFile CLI path for save-browse subcommand + SINGLEFILE_PATH: "${SINGLEFILE_PATH:-single-file}" # PocketBase POCKETBASE_URL: "http://pocketbase:8090" POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" diff --git a/scraper/Dockerfile b/scraper/Dockerfile index a16a59d..81bd8a5 100644 --- a/scraper/Dockerfile +++ b/scraper/Dockerfile @@ -12,15 +12,30 @@ COPY . . RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ go build -ldflags="-s -w" -o /scraper ./cmd/scraper +# ── single-file binary stage ─────────────────────────────────────────────────── +# Download the official pre-compiled single-file-cli binary (Deno-compiled, +# statically linked for Linux x86_64 glibc). We pin to a specific version so +# image builds are reproducible; bump this ARG to upgrade. +FROM alpine:3.20 AS singlefile-downloader +ARG SINGLEFILE_VERSION=2.0.83 +RUN apk add --no-cache wget ca-certificates && \ + wget -q -O /single-file \ + "https://github.com/gildas-lormeau/single-file-cli/releases/download/v${SINGLEFILE_VERSION}/single-file-x86_64-linux" && \ + chmod +x /single-file + # ── Runtime stage ────────────────────────────────────────────────────────────── FROM alpine:3.20 -# ca-certificates is required for HTTPS requests to novelfire.net. -RUN apk add --no-cache ca-certificates tzdata +# ca-certificates: HTTPS to novelfire.net +# gcompat: glibc compatibility shim so the glibc-linked single-file binary +# runs on musl/Alpine without needing a full glibc image. +# libstdc++: required by the Deno runtime embedded in single-file +RUN apk add --no-cache ca-certificates tzdata gcompat libstdc++ WORKDIR /app COPY --from=builder /scraper /app/scraper +COPY --from=singlefile-downloader /single-file /usr/local/bin/single-file # Create the default static output directory. RUN mkdir -p /app/static/books @@ -36,6 +51,7 @@ ENV BROWSERLESS_STRATEGY=content ENV SCRAPER_WORKERS=0 ENV SCRAPER_STATIC_ROOT=/app/static/books ENV SCRAPER_HTTP_ADDR=:8080 +ENV SINGLEFILE_PATH=/usr/local/bin/single-file EXPOSE 8080 diff --git a/scraper/cmd/scraper/main.go b/scraper/cmd/scraper/main.go index baee1fa..6465923 100644 --- a/scraper/cmd/scraper/main.go +++ b/scraper/cmd/scraper/main.go @@ -35,6 +35,7 @@ import ( "fmt" "log/slog" "os" + "os/exec" "os/signal" "runtime" "strconv" @@ -106,6 +107,7 @@ func run(log *slog.Logger) error { PublicUseSSL: strings.ToLower(os.Getenv("MINIO_PUBLIC_USE_SSL")) != "false", BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), + BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"), } pbCfg := storage.PocketBaseConfig{ BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"), @@ -198,11 +200,140 @@ func run(log *slog.Logger) error { srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice) return srv.ListenAndServe(ctx) + case "save-browse": + return runSaveBrowse(ctx, args[1:], store, log) + default: - return fmt.Errorf("unknown command %q; use 'run' or 'serve'", cmd) + return fmt.Errorf("unknown command %q; use 'run', 'refresh', 'serve', or 'save-browse'", cmd) } } +// runSaveBrowse implements the `save-browse` subcommand. +// It iterates over browse pages on novelfire.net, captures each using +// SingleFile CLI (connected to the existing Browserless instance), and +// stores the resulting self-contained HTML in the MinIO browse bucket. +// +// Flags (all optional): +// +// --genre genre slug (default: all) +// --sort sort order (default: popular) +// --status status (default: all) +// --type novel type (default: all-novel) +// --max-pages max pages (default: 5) +func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log *slog.Logger) error { + // Parse flags manually to avoid importing flag package. + genre := "all" + sortBy := "popular" + status := "all" + novelType := "all-novel" + maxPages := 5 + + for i := 0; i < len(args); i++ { + switch args[i] { + case "--genre": + if i+1 < len(args) { + genre = args[i+1] + i++ + } + case "--sort": + if i+1 < len(args) { + sortBy = args[i+1] + i++ + } + case "--status": + if i+1 < len(args) { + status = args[i+1] + i++ + } + case "--type": + if i+1 < len(args) { + novelType = args[i+1] + i++ + } + case "--max-pages": + if i+1 < len(args) { + if n, err := strconv.Atoi(args[i+1]); err == nil && n > 0 { + maxPages = n + } + i++ + } + } + } + + singleFilePath := envOr("SINGLEFILE_PATH", "single-file") + browserlessURL := envOr("BROWSERLESS_URL", "http://localhost:3030") + // SingleFile expects a WebSocket CDP endpoint. + // Browserless exposes /chromium at the WS root. + wsEndpoint := strings.Replace(browserlessURL, "http://", "ws://", 1) + wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1) + + log.Info("save-browse: starting", + "genre", genre, "sort", sortBy, "status", status, + "type", novelType, "max_pages", maxPages, + "singlefile", singleFilePath, + "browserless_ws", wsEndpoint, + ) + + tmpDir, err := os.MkdirTemp("", "libnovel-browse-*") + if err != nil { + return fmt.Errorf("save-browse: create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + const novelFireBase = "https://novelfire.net" + + for page := 1; page <= maxPages; page++ { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d", + novelFireBase, genre, sortBy, status, novelType, page) + + key := store.BrowsePageKey(genre, sortBy, status, novelType, page) + + outFile := fmt.Sprintf("%s/page-%d.html", tmpDir, page) + + log.Info("save-browse: capturing page", "page", page, "url", pageURL) + + //nolint:gosec // singleFilePath and pageURL are config/URL values, not user input. + cmd := exec.CommandContext(ctx, singleFilePath, + pageURL, + "--browser-server="+wsEndpoint, + "--output="+outFile, + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if runErr := cmd.Run(); runErr != nil { + log.Warn("save-browse: SingleFile failed, skipping page", + "page", page, "err", runErr) + continue + } + + htmlBytes, readErr := os.ReadFile(outFile) + if readErr != nil { + log.Warn("save-browse: failed to read output file", + "page", page, "file", outFile, "err", readErr) + continue + } + + if putErr := store.SaveBrowsePage(ctx, key, string(htmlBytes)); putErr != nil { + log.Warn("save-browse: failed to store snapshot in MinIO", + "page", page, "key", key, "err", putErr) + continue + } + + log.Info("save-browse: snapshot stored", "page", page, "key", key, + "bytes", len(htmlBytes)) + } + + log.Info("save-browse: done") + return nil +} + func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient { switch strategy { case browser.StrategyScrape: @@ -230,6 +361,12 @@ Commands: run [--url ] One-shot: scrape full catalogue, or a single book refresh Re-scrape a book from its saved source_url serve Start HTTP server (POST /scrape, POST /scrape/book) + save-browse Capture browse pages via SingleFile → MinIO + --genre genre filter (default: all) + --sort sort order (default: popular) + --status status filter (default: all) + --type novel type (default: all-novel) + --max-pages pages to capture (default: 5) Environment variables: BROWSERLESS_URL Browserless base URL (default: http://localhost:3030) @@ -242,6 +379,8 @@ Environment variables: SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled) KOKORO_VOICE Default TTS voice (default: af_bella) + MINIO_BUCKET_BROWSE Browse snapshots bucket (default: libnovel-browse) + SINGLEFILE_PATH Path to single-file CLI (default: single-file) LOG_LEVEL debug|info|warn|error (default: info) `, runtime.NumCPU()) } diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index 03d32ec..6f0d4d6 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -25,6 +25,8 @@ import ( "io" "log/slog" "net/http" + "os" + "os/exec" "strconv" "strings" "sync" @@ -48,6 +50,10 @@ type Server struct { kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 kokoroVoice string // default voice, e.g. af_bella + // SingleFile CLI settings for browse-page snapshots. + singleFilePath string // path to single-file binary, e.g. /usr/local/bin/single-file + browserlessURL string // Browserless base URL, e.g. http://browserless:3000 + // voiceMu guards cachedVoices. voiceMu sync.RWMutex cachedVoices []string // populated on first request from Kokoro /v1/audio/voices @@ -57,19 +63,27 @@ type Server struct { // audioInFlight deduplicates concurrent generation requests for the same key. audioMu sync.Mutex audioInFlight map[string]chan struct{} // cacheKey → closed when done + + // browseMu guards browseInFlight — keys of MinIO objects currently being + // captured by a background SingleFile goroutine. + browseMu sync.Mutex + browseInFlight map[string]struct{} } // New creates a new Server. func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server { return &Server{ - addr: addr, - oCfg: oCfg, - novel: novel, - log: log, - store: store, - kokoroURL: kokoroURL, - kokoroVoice: kokoroVoice, - audioInFlight: make(map[string]chan struct{}), + addr: addr, + oCfg: oCfg, + novel: novel, + log: log, + store: store, + kokoroURL: kokoroURL, + kokoroVoice: kokoroVoice, + singleFilePath: os.Getenv("SINGLEFILE_PATH"), + browserlessURL: os.Getenv("BROWSERLESS_URL"), + audioInFlight: make(map[string]chan struct{}), + browseInFlight: make(map[string]struct{}), } } @@ -815,6 +829,9 @@ const novelFireBase = "https://novelfire.net" // type (default "all-novel") // // Returns JSON: {"novels":[...], "page": N, "hasNext": bool} +// +// Cache strategy: check MinIO browse bucket first; if a snapshot exists, +// parse and return it. Otherwise fetch live from novelfire.net. func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() page := q.Get("page") @@ -838,13 +855,34 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { novelType = "all-novel" } - // Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page} - targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s", - novelFireBase, genre, sortBy, status, novelType, page) + pageNum, _ := strconv.Atoi(page) + if pageNum <= 0 { + pageNum = 1 + } ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) defer cancel() + // ── Cache-first: try MinIO snapshot ────────────────────────────────── + cacheKey := s.store.BrowsePageKey(genre, sortBy, status, novelType, pageNum) + if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok { + novels, hasNext := parseBrowsePage(strings.NewReader(html)) + s.log.Debug("browse: served from cache", "key", cacheKey) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "public, max-age=300") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "novels": novels, + "page": pageNum, + "hasNext": hasNext, + }) + return + } + + // ── Live fallback: fetch from novelfire.net ─────────────────────────── + // Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page} + targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s", + novelFireBase, genre, sortBy, status, novelType, page) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) if err != nil { http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError) @@ -867,7 +905,11 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { } novels, hasNext := parseBrowsePage(resp.Body) - pageNum, _ := strconv.Atoi(page) + + // ── Background: populate MinIO cache via SingleFile ─────────────────── + // Fire-and-forget: capture the JS-rendered page with SingleFile and store + // it in MinIO so the next request is served from cache. + s.triggerBrowseSnapshot(cacheKey, targetURL) w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "public, max-age=300") @@ -878,6 +920,80 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { }) } +// triggerBrowseSnapshot fires a background goroutine that uses SingleFile CLI +// to capture the fully-rendered novelfire browse page and store it in MinIO. +// It is a no-op when: +// - SINGLEFILE_PATH is not set (SingleFile not installed) +// - a capture for this cache key is already in progress +// +// The goroutine uses a fresh context so it outlives the HTTP request. +func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) { + if s.singleFilePath == "" { + return + } + + s.browseMu.Lock() + if _, inflight := s.browseInFlight[cacheKey]; inflight { + s.browseMu.Unlock() + return + } + s.browseInFlight[cacheKey] = struct{}{} + s.browseMu.Unlock() + + go func() { + defer func() { + s.browseMu.Lock() + delete(s.browseInFlight, cacheKey) + s.browseMu.Unlock() + }() + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + // Convert http(s) → ws(s) for the SingleFile --browser-server flag. + wsEndpoint := s.browserlessURL + wsEndpoint = strings.Replace(wsEndpoint, "http://", "ws://", 1) + wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1) + + tmpFile, err := os.CreateTemp("", "libnovel-browse-*.html") + if err != nil { + s.log.Warn("triggerBrowseSnapshot: create temp file failed", "key", cacheKey, "err", err) + return + } + tmpPath := tmpFile.Name() + tmpFile.Close() + defer os.Remove(tmpPath) + + //nolint:gosec + cmd := exec.CommandContext(ctx, s.singleFilePath, + pageURL, + "--browser-server="+wsEndpoint, + "--output="+tmpPath, + ) + if out, runErr := cmd.CombinedOutput(); runErr != nil { + s.log.Warn("triggerBrowseSnapshot: SingleFile failed", + "key", cacheKey, "err", runErr, "output", string(out)) + return + } + + htmlBytes, readErr := os.ReadFile(tmpPath) + if readErr != nil { + s.log.Warn("triggerBrowseSnapshot: read output failed", + "key", cacheKey, "err", readErr) + return + } + + if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil { + s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed", + "key", cacheKey, "err", putErr) + return + } + + s.log.Info("triggerBrowseSnapshot: cached browse page", + "key", cacheKey, "bytes", len(htmlBytes)) + }() +} + // parseBrowsePage parses the novelfire HTML and extracts novel listings. // Returns novels and whether a "next page" link was found. func parseBrowsePage(r io.Reader) ([]NovelListing, bool) { diff --git a/scraper/internal/storage/hybrid.go b/scraper/internal/storage/hybrid.go index f33cd7d..a3ad2f2 100644 --- a/scraper/internal/storage/hybrid.go +++ b/scraper/internal/storage/hybrid.go @@ -304,6 +304,20 @@ func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time return h.minio.PresignAudio(ctx, key, expires) } +// ─── Browse page snapshots ──────────────────────────────────────────────────── + +func (h *HybridStore) SaveBrowsePage(ctx context.Context, key, html string) error { + return h.minio.PutBrowsePage(ctx, key, html) +} + +func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bool, error) { + return h.minio.GetBrowsePage(ctx, key) +} + +func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string { + return BrowsePageKey(genre, sortBy, status, novelType, page) +} + // ─── Scraping tasks ─────────────────────────────────────────────────────────── func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) { diff --git a/scraper/internal/storage/minio.go b/scraper/internal/storage/minio.go index 83ec2c1..cd137f4 100644 --- a/scraper/internal/storage/minio.go +++ b/scraper/internal/storage/minio.go @@ -22,6 +22,7 @@ type MinioConfig struct { PublicUseSSL bool // TLS for the public endpoint (usually true in prod) BucketChapters string // e.g. "libnovel-chapters" BucketAudio string // e.g. "libnovel-audio" + BucketBrowse string // e.g. "libnovel-browse" } // MinioClient wraps a minio.Client and exposes object operations for @@ -58,7 +59,10 @@ func NewMinioClient(ctx context.Context, cfg MinioConfig) (*MinioClient, error) } mc := &MinioClient{c: c, pub: pub, cfg: cfg} - for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio} { + for _, bucket := range []string{cfg.BucketChapters, cfg.BucketAudio, cfg.BucketBrowse} { + if bucket == "" { + continue + } if err := mc.ensureBucket(ctx, bucket); err != nil { return nil, err } @@ -206,6 +210,51 @@ func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time return u.String(), nil } +// ─── Browse page snapshots ──────────────────────────────────────────────────── + +// BrowsePageKey returns the MinIO object key for a cached browse-page snapshot. +// Layout: {genre}/{sort}/{status}/{novelType}/page-{n}.html +func BrowsePageKey(genre, sortBy, status, novelType string, page int) string { + return fmt.Sprintf("%s/%s/%s/%s/page-%d.html", genre, sortBy, status, novelType, page) +} + +// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket. +func (m *MinioClient) PutBrowsePage(ctx context.Context, key, html string) error { + data := []byte(html) + _, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "text/html; charset=utf-8"}) + if err != nil { + return fmt.Errorf("minio: put browse page %s: %w", key, err) + } + return nil +} + +// GetBrowsePage retrieves a SingleFile HTML snapshot from the browse bucket. +// Returns ("", false, nil) when the object does not exist. +func (m *MinioClient) GetBrowsePage(ctx context.Context, key string) (string, bool, error) { + obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{}) + if err != nil { + return "", false, fmt.Errorf("minio: get browse page %s: %w", key, err) + } + defer obj.Close() + // Check whether the object actually exists by inspecting the Stat. + if _, statErr := obj.Stat(); statErr != nil { + return "", false, nil // not found + } + data, err := io.ReadAll(obj) + if err != nil { + return "", false, fmt.Errorf("minio: read browse page %s: %w", key, err) + } + return string(data), true, nil +} + +// BrowsePageExists returns true if a snapshot object is present in the browse bucket. +func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool { + _, err := m.c.StatObject(ctx, m.cfg.BucketBrowse, key, minio.StatObjectOptions{}) + return err == nil +} + // ─── helpers ────────────────────────────────────────────────────────────────── // sanitiseVoice converts a voice name to a filename-safe string. diff --git a/scraper/internal/storage/store.go b/scraper/internal/storage/store.go index d2360df..d80860e 100644 --- a/scraper/internal/storage/store.go +++ b/scraper/internal/storage/store.go @@ -139,6 +139,16 @@ type Store interface { // PresignAudio returns a presigned GET URL for an audio object. PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) + // ── Browse page snapshots (MinIO) ────────────────────────────────────── + + // SaveBrowsePage stores a SingleFile HTML snapshot for the given cache key. + SaveBrowsePage(ctx context.Context, key, html string) error + // GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil) + // when no snapshot exists for the key. + GetBrowsePage(ctx context.Context, key string) (string, bool, error) + // BrowsePageKey returns the MinIO object key for the given browse params. + BrowsePageKey(genre, sortBy, status, novelType string, page int) string + // ── Scraping tasks ───────────────────────────────────────────────────── // CreateScrapeTask inserts a new scraping_tasks record with status="running" diff --git a/scraper/scraper b/scraper/scraper deleted file mode 100755 index 9e8e1a6..0000000 Binary files a/scraper/scraper and /dev/null differ