diff --git a/scraper/Dockerfile b/scraper/Dockerfile index 20c222b..561c518 100644 --- a/scraper/Dockerfile +++ b/scraper/Dockerfile @@ -13,20 +13,12 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ go build -ldflags="-s -w" -o /scraper ./cmd/scraper # ── Runtime stage ────────────────────────────────────────────────────────────── -# Use node:22-alpine so single-file-cli (npm package) runs natively without -# any glibc shims. The pre-compiled binary release requires glibc symbols -# (e.g. __res_init) that Alpine's gcompat shim does not provide. -FROM node:22-alpine +FROM alpine:3.21 # ca-certificates: HTTPS to novelfire.net # tzdata: timezone data RUN apk add --no-cache ca-certificates tzdata -# Install single-file-cli as a global npm package. -# It runs via Node.js (no Deno/glibc needed) and connects to an external -# Chromium via the CDP --browser-server flag. -RUN npm install -g single-file-cli - WORKDIR /app COPY --from=builder /scraper /app/scraper @@ -40,12 +32,9 @@ RUN chown -R scraper:scraper /app USER scraper # ── Configuration ───────────────────────────────────────────────────────────── -ENV BROWSERLESS_URL=http://browserless:3030 -ENV BROWSERLESS_STRATEGY=content ENV SCRAPER_WORKERS=0 ENV SCRAPER_STATIC_ROOT=/app/static/books ENV SCRAPER_HTTP_ADDR=:8080 -ENV SINGLEFILE_PATH=single-file EXPOSE 8080 diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index 946b804..322baaf 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -25,8 +25,6 @@ import ( "io" "log/slog" "net/http" - "os" - "os/exec" "strconv" "strings" "sync" @@ -50,10 +48,6 @@ type Server struct { kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880 kokoroVoice string // default voice, e.g. af_bella - // SingleFile CLI settings for browse-page snapshots. - singleFilePath string // path to single-file binary, e.g. /usr/local/bin/single-file - browserlessURL string // Browserless base URL, e.g. http://browserless:3000 - // voiceMu guards cachedVoices. voiceMu sync.RWMutex cachedVoices []string // populated on first request from Kokoro /v1/audio/voices @@ -64,10 +58,23 @@ type Server struct { audioMu sync.Mutex audioInFlight map[string]chan struct{} // cacheKey → closed when done - // browseMu guards browseInFlight — keys of MinIO objects currently being - // captured by a background SingleFile goroutine. + // browseMu guards browseInFlight — keys currently being refreshed + // in the background. browseMu sync.Mutex browseInFlight map[string]struct{} + + // browseMemCache is a short-lived in-process cache for browse results. + // It is populated whenever a live upstream fetch succeeds and used as a + // last-resort fallback when both MinIO and the upstream are unavailable. + // Key: the MinIO cache key (same as used for BrowseHTMLKey). + browseMemCacheMu sync.RWMutex + browseMemCache map[string]browseCacheEntry +} + +type browseCacheEntry struct { + novels []NovelListing + hasNext bool + cachedAt time.Time } // New creates a new Server. @@ -80,10 +87,9 @@ func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log store: store, kokoroURL: kokoroURL, kokoroVoice: kokoroVoice, - singleFilePath: os.Getenv("SINGLEFILE_PATH"), - browserlessURL: os.Getenv("BROWSERLESS_URL"), audioInFlight: make(map[string]chan struct{}), browseInFlight: make(map[string]struct{}), + browseMemCache: make(map[string]browseCacheEntry), } } @@ -192,6 +198,11 @@ func (s *Server) ListenAndServe(ctx context.Context) error { // has playable previews without requiring a manual trigger. go s.warmVoiceSamples(ctx) + // Warm the browse cache on startup: if page 1 is not cached in MinIO yet, + // trigger a background SingleFile snapshot immediately so the first user + // request is served from cache rather than hitting novelfire.net live. + go s.warmBrowseCache() + select { case <-ctx.Done(): shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -222,7 +233,7 @@ func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) { items = []storage.RankingItem{} } // Rewrite cover keys to proxy URLs. - // Keys stored by triggerBrowseSnapshot look like: + // Keys stored by triggerDirectScrape look like: // "novelfire.net/assets/book-covers/shadow-slave.jpg" // We expose them as: // "/api/cover/novelfire.net/shadow-slave" @@ -968,6 +979,11 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 { novels, hasNext := parseBrowsePage(strings.NewReader(html)) s.log.Debug("browse: served from cache", "key", cacheKey) + // Still fire background ranking population in case PocketBase ranking + // records are missing (e.g. after a schema reset / fresh deploy). + targetURLForRanking := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s", + novelFireBase, genre, sortBy, status, novelType, page) + s.triggerDirectScrape(cacheKey, targetURLForRanking) w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "public, max-age=300") _ = json.NewEncoder(w).Encode(map[string]interface{}{ @@ -1005,7 +1021,10 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.9") - req.Header.Set("Accept-Encoding", "gzip, deflate, br") + // Do NOT set Accept-Encoding manually: Go's http.Transport handles + // transparent gzip decompression only when it adds the header itself. + // If we set it explicitly, Transport disables auto-decompression and + // parseBrowsePage receives raw gzip bytes instead of HTML. req.Header.Set("Cache-Control", "no-cache") req.Header.Set("Pragma", "no-cache") @@ -1031,14 +1050,41 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { } if fetchErr != nil { s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr) + // ── In-memory fallback: use cached result from a prior successful fetch ── + s.browseMemCacheMu.RLock() + entry, memHit := s.browseMemCache[cacheKey] + s.browseMemCacheMu.RUnlock() + if memHit { + s.log.Warn("browse: upstream unavailable, serving stale in-memory cache", + "key", cacheKey, "age", time.Since(entry.cachedAt).Round(time.Second)) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "public, max-age=60") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "novels": entry.novels, + "page": pageNum, + "hasNext": entry.hasNext, + }) + return + } http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway) return } - // ── Background: populate MinIO cache via SingleFile ─────────────────── - // Fire-and-forget: capture the JS-rendered page with SingleFile, store - // it in MinIO, then parse it to populate the ranking collection. - s.triggerBrowseSnapshot(cacheKey, targetURL) + // ── Populate in-memory cache with the fresh upstream result ────────── + if len(novels) > 0 { + s.browseMemCacheMu.Lock() + s.browseMemCache[cacheKey] = browseCacheEntry{ + novels: novels, + hasNext: hasNext, + cachedAt: time.Now(), + } + s.browseMemCacheMu.Unlock() + } + + // ── Background: fetch and cache page directly from novelfire.net ───── + // Fire-and-forget: stores raw HTML in MinIO and populates the ranking + // collection in PocketBase (no browser/SingleFile needed). + s.triggerDirectScrape(cacheKey, targetURL) w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "public, max-age=300") @@ -1049,25 +1095,20 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { }) } -// triggerBrowseSnapshot fires a background goroutine that: -// 1. Runs SingleFile CLI to capture the fully-rendered novelfire browse page -// and stores the self-contained HTML at {domain}/html/page-N.html in MinIO. -// 2. Parses the stored HTML to extract novel listings. -// 3. For each listing, upserts a ranking record in PocketBase (rank, slug, +// triggerDirectScrape fires a background goroutine that: +// 1. Fetches pageURL directly from novelfire.net using Go's HTTP client +// (no browser/SingleFile needed — the page is server-rendered HTML). +// 2. Stores the raw HTML in MinIO at cacheKey so future requests are served +// from cache without hitting the origin. +// 3. Parses the HTML to extract novel listings. +// 4. For each listing, upserts a ranking record in PocketBase (rank, slug, // title, cover key, source_url). -// 4. Fires a separate goroutine per cover image to download and store it at +// 5. Fires a separate goroutine per cover image to download and store it at // {domain}/assets/book-covers/{slug}.jpg in MinIO. // -// It is a no-op when: -// - SINGLEFILE_PATH is not set (SingleFile not installed) -// - a capture for this cache key is already in progress -// +// It is a no-op when a refresh for this cache key is already in progress. // The goroutine uses a fresh context so it outlives the HTTP request. -func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) { - if s.singleFilePath == "" { - return - } - +func (s *Server) triggerDirectScrape(cacheKey, pageURL string) { s.browseMu.Lock() if _, inflight := s.browseInFlight[cacheKey]; inflight { s.browseMu.Unlock() @@ -1083,64 +1124,56 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) { s.browseMu.Unlock() }() - ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - // Convert http(s) → ws(s) for the SingleFile --browser-server flag. - wsEndpoint := s.browserlessURL - wsEndpoint = strings.Replace(wsEndpoint, "http://", "ws://", 1) - wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1) - - tmpFile, err := os.CreateTemp("", "libnovel-browse-*.html") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) if err != nil { - s.log.Warn("triggerBrowseSnapshot: create temp file failed", "key", cacheKey, "err", err) + s.log.Warn("triggerDirectScrape: build request failed", "key", cacheKey, "err", err) return } - tmpPath := tmpFile.Name() - tmpFile.Close() - defer os.Remove(tmpPath) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") - //nolint:gosec - cmd := exec.CommandContext(ctx, s.singleFilePath, - pageURL, - "--browser-server="+wsEndpoint, - "--output="+tmpPath, - ) - if out, runErr := cmd.CombinedOutput(); runErr != nil { - s.log.Warn("triggerBrowseSnapshot: SingleFile failed", - "key", cacheKey, "err", runErr, "output", string(out)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + s.log.Warn("triggerDirectScrape: fetch failed", "key", cacheKey, "err", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + s.log.Warn("triggerDirectScrape: non-200 response", "key", cacheKey, "status", resp.StatusCode) return } - htmlBytes, readErr := os.ReadFile(tmpPath) + htmlBytes, readErr := io.ReadAll(resp.Body) if readErr != nil { - s.log.Warn("triggerBrowseSnapshot: read output failed", - "key", cacheKey, "err", readErr) + s.log.Warn("triggerDirectScrape: read body failed", "key", cacheKey, "err", readErr) return } if len(htmlBytes) == 0 { - s.log.Warn("triggerBrowseSnapshot: SingleFile produced empty output, skipping cache", - "key", cacheKey) + s.log.Warn("triggerDirectScrape: empty response body", "key", cacheKey) return } - // Store the HTML snapshot. + // Store the HTML in MinIO so subsequent requests are cache-hits. if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil { - s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed", - "key", cacheKey, "err", putErr) - return + s.log.Warn("triggerDirectScrape: SaveBrowsePage failed", "key", cacheKey, "err", putErr) + // Non-fatal: continue to populate PocketBase/covers even if MinIO write fails. + } else { + s.log.Info("triggerDirectScrape: cached browse page", "key", cacheKey, "bytes", len(htmlBytes)) } - s.log.Info("triggerBrowseSnapshot: cached browse page", - "key", cacheKey, "bytes", len(htmlBytes)) - // Parse the stored HTML to extract novel listings. + // Parse to extract novel listings. novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes))) if len(novels) == 0 { + s.log.Warn("triggerDirectScrape: no novels parsed", "key", cacheKey) return } - // Upsert each novel into the ranking PocketBase collection and - // kick off a background cover download. + // Upsert each novel into PocketBase ranking and kick off cover downloads. for i, novel := range novels { rank := i + 1 coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug) @@ -1153,21 +1186,37 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) { SourceURL: novel.URL, } if werr := s.store.WriteRankingItem(ctx, item); werr != nil { - s.log.Warn("triggerBrowseSnapshot: WriteRankingItem failed", + s.log.Warn("triggerDirectScrape: WriteRankingItem failed", "slug", novel.Slug, "err", werr) } - // Download and store the cover image in a separate goroutine. - coverURL := novel.Cover - if coverURL != "" { - go s.downloadAndStoreCover(coverKey, coverURL) + if novel.Cover != "" { + go s.downloadAndStoreCover(coverKey, novel.Cover) } } - s.log.Info("triggerBrowseSnapshot: ranking populated", "count", len(novels), "key", cacheKey) + s.log.Info("triggerDirectScrape: ranking populated", "count", len(novels), "key", cacheKey) }() } +// warmBrowseCache checks whether the browse cache for page 1 is populated in +// MinIO and, if not, triggers a background direct scrape. This is called +// once on server startup so the first user request is likely served from cache. +func (s *Server) warmBrowseCache() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + cacheKey := s.store.BrowseHTMLKey(novelFireDomain, 1) + if _, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok { + s.log.Debug("warmBrowseCache: page 1 already cached, skipping") + return + } + + targetURL := fmt.Sprintf("%s/genre-all/sort-popular/status-all/all-novel?page=1", novelFireBase) + s.log.Info("warmBrowseCache: page 1 not cached, triggering background scrape") + s.triggerDirectScrape(cacheKey, targetURL) +} + // downloadAndStoreCover fetches a cover image URL and stores it in MinIO under // the given key. Errors are logged but not propagated — this is best-effort. func (s *Server) downloadAndStoreCover(key, imageURL string) { diff --git a/ui/src/routes/browse/+page.svelte b/ui/src/routes/browse/+page.svelte index 104f3ef..51210dc 100644 --- a/ui/src/routes/browse/+page.svelte +++ b/ui/src/routes/browse/+page.svelte @@ -250,7 +250,10 @@
{#each data.novels as novel} -
+
{#if novel.cover} @@ -302,7 +305,7 @@ Error {:else}
{/if}
-
+ {/each}