kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

fix(browse): replace SingleFile with direct Go HTTP fetch; fix ranking schema and cache-hit ranking gap

Browse Source 
- Remove SingleFile CLI and Node.js from Dockerfile; runtime base reverted to alpine:3.21
- Replace triggerBrowseSnapshot with triggerDirectScrape: plain Go HTTP GET to novelfire.net
  (page is server-rendered, no browser needed)
- Fix Accept-Encoding bug: stop setting header manually so Go transport auto-decompresses gzip
- Add in-memory browse cache fallback (browseMemCache) for when MinIO and upstream both fail
- Add warmBrowseCache() startup goroutine
- Call triggerDirectScrape on MinIO cache hits too, so PocketBase ranking records are
  repopulated after a schema reset or fresh deploy without waiting for a cache miss
- Fix grid view cards: wrap in <a> instead of <div> so clicking navigates to book detail
- Remove SINGLEFILE_PATH and BROWSERLESS_URL env vars from Dockerfile
This commit is contained in:
Admin
2026-03-04 21:13:59 +05:00
parent b0547c1b43
commit 7b48707cd9
3 changed files with 127 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
scraper/internal/server/server.go
View File

@@ -25,8 +25,6 @@ import (
"io"
"log/slog"
"net/http"
"os"
"os/exec"
"strconv"
"strings"
"sync"
@@ -50,10 +48,6 @@ type Server struct {
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
kokoroVoice string // default voice, e.g. af_bella
// SingleFile CLI settings for browse-page snapshots.
singleFilePath string // path to single-file binary, e.g. /usr/local/bin/single-file
browserlessURL string // Browserless base URL, e.g. http://browserless:3000
// voiceMu guards cachedVoices.
voiceMu sync.RWMutex
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
@@ -64,10 +58,23 @@ type Server struct {
audioMu sync.Mutex
audioInFlight map[string]chan struct{} // cacheKey → closed when done
// browseMu guards browseInFlight — keys of MinIO objects currently being
// captured by a background SingleFile goroutine.
// browseMu guards browseInFlight — keys currently being refreshed
// in the background.
browseMu sync.Mutex
browseInFlight map[string]struct{}
// browseMemCache is a short-lived in-process cache for browse results.
// It is populated whenever a live upstream fetch succeeds and used as a
// last-resort fallback when both MinIO and the upstream are unavailable.
// Key: the MinIO cache key (same as used for BrowseHTMLKey).
browseMemCacheMu sync.RWMutex
browseMemCache map[string]browseCacheEntry
}
type browseCacheEntry struct {
novels []NovelListing
hasNext bool
cachedAt time.Time
}
// New creates a new Server.
@@ -80,10 +87,9 @@ func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log
store: store,
kokoroURL: kokoroURL,
kokoroVoice: kokoroVoice,
singleFilePath: os.Getenv("SINGLEFILE_PATH"),
browserlessURL: os.Getenv("BROWSERLESS_URL"),
audioInFlight: make(map[string]chan struct{}),
browseInFlight: make(map[string]struct{}),
browseMemCache: make(map[string]browseCacheEntry),
}
}
@@ -192,6 +198,11 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
// has playable previews without requiring a manual trigger.
go s.warmVoiceSamples(ctx)
// Warm the browse cache on startup: if page 1 is not cached in MinIO yet,
// trigger a background SingleFile snapshot immediately so the first user
// request is served from cache rather than hitting novelfire.net live.
go s.warmBrowseCache()
select {
case <-ctx.Done():
shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
@@ -222,7 +233,7 @@ func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
items = []storage.RankingItem{}
}
// Rewrite cover keys to proxy URLs.
// Keys stored by triggerBrowseSnapshot look like:
// Keys stored by triggerDirectScrape look like:
// "novelfire.net/assets/book-covers/shadow-slave.jpg"
// We expose them as:
// "/api/cover/novelfire.net/shadow-slave"
@@ -968,6 +979,11 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 {
novels, hasNext := parseBrowsePage(strings.NewReader(html))
s.log.Debug("browse: served from cache", "key", cacheKey)
// Still fire background ranking population in case PocketBase ranking
// records are missing (e.g. after a schema reset / fresh deploy).
targetURLForRanking := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
novelFireBase, genre, sortBy, status, novelType, page)
s.triggerDirectScrape(cacheKey, targetURLForRanking)
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "public, max-age=300")
_ = json.NewEncoder(w).Encode(map[string]interface{}{
@@ -1005,7 +1021,10 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
req.Header.Set("Accept-Encoding", "gzip, deflate, br")
// Do NOT set Accept-Encoding manually: Go's http.Transport handles
// transparent gzip decompression only when it adds the header itself.
// If we set it explicitly, Transport disables auto-decompression and
// parseBrowsePage receives raw gzip bytes instead of HTML.
req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("Pragma", "no-cache")
@@ -1031,14 +1050,41 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
}
if fetchErr != nil {
s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr)
// ── In-memory fallback: use cached result from a prior successful fetch ──
s.browseMemCacheMu.RLock()
entry, memHit := s.browseMemCache[cacheKey]
s.browseMemCacheMu.RUnlock()
if memHit {
s.log.Warn("browse: upstream unavailable, serving stale in-memory cache",
"key", cacheKey, "age", time.Since(entry.cachedAt).Round(time.Second))
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "public, max-age=60")
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"novels": entry.novels,
"page": pageNum,
"hasNext": entry.hasNext,
})
return
}
http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway)
return
}
// ── Background: populate MinIO cache via SingleFile ───────────────────
// Fire-and-forget: capture the JS-rendered page with SingleFile, store
// it in MinIO, then parse it to populate the ranking collection.
s.triggerBrowseSnapshot(cacheKey, targetURL)
// ── Populate in-memory cache with the fresh upstream result ──────────
if len(novels) > 0 {
s.browseMemCacheMu.Lock()
s.browseMemCache[cacheKey] = browseCacheEntry{
novels: novels,
hasNext: hasNext,
cachedAt: time.Now(),
}
s.browseMemCacheMu.Unlock()
}
// ── Background: fetch and cache page directly from novelfire.net ─────
// Fire-and-forget: stores raw HTML in MinIO and populates the ranking
// collection in PocketBase (no browser/SingleFile needed).
s.triggerDirectScrape(cacheKey, targetURL)
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "public, max-age=300")
@@ -1049,25 +1095,20 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
})
}
// triggerBrowseSnapshot fires a background goroutine that:
// 1. Runs SingleFile CLI to capture the fully-rendered novelfire browse page
// and stores the self-contained HTML at {domain}/html/page-N.html in MinIO.
// 2. Parses the stored HTML to extract novel listings.
// 3. For each listing, upserts a ranking record in PocketBase (rank, slug,
// triggerDirectScrape fires a background goroutine that:
// 1. Fetches pageURL directly from novelfire.net using Go's HTTP client
// (no browser/SingleFile needed — the page is server-rendered HTML).
// 2. Stores the raw HTML in MinIO at cacheKey so future requests are served
// from cache without hitting the origin.
// 3. Parses the HTML to extract novel listings.
// 4. For each listing, upserts a ranking record in PocketBase (rank, slug,
// title, cover key, source_url).
// 4. Fires a separate goroutine per cover image to download and store it at
// 5. Fires a separate goroutine per cover image to download and store it at
// {domain}/assets/book-covers/{slug}.jpg in MinIO.
//
// It is a no-op when:
// - SINGLEFILE_PATH is not set (SingleFile not installed)
// - a capture for this cache key is already in progress
//
// It is a no-op when a refresh for this cache key is already in progress.
// The goroutine uses a fresh context so it outlives the HTTP request.
func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
if s.singleFilePath == "" {
return
}
func (s *Server) triggerDirectScrape(cacheKey, pageURL string) {
s.browseMu.Lock()
if _, inflight := s.browseInFlight[cacheKey]; inflight {
s.browseMu.Unlock()
@@ -1083,64 +1124,56 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
s.browseMu.Unlock()
}()
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Convert http(s) → ws(s) for the SingleFile --browser-server flag.
wsEndpoint := s.browserlessURL
wsEndpoint = strings.Replace(wsEndpoint, "http://", "ws://", 1)
wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)
tmpFile, err := os.CreateTemp("", "libnovel-browse-*.html")
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
if err != nil {
s.log.Warn("triggerBrowseSnapshot: create temp file failed", "key", cacheKey, "err", err)
s.log.Warn("triggerDirectScrape: build request failed", "key", cacheKey, "err", err)
return
}
tmpPath := tmpFile.Name()
tmpFile.Close()
defer os.Remove(tmpPath)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
//nolint:gosec
cmd := exec.CommandContext(ctx, s.singleFilePath,
pageURL,
"--browser-server="+wsEndpoint,
"--output="+tmpPath,
)
if out, runErr := cmd.CombinedOutput(); runErr != nil {
s.log.Warn("triggerBrowseSnapshot: SingleFile failed",
"key", cacheKey, "err", runErr, "output", string(out))
resp, err := http.DefaultClient.Do(req)
if err != nil {
s.log.Warn("triggerDirectScrape: fetch failed", "key", cacheKey, "err", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
s.log.Warn("triggerDirectScrape: non-200 response", "key", cacheKey, "status", resp.StatusCode)
return
}
htmlBytes, readErr := os.ReadFile(tmpPath)
htmlBytes, readErr := io.ReadAll(resp.Body)
if readErr != nil {
s.log.Warn("triggerBrowseSnapshot: read output failed",
"key", cacheKey, "err", readErr)
s.log.Warn("triggerDirectScrape: read body failed", "key", cacheKey, "err", readErr)
return
}
if len(htmlBytes) == 0 {
s.log.Warn("triggerBrowseSnapshot: SingleFile produced empty output, skipping cache",
"key", cacheKey)
s.log.Warn("triggerDirectScrape: empty response body", "key", cacheKey)
return
}
// Store the HTML snapshot.
// Store the HTML in MinIO so subsequent requests are cache-hits.
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
"key", cacheKey, "err", putErr)
return
s.log.Warn("triggerDirectScrape: SaveBrowsePage failed", "key", cacheKey, "err", putErr)
// Non-fatal: continue to populate PocketBase/covers even if MinIO write fails.
} else {
s.log.Info("triggerDirectScrape: cached browse page", "key", cacheKey, "bytes", len(htmlBytes))
}
s.log.Info("triggerBrowseSnapshot: cached browse page",
"key", cacheKey, "bytes", len(htmlBytes))
// Parse the stored HTML to extract novel listings.
// Parse to extract novel listings.
novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
if len(novels) == 0 {
s.log.Warn("triggerDirectScrape: no novels parsed", "key", cacheKey)
return
}
// Upsert each novel into the ranking PocketBase collection and
// kick off a background cover download.
// Upsert each novel into PocketBase ranking and kick off cover downloads.
for i, novel := range novels {
rank := i + 1
coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
@@ -1153,21 +1186,37 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
SourceURL: novel.URL,
}
if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
s.log.Warn("triggerBrowseSnapshot: WriteRankingItem failed",
s.log.Warn("triggerDirectScrape: WriteRankingItem failed",
"slug", novel.Slug, "err", werr)
}
// Download and store the cover image in a separate goroutine.
coverURL := novel.Cover
if coverURL != "" {
go s.downloadAndStoreCover(coverKey, coverURL)
if novel.Cover != "" {
go s.downloadAndStoreCover(coverKey, novel.Cover)
}
}
s.log.Info("triggerBrowseSnapshot: ranking populated", "count", len(novels), "key", cacheKey)
s.log.Info("triggerDirectScrape: ranking populated", "count", len(novels), "key", cacheKey)
}()
}
// warmBrowseCache checks whether the browse cache for page 1 is populated in
// MinIO and, if not, triggers a background direct scrape. This is called
// once on server startup so the first user request is likely served from cache.
func (s *Server) warmBrowseCache() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
cacheKey := s.store.BrowseHTMLKey(novelFireDomain, 1)
if _, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
s.log.Debug("warmBrowseCache: page 1 already cached, skipping")
return
}
targetURL := fmt.Sprintf("%s/genre-all/sort-popular/status-all/all-novel?page=1", novelFireBase)
s.log.Info("warmBrowseCache: page 1 not cached, triggering background scrape")
s.triggerDirectScrape(cacheKey, targetURL)
}
// downloadAndStoreCover fetches a cover image URL and stores it in MinIO under
// the given key. Errors are logged but not propagated — this is best-effort.
func (s *Server) downloadAndStoreCover(key, imageURL string) {