diff --git a/scraper/cmd/scraper/main.go b/scraper/cmd/scraper/main.go
index 6465923..cf2ed18 100644
--- a/scraper/cmd/scraper/main.go
+++ b/scraper/cmd/scraper/main.go
@@ -33,7 +33,9 @@ package main
import (
"context"
"fmt"
+ "io"
"log/slog"
+ "net/http"
"os"
"os/exec"
"os/signal"
@@ -212,6 +214,8 @@ func run(log *slog.Logger) error {
// It iterates over browse pages on novelfire.net, captures each using
// SingleFile CLI (connected to the existing Browserless instance), and
// stores the resulting self-contained HTML in the MinIO browse bucket.
+// After storing each page it parses the HTML, upserts ranking records in
+// PocketBase, and fires background goroutines to download cover images.
//
// Flags (all optional):
//
@@ -281,6 +285,7 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
defer os.RemoveAll(tmpDir)
const novelFireBase = "https://novelfire.net"
+ const novelFireDomain = "novelfire.net"
for page := 1; page <= maxPages; page++ {
select {
@@ -292,7 +297,8 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
novelFireBase, genre, sortBy, status, novelType, page)
- key := store.BrowsePageKey(genre, sortBy, status, novelType, page)
+ // Use the new domain-based key layout: {domain}/html/page-{n}.html
+ key := store.BrowseHTMLKey(novelFireDomain, page)
outFile := fmt.Sprintf("%s/page-%d.html", tmpDir, page)
@@ -328,12 +334,205 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
log.Info("save-browse: snapshot stored", "page", page, "key", key,
"bytes", len(htmlBytes))
+
+ // Parse the stored HTML and populate the ranking collection.
+ novels := parseSaveBrowseListings(htmlBytes, novelFireBase)
+ for i, novel := range novels {
+ rank := i + 1
+ coverKey := store.BrowseCoverKey(novelFireDomain, novel.slug)
+
+ item := storage.RankingItem{
+ Rank: rank,
+ Slug: novel.slug,
+ Title: novel.title,
+ Cover: coverKey,
+ SourceURL: novel.url,
+ }
+ if werr := store.WriteRankingItem(ctx, item); werr != nil {
+ log.Warn("save-browse: WriteRankingItem failed",
+ "slug", novel.slug, "err", werr)
+ }
+
+ // Download cover image in the background (best-effort).
+ if novel.coverURL != "" {
+ go downloadAndStoreCoverCLI(store, log, coverKey, novel.coverURL)
+ }
+ }
+ if len(novels) > 0 {
+ log.Info("save-browse: ranking populated", "page", page, "count", len(novels))
+ }
}
log.Info("save-browse: done")
return nil
}
+// novelListingCLI is a minimal novel listing used within the CLI command.
+type novelListingCLI struct {
+ slug string
+ title string
+ url string
+ coverURL string
+}
+
+// parseSaveBrowseListings extracts novel listings from raw HTML bytes.
+// It reuses the same parsing logic as the server's parseBrowsePage but
+// operates on []byte to avoid importing the server package.
+func parseSaveBrowseListings(htmlBytes []byte, novelFireBase string) []novelListingCLI {
+ type listing = novelListingCLI
+
+ // Minimal tokeniser-based walk to find blocks.
+ // We use the golang.org/x/net/html parser via a local import.
+ // Because main.go already imports golang.org/x/net/html indirectly through
+ // the server package build, we do a simple line-scan here instead to keep
+ // the dependency surface small.
+ //
+ // Strategy: scan for href="/book/{slug}", img data-src/src, h4.novel-title text.
+ var novels []listing
+
+ lines := strings.Split(string(htmlBytes), "\n")
+ var cur listing
+ inNovelItem := false
+
+ for _, line := range lines {
+ trimmed := strings.TrimSpace(line)
+
+ // Detect start of a novel-item list element.
+ if strings.Contains(trimmed, `class="novel-item"`) || strings.Contains(trimmed, "novel-item") && strings.HasPrefix(trimmed, "" && cur.slug != "" {
+ novels = append(novels, cur)
+ inNovelItem = false
+ cur = listing{}
+ continue
+ }
+
+ // Extract slug from href="/book/{slug}".
+ if cur.slug == "" {
+ if idx := strings.Index(trimmed, `href="/book/`); idx >= 0 {
+ rest := trimmed[idx+len(`href="/book/`):]
+ if end := strings.IndexAny(rest, `"/ `); end > 0 {
+ cur.slug = rest[:end]
+ cur.url = novelFireBase + "/book/" + cur.slug
+ } else if end := strings.Index(rest, `"`); end > 0 {
+ cur.slug = strings.TrimSuffix(rest[:end], "/")
+ cur.url = novelFireBase + "/book/" + cur.slug
+ }
+ }
+ }
+
+ // Extract cover URL from data-src or src on img tags.
+ if cur.coverURL == "" && strings.Contains(trimmed, "Title Here
+ if start := strings.Index(trimmed, ">"); start >= 0 {
+ rest := trimmed[start+1:]
+ if end := strings.Index(rest, "<"); end > 0 {
+ title := strings.TrimSpace(rest[:end])
+ if title != "" {
+ cur.title = title
+ }
+ }
+ }
+ }
+ }
+
+ // Flush any open item that wasn't closed by (e.g. last item in file).
+ if inNovelItem && cur.slug != "" {
+ novels = append(novels, cur)
+ }
+
+ return novels
+}
+
+// extractAttr extracts an HTML attribute value from a raw tag string.
+// e.g. extractAttr(`![]()
`, "data-src") → "foo.jpg"
+func extractAttr(tag, attr string) string {
+ needle := attr + `="`
+ idx := strings.Index(tag, needle)
+ if idx < 0 {
+ return ""
+ }
+ rest := tag[idx+len(needle):]
+ end := strings.Index(rest, `"`)
+ if end < 0 {
+ return ""
+ }
+ return rest[:end]
+}
+
+// resolveURL ensures the URL is absolute, prepending novelFireBase if needed.
+func resolveURL(src, base string) string {
+ if strings.HasPrefix(src, "http") {
+ return src
+ }
+ return base + src
+}
+
+// downloadAndStoreCoverCLI fetches a cover image and stores it in MinIO.
+// Errors are logged but not propagated — this is a best-effort background task.
+func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageURL string) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ // Skip if already stored.
+ if _, _, ok, _ := store.GetBrowseAsset(ctx, key); ok {
+ return
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
+ if err != nil {
+ log.Warn("save-browse: cover build request failed", "url", imageURL, "err", err)
+ return
+ }
+ req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ log.Warn("save-browse: cover fetch failed", "url", imageURL, "err", err)
+ return
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ log.Warn("save-browse: cover non-200", "url", imageURL, "status", resp.StatusCode)
+ return
+ }
+
+ data, readErr := io.ReadAll(resp.Body)
+ if readErr != nil {
+ log.Warn("save-browse: cover read body failed", "url", imageURL, "err", readErr)
+ return
+ }
+
+ contentType := resp.Header.Get("Content-Type")
+ if contentType == "" {
+ contentType = "image/jpeg"
+ }
+
+ if putErr := store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
+ log.Warn("save-browse: SaveBrowseAsset failed", "key", key, "err", putErr)
+ return
+ }
+ log.Debug("save-browse: cover stored", "key", key, "bytes", len(data))
+}
+
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
switch strategy {
case browser.StrategyScrape:
diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go
index 6f0d4d6..29c2611 100644
--- a/scraper/internal/server/server.go
+++ b/scraper/internal/server/server.go
@@ -137,6 +137,8 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /api/browse", s.handleBrowse)
// Ranking API
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
+ // Cover image proxy (serves images stored in browse MinIO bucket)
+ mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
// Scrape status
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
@@ -191,6 +193,9 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
}
// handleGetRanking returns all ranking items sorted by rank ascending.
+// Cover fields that hold a MinIO object key (e.g. "novelfire.net/assets/book-covers/slug.jpg")
+// are rewritten to a /api/cover/{key} proxy URL so the UI can fetch them
+// without knowing about the internal MinIO topology.
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
items, err := s.store.ReadRankingItems(r.Context())
if err != nil {
@@ -198,13 +203,70 @@ func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
return
}
- w.Header().Set("Content-Type", "application/json")
if items == nil {
items = []storage.RankingItem{}
}
+ // Rewrite cover keys to proxy URLs.
+ // Keys stored by triggerBrowseSnapshot look like:
+ // "novelfire.net/assets/book-covers/shadow-slave.jpg"
+ // We expose them as:
+ // "/api/cover/novelfire.net/shadow-slave"
+ // (the handler strips the domain and slug from the path, reconstructs the key)
+ for i := range items {
+ cover := items[i].Cover
+ if cover != "" && !strings.HasPrefix(cover, "http") {
+ // cover is a MinIO key; extract domain + slug for the proxy path.
+ // Key format: {domain}/assets/book-covers/{slug}.jpg
+ parts := strings.SplitN(cover, "/assets/book-covers/", 2)
+ if len(parts) == 2 {
+ domain := parts[0]
+ slug := strings.TrimSuffix(parts[1], ".jpg")
+ items[i].Cover = "/api/cover/" + domain + "/" + slug
+ }
+ }
+ }
+ w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(items)
}
+// handleGetCover proxies a cover image stored in the MinIO browse bucket.
+// Route: GET /api/cover/{domain}/{slug}
+// It reconstructs the MinIO key as {domain}/assets/book-covers/{slug}.jpg,
+// fetches the object, and streams it to the client.
+// Returns 404 if not yet downloaded, allowing the UI to fall back to the
+// original source URL.
+func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
+ domain := r.PathValue("domain")
+ slug := r.PathValue("slug")
+ if domain == "" || slug == "" {
+ http.Error(w, "missing domain or slug", http.StatusBadRequest)
+ return
+ }
+
+ key := s.store.BrowseCoverKey(domain, slug)
+
+ ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+ defer cancel()
+
+ data, contentType, ok, err := s.store.GetBrowseAsset(ctx, key)
+ if err != nil {
+ s.log.Warn("handleGetCover: GetBrowseAsset error", "key", key, "err", err)
+ http.Error(w, "storage error", http.StatusInternalServerError)
+ return
+ }
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+
+ if contentType == "" {
+ contentType = "image/jpeg"
+ }
+ w.Header().Set("Content-Type", contentType)
+ w.Header().Set("Cache-Control", "public, max-age=86400")
+ _, _ = w.Write(data)
+}
+
// ─── Session cookie helpers ───────────────────────────────────────────────────
const sessionCookieName = "libnovel_session"
@@ -818,6 +880,7 @@ type NovelListing struct {
}
const novelFireBase = "https://novelfire.net"
+const novelFireDomain = "novelfire.net"
// handleBrowse handles GET /api/browse.
// Query params:
@@ -830,8 +893,10 @@ const novelFireBase = "https://novelfire.net"
//
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
//
-// Cache strategy: check MinIO browse bucket first; if a snapshot exists,
-// parse and return it. Otherwise fetch live from novelfire.net.
+// Cache strategy: check MinIO browse bucket first (key: {domain}/html/page-N.html);
+// if a snapshot exists, parse it and return structured JSON.
+// On a cache miss, fetch live from novelfire.net, return the result, and
+// trigger a background SingleFile snapshot + ranking population.
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
page := q.Get("page")
@@ -863,8 +928,8 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
defer cancel()
- // ── Cache-first: try MinIO snapshot ──────────────────────────────────
- cacheKey := s.store.BrowsePageKey(genre, sortBy, status, novelType, pageNum)
+ // ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
+ cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
novels, hasNext := parseBrowsePage(strings.NewReader(html))
s.log.Debug("browse: served from cache", "key", cacheKey)
@@ -907,8 +972,8 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
novels, hasNext := parseBrowsePage(resp.Body)
// ── Background: populate MinIO cache via SingleFile ───────────────────
- // Fire-and-forget: capture the JS-rendered page with SingleFile and store
- // it in MinIO so the next request is served from cache.
+ // Fire-and-forget: capture the JS-rendered page with SingleFile, store
+ // it in MinIO, then parse it to populate the ranking collection.
s.triggerBrowseSnapshot(cacheKey, targetURL)
w.Header().Set("Content-Type", "application/json")
@@ -920,8 +985,15 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
})
}
-// triggerBrowseSnapshot fires a background goroutine that uses SingleFile CLI
-// to capture the fully-rendered novelfire browse page and store it in MinIO.
+// triggerBrowseSnapshot fires a background goroutine that:
+// 1. Runs SingleFile CLI to capture the fully-rendered novelfire browse page
+// and stores the self-contained HTML at {domain}/html/page-N.html in MinIO.
+// 2. Parses the stored HTML to extract novel listings.
+// 3. For each listing, upserts a ranking record in PocketBase (rank, slug,
+// title, cover key, source_url).
+// 4. Fires a separate goroutine per cover image to download and store it at
+// {domain}/assets/book-covers/{slug}.jpg in MinIO.
+//
// It is a no-op when:
// - SINGLEFILE_PATH is not set (SingleFile not installed)
// - a capture for this cache key is already in progress
@@ -983,17 +1055,98 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
return
}
+ // Store the HTML snapshot.
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
"key", cacheKey, "err", putErr)
return
}
-
s.log.Info("triggerBrowseSnapshot: cached browse page",
"key", cacheKey, "bytes", len(htmlBytes))
+
+ // Parse the stored HTML to extract novel listings.
+ novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
+ if len(novels) == 0 {
+ return
+ }
+
+ // Upsert each novel into the ranking PocketBase collection and
+ // kick off a background cover download.
+ for i, novel := range novels {
+ rank := i + 1
+ coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
+
+ item := storage.RankingItem{
+ Rank: rank,
+ Slug: novel.Slug,
+ Title: novel.Title,
+ Cover: coverKey, // stored as MinIO key; UI fetches via /api/cover/...
+ SourceURL: novel.URL,
+ }
+ if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
+ s.log.Warn("triggerBrowseSnapshot: WriteRankingItem failed",
+ "slug", novel.Slug, "err", werr)
+ }
+
+ // Download and store the cover image in a separate goroutine.
+ coverURL := novel.Cover
+ if coverURL != "" {
+ go s.downloadAndStoreCover(coverKey, coverURL)
+ }
+ }
+
+ s.log.Info("triggerBrowseSnapshot: ranking populated", "count", len(novels), "key", cacheKey)
}()
}
+// downloadAndStoreCover fetches a cover image URL and stores it in MinIO under
+// the given key. Errors are logged but not propagated — this is best-effort.
+func (s *Server) downloadAndStoreCover(key, imageURL string) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ // Skip if already stored.
+ if _, _, ok, _ := s.store.GetBrowseAsset(ctx, key); ok {
+ return
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
+ if err != nil {
+ s.log.Warn("downloadAndStoreCover: build request failed", "url", imageURL, "err", err)
+ return
+ }
+ req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ s.log.Warn("downloadAndStoreCover: fetch failed", "url", imageURL, "err", err)
+ return
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ s.log.Warn("downloadAndStoreCover: non-200 response", "url", imageURL, "status", resp.StatusCode)
+ return
+ }
+
+ data, readErr := io.ReadAll(resp.Body)
+ if readErr != nil {
+ s.log.Warn("downloadAndStoreCover: read body failed", "url", imageURL, "err", readErr)
+ return
+ }
+
+ contentType := resp.Header.Get("Content-Type")
+ if contentType == "" {
+ contentType = "image/jpeg"
+ }
+
+ if putErr := s.store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
+ s.log.Warn("downloadAndStoreCover: SaveBrowseAsset failed", "key", key, "err", putErr)
+ return
+ }
+ s.log.Debug("downloadAndStoreCover: stored cover", "key", key, "bytes", len(data))
+}
+
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
// Returns novels and whether a "next page" link was found.
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
diff --git a/scraper/internal/storage/hybrid.go b/scraper/internal/storage/hybrid.go
index 01ac07d..f7d7345 100644
--- a/scraper/internal/storage/hybrid.go
+++ b/scraper/internal/storage/hybrid.go
@@ -317,8 +317,20 @@ func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bo
return h.minio.GetBrowsePage(ctx, key)
}
-func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
- return BrowsePageKey(genre, sortBy, status, novelType, page)
+func (h *HybridStore) BrowseHTMLKey(domain string, page int) string {
+ return BrowseHTMLKey(domain, page)
+}
+
+func (h *HybridStore) BrowseCoverKey(domain, slug string) string {
+ return BrowseCoverKey(domain, slug)
+}
+
+func (h *HybridStore) SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
+ return h.minio.PutBrowseAsset(ctx, key, data, contentType)
+}
+
+func (h *HybridStore) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
+ return h.minio.GetBrowseAsset(ctx, key)
}
// ─── Scraping tasks ───────────────────────────────────────────────────────────
diff --git a/scraper/internal/storage/minio.go b/scraper/internal/storage/minio.go
index cd137f4..c44e95f 100644
--- a/scraper/internal/storage/minio.go
+++ b/scraper/internal/storage/minio.go
@@ -211,11 +211,26 @@ func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time
}
// ─── Browse page snapshots ────────────────────────────────────────────────────
+//
+// New bucket layout (libnovel-browse):
+//
+// {domain}/html/page-{n}.html — SingleFile HTML snapshot
+// {domain}/assets/book-covers/{slug}.jpg — downloaded cover image
+//
+// The domain segment is derived from the source URL hostname
+// (e.g. "novelfire.net"). This makes the bucket self-describing and
+// extensible to multiple sources.
-// BrowsePageKey returns the MinIO object key for a cached browse-page snapshot.
-// Layout: {genre}/{sort}/{status}/{novelType}/page-{n}.html
-func BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
- return fmt.Sprintf("%s/%s/%s/%s/page-%d.html", genre, sortBy, status, novelType, page)
+// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
+// Layout: {domain}/html/page-{n}.html
+func BrowseHTMLKey(domain string, page int) string {
+ return fmt.Sprintf("%s/html/page-%d.html", domain, page)
+}
+
+// BrowseCoverKey returns the MinIO object key for a cached book cover image.
+// Layout: {domain}/assets/book-covers/{slug}.jpg
+func BrowseCoverKey(domain, slug string) string {
+ return fmt.Sprintf("%s/assets/book-covers/%s.jpg", domain, slug)
}
// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket.
@@ -255,6 +270,37 @@ func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool {
return err == nil
}
+// PutBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
+// contentType should be the MIME type, e.g. "image/jpeg".
+func (m *MinioClient) PutBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
+ _, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
+ bytes.NewReader(data), int64(len(data)),
+ minio.PutObjectOptions{ContentType: contentType})
+ if err != nil {
+ return fmt.Errorf("minio: put browse asset %s: %w", key, err)
+ }
+ return nil
+}
+
+// GetBrowseAsset retrieves a binary asset from the browse bucket.
+// Returns (nil, false, nil) when the object does not exist.
+func (m *MinioClient) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
+ obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
+ if err != nil {
+ return nil, "", false, fmt.Errorf("minio: get browse asset %s: %w", key, err)
+ }
+ defer obj.Close()
+ info, statErr := obj.Stat()
+ if statErr != nil {
+ return nil, "", false, nil // not found
+ }
+ data, err := io.ReadAll(obj)
+ if err != nil {
+ return nil, "", false, fmt.Errorf("minio: read browse asset %s: %w", key, err)
+ }
+ return data, info.ContentType, true, nil
+}
+
// ─── helpers ──────────────────────────────────────────────────────────────────
// sanitiseVoice converts a voice name to a filename-safe string.
diff --git a/scraper/internal/storage/store.go b/scraper/internal/storage/store.go
index d80860e..090bc65 100644
--- a/scraper/internal/storage/store.go
+++ b/scraper/internal/storage/store.go
@@ -146,8 +146,17 @@ type Store interface {
// GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil)
// when no snapshot exists for the key.
GetBrowsePage(ctx context.Context, key string) (string, bool, error)
- // BrowsePageKey returns the MinIO object key for the given browse params.
- BrowsePageKey(genre, sortBy, status, novelType string, page int) string
+ // BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
+ // Layout: {domain}/html/page-{n}.html
+ BrowseHTMLKey(domain string, page int) string
+ // BrowseCoverKey returns the MinIO object key for a cached book cover image.
+ // Layout: {domain}/assets/book-covers/{slug}.jpg
+ BrowseCoverKey(domain, slug string) string
+ // SaveBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
+ SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error
+ // GetBrowseAsset retrieves a binary asset from the browse bucket.
+ // Returns (nil, "", false, nil) when the object does not exist.
+ GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error)
// ── Scraping tasks ─────────────────────────────────────────────────────
`, "data-src") → "foo.jpg"
+func extractAttr(tag, attr string) string {
+ needle := attr + `="`
+ idx := strings.Index(tag, needle)
+ if idx < 0 {
+ return ""
+ }
+ rest := tag[idx+len(needle):]
+ end := strings.Index(rest, `"`)
+ if end < 0 {
+ return ""
+ }
+ return rest[:end]
+}
+
+// resolveURL ensures the URL is absolute, prepending novelFireBase if needed.
+func resolveURL(src, base string) string {
+ if strings.HasPrefix(src, "http") {
+ return src
+ }
+ return base + src
+}
+
+// downloadAndStoreCoverCLI fetches a cover image and stores it in MinIO.
+// Errors are logged but not propagated — this is a best-effort background task.
+func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageURL string) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ // Skip if already stored.
+ if _, _, ok, _ := store.GetBrowseAsset(ctx, key); ok {
+ return
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
+ if err != nil {
+ log.Warn("save-browse: cover build request failed", "url", imageURL, "err", err)
+ return
+ }
+ req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ log.Warn("save-browse: cover fetch failed", "url", imageURL, "err", err)
+ return
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ log.Warn("save-browse: cover non-200", "url", imageURL, "status", resp.StatusCode)
+ return
+ }
+
+ data, readErr := io.ReadAll(resp.Body)
+ if readErr != nil {
+ log.Warn("save-browse: cover read body failed", "url", imageURL, "err", readErr)
+ return
+ }
+
+ contentType := resp.Header.Get("Content-Type")
+ if contentType == "" {
+ contentType = "image/jpeg"
+ }
+
+ if putErr := store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
+ log.Warn("save-browse: SaveBrowseAsset failed", "key", key, "err", putErr)
+ return
+ }
+ log.Debug("save-browse: cover stored", "key", key, "bytes", len(data))
+}
+
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
switch strategy {
case browser.StrategyScrape:
diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go
index 6f0d4d6..29c2611 100644
--- a/scraper/internal/server/server.go
+++ b/scraper/internal/server/server.go
@@ -137,6 +137,8 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /api/browse", s.handleBrowse)
// Ranking API
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
+ // Cover image proxy (serves images stored in browse MinIO bucket)
+ mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
// Scrape status
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
@@ -191,6 +193,9 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
}
// handleGetRanking returns all ranking items sorted by rank ascending.
+// Cover fields that hold a MinIO object key (e.g. "novelfire.net/assets/book-covers/slug.jpg")
+// are rewritten to a /api/cover/{key} proxy URL so the UI can fetch them
+// without knowing about the internal MinIO topology.
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
items, err := s.store.ReadRankingItems(r.Context())
if err != nil {
@@ -198,13 +203,70 @@ func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
return
}
- w.Header().Set("Content-Type", "application/json")
if items == nil {
items = []storage.RankingItem{}
}
+ // Rewrite cover keys to proxy URLs.
+ // Keys stored by triggerBrowseSnapshot look like:
+ // "novelfire.net/assets/book-covers/shadow-slave.jpg"
+ // We expose them as:
+ // "/api/cover/novelfire.net/shadow-slave"
+ // (the handler strips the domain and slug from the path, reconstructs the key)
+ for i := range items {
+ cover := items[i].Cover
+ if cover != "" && !strings.HasPrefix(cover, "http") {
+ // cover is a MinIO key; extract domain + slug for the proxy path.
+ // Key format: {domain}/assets/book-covers/{slug}.jpg
+ parts := strings.SplitN(cover, "/assets/book-covers/", 2)
+ if len(parts) == 2 {
+ domain := parts[0]
+ slug := strings.TrimSuffix(parts[1], ".jpg")
+ items[i].Cover = "/api/cover/" + domain + "/" + slug
+ }
+ }
+ }
+ w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(items)
}
+// handleGetCover proxies a cover image stored in the MinIO browse bucket.
+// Route: GET /api/cover/{domain}/{slug}
+// It reconstructs the MinIO key as {domain}/assets/book-covers/{slug}.jpg,
+// fetches the object, and streams it to the client.
+// Returns 404 if not yet downloaded, allowing the UI to fall back to the
+// original source URL.
+func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
+ domain := r.PathValue("domain")
+ slug := r.PathValue("slug")
+ if domain == "" || slug == "" {
+ http.Error(w, "missing domain or slug", http.StatusBadRequest)
+ return
+ }
+
+ key := s.store.BrowseCoverKey(domain, slug)
+
+ ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+ defer cancel()
+
+ data, contentType, ok, err := s.store.GetBrowseAsset(ctx, key)
+ if err != nil {
+ s.log.Warn("handleGetCover: GetBrowseAsset error", "key", key, "err", err)
+ http.Error(w, "storage error", http.StatusInternalServerError)
+ return
+ }
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+
+ if contentType == "" {
+ contentType = "image/jpeg"
+ }
+ w.Header().Set("Content-Type", contentType)
+ w.Header().Set("Cache-Control", "public, max-age=86400")
+ _, _ = w.Write(data)
+}
+
// ─── Session cookie helpers ───────────────────────────────────────────────────
const sessionCookieName = "libnovel_session"
@@ -818,6 +880,7 @@ type NovelListing struct {
}
const novelFireBase = "https://novelfire.net"
+const novelFireDomain = "novelfire.net"
// handleBrowse handles GET /api/browse.
// Query params:
@@ -830,8 +893,10 @@ const novelFireBase = "https://novelfire.net"
//
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
//
-// Cache strategy: check MinIO browse bucket first; if a snapshot exists,
-// parse and return it. Otherwise fetch live from novelfire.net.
+// Cache strategy: check MinIO browse bucket first (key: {domain}/html/page-N.html);
+// if a snapshot exists, parse it and return structured JSON.
+// On a cache miss, fetch live from novelfire.net, return the result, and
+// trigger a background SingleFile snapshot + ranking population.
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
page := q.Get("page")
@@ -863,8 +928,8 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
defer cancel()
- // ── Cache-first: try MinIO snapshot ──────────────────────────────────
- cacheKey := s.store.BrowsePageKey(genre, sortBy, status, novelType, pageNum)
+ // ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
+ cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
novels, hasNext := parseBrowsePage(strings.NewReader(html))
s.log.Debug("browse: served from cache", "key", cacheKey)
@@ -907,8 +972,8 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
novels, hasNext := parseBrowsePage(resp.Body)
// ── Background: populate MinIO cache via SingleFile ───────────────────
- // Fire-and-forget: capture the JS-rendered page with SingleFile and store
- // it in MinIO so the next request is served from cache.
+ // Fire-and-forget: capture the JS-rendered page with SingleFile, store
+ // it in MinIO, then parse it to populate the ranking collection.
s.triggerBrowseSnapshot(cacheKey, targetURL)
w.Header().Set("Content-Type", "application/json")
@@ -920,8 +985,15 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
})
}
-// triggerBrowseSnapshot fires a background goroutine that uses SingleFile CLI
-// to capture the fully-rendered novelfire browse page and store it in MinIO.
+// triggerBrowseSnapshot fires a background goroutine that:
+// 1. Runs SingleFile CLI to capture the fully-rendered novelfire browse page
+// and stores the self-contained HTML at {domain}/html/page-N.html in MinIO.
+// 2. Parses the stored HTML to extract novel listings.
+// 3. For each listing, upserts a ranking record in PocketBase (rank, slug,
+// title, cover key, source_url).
+// 4. Fires a separate goroutine per cover image to download and store it at
+// {domain}/assets/book-covers/{slug}.jpg in MinIO.
+//
// It is a no-op when:
// - SINGLEFILE_PATH is not set (SingleFile not installed)
// - a capture for this cache key is already in progress
@@ -983,17 +1055,98 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
return
}
+ // Store the HTML snapshot.
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
"key", cacheKey, "err", putErr)
return
}
-
s.log.Info("triggerBrowseSnapshot: cached browse page",
"key", cacheKey, "bytes", len(htmlBytes))
+
+ // Parse the stored HTML to extract novel listings.
+ novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
+ if len(novels) == 0 {
+ return
+ }
+
+ // Upsert each novel into the ranking PocketBase collection and
+ // kick off a background cover download.
+ for i, novel := range novels {
+ rank := i + 1
+ coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
+
+ item := storage.RankingItem{
+ Rank: rank,
+ Slug: novel.Slug,
+ Title: novel.Title,
+ Cover: coverKey, // stored as MinIO key; UI fetches via /api/cover/...
+ SourceURL: novel.URL,
+ }
+ if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
+ s.log.Warn("triggerBrowseSnapshot: WriteRankingItem failed",
+ "slug", novel.Slug, "err", werr)
+ }
+
+ // Download and store the cover image in a separate goroutine.
+ coverURL := novel.Cover
+ if coverURL != "" {
+ go s.downloadAndStoreCover(coverKey, coverURL)
+ }
+ }
+
+ s.log.Info("triggerBrowseSnapshot: ranking populated", "count", len(novels), "key", cacheKey)
}()
}
+// downloadAndStoreCover fetches a cover image URL and stores it in MinIO under
+// the given key. Errors are logged but not propagated — this is best-effort.
+func (s *Server) downloadAndStoreCover(key, imageURL string) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ // Skip if already stored.
+ if _, _, ok, _ := s.store.GetBrowseAsset(ctx, key); ok {
+ return
+ }
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
+ if err != nil {
+ s.log.Warn("downloadAndStoreCover: build request failed", "url", imageURL, "err", err)
+ return
+ }
+ req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ s.log.Warn("downloadAndStoreCover: fetch failed", "url", imageURL, "err", err)
+ return
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ s.log.Warn("downloadAndStoreCover: non-200 response", "url", imageURL, "status", resp.StatusCode)
+ return
+ }
+
+ data, readErr := io.ReadAll(resp.Body)
+ if readErr != nil {
+ s.log.Warn("downloadAndStoreCover: read body failed", "url", imageURL, "err", readErr)
+ return
+ }
+
+ contentType := resp.Header.Get("Content-Type")
+ if contentType == "" {
+ contentType = "image/jpeg"
+ }
+
+ if putErr := s.store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
+ s.log.Warn("downloadAndStoreCover: SaveBrowseAsset failed", "key", key, "err", putErr)
+ return
+ }
+ s.log.Debug("downloadAndStoreCover: stored cover", "key", key, "bytes", len(data))
+}
+
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
// Returns novels and whether a "next page" link was found.
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
diff --git a/scraper/internal/storage/hybrid.go b/scraper/internal/storage/hybrid.go
index 01ac07d..f7d7345 100644
--- a/scraper/internal/storage/hybrid.go
+++ b/scraper/internal/storage/hybrid.go
@@ -317,8 +317,20 @@ func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bo
return h.minio.GetBrowsePage(ctx, key)
}
-func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
- return BrowsePageKey(genre, sortBy, status, novelType, page)
+func (h *HybridStore) BrowseHTMLKey(domain string, page int) string {
+ return BrowseHTMLKey(domain, page)
+}
+
+func (h *HybridStore) BrowseCoverKey(domain, slug string) string {
+ return BrowseCoverKey(domain, slug)
+}
+
+func (h *HybridStore) SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
+ return h.minio.PutBrowseAsset(ctx, key, data, contentType)
+}
+
+func (h *HybridStore) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
+ return h.minio.GetBrowseAsset(ctx, key)
}
// ─── Scraping tasks ───────────────────────────────────────────────────────────
diff --git a/scraper/internal/storage/minio.go b/scraper/internal/storage/minio.go
index cd137f4..c44e95f 100644
--- a/scraper/internal/storage/minio.go
+++ b/scraper/internal/storage/minio.go
@@ -211,11 +211,26 @@ func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time
}
// ─── Browse page snapshots ────────────────────────────────────────────────────
+//
+// New bucket layout (libnovel-browse):
+//
+// {domain}/html/page-{n}.html — SingleFile HTML snapshot
+// {domain}/assets/book-covers/{slug}.jpg — downloaded cover image
+//
+// The domain segment is derived from the source URL hostname
+// (e.g. "novelfire.net"). This makes the bucket self-describing and
+// extensible to multiple sources.
-// BrowsePageKey returns the MinIO object key for a cached browse-page snapshot.
-// Layout: {genre}/{sort}/{status}/{novelType}/page-{n}.html
-func BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
- return fmt.Sprintf("%s/%s/%s/%s/page-%d.html", genre, sortBy, status, novelType, page)
+// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
+// Layout: {domain}/html/page-{n}.html
+func BrowseHTMLKey(domain string, page int) string {
+ return fmt.Sprintf("%s/html/page-%d.html", domain, page)
+}
+
+// BrowseCoverKey returns the MinIO object key for a cached book cover image.
+// Layout: {domain}/assets/book-covers/{slug}.jpg
+func BrowseCoverKey(domain, slug string) string {
+ return fmt.Sprintf("%s/assets/book-covers/%s.jpg", domain, slug)
}
// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket.
@@ -255,6 +270,37 @@ func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool {
return err == nil
}
+// PutBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
+// contentType should be the MIME type, e.g. "image/jpeg".
+func (m *MinioClient) PutBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
+ _, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
+ bytes.NewReader(data), int64(len(data)),
+ minio.PutObjectOptions{ContentType: contentType})
+ if err != nil {
+ return fmt.Errorf("minio: put browse asset %s: %w", key, err)
+ }
+ return nil
+}
+
+// GetBrowseAsset retrieves a binary asset from the browse bucket.
+// Returns (nil, false, nil) when the object does not exist.
+func (m *MinioClient) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
+ obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
+ if err != nil {
+ return nil, "", false, fmt.Errorf("minio: get browse asset %s: %w", key, err)
+ }
+ defer obj.Close()
+ info, statErr := obj.Stat()
+ if statErr != nil {
+ return nil, "", false, nil // not found
+ }
+ data, err := io.ReadAll(obj)
+ if err != nil {
+ return nil, "", false, fmt.Errorf("minio: read browse asset %s: %w", key, err)
+ }
+ return data, info.ContentType, true, nil
+}
+
// ─── helpers ──────────────────────────────────────────────────────────────────
// sanitiseVoice converts a voice name to a filename-safe string.
diff --git a/scraper/internal/storage/store.go b/scraper/internal/storage/store.go
index d80860e..090bc65 100644
--- a/scraper/internal/storage/store.go
+++ b/scraper/internal/storage/store.go
@@ -146,8 +146,17 @@ type Store interface {
// GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil)
// when no snapshot exists for the key.
GetBrowsePage(ctx context.Context, key string) (string, bool, error)
- // BrowsePageKey returns the MinIO object key for the given browse params.
- BrowsePageKey(genre, sortBy, status, novelType string, page int) string
+ // BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
+ // Layout: {domain}/html/page-{n}.html
+ BrowseHTMLKey(domain string, page int) string
+ // BrowseCoverKey returns the MinIO object key for a cached book cover image.
+ // Layout: {domain}/assets/book-covers/{slug}.jpg
+ BrowseCoverKey(domain, slug string) string
+ // SaveBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
+ SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error
+ // GetBrowseAsset retrieves a binary asset from the browse bucket.
+ // Returns (nil, "", false, nil) when the object does not exist.
+ GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error)
// ── Scraping tasks ─────────────────────────────────────────────────────