kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

feat(scraper): rewrite browse storage to domain/html+assets structure, populate ranking from snapshot

Browse Source 
- Replace BrowsePageKey(genre/sort/status/type/page) with BrowseHTMLKey(domain, page) -> {domain}/html/page-{n}.html
- Add BrowseCoverKey(domain, slug) -> {domain}/assets/book-covers/{slug}.jpg
- Add SaveBrowseAsset/GetBrowseAsset for binary assets in browse bucket
- Rewrite triggerBrowseSnapshot: after storing HTML, parse it, upsert ranking records with MinIO cover keys, fire per-novel cover download goroutines
- Add handleGetCover endpoint (GET /api/cover/{domain}/{slug}) to proxy cover images from MinIO
- handleGetRanking rewrites MinIO cover keys to /api/cover/... proxy URLs
- Update save-browse CLI to use BrowseHTMLKey, populate ranking, and download covers
This commit is contained in:
Admin
2026-03-04 15:13:09 +05:00
parent bf2ffa54db
commit 11a846d043
5 changed files with 438 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
scraper/internal/server/server.go
View File

@@ -137,6 +137,8 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /api/browse", s.handleBrowse)
// Ranking API
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
// Cover image proxy (serves images stored in browse MinIO bucket)
mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
// Scrape status
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
@@ -191,6 +193,9 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
}
// handleGetRanking returns all ranking items sorted by rank ascending.
// Cover fields that hold a MinIO object key (e.g. "novelfire.net/assets/book-covers/slug.jpg")
// are rewritten to a /api/cover/{key} proxy URL so the UI can fetch them
// without knowing about the internal MinIO topology.
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
items, err := s.store.ReadRankingItems(r.Context())
if err != nil {
@@ -198,13 +203,70 @@ func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
if items == nil {
items = []storage.RankingItem{}
}
// Rewrite cover keys to proxy URLs.
// Keys stored by triggerBrowseSnapshot look like:
// "novelfire.net/assets/book-covers/shadow-slave.jpg"
// We expose them as:
// "/api/cover/novelfire.net/shadow-slave"
// (the handler strips the domain and slug from the path, reconstructs the key)
for i := range items {
cover := items[i].Cover
if cover != "" && !strings.HasPrefix(cover, "http") {
// cover is a MinIO key; extract domain + slug for the proxy path.
// Key format: {domain}/assets/book-covers/{slug}.jpg
parts := strings.SplitN(cover, "/assets/book-covers/", 2)
if len(parts) == 2 {
domain := parts[0]
slug := strings.TrimSuffix(parts[1], ".jpg")
items[i].Cover = "/api/cover/" + domain + "/" + slug
}
}
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(items)
}
// handleGetCover proxies a cover image stored in the MinIO browse bucket.
// Route: GET /api/cover/{domain}/{slug}
// It reconstructs the MinIO key as {domain}/assets/book-covers/{slug}.jpg,
// fetches the object, and streams it to the client.
// Returns 404 if not yet downloaded, allowing the UI to fall back to the
// original source URL.
func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
domain := r.PathValue("domain")
slug := r.PathValue("slug")
if domain == "" || slug == "" {
http.Error(w, "missing domain or slug", http.StatusBadRequest)
return
}
key := s.store.BrowseCoverKey(domain, slug)
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
data, contentType, ok, err := s.store.GetBrowseAsset(ctx, key)
if err != nil {
s.log.Warn("handleGetCover: GetBrowseAsset error", "key", key, "err", err)
http.Error(w, "storage error", http.StatusInternalServerError)
return
}
if !ok {
http.NotFound(w, r)
return
}
if contentType == "" {
contentType = "image/jpeg"
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("Cache-Control", "public, max-age=86400")
_, _ = w.Write(data)
}
// ─── Session cookie helpers ───────────────────────────────────────────────────
const sessionCookieName = "libnovel_session"
@@ -818,6 +880,7 @@ type NovelListing struct {
}
const novelFireBase = "https://novelfire.net"
const novelFireDomain = "novelfire.net"
// handleBrowse handles GET /api/browse.
// Query params:
@@ -830,8 +893,10 @@ const novelFireBase = "https://novelfire.net"
//
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
//
// Cache strategy: check MinIO browse bucket first; if a snapshot exists,
// parse and return it. Otherwise fetch live from novelfire.net.
// Cache strategy: check MinIO browse bucket first (key: {domain}/html/page-N.html);
// if a snapshot exists, parse it and return structured JSON.
// On a cache miss, fetch live from novelfire.net, return the result, and
// trigger a background SingleFile snapshot + ranking population.
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
page := q.Get("page")
@@ -863,8 +928,8 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
defer cancel()
// ── Cache-first: try MinIO snapshot ──────────────────────────────────
cacheKey := s.store.BrowsePageKey(genre, sortBy, status, novelType, pageNum)
// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
novels, hasNext := parseBrowsePage(strings.NewReader(html))
s.log.Debug("browse: served from cache", "key", cacheKey)
@@ -907,8 +972,8 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
novels, hasNext := parseBrowsePage(resp.Body)
// ── Background: populate MinIO cache via SingleFile ───────────────────
// Fire-and-forget: capture the JS-rendered page with SingleFile and store
// it in MinIO so the next request is served from cache.
// Fire-and-forget: capture the JS-rendered page with SingleFile, store
// it in MinIO, then parse it to populate the ranking collection.
s.triggerBrowseSnapshot(cacheKey, targetURL)
w.Header().Set("Content-Type", "application/json")
@@ -920,8 +985,15 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
})
}
// triggerBrowseSnapshot fires a background goroutine that uses SingleFile CLI
// to capture the fully-rendered novelfire browse page and store it in MinIO.
// triggerBrowseSnapshot fires a background goroutine that:
// 1. Runs SingleFile CLI to capture the fully-rendered novelfire browse page
// and stores the self-contained HTML at {domain}/html/page-N.html in MinIO.
// 2. Parses the stored HTML to extract novel listings.
// 3. For each listing, upserts a ranking record in PocketBase (rank, slug,
// title, cover key, source_url).
// 4. Fires a separate goroutine per cover image to download and store it at
// {domain}/assets/book-covers/{slug}.jpg in MinIO.
//
// It is a no-op when:
// - SINGLEFILE_PATH is not set (SingleFile not installed)
// - a capture for this cache key is already in progress
@@ -983,17 +1055,98 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
return
}
// Store the HTML snapshot.
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
"key", cacheKey, "err", putErr)
return
}
s.log.Info("triggerBrowseSnapshot: cached browse page",
"key", cacheKey, "bytes", len(htmlBytes))
// Parse the stored HTML to extract novel listings.
novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
if len(novels) == 0 {
return
}
// Upsert each novel into the ranking PocketBase collection and
// kick off a background cover download.
for i, novel := range novels {
rank := i + 1
coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
item := storage.RankingItem{
Rank: rank,
Slug: novel.Slug,
Title: novel.Title,
Cover: coverKey, // stored as MinIO key; UI fetches via /api/cover/...
SourceURL: novel.URL,
}
if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
s.log.Warn("triggerBrowseSnapshot: WriteRankingItem failed",
"slug", novel.Slug, "err", werr)
}
// Download and store the cover image in a separate goroutine.
coverURL := novel.Cover
if coverURL != "" {
go s.downloadAndStoreCover(coverKey, coverURL)
}
}
s.log.Info("triggerBrowseSnapshot: ranking populated", "count", len(novels), "key", cacheKey)
}()
}
// downloadAndStoreCover fetches a cover image URL and stores it in MinIO under
// the given key. Errors are logged but not propagated — this is best-effort.
func (s *Server) downloadAndStoreCover(key, imageURL string) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Skip if already stored.
if _, _, ok, _ := s.store.GetBrowseAsset(ctx, key); ok {
return
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
if err != nil {
s.log.Warn("downloadAndStoreCover: build request failed", "url", imageURL, "err", err)
return
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
resp, err := http.DefaultClient.Do(req)
if err != nil {
s.log.Warn("downloadAndStoreCover: fetch failed", "url", imageURL, "err", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
s.log.Warn("downloadAndStoreCover: non-200 response", "url", imageURL, "status", resp.StatusCode)
return
}
data, readErr := io.ReadAll(resp.Body)
if readErr != nil {
s.log.Warn("downloadAndStoreCover: read body failed", "url", imageURL, "err", readErr)
return
}
contentType := resp.Header.Get("Content-Type")
if contentType == "" {
contentType = "image/jpeg"
}
if putErr := s.store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
s.log.Warn("downloadAndStoreCover: SaveBrowseAsset failed", "key", key, "err", putErr)
return
}
s.log.Debug("downloadAndStoreCover: stored cover", "key", key, "bytes", len(data))
}
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
// Returns novels and whether a "next page" link was found.
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {