feat(scraper): rewrite browse storage to domain/html+assets structure, populate ranking from snapshot

- Replace BrowsePageKey(genre/sort/status/type/page) with BrowseHTMLKey(domain, page) -> {domain}/html/page-{n}.html
- Add BrowseCoverKey(domain, slug) -> {domain}/assets/book-covers/{slug}.jpg
- Add SaveBrowseAsset/GetBrowseAsset for binary assets in browse bucket
- Rewrite triggerBrowseSnapshot: after storing HTML, parse it, upsert ranking records with MinIO cover keys, fire per-novel cover download goroutines
- Add handleGetCover endpoint (GET /api/cover/{domain}/{slug}) to proxy cover images from MinIO
- handleGetRanking rewrites MinIO cover keys to /api/cover/... proxy URLs
- Update save-browse CLI to use BrowseHTMLKey, populate ranking, and download covers
This commit is contained in:
Admin
2026-03-04 15:13:09 +05:00
parent bf2ffa54db
commit 11a846d043
5 changed files with 438 additions and 19 deletions

View File

@@ -33,7 +33,9 @@ package main
import (
"context"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"os/exec"
"os/signal"
@@ -212,6 +214,8 @@ func run(log *slog.Logger) error {
// It iterates over browse pages on novelfire.net, captures each using
// SingleFile CLI (connected to the existing Browserless instance), and
// stores the resulting self-contained HTML in the MinIO browse bucket.
// After storing each page it parses the HTML, upserts ranking records in
// PocketBase, and fires background goroutines to download cover images.
//
// Flags (all optional):
//
@@ -281,6 +285,7 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
defer os.RemoveAll(tmpDir)
const novelFireBase = "https://novelfire.net"
const novelFireDomain = "novelfire.net"
for page := 1; page <= maxPages; page++ {
select {
@@ -292,7 +297,8 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
novelFireBase, genre, sortBy, status, novelType, page)
key := store.BrowsePageKey(genre, sortBy, status, novelType, page)
// Use the new domain-based key layout: {domain}/html/page-{n}.html
key := store.BrowseHTMLKey(novelFireDomain, page)
outFile := fmt.Sprintf("%s/page-%d.html", tmpDir, page)
@@ -328,12 +334,205 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
log.Info("save-browse: snapshot stored", "page", page, "key", key,
"bytes", len(htmlBytes))
// Parse the stored HTML and populate the ranking collection.
novels := parseSaveBrowseListings(htmlBytes, novelFireBase)
for i, novel := range novels {
rank := i + 1
coverKey := store.BrowseCoverKey(novelFireDomain, novel.slug)
item := storage.RankingItem{
Rank: rank,
Slug: novel.slug,
Title: novel.title,
Cover: coverKey,
SourceURL: novel.url,
}
if werr := store.WriteRankingItem(ctx, item); werr != nil {
log.Warn("save-browse: WriteRankingItem failed",
"slug", novel.slug, "err", werr)
}
// Download cover image in the background (best-effort).
if novel.coverURL != "" {
go downloadAndStoreCoverCLI(store, log, coverKey, novel.coverURL)
}
}
if len(novels) > 0 {
log.Info("save-browse: ranking populated", "page", page, "count", len(novels))
}
}
log.Info("save-browse: done")
return nil
}
// novelListingCLI is a minimal novel listing used within the CLI command.
type novelListingCLI struct {
slug string
title string
url string
coverURL string
}
// parseSaveBrowseListings extracts novel listings from raw HTML bytes.
// It reuses the same parsing logic as the server's parseBrowsePage but
// operates on []byte to avoid importing the server package.
func parseSaveBrowseListings(htmlBytes []byte, novelFireBase string) []novelListingCLI {
type listing = novelListingCLI
// Minimal tokeniser-based walk to find <li class="novel-item"> blocks.
// We use the golang.org/x/net/html parser via a local import.
// Because main.go already imports golang.org/x/net/html indirectly through
// the server package build, we do a simple line-scan here instead to keep
// the dependency surface small.
//
// Strategy: scan for href="/book/{slug}", img data-src/src, h4.novel-title text.
var novels []listing
lines := strings.Split(string(htmlBytes), "\n")
var cur listing
inNovelItem := false
for _, line := range lines {
trimmed := strings.TrimSpace(line)
// Detect start of a novel-item list element.
if strings.Contains(trimmed, `class="novel-item"`) || strings.Contains(trimmed, "novel-item") && strings.HasPrefix(trimmed, "<li") {
inNovelItem = true
cur = listing{}
}
if !inNovelItem {
continue
}
// Detect end of list element.
if trimmed == "</li>" && cur.slug != "" {
novels = append(novels, cur)
inNovelItem = false
cur = listing{}
continue
}
// Extract slug from href="/book/{slug}".
if cur.slug == "" {
if idx := strings.Index(trimmed, `href="/book/`); idx >= 0 {
rest := trimmed[idx+len(`href="/book/`):]
if end := strings.IndexAny(rest, `"/ `); end > 0 {
cur.slug = rest[:end]
cur.url = novelFireBase + "/book/" + cur.slug
} else if end := strings.Index(rest, `"`); end > 0 {
cur.slug = strings.TrimSuffix(rest[:end], "/")
cur.url = novelFireBase + "/book/" + cur.slug
}
}
}
// Extract cover URL from data-src or src on img tags.
if cur.coverURL == "" && strings.Contains(trimmed, "<img") {
if src := extractAttr(trimmed, "data-src"); src != "" {
cur.coverURL = resolveURL(src, novelFireBase)
} else if src := extractAttr(trimmed, "src"); src != "" && !strings.Contains(src, "data:") {
cur.coverURL = resolveURL(src, novelFireBase)
}
}
// Extract title from novel-title element.
if cur.title == "" && strings.Contains(trimmed, "novel-title") {
// Try to grab inner text: <h4 class="novel-title">Title Here</h4>
if start := strings.Index(trimmed, ">"); start >= 0 {
rest := trimmed[start+1:]
if end := strings.Index(rest, "<"); end > 0 {
title := strings.TrimSpace(rest[:end])
if title != "" {
cur.title = title
}
}
}
}
}
// Flush any open item that wasn't closed by </li> (e.g. last item in file).
if inNovelItem && cur.slug != "" {
novels = append(novels, cur)
}
return novels
}
// extractAttr extracts an HTML attribute value from a raw tag string.
// e.g. extractAttr(`<img data-src="foo.jpg">`, "data-src") → "foo.jpg"
func extractAttr(tag, attr string) string {
needle := attr + `="`
idx := strings.Index(tag, needle)
if idx < 0 {
return ""
}
rest := tag[idx+len(needle):]
end := strings.Index(rest, `"`)
if end < 0 {
return ""
}
return rest[:end]
}
// resolveURL ensures the URL is absolute, prepending novelFireBase if needed.
func resolveURL(src, base string) string {
if strings.HasPrefix(src, "http") {
return src
}
return base + src
}
// downloadAndStoreCoverCLI fetches a cover image and stores it in MinIO.
// Errors are logged but not propagated — this is a best-effort background task.
func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageURL string) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Skip if already stored.
if _, _, ok, _ := store.GetBrowseAsset(ctx, key); ok {
return
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
if err != nil {
log.Warn("save-browse: cover build request failed", "url", imageURL, "err", err)
return
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Warn("save-browse: cover fetch failed", "url", imageURL, "err", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Warn("save-browse: cover non-200", "url", imageURL, "status", resp.StatusCode)
return
}
data, readErr := io.ReadAll(resp.Body)
if readErr != nil {
log.Warn("save-browse: cover read body failed", "url", imageURL, "err", readErr)
return
}
contentType := resp.Header.Get("Content-Type")
if contentType == "" {
contentType = "image/jpeg"
}
if putErr := store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
log.Warn("save-browse: SaveBrowseAsset failed", "key", key, "err", putErr)
return
}
log.Debug("save-browse: cover stored", "key", key, "bytes", len(data))
}
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
switch strategy {
case browser.StrategyScrape:

View File

@@ -137,6 +137,8 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /api/browse", s.handleBrowse)
// Ranking API
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
// Cover image proxy (serves images stored in browse MinIO bucket)
mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
// Scrape status
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
@@ -191,6 +193,9 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
}
// handleGetRanking returns all ranking items sorted by rank ascending.
// Cover fields that hold a MinIO object key (e.g. "novelfire.net/assets/book-covers/slug.jpg")
// are rewritten to a /api/cover/{key} proxy URL so the UI can fetch them
// without knowing about the internal MinIO topology.
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
items, err := s.store.ReadRankingItems(r.Context())
if err != nil {
@@ -198,13 +203,70 @@ func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
if items == nil {
items = []storage.RankingItem{}
}
// Rewrite cover keys to proxy URLs.
// Keys stored by triggerBrowseSnapshot look like:
// "novelfire.net/assets/book-covers/shadow-slave.jpg"
// We expose them as:
// "/api/cover/novelfire.net/shadow-slave"
// (the handler strips the domain and slug from the path, reconstructs the key)
for i := range items {
cover := items[i].Cover
if cover != "" && !strings.HasPrefix(cover, "http") {
// cover is a MinIO key; extract domain + slug for the proxy path.
// Key format: {domain}/assets/book-covers/{slug}.jpg
parts := strings.SplitN(cover, "/assets/book-covers/", 2)
if len(parts) == 2 {
domain := parts[0]
slug := strings.TrimSuffix(parts[1], ".jpg")
items[i].Cover = "/api/cover/" + domain + "/" + slug
}
}
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(items)
}
// handleGetCover proxies a cover image stored in the MinIO browse bucket.
// Route: GET /api/cover/{domain}/{slug}
// It reconstructs the MinIO key as {domain}/assets/book-covers/{slug}.jpg,
// fetches the object, and streams it to the client.
// Returns 404 if not yet downloaded, allowing the UI to fall back to the
// original source URL.
func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
domain := r.PathValue("domain")
slug := r.PathValue("slug")
if domain == "" || slug == "" {
http.Error(w, "missing domain or slug", http.StatusBadRequest)
return
}
key := s.store.BrowseCoverKey(domain, slug)
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
data, contentType, ok, err := s.store.GetBrowseAsset(ctx, key)
if err != nil {
s.log.Warn("handleGetCover: GetBrowseAsset error", "key", key, "err", err)
http.Error(w, "storage error", http.StatusInternalServerError)
return
}
if !ok {
http.NotFound(w, r)
return
}
if contentType == "" {
contentType = "image/jpeg"
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("Cache-Control", "public, max-age=86400")
_, _ = w.Write(data)
}
// ─── Session cookie helpers ───────────────────────────────────────────────────
const sessionCookieName = "libnovel_session"
@@ -818,6 +880,7 @@ type NovelListing struct {
}
const novelFireBase = "https://novelfire.net"
const novelFireDomain = "novelfire.net"
// handleBrowse handles GET /api/browse.
// Query params:
@@ -830,8 +893,10 @@ const novelFireBase = "https://novelfire.net"
//
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
//
// Cache strategy: check MinIO browse bucket first; if a snapshot exists,
// parse and return it. Otherwise fetch live from novelfire.net.
// Cache strategy: check MinIO browse bucket first (key: {domain}/html/page-N.html);
// if a snapshot exists, parse it and return structured JSON.
// On a cache miss, fetch live from novelfire.net, return the result, and
// trigger a background SingleFile snapshot + ranking population.
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
page := q.Get("page")
@@ -863,8 +928,8 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
defer cancel()
// ── Cache-first: try MinIO snapshot ──────────────────────────────────
cacheKey := s.store.BrowsePageKey(genre, sortBy, status, novelType, pageNum)
// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
novels, hasNext := parseBrowsePage(strings.NewReader(html))
s.log.Debug("browse: served from cache", "key", cacheKey)
@@ -907,8 +972,8 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
novels, hasNext := parseBrowsePage(resp.Body)
// ── Background: populate MinIO cache via SingleFile ───────────────────
// Fire-and-forget: capture the JS-rendered page with SingleFile and store
// it in MinIO so the next request is served from cache.
// Fire-and-forget: capture the JS-rendered page with SingleFile, store
// it in MinIO, then parse it to populate the ranking collection.
s.triggerBrowseSnapshot(cacheKey, targetURL)
w.Header().Set("Content-Type", "application/json")
@@ -920,8 +985,15 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
})
}
// triggerBrowseSnapshot fires a background goroutine that uses SingleFile CLI
// to capture the fully-rendered novelfire browse page and store it in MinIO.
// triggerBrowseSnapshot fires a background goroutine that:
// 1. Runs SingleFile CLI to capture the fully-rendered novelfire browse page
// and stores the self-contained HTML at {domain}/html/page-N.html in MinIO.
// 2. Parses the stored HTML to extract novel listings.
// 3. For each listing, upserts a ranking record in PocketBase (rank, slug,
// title, cover key, source_url).
// 4. Fires a separate goroutine per cover image to download and store it at
// {domain}/assets/book-covers/{slug}.jpg in MinIO.
//
// It is a no-op when:
// - SINGLEFILE_PATH is not set (SingleFile not installed)
// - a capture for this cache key is already in progress
@@ -983,17 +1055,98 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
return
}
// Store the HTML snapshot.
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
"key", cacheKey, "err", putErr)
return
}
s.log.Info("triggerBrowseSnapshot: cached browse page",
"key", cacheKey, "bytes", len(htmlBytes))
// Parse the stored HTML to extract novel listings.
novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
if len(novels) == 0 {
return
}
// Upsert each novel into the ranking PocketBase collection and
// kick off a background cover download.
for i, novel := range novels {
rank := i + 1
coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
item := storage.RankingItem{
Rank: rank,
Slug: novel.Slug,
Title: novel.Title,
Cover: coverKey, // stored as MinIO key; UI fetches via /api/cover/...
SourceURL: novel.URL,
}
if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
s.log.Warn("triggerBrowseSnapshot: WriteRankingItem failed",
"slug", novel.Slug, "err", werr)
}
// Download and store the cover image in a separate goroutine.
coverURL := novel.Cover
if coverURL != "" {
go s.downloadAndStoreCover(coverKey, coverURL)
}
}
s.log.Info("triggerBrowseSnapshot: ranking populated", "count", len(novels), "key", cacheKey)
}()
}
// downloadAndStoreCover fetches a cover image URL and stores it in MinIO under
// the given key. Errors are logged but not propagated — this is best-effort.
func (s *Server) downloadAndStoreCover(key, imageURL string) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Skip if already stored.
if _, _, ok, _ := s.store.GetBrowseAsset(ctx, key); ok {
return
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
if err != nil {
s.log.Warn("downloadAndStoreCover: build request failed", "url", imageURL, "err", err)
return
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
resp, err := http.DefaultClient.Do(req)
if err != nil {
s.log.Warn("downloadAndStoreCover: fetch failed", "url", imageURL, "err", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
s.log.Warn("downloadAndStoreCover: non-200 response", "url", imageURL, "status", resp.StatusCode)
return
}
data, readErr := io.ReadAll(resp.Body)
if readErr != nil {
s.log.Warn("downloadAndStoreCover: read body failed", "url", imageURL, "err", readErr)
return
}
contentType := resp.Header.Get("Content-Type")
if contentType == "" {
contentType = "image/jpeg"
}
if putErr := s.store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
s.log.Warn("downloadAndStoreCover: SaveBrowseAsset failed", "key", key, "err", putErr)
return
}
s.log.Debug("downloadAndStoreCover: stored cover", "key", key, "bytes", len(data))
}
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
// Returns novels and whether a "next page" link was found.
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {

View File

@@ -317,8 +317,20 @@ func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bo
return h.minio.GetBrowsePage(ctx, key)
}
func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
return BrowsePageKey(genre, sortBy, status, novelType, page)
func (h *HybridStore) BrowseHTMLKey(domain string, page int) string {
return BrowseHTMLKey(domain, page)
}
func (h *HybridStore) BrowseCoverKey(domain, slug string) string {
return BrowseCoverKey(domain, slug)
}
func (h *HybridStore) SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
return h.minio.PutBrowseAsset(ctx, key, data, contentType)
}
func (h *HybridStore) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
return h.minio.GetBrowseAsset(ctx, key)
}
// ─── Scraping tasks ───────────────────────────────────────────────────────────

View File

@@ -211,11 +211,26 @@ func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time
}
// ─── Browse page snapshots ────────────────────────────────────────────────────
//
// New bucket layout (libnovel-browse):
//
// {domain}/html/page-{n}.html — SingleFile HTML snapshot
// {domain}/assets/book-covers/{slug}.jpg — downloaded cover image
//
// The domain segment is derived from the source URL hostname
// (e.g. "novelfire.net"). This makes the bucket self-describing and
// extensible to multiple sources.
// BrowsePageKey returns the MinIO object key for a cached browse-page snapshot.
// Layout: {genre}/{sort}/{status}/{novelType}/page-{n}.html
func BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
return fmt.Sprintf("%s/%s/%s/%s/page-%d.html", genre, sortBy, status, novelType, page)
// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
// Layout: {domain}/html/page-{n}.html
func BrowseHTMLKey(domain string, page int) string {
return fmt.Sprintf("%s/html/page-%d.html", domain, page)
}
// BrowseCoverKey returns the MinIO object key for a cached book cover image.
// Layout: {domain}/assets/book-covers/{slug}.jpg
func BrowseCoverKey(domain, slug string) string {
return fmt.Sprintf("%s/assets/book-covers/%s.jpg", domain, slug)
}
// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket.
@@ -255,6 +270,37 @@ func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool {
return err == nil
}
// PutBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
// contentType should be the MIME type, e.g. "image/jpeg".
func (m *MinioClient) PutBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
_, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: contentType})
if err != nil {
return fmt.Errorf("minio: put browse asset %s: %w", key, err)
}
return nil
}
// GetBrowseAsset retrieves a binary asset from the browse bucket.
// Returns (nil, false, nil) when the object does not exist.
func (m *MinioClient) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
if err != nil {
return nil, "", false, fmt.Errorf("minio: get browse asset %s: %w", key, err)
}
defer obj.Close()
info, statErr := obj.Stat()
if statErr != nil {
return nil, "", false, nil // not found
}
data, err := io.ReadAll(obj)
if err != nil {
return nil, "", false, fmt.Errorf("minio: read browse asset %s: %w", key, err)
}
return data, info.ContentType, true, nil
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// sanitiseVoice converts a voice name to a filename-safe string.

View File

@@ -146,8 +146,17 @@ type Store interface {
// GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil)
// when no snapshot exists for the key.
GetBrowsePage(ctx context.Context, key string) (string, bool, error)
// BrowsePageKey returns the MinIO object key for the given browse params.
BrowsePageKey(genre, sortBy, status, novelType string, page int) string
// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
// Layout: {domain}/html/page-{n}.html
BrowseHTMLKey(domain string, page int) string
// BrowseCoverKey returns the MinIO object key for a cached book cover image.
// Layout: {domain}/assets/book-covers/{slug}.jpg
BrowseCoverKey(domain, slug string) string
// SaveBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error
// GetBrowseAsset retrieves a binary asset from the browse bucket.
// Returns (nil, "", false, nil) when the object does not exist.
GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error)
// ── Scraping tasks ─────────────────────────────────────────────────────