feat(scraper): rewrite browse storage to domain/html+assets structure, populate ranking from snapshot
- Replace BrowsePageKey(genre/sort/status/type/page) with BrowseHTMLKey(domain, page) -> {domain}/html/page-{n}.html
- Add BrowseCoverKey(domain, slug) -> {domain}/assets/book-covers/{slug}.jpg
- Add SaveBrowseAsset/GetBrowseAsset for binary assets in browse bucket
- Rewrite triggerBrowseSnapshot: after storing HTML, parse it, upsert ranking records with MinIO cover keys, fire per-novel cover download goroutines
- Add handleGetCover endpoint (GET /api/cover/{domain}/{slug}) to proxy cover images from MinIO
- handleGetRanking rewrites MinIO cover keys to /api/cover/... proxy URLs
- Update save-browse CLI to use BrowseHTMLKey, populate ranking, and download covers
This commit is contained in:
@@ -33,7 +33,9 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
@@ -212,6 +214,8 @@ func run(log *slog.Logger) error {
|
||||
// It iterates over browse pages on novelfire.net, captures each using
|
||||
// SingleFile CLI (connected to the existing Browserless instance), and
|
||||
// stores the resulting self-contained HTML in the MinIO browse bucket.
|
||||
// After storing each page it parses the HTML, upserts ranking records in
|
||||
// PocketBase, and fires background goroutines to download cover images.
|
||||
//
|
||||
// Flags (all optional):
|
||||
//
|
||||
@@ -281,6 +285,7 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
const novelFireBase = "https://novelfire.net"
|
||||
const novelFireDomain = "novelfire.net"
|
||||
|
||||
for page := 1; page <= maxPages; page++ {
|
||||
select {
|
||||
@@ -292,7 +297,8 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
|
||||
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
|
||||
key := store.BrowsePageKey(genre, sortBy, status, novelType, page)
|
||||
// Use the new domain-based key layout: {domain}/html/page-{n}.html
|
||||
key := store.BrowseHTMLKey(novelFireDomain, page)
|
||||
|
||||
outFile := fmt.Sprintf("%s/page-%d.html", tmpDir, page)
|
||||
|
||||
@@ -328,12 +334,205 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
|
||||
|
||||
log.Info("save-browse: snapshot stored", "page", page, "key", key,
|
||||
"bytes", len(htmlBytes))
|
||||
|
||||
// Parse the stored HTML and populate the ranking collection.
|
||||
novels := parseSaveBrowseListings(htmlBytes, novelFireBase)
|
||||
for i, novel := range novels {
|
||||
rank := i + 1
|
||||
coverKey := store.BrowseCoverKey(novelFireDomain, novel.slug)
|
||||
|
||||
item := storage.RankingItem{
|
||||
Rank: rank,
|
||||
Slug: novel.slug,
|
||||
Title: novel.title,
|
||||
Cover: coverKey,
|
||||
SourceURL: novel.url,
|
||||
}
|
||||
if werr := store.WriteRankingItem(ctx, item); werr != nil {
|
||||
log.Warn("save-browse: WriteRankingItem failed",
|
||||
"slug", novel.slug, "err", werr)
|
||||
}
|
||||
|
||||
// Download cover image in the background (best-effort).
|
||||
if novel.coverURL != "" {
|
||||
go downloadAndStoreCoverCLI(store, log, coverKey, novel.coverURL)
|
||||
}
|
||||
}
|
||||
if len(novels) > 0 {
|
||||
log.Info("save-browse: ranking populated", "page", page, "count", len(novels))
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("save-browse: done")
|
||||
return nil
|
||||
}
|
||||
|
||||
// novelListingCLI is a minimal novel listing used within the CLI command.
|
||||
type novelListingCLI struct {
|
||||
slug string
|
||||
title string
|
||||
url string
|
||||
coverURL string
|
||||
}
|
||||
|
||||
// parseSaveBrowseListings extracts novel listings from raw HTML bytes.
|
||||
// It reuses the same parsing logic as the server's parseBrowsePage but
|
||||
// operates on []byte to avoid importing the server package.
|
||||
func parseSaveBrowseListings(htmlBytes []byte, novelFireBase string) []novelListingCLI {
|
||||
type listing = novelListingCLI
|
||||
|
||||
// Minimal tokeniser-based walk to find <li class="novel-item"> blocks.
|
||||
// We use the golang.org/x/net/html parser via a local import.
|
||||
// Because main.go already imports golang.org/x/net/html indirectly through
|
||||
// the server package build, we do a simple line-scan here instead to keep
|
||||
// the dependency surface small.
|
||||
//
|
||||
// Strategy: scan for href="/book/{slug}", img data-src/src, h4.novel-title text.
|
||||
var novels []listing
|
||||
|
||||
lines := strings.Split(string(htmlBytes), "\n")
|
||||
var cur listing
|
||||
inNovelItem := false
|
||||
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
|
||||
// Detect start of a novel-item list element.
|
||||
if strings.Contains(trimmed, `class="novel-item"`) || strings.Contains(trimmed, "novel-item") && strings.HasPrefix(trimmed, "<li") {
|
||||
inNovelItem = true
|
||||
cur = listing{}
|
||||
}
|
||||
|
||||
if !inNovelItem {
|
||||
continue
|
||||
}
|
||||
|
||||
// Detect end of list element.
|
||||
if trimmed == "</li>" && cur.slug != "" {
|
||||
novels = append(novels, cur)
|
||||
inNovelItem = false
|
||||
cur = listing{}
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract slug from href="/book/{slug}".
|
||||
if cur.slug == "" {
|
||||
if idx := strings.Index(trimmed, `href="/book/`); idx >= 0 {
|
||||
rest := trimmed[idx+len(`href="/book/`):]
|
||||
if end := strings.IndexAny(rest, `"/ `); end > 0 {
|
||||
cur.slug = rest[:end]
|
||||
cur.url = novelFireBase + "/book/" + cur.slug
|
||||
} else if end := strings.Index(rest, `"`); end > 0 {
|
||||
cur.slug = strings.TrimSuffix(rest[:end], "/")
|
||||
cur.url = novelFireBase + "/book/" + cur.slug
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract cover URL from data-src or src on img tags.
|
||||
if cur.coverURL == "" && strings.Contains(trimmed, "<img") {
|
||||
if src := extractAttr(trimmed, "data-src"); src != "" {
|
||||
cur.coverURL = resolveURL(src, novelFireBase)
|
||||
} else if src := extractAttr(trimmed, "src"); src != "" && !strings.Contains(src, "data:") {
|
||||
cur.coverURL = resolveURL(src, novelFireBase)
|
||||
}
|
||||
}
|
||||
|
||||
// Extract title from novel-title element.
|
||||
if cur.title == "" && strings.Contains(trimmed, "novel-title") {
|
||||
// Try to grab inner text: <h4 class="novel-title">Title Here</h4>
|
||||
if start := strings.Index(trimmed, ">"); start >= 0 {
|
||||
rest := trimmed[start+1:]
|
||||
if end := strings.Index(rest, "<"); end > 0 {
|
||||
title := strings.TrimSpace(rest[:end])
|
||||
if title != "" {
|
||||
cur.title = title
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush any open item that wasn't closed by </li> (e.g. last item in file).
|
||||
if inNovelItem && cur.slug != "" {
|
||||
novels = append(novels, cur)
|
||||
}
|
||||
|
||||
return novels
|
||||
}
|
||||
|
||||
// extractAttr extracts an HTML attribute value from a raw tag string.
|
||||
// e.g. extractAttr(`<img data-src="foo.jpg">`, "data-src") → "foo.jpg"
|
||||
func extractAttr(tag, attr string) string {
|
||||
needle := attr + `="`
|
||||
idx := strings.Index(tag, needle)
|
||||
if idx < 0 {
|
||||
return ""
|
||||
}
|
||||
rest := tag[idx+len(needle):]
|
||||
end := strings.Index(rest, `"`)
|
||||
if end < 0 {
|
||||
return ""
|
||||
}
|
||||
return rest[:end]
|
||||
}
|
||||
|
||||
// resolveURL ensures the URL is absolute, prepending novelFireBase if needed.
|
||||
func resolveURL(src, base string) string {
|
||||
if strings.HasPrefix(src, "http") {
|
||||
return src
|
||||
}
|
||||
return base + src
|
||||
}
|
||||
|
||||
// downloadAndStoreCoverCLI fetches a cover image and stores it in MinIO.
|
||||
// Errors are logged but not propagated — this is a best-effort background task.
|
||||
func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageURL string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Skip if already stored.
|
||||
if _, _, ok, _ := store.GetBrowseAsset(ctx, key); ok {
|
||||
return
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
||||
if err != nil {
|
||||
log.Warn("save-browse: cover build request failed", "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
log.Warn("save-browse: cover fetch failed", "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Warn("save-browse: cover non-200", "url", imageURL, "status", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
data, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
log.Warn("save-browse: cover read body failed", "url", imageURL, "err", readErr)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
|
||||
if putErr := store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
|
||||
log.Warn("save-browse: SaveBrowseAsset failed", "key", key, "err", putErr)
|
||||
return
|
||||
}
|
||||
log.Debug("save-browse: cover stored", "key", key, "bytes", len(data))
|
||||
}
|
||||
|
||||
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
|
||||
switch strategy {
|
||||
case browser.StrategyScrape:
|
||||
|
||||
@@ -137,6 +137,8 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
||||
// Ranking API
|
||||
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
|
||||
// Cover image proxy (serves images stored in browse MinIO bucket)
|
||||
mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
|
||||
// Scrape status
|
||||
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
|
||||
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
|
||||
@@ -191,6 +193,9 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
|
||||
// handleGetRanking returns all ranking items sorted by rank ascending.
|
||||
// Cover fields that hold a MinIO object key (e.g. "novelfire.net/assets/book-covers/slug.jpg")
|
||||
// are rewritten to a /api/cover/{key} proxy URL so the UI can fetch them
|
||||
// without knowing about the internal MinIO topology.
|
||||
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
|
||||
items, err := s.store.ReadRankingItems(r.Context())
|
||||
if err != nil {
|
||||
@@ -198,13 +203,70 @@ func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if items == nil {
|
||||
items = []storage.RankingItem{}
|
||||
}
|
||||
// Rewrite cover keys to proxy URLs.
|
||||
// Keys stored by triggerBrowseSnapshot look like:
|
||||
// "novelfire.net/assets/book-covers/shadow-slave.jpg"
|
||||
// We expose them as:
|
||||
// "/api/cover/novelfire.net/shadow-slave"
|
||||
// (the handler strips the domain and slug from the path, reconstructs the key)
|
||||
for i := range items {
|
||||
cover := items[i].Cover
|
||||
if cover != "" && !strings.HasPrefix(cover, "http") {
|
||||
// cover is a MinIO key; extract domain + slug for the proxy path.
|
||||
// Key format: {domain}/assets/book-covers/{slug}.jpg
|
||||
parts := strings.SplitN(cover, "/assets/book-covers/", 2)
|
||||
if len(parts) == 2 {
|
||||
domain := parts[0]
|
||||
slug := strings.TrimSuffix(parts[1], ".jpg")
|
||||
items[i].Cover = "/api/cover/" + domain + "/" + slug
|
||||
}
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(items)
|
||||
}
|
||||
|
||||
// handleGetCover proxies a cover image stored in the MinIO browse bucket.
|
||||
// Route: GET /api/cover/{domain}/{slug}
|
||||
// It reconstructs the MinIO key as {domain}/assets/book-covers/{slug}.jpg,
|
||||
// fetches the object, and streams it to the client.
|
||||
// Returns 404 if not yet downloaded, allowing the UI to fall back to the
|
||||
// original source URL.
|
||||
func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
|
||||
domain := r.PathValue("domain")
|
||||
slug := r.PathValue("slug")
|
||||
if domain == "" || slug == "" {
|
||||
http.Error(w, "missing domain or slug", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
key := s.store.BrowseCoverKey(domain, slug)
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
data, contentType, ok, err := s.store.GetBrowseAsset(ctx, key)
|
||||
if err != nil {
|
||||
s.log.Warn("handleGetCover: GetBrowseAsset error", "key", key, "err", err)
|
||||
http.Error(w, "storage error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Cache-Control", "public, max-age=86400")
|
||||
_, _ = w.Write(data)
|
||||
}
|
||||
|
||||
// ─── Session cookie helpers ───────────────────────────────────────────────────
|
||||
|
||||
const sessionCookieName = "libnovel_session"
|
||||
@@ -818,6 +880,7 @@ type NovelListing struct {
|
||||
}
|
||||
|
||||
const novelFireBase = "https://novelfire.net"
|
||||
const novelFireDomain = "novelfire.net"
|
||||
|
||||
// handleBrowse handles GET /api/browse.
|
||||
// Query params:
|
||||
@@ -830,8 +893,10 @@ const novelFireBase = "https://novelfire.net"
|
||||
//
|
||||
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
|
||||
//
|
||||
// Cache strategy: check MinIO browse bucket first; if a snapshot exists,
|
||||
// parse and return it. Otherwise fetch live from novelfire.net.
|
||||
// Cache strategy: check MinIO browse bucket first (key: {domain}/html/page-N.html);
|
||||
// if a snapshot exists, parse it and return structured JSON.
|
||||
// On a cache miss, fetch live from novelfire.net, return the result, and
|
||||
// trigger a background SingleFile snapshot + ranking population.
|
||||
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page := q.Get("page")
|
||||
@@ -863,8 +928,8 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// ── Cache-first: try MinIO snapshot ──────────────────────────────────
|
||||
cacheKey := s.store.BrowsePageKey(genre, sortBy, status, novelType, pageNum)
|
||||
// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
|
||||
cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
|
||||
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
|
||||
novels, hasNext := parseBrowsePage(strings.NewReader(html))
|
||||
s.log.Debug("browse: served from cache", "key", cacheKey)
|
||||
@@ -907,8 +972,8 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
novels, hasNext := parseBrowsePage(resp.Body)
|
||||
|
||||
// ── Background: populate MinIO cache via SingleFile ───────────────────
|
||||
// Fire-and-forget: capture the JS-rendered page with SingleFile and store
|
||||
// it in MinIO so the next request is served from cache.
|
||||
// Fire-and-forget: capture the JS-rendered page with SingleFile, store
|
||||
// it in MinIO, then parse it to populate the ranking collection.
|
||||
s.triggerBrowseSnapshot(cacheKey, targetURL)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
@@ -920,8 +985,15 @@ func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
// triggerBrowseSnapshot fires a background goroutine that uses SingleFile CLI
|
||||
// to capture the fully-rendered novelfire browse page and store it in MinIO.
|
||||
// triggerBrowseSnapshot fires a background goroutine that:
|
||||
// 1. Runs SingleFile CLI to capture the fully-rendered novelfire browse page
|
||||
// and stores the self-contained HTML at {domain}/html/page-N.html in MinIO.
|
||||
// 2. Parses the stored HTML to extract novel listings.
|
||||
// 3. For each listing, upserts a ranking record in PocketBase (rank, slug,
|
||||
// title, cover key, source_url).
|
||||
// 4. Fires a separate goroutine per cover image to download and store it at
|
||||
// {domain}/assets/book-covers/{slug}.jpg in MinIO.
|
||||
//
|
||||
// It is a no-op when:
|
||||
// - SINGLEFILE_PATH is not set (SingleFile not installed)
|
||||
// - a capture for this cache key is already in progress
|
||||
@@ -983,17 +1055,98 @@ func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
|
||||
return
|
||||
}
|
||||
|
||||
// Store the HTML snapshot.
|
||||
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
|
||||
s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
|
||||
"key", cacheKey, "err", putErr)
|
||||
return
|
||||
}
|
||||
|
||||
s.log.Info("triggerBrowseSnapshot: cached browse page",
|
||||
"key", cacheKey, "bytes", len(htmlBytes))
|
||||
|
||||
// Parse the stored HTML to extract novel listings.
|
||||
novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
|
||||
if len(novels) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Upsert each novel into the ranking PocketBase collection and
|
||||
// kick off a background cover download.
|
||||
for i, novel := range novels {
|
||||
rank := i + 1
|
||||
coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
|
||||
|
||||
item := storage.RankingItem{
|
||||
Rank: rank,
|
||||
Slug: novel.Slug,
|
||||
Title: novel.Title,
|
||||
Cover: coverKey, // stored as MinIO key; UI fetches via /api/cover/...
|
||||
SourceURL: novel.URL,
|
||||
}
|
||||
if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
|
||||
s.log.Warn("triggerBrowseSnapshot: WriteRankingItem failed",
|
||||
"slug", novel.Slug, "err", werr)
|
||||
}
|
||||
|
||||
// Download and store the cover image in a separate goroutine.
|
||||
coverURL := novel.Cover
|
||||
if coverURL != "" {
|
||||
go s.downloadAndStoreCover(coverKey, coverURL)
|
||||
}
|
||||
}
|
||||
|
||||
s.log.Info("triggerBrowseSnapshot: ranking populated", "count", len(novels), "key", cacheKey)
|
||||
}()
|
||||
}
|
||||
|
||||
// downloadAndStoreCover fetches a cover image URL and stores it in MinIO under
|
||||
// the given key. Errors are logged but not propagated — this is best-effort.
|
||||
func (s *Server) downloadAndStoreCover(key, imageURL string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Skip if already stored.
|
||||
if _, _, ok, _ := s.store.GetBrowseAsset(ctx, key); ok {
|
||||
return
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
||||
if err != nil {
|
||||
s.log.Warn("downloadAndStoreCover: build request failed", "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
s.log.Warn("downloadAndStoreCover: fetch failed", "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
s.log.Warn("downloadAndStoreCover: non-200 response", "url", imageURL, "status", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
data, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
s.log.Warn("downloadAndStoreCover: read body failed", "url", imageURL, "err", readErr)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
|
||||
if putErr := s.store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
|
||||
s.log.Warn("downloadAndStoreCover: SaveBrowseAsset failed", "key", key, "err", putErr)
|
||||
return
|
||||
}
|
||||
s.log.Debug("downloadAndStoreCover: stored cover", "key", key, "bytes", len(data))
|
||||
}
|
||||
|
||||
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
|
||||
// Returns novels and whether a "next page" link was found.
|
||||
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
|
||||
|
||||
@@ -317,8 +317,20 @@ func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bo
|
||||
return h.minio.GetBrowsePage(ctx, key)
|
||||
}
|
||||
|
||||
func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
|
||||
return BrowsePageKey(genre, sortBy, status, novelType, page)
|
||||
func (h *HybridStore) BrowseHTMLKey(domain string, page int) string {
|
||||
return BrowseHTMLKey(domain, page)
|
||||
}
|
||||
|
||||
func (h *HybridStore) BrowseCoverKey(domain, slug string) string {
|
||||
return BrowseCoverKey(domain, slug)
|
||||
}
|
||||
|
||||
func (h *HybridStore) SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
|
||||
return h.minio.PutBrowseAsset(ctx, key, data, contentType)
|
||||
}
|
||||
|
||||
func (h *HybridStore) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
|
||||
return h.minio.GetBrowseAsset(ctx, key)
|
||||
}
|
||||
|
||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
@@ -211,11 +211,26 @@ func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time
|
||||
}
|
||||
|
||||
// ─── Browse page snapshots ────────────────────────────────────────────────────
|
||||
//
|
||||
// New bucket layout (libnovel-browse):
|
||||
//
|
||||
// {domain}/html/page-{n}.html — SingleFile HTML snapshot
|
||||
// {domain}/assets/book-covers/{slug}.jpg — downloaded cover image
|
||||
//
|
||||
// The domain segment is derived from the source URL hostname
|
||||
// (e.g. "novelfire.net"). This makes the bucket self-describing and
|
||||
// extensible to multiple sources.
|
||||
|
||||
// BrowsePageKey returns the MinIO object key for a cached browse-page snapshot.
|
||||
// Layout: {genre}/{sort}/{status}/{novelType}/page-{n}.html
|
||||
func BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
|
||||
return fmt.Sprintf("%s/%s/%s/%s/page-%d.html", genre, sortBy, status, novelType, page)
|
||||
// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
|
||||
// Layout: {domain}/html/page-{n}.html
|
||||
func BrowseHTMLKey(domain string, page int) string {
|
||||
return fmt.Sprintf("%s/html/page-%d.html", domain, page)
|
||||
}
|
||||
|
||||
// BrowseCoverKey returns the MinIO object key for a cached book cover image.
|
||||
// Layout: {domain}/assets/book-covers/{slug}.jpg
|
||||
func BrowseCoverKey(domain, slug string) string {
|
||||
return fmt.Sprintf("%s/assets/book-covers/%s.jpg", domain, slug)
|
||||
}
|
||||
|
||||
// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket.
|
||||
@@ -255,6 +270,37 @@ func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// PutBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
|
||||
// contentType should be the MIME type, e.g. "image/jpeg".
|
||||
func (m *MinioClient) PutBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: contentType})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put browse asset %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBrowseAsset retrieves a binary asset from the browse bucket.
|
||||
// Returns (nil, false, nil) when the object does not exist.
|
||||
func (m *MinioClient) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, "", false, fmt.Errorf("minio: get browse asset %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
info, statErr := obj.Stat()
|
||||
if statErr != nil {
|
||||
return nil, "", false, nil // not found
|
||||
}
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return nil, "", false, fmt.Errorf("minio: read browse asset %s: %w", key, err)
|
||||
}
|
||||
return data, info.ContentType, true, nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// sanitiseVoice converts a voice name to a filename-safe string.
|
||||
|
||||
@@ -146,8 +146,17 @@ type Store interface {
|
||||
// GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil)
|
||||
// when no snapshot exists for the key.
|
||||
GetBrowsePage(ctx context.Context, key string) (string, bool, error)
|
||||
// BrowsePageKey returns the MinIO object key for the given browse params.
|
||||
BrowsePageKey(genre, sortBy, status, novelType string, page int) string
|
||||
// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
|
||||
// Layout: {domain}/html/page-{n}.html
|
||||
BrowseHTMLKey(domain string, page int) string
|
||||
// BrowseCoverKey returns the MinIO object key for a cached book cover image.
|
||||
// Layout: {domain}/assets/book-covers/{slug}.jpg
|
||||
BrowseCoverKey(domain, slug string) string
|
||||
// SaveBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
|
||||
SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error
|
||||
// GetBrowseAsset retrieves a binary asset from the browse bucket.
|
||||
// Returns (nil, "", false, nil) when the object does not exist.
|
||||
GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error)
|
||||
|
||||
// ── Scraping tasks ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
Reference in New Issue
Block a user