feat(scraper): rewrite browse storage to domain/html+assets structure, populate ranking from snapshot

- Replace BrowsePageKey(genre/sort/status/type/page) with BrowseHTMLKey(domain, page) -> {domain}/html/page-{n}.html
- Add BrowseCoverKey(domain, slug) -> {domain}/assets/book-covers/{slug}.jpg
- Add SaveBrowseAsset/GetBrowseAsset for binary assets in browse bucket
- Rewrite triggerBrowseSnapshot: after storing HTML, parse it, upsert ranking records with MinIO cover keys, fire per-novel cover download goroutines
- Add handleGetCover endpoint (GET /api/cover/{domain}/{slug}) to proxy cover images from MinIO
- handleGetRanking rewrites MinIO cover keys to /api/cover/... proxy URLs
- Update save-browse CLI to use BrowseHTMLKey, populate ranking, and download covers
This commit is contained in:
Admin
2026-03-04 15:13:09 +05:00
parent bf2ffa54db
commit 11a846d043
5 changed files with 438 additions and 19 deletions

View File

@@ -317,8 +317,20 @@ func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bo
return h.minio.GetBrowsePage(ctx, key)
}
func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
return BrowsePageKey(genre, sortBy, status, novelType, page)
func (h *HybridStore) BrowseHTMLKey(domain string, page int) string {
return BrowseHTMLKey(domain, page)
}
func (h *HybridStore) BrowseCoverKey(domain, slug string) string {
return BrowseCoverKey(domain, slug)
}
func (h *HybridStore) SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
return h.minio.PutBrowseAsset(ctx, key, data, contentType)
}
func (h *HybridStore) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
return h.minio.GetBrowseAsset(ctx, key)
}
// ─── Scraping tasks ───────────────────────────────────────────────────────────

View File

@@ -211,11 +211,26 @@ func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time
}
// ─── Browse page snapshots ────────────────────────────────────────────────────
//
// New bucket layout (libnovel-browse):
//
// {domain}/html/page-{n}.html — SingleFile HTML snapshot
// {domain}/assets/book-covers/{slug}.jpg — downloaded cover image
//
// The domain segment is derived from the source URL hostname
// (e.g. "novelfire.net"). This makes the bucket self-describing and
// extensible to multiple sources.
// BrowsePageKey returns the MinIO object key for a cached browse-page snapshot.
// Layout: {genre}/{sort}/{status}/{novelType}/page-{n}.html
func BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
return fmt.Sprintf("%s/%s/%s/%s/page-%d.html", genre, sortBy, status, novelType, page)
// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
// Layout: {domain}/html/page-{n}.html
func BrowseHTMLKey(domain string, page int) string {
return fmt.Sprintf("%s/html/page-%d.html", domain, page)
}
// BrowseCoverKey returns the MinIO object key for a cached book cover image.
// Layout: {domain}/assets/book-covers/{slug}.jpg
func BrowseCoverKey(domain, slug string) string {
return fmt.Sprintf("%s/assets/book-covers/%s.jpg", domain, slug)
}
// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket.
@@ -255,6 +270,37 @@ func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool {
return err == nil
}
// PutBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
// contentType should be the MIME type, e.g. "image/jpeg".
func (m *MinioClient) PutBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
_, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: contentType})
if err != nil {
return fmt.Errorf("minio: put browse asset %s: %w", key, err)
}
return nil
}
// GetBrowseAsset retrieves a binary asset from the browse bucket.
// Returns (nil, false, nil) when the object does not exist.
func (m *MinioClient) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
if err != nil {
return nil, "", false, fmt.Errorf("minio: get browse asset %s: %w", key, err)
}
defer obj.Close()
info, statErr := obj.Stat()
if statErr != nil {
return nil, "", false, nil // not found
}
data, err := io.ReadAll(obj)
if err != nil {
return nil, "", false, fmt.Errorf("minio: read browse asset %s: %w", key, err)
}
return data, info.ContentType, true, nil
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// sanitiseVoice converts a voice name to a filename-safe string.

View File

@@ -146,8 +146,17 @@ type Store interface {
// GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil)
// when no snapshot exists for the key.
GetBrowsePage(ctx context.Context, key string) (string, bool, error)
// BrowsePageKey returns the MinIO object key for the given browse params.
BrowsePageKey(genre, sortBy, status, novelType string, page int) string
// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
// Layout: {domain}/html/page-{n}.html
BrowseHTMLKey(domain string, page int) string
// BrowseCoverKey returns the MinIO object key for a cached book cover image.
// Layout: {domain}/assets/book-covers/{slug}.jpg
BrowseCoverKey(domain, slug string) string
// SaveBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error
// GetBrowseAsset retrieves a binary asset from the browse bucket.
// Returns (nil, "", false, nil) when the object does not exist.
GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error)
// ── Scraping tasks ─────────────────────────────────────────────────────