feat(scraper): rewrite browse storage to domain/html+assets structure, populate ranking from snapshot
- Replace BrowsePageKey(genre/sort/status/type/page) with BrowseHTMLKey(domain, page) -> {domain}/html/page-{n}.html
- Add BrowseCoverKey(domain, slug) -> {domain}/assets/book-covers/{slug}.jpg
- Add SaveBrowseAsset/GetBrowseAsset for binary assets in browse bucket
- Rewrite triggerBrowseSnapshot: after storing HTML, parse it, upsert ranking records with MinIO cover keys, fire per-novel cover download goroutines
- Add handleGetCover endpoint (GET /api/cover/{domain}/{slug}) to proxy cover images from MinIO
- handleGetRanking rewrites MinIO cover keys to /api/cover/... proxy URLs
- Update save-browse CLI to use BrowseHTMLKey, populate ranking, and download covers
This commit is contained in:
@@ -317,8 +317,20 @@ func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bo
|
||||
return h.minio.GetBrowsePage(ctx, key)
|
||||
}
|
||||
|
||||
func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
|
||||
return BrowsePageKey(genre, sortBy, status, novelType, page)
|
||||
func (h *HybridStore) BrowseHTMLKey(domain string, page int) string {
|
||||
return BrowseHTMLKey(domain, page)
|
||||
}
|
||||
|
||||
func (h *HybridStore) BrowseCoverKey(domain, slug string) string {
|
||||
return BrowseCoverKey(domain, slug)
|
||||
}
|
||||
|
||||
func (h *HybridStore) SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
|
||||
return h.minio.PutBrowseAsset(ctx, key, data, contentType)
|
||||
}
|
||||
|
||||
func (h *HybridStore) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
|
||||
return h.minio.GetBrowseAsset(ctx, key)
|
||||
}
|
||||
|
||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
@@ -211,11 +211,26 @@ func (m *MinioClient) PresignAudio(ctx context.Context, key string, expires time
|
||||
}
|
||||
|
||||
// ─── Browse page snapshots ────────────────────────────────────────────────────
|
||||
//
|
||||
// New bucket layout (libnovel-browse):
|
||||
//
|
||||
// {domain}/html/page-{n}.html — SingleFile HTML snapshot
|
||||
// {domain}/assets/book-covers/{slug}.jpg — downloaded cover image
|
||||
//
|
||||
// The domain segment is derived from the source URL hostname
|
||||
// (e.g. "novelfire.net"). This makes the bucket self-describing and
|
||||
// extensible to multiple sources.
|
||||
|
||||
// BrowsePageKey returns the MinIO object key for a cached browse-page snapshot.
|
||||
// Layout: {genre}/{sort}/{status}/{novelType}/page-{n}.html
|
||||
func BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
|
||||
return fmt.Sprintf("%s/%s/%s/%s/page-%d.html", genre, sortBy, status, novelType, page)
|
||||
// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
|
||||
// Layout: {domain}/html/page-{n}.html
|
||||
func BrowseHTMLKey(domain string, page int) string {
|
||||
return fmt.Sprintf("%s/html/page-%d.html", domain, page)
|
||||
}
|
||||
|
||||
// BrowseCoverKey returns the MinIO object key for a cached book cover image.
|
||||
// Layout: {domain}/assets/book-covers/{slug}.jpg
|
||||
func BrowseCoverKey(domain, slug string) string {
|
||||
return fmt.Sprintf("%s/assets/book-covers/%s.jpg", domain, slug)
|
||||
}
|
||||
|
||||
// PutBrowsePage stores a SingleFile HTML snapshot in the browse bucket.
|
||||
@@ -255,6 +270,37 @@ func (m *MinioClient) BrowsePageExists(ctx context.Context, key string) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// PutBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
|
||||
// contentType should be the MIME type, e.g. "image/jpeg".
|
||||
func (m *MinioClient) PutBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
|
||||
_, err := m.c.PutObject(ctx, m.cfg.BucketBrowse, key,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: contentType})
|
||||
if err != nil {
|
||||
return fmt.Errorf("minio: put browse asset %s: %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBrowseAsset retrieves a binary asset from the browse bucket.
|
||||
// Returns (nil, false, nil) when the object does not exist.
|
||||
func (m *MinioClient) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
|
||||
obj, err := m.c.GetObject(ctx, m.cfg.BucketBrowse, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, "", false, fmt.Errorf("minio: get browse asset %s: %w", key, err)
|
||||
}
|
||||
defer obj.Close()
|
||||
info, statErr := obj.Stat()
|
||||
if statErr != nil {
|
||||
return nil, "", false, nil // not found
|
||||
}
|
||||
data, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return nil, "", false, fmt.Errorf("minio: read browse asset %s: %w", key, err)
|
||||
}
|
||||
return data, info.ContentType, true, nil
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// sanitiseVoice converts a voice name to a filename-safe string.
|
||||
|
||||
@@ -146,8 +146,17 @@ type Store interface {
|
||||
// GetBrowsePage retrieves a cached HTML snapshot. Returns ("", false, nil)
|
||||
// when no snapshot exists for the key.
|
||||
GetBrowsePage(ctx context.Context, key string) (string, bool, error)
|
||||
// BrowsePageKey returns the MinIO object key for the given browse params.
|
||||
BrowsePageKey(genre, sortBy, status, novelType string, page int) string
|
||||
// BrowseHTMLKey returns the MinIO object key for a SingleFile HTML snapshot.
|
||||
// Layout: {domain}/html/page-{n}.html
|
||||
BrowseHTMLKey(domain string, page int) string
|
||||
// BrowseCoverKey returns the MinIO object key for a cached book cover image.
|
||||
// Layout: {domain}/assets/book-covers/{slug}.jpg
|
||||
BrowseCoverKey(domain, slug string) string
|
||||
// SaveBrowseAsset stores a binary asset (e.g. a cover image) in the browse bucket.
|
||||
SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error
|
||||
// GetBrowseAsset retrieves a binary asset from the browse bucket.
|
||||
// Returns (nil, "", false, nil) when the object does not exist.
|
||||
GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error)
|
||||
|
||||
// ── Scraping tasks ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
Reference in New Issue
Block a user