feat(scraper): rewrite browse storage to domain/html+assets structure, populate ranking from snapshot
- Replace BrowsePageKey(genre/sort/status/type/page) with BrowseHTMLKey(domain, page) -> {domain}/html/page-{n}.html
- Add BrowseCoverKey(domain, slug) -> {domain}/assets/book-covers/{slug}.jpg
- Add SaveBrowseAsset/GetBrowseAsset for binary assets in browse bucket
- Rewrite triggerBrowseSnapshot: after storing HTML, parse it, upsert ranking records with MinIO cover keys, fire per-novel cover download goroutines
- Add handleGetCover endpoint (GET /api/cover/{domain}/{slug}) to proxy cover images from MinIO
- handleGetRanking rewrites MinIO cover keys to /api/cover/... proxy URLs
- Update save-browse CLI to use BrowseHTMLKey, populate ranking, and download covers
This commit is contained in:
@@ -317,8 +317,20 @@ func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bo
|
||||
return h.minio.GetBrowsePage(ctx, key)
|
||||
}
|
||||
|
||||
func (h *HybridStore) BrowsePageKey(genre, sortBy, status, novelType string, page int) string {
|
||||
return BrowsePageKey(genre, sortBy, status, novelType, page)
|
||||
func (h *HybridStore) BrowseHTMLKey(domain string, page int) string {
|
||||
return BrowseHTMLKey(domain, page)
|
||||
}
|
||||
|
||||
func (h *HybridStore) BrowseCoverKey(domain, slug string) string {
|
||||
return BrowseCoverKey(domain, slug)
|
||||
}
|
||||
|
||||
func (h *HybridStore) SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error {
|
||||
return h.minio.PutBrowseAsset(ctx, key, data, contentType)
|
||||
}
|
||||
|
||||
func (h *HybridStore) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) {
|
||||
return h.minio.GetBrowseAsset(ctx, key)
|
||||
}
|
||||
|
||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user