feat(scraper): rewrite browse storage to domain/html+assets structure, populate ranking from snapshot
- Replace BrowsePageKey(genre/sort/status/type/page) with BrowseHTMLKey(domain, page) -> {domain}/html/page-{n}.html
- Add BrowseCoverKey(domain, slug) -> {domain}/assets/book-covers/{slug}.jpg
- Add SaveBrowseAsset/GetBrowseAsset for binary assets in browse bucket
- Rewrite triggerBrowseSnapshot: after storing HTML, parse it, upsert ranking records with MinIO cover keys, fire per-novel cover download goroutines
- Add handleGetCover endpoint (GET /api/cover/{domain}/{slug}) to proxy cover images from MinIO
- handleGetRanking rewrites MinIO cover keys to /api/cover/... proxy URLs
- Update save-browse CLI to use BrowseHTMLKey, populate ranking, and download covers
This commit is contained in:
Admin
@@ -33,7 +33,9 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
@@ -212,6 +214,8 @@ func run(log *slog.Logger) error {
|
||||
// It iterates over browse pages on novelfire.net, captures each using
|
||||
// SingleFile CLI (connected to the existing Browserless instance), and
|
||||
// stores the resulting self-contained HTML in the MinIO browse bucket.
|
||||
// After storing each page it parses the HTML, upserts ranking records in
|
||||
// PocketBase, and fires background goroutines to download cover images.
|
||||
//
|
||||
// Flags (all optional):
|
||||
//
|
||||
@@ -281,6 +285,7 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
|
||||
defer os.RemoveAll(tmpDir)
|
||||
|
||||
const novelFireBase = "https://novelfire.net"
|
||||
const novelFireDomain = "novelfire.net"
|
||||
|
||||
for page := 1; page <= maxPages; page++ {
|
||||
select {
|
||||
@@ -292,7 +297,8 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
|
||||
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
|
||||
key := store.BrowsePageKey(genre, sortBy, status, novelType, page)
|
||||
// Use the new domain-based key layout: {domain}/html/page-{n}.html
|
||||
key := store.BrowseHTMLKey(novelFireDomain, page)
|
||||
|
||||
outFile := fmt.Sprintf("%s/page-%d.html", tmpDir, page)
|
||||
|
||||
@@ -328,12 +334,205 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
|
||||
|
||||
log.Info("save-browse: snapshot stored", "page", page, "key", key,
|
||||
"bytes", len(htmlBytes))
|
||||
|
||||
// Parse the stored HTML and populate the ranking collection.
|
||||
novels := parseSaveBrowseListings(htmlBytes, novelFireBase)
|
||||
for i, novel := range novels {
|
||||
rank := i + 1
|
||||
coverKey := store.BrowseCoverKey(novelFireDomain, novel.slug)
|
||||
|
||||
item := storage.RankingItem{
|
||||
Rank: rank,
|
||||
Slug: novel.slug,
|
||||
Title: novel.title,
|
||||
Cover: coverKey,
|
||||
SourceURL: novel.url,
|
||||
}
|
||||
if werr := store.WriteRankingItem(ctx, item); werr != nil {
|
||||
log.Warn("save-browse: WriteRankingItem failed",
|
||||
"slug", novel.slug, "err", werr)
|
||||
}
|
||||
|
||||
// Download cover image in the background (best-effort).
|
||||
if novel.coverURL != "" {
|
||||
go downloadAndStoreCoverCLI(store, log, coverKey, novel.coverURL)
|
||||
}
|
||||
}
|
||||
if len(novels) > 0 {
|
||||
log.Info("save-browse: ranking populated", "page", page, "count", len(novels))
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("save-browse: done")
|
||||
return nil
|
||||
}
|
||||
|
||||
// novelListingCLI is a minimal novel listing used within the CLI command.
|
||||
type novelListingCLI struct {
|
||||
slug string
|
||||
title string
|
||||
url string
|
||||
coverURL string
|
||||
}
|
||||
|
||||
// parseSaveBrowseListings extracts novel listings from raw HTML bytes.
|
||||
// It reuses the same parsing logic as the server's parseBrowsePage but
|
||||
// operates on []byte to avoid importing the server package.
|
||||
func parseSaveBrowseListings(htmlBytes []byte, novelFireBase string) []novelListingCLI {
|
||||
type listing = novelListingCLI
|
||||
|
||||
// Minimal tokeniser-based walk to find <li class="novel-item"> blocks.
|
||||
// We use the golang.org/x/net/html parser via a local import.
|
||||
// Because main.go already imports golang.org/x/net/html indirectly through
|
||||
// the server package build, we do a simple line-scan here instead to keep
|
||||
// the dependency surface small.
|
||||
//
|
||||
// Strategy: scan for href="/book/{slug}", img data-src/src, h4.novel-title text.
|
||||
var novels []listing
|
||||
|
||||
lines := strings.Split(string(htmlBytes), "\n")
|
||||
var cur listing
|
||||
inNovelItem := false
|
||||
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
|
||||
// Detect start of a novel-item list element.
|
||||
if strings.Contains(trimmed, `class="novel-item"`) || strings.Contains(trimmed, "novel-item") && strings.HasPrefix(trimmed, "<li") {
|
||||
inNovelItem = true
|
||||
cur = listing{}
|
||||
}
|
||||
|
||||
if !inNovelItem {
|
||||
continue
|
||||
}
|
||||
|
||||
// Detect end of list element.
|
||||
if trimmed == "</li>" && cur.slug != "" {
|
||||
novels = append(novels, cur)
|
||||
inNovelItem = false
|
||||
cur = listing{}
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract slug from href="/book/{slug}".
|
||||
if cur.slug == "" {
|
||||
if idx := strings.Index(trimmed, `href="/book/`); idx >= 0 {
|
||||
rest := trimmed[idx+len(`href="/book/`):]
|
||||
if end := strings.IndexAny(rest, `"/ `); end > 0 {
|
||||
cur.slug = rest[:end]
|
||||
cur.url = novelFireBase + "/book/" + cur.slug
|
||||
} else if end := strings.Index(rest, `"`); end > 0 {
|
||||
cur.slug = strings.TrimSuffix(rest[:end], "/")
|
||||
cur.url = novelFireBase + "/book/" + cur.slug
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract cover URL from data-src or src on img tags.
|
||||
if cur.coverURL == "" && strings.Contains(trimmed, "<img") {
|
||||
if src := extractAttr(trimmed, "data-src"); src != "" {
|
||||
cur.coverURL = resolveURL(src, novelFireBase)
|
||||
} else if src := extractAttr(trimmed, "src"); src != "" && !strings.Contains(src, "data:") {
|
||||
cur.coverURL = resolveURL(src, novelFireBase)
|
||||
}
|
||||
}
|
||||
|
||||
// Extract title from novel-title element.
|
||||
if cur.title == "" && strings.Contains(trimmed, "novel-title") {
|
||||
// Try to grab inner text: <h4 class="novel-title">Title Here</h4>
|
||||
if start := strings.Index(trimmed, ">"); start >= 0 {
|
||||
rest := trimmed[start+1:]
|
||||
if end := strings.Index(rest, "<"); end > 0 {
|
||||
title := strings.TrimSpace(rest[:end])
|
||||
if title != "" {
|
||||
cur.title = title
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush any open item that wasn't closed by </li> (e.g. last item in file).
|
||||
if inNovelItem && cur.slug != "" {
|
||||
novels = append(novels, cur)
|
||||
}
|
||||
|
||||
return novels
|
||||
}
|
||||
|
||||
// extractAttr extracts an HTML attribute value from a raw tag string.
|
||||
// e.g. extractAttr(`<img data-src="foo.jpg">`, "data-src") → "foo.jpg"
|
||||
func extractAttr(tag, attr string) string {
|
||||
needle := attr + `="`
|
||||
idx := strings.Index(tag, needle)
|
||||
if idx < 0 {
|
||||
return ""
|
||||
}
|
||||
rest := tag[idx+len(needle):]
|
||||
end := strings.Index(rest, `"`)
|
||||
if end < 0 {
|
||||
return ""
|
||||
}
|
||||
return rest[:end]
|
||||
}
|
||||
|
||||
// resolveURL ensures the URL is absolute, prepending novelFireBase if needed.
|
||||
func resolveURL(src, base string) string {
|
||||
if strings.HasPrefix(src, "http") {
|
||||
return src
|
||||
}
|
||||
return base + src
|
||||
}
|
||||
|
||||
// downloadAndStoreCoverCLI fetches a cover image and stores it in MinIO.
|
||||
// Errors are logged but not propagated — this is a best-effort background task.
|
||||
func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageURL string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Skip if already stored.
|
||||
if _, _, ok, _ := store.GetBrowseAsset(ctx, key); ok {
|
||||
return
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
||||
if err != nil {
|
||||
log.Warn("save-browse: cover build request failed", "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
log.Warn("save-browse: cover fetch failed", "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Warn("save-browse: cover non-200", "url", imageURL, "status", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
data, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
log.Warn("save-browse: cover read body failed", "url", imageURL, "err", readErr)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
|
||||
if putErr := store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
|
||||
log.Warn("save-browse: SaveBrowseAsset failed", "key", key, "err", putErr)
|
||||
return
|
||||
}
|
||||
log.Debug("save-browse: cover stored", "key", key, "bytes", len(data))
|
||||
}
|
||||
|
||||
func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.BrowserClient {
|
||||
switch strategy {
|
||||
case browser.StrategyScrape:
|
||||
|
||||
Reference in New Issue
Block a user