refactor: audit, split server.go, add unit tests, and fix latent bugs
- Remove dead code: browser cdp/content_scrape strategies, writer package, printUsage, downloadAndStoreCoverCLI in main.go - Fix bugs: defer-in-loop in pocketbase deleteWhere, listAll() pagination hard cap removed, splitChapterTitle off-by-one in date extraction - Split server.go (~1700 lines) into focused handler files: handlers_audio, handlers_browse, handlers_progress, handlers_ranking, handlers_scrape - Export htmlutil.AttrVal/TextContent/ResolveURL; add storage/coverutil.go to consolidate duplicate helpers - Flatten deeply nested conditionals: voices() early-return guards, ScrapeCatalogue next-link double attr scan, chapterNumberFromKey dead strings.Cut line, splitChapterTitle double-nested unit/suffix loop - Add unit tests: htmlutil (9 funcs), novelfire ScrapeMetadata (3 cases), orchestrator Run (5 cases), storage chapterNumberFromKey/splitChapterTitle (22 cases); all pass with go build/vet/test clean
This commit is contained in:
59
scraper/internal/storage/coverutil.go
Normal file
59
scraper/internal/storage/coverutil.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DownloadAndStoreCover fetches the image at imageURL and stores it in the
|
||||
// store under key. Errors are logged but not returned — this is best-effort.
|
||||
// If the asset is already present the download is skipped.
|
||||
func DownloadAndStoreCover(store Store, log *slog.Logger, key, imageURL string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Skip if already stored.
|
||||
if _, _, ok, _ := store.GetBrowseAsset(ctx, key); ok {
|
||||
return
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
||||
if err != nil {
|
||||
log.Warn("cover: build request failed", "key", key, "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
log.Warn("cover: fetch failed", "key", key, "url", imageURL, "err", fmt.Errorf("%w", err))
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Warn("cover: non-200 response", "key", key, "url", imageURL, "status", resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Warn("cover: read body failed", "key", key, "url", imageURL, "err", err)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "image/jpeg"
|
||||
}
|
||||
|
||||
if err := store.SaveBrowseAsset(ctx, key, data, contentType); err != nil {
|
||||
log.Warn("cover: SaveBrowseAsset failed", "key", key, "err", err)
|
||||
return
|
||||
}
|
||||
log.Debug("cover: stored", "key", key, "bytes", len(data))
|
||||
}
|
||||
Reference in New Issue
Block a user