kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

refactor: audit, split server.go, add unit tests, and fix latent bugs

Browse Source 
- Remove dead code: browser cdp/content_scrape strategies, writer package,
  printUsage, downloadAndStoreCoverCLI in main.go
- Fix bugs: defer-in-loop in pocketbase deleteWhere, listAll() pagination
  hard cap removed, splitChapterTitle off-by-one in date extraction
- Split server.go (~1700 lines) into focused handler files:
  handlers_audio, handlers_browse, handlers_progress, handlers_ranking,
  handlers_scrape
- Export htmlutil.AttrVal/TextContent/ResolveURL; add storage/coverutil.go
  to consolidate duplicate helpers
- Flatten deeply nested conditionals: voices() early-return guards,
  ScrapeCatalogue next-link double attr scan, chapterNumberFromKey dead
  strings.Cut line, splitChapterTitle double-nested unit/suffix loop
- Add unit tests: htmlutil (9 funcs), novelfire ScrapeMetadata (3 cases),
  orchestrator Run (5 cases), storage chapterNumberFromKey/splitChapterTitle
  (22 cases); all pass with go build/vet/test clean
This commit is contained in:
Admin
2026-03-04 22:14:23 +05:00
parent 7b48707cd9
commit fb6b364382
26 changed files with 2359 additions and 2392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
scraper/cmd/scraper/main.go
View File

@@ -10,10 +10,6 @@
//
// Environment variables:
//
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp | direct (default: direct; chapter list+text always use direct HTTP)
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
@@ -33,9 +29,7 @@ package main
import (
"context"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"os/exec"
"os/signal"
@@ -48,6 +42,7 @@ import (
"github.com/libnovel/scraper/internal/browser"
"github.com/libnovel/scraper/internal/novelfire"
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/scraper/htmlutil"
"github.com/libnovel/scraper/internal/server"
"github.com/libnovel/scraper/internal/storage"
)
@@ -343,7 +338,7 @@ func runSaveBrowse(ctx context.Context, args []string, store storage.Store, log
// Download cover image in the background (best-effort).
if novel.coverURL != "" {
go downloadAndStoreCoverCLI(store, log, coverKey, novel.coverURL)
go storage.DownloadAndStoreCover(store, log, coverKey, novel.coverURL)
}
}
if len(novels) > 0 {
@@ -420,9 +415,9 @@ func parseSaveBrowseListings(htmlBytes []byte, novelFireBase string) []novelList
// Extract cover URL from data-src or src on img tags.
if cur.coverURL == "" && strings.Contains(trimmed, "<img") {
if src := extractAttr(trimmed, "data-src"); src != "" {
cur.coverURL = resolveURL(src, novelFireBase)
cur.coverURL = htmlutil.ResolveURL(novelFireBase, src)
} else if src := extractAttr(trimmed, "src"); src != "" && !strings.Contains(src, "data:") {
cur.coverURL = resolveURL(src, novelFireBase)
cur.coverURL = htmlutil.ResolveURL(novelFireBase, src)
}
}
@@ -465,62 +460,6 @@ func extractAttr(tag, attr string) string {
return rest[:end]
}
// resolveURL ensures the URL is absolute, prepending novelFireBase if needed.
func resolveURL(src, base string) string {
if strings.HasPrefix(src, "http") {
return src
}
return base + src
}
// downloadAndStoreCoverCLI fetches a cover image and stores it in MinIO.
// Errors are logged but not propagated — this is a best-effort background task.
func downloadAndStoreCoverCLI(store storage.Store, log *slog.Logger, key, imageURL string) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Skip if already stored.
if _, _, ok, _ := store.GetBrowseAsset(ctx, key); ok {
return
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
if err != nil {
log.Warn("save-browse: cover build request failed", "url", imageURL, "err", err)
return
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Warn("save-browse: cover fetch failed", "url", imageURL, "err", err)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Warn("save-browse: cover non-200", "url", imageURL, "status", resp.StatusCode)
return
}
data, readErr := io.ReadAll(resp.Body)
if readErr != nil {
log.Warn("save-browse: cover read body failed", "url", imageURL, "err", readErr)
return
}
contentType := resp.Header.Get("Content-Type")
if contentType == "" {
contentType = "image/jpeg"
}
if putErr := store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
log.Warn("save-browse: SaveBrowseAsset failed", "key", key, "err", putErr)
return
}
log.Debug("save-browse: cover stored", "key", key, "bytes", len(data))
}
func envOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
@@ -543,17 +482,22 @@ Commands:
--max-pages <n> pages to capture (default: 5)
Environment variables:
BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
BROWSERLESS_TOKEN API token (default: "")
BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct; chapter list+text always use direct HTTP)
BROWSERLESS_MAX_CONCURRENT Max simultaneous sessions (default: 5)
BROWSERLESS_TIMEOUT HTTP request timeout sec (default: 90)
SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d)
SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
SCRAPER_TIMEOUT HTTP request timeout sec (default: 90)
KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled)
KOKORO_VOICE Default TTS voice (default: af_bella)
POCKETBASE_URL PocketBase base URL (default: http://localhost:8090)
POCKETBASE_ADMIN_EMAIL PocketBase admin email (default: admin@libnovel.local)
POCKETBASE_ADMIN_PASSWORD PocketBase admin password (default: changeme123)
MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000)
MINIO_ACCESS_KEY MinIO access key (default: admin)
MINIO_SECRET_KEY MinIO secret key (default: changeme123)
MINIO_USE_SSL MinIO TLS (default: false)
MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters)
MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio)
MINIO_BUCKET_BROWSE Browse snapshots bucket (default: libnovel-browse)
BROWSERLESS_URL Browserless WS endpoint (default: http://localhost:3030)
SINGLEFILE_PATH Path to single-file CLI (default: single-file)
LOG_LEVEL debug|info|warn|error (default: info)
`, runtime.NumCPU())