kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

steps 6-8: wire HybridStore into orchestrator, server, and main

Browse Source 
- Add storage/hybrid.go: HybridStore composing PocketBase + MinIO backends
- Rewrite orchestrator to accept storage.Store instead of *writer.Writer
- Replace *writer.Writer with storage.Store in server.go and ui.go
- Wire audio cache, reading progress, chapter reads/writes through store
- Add rankingCacheAdapter in main.go to bridge context-free RankingPageCacher
  interface to HybridStore's context-aware methods
This commit is contained in:
Admin
2026-03-02 14:44:02 +05:00
parent 9add9033b9
commit 18e76c9668
5 changed files with 443 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
scraper/cmd/scraper/main.go
View File

@@ -10,16 +10,24 @@
//
// Environment variables:
//
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030)
// BROWSERLESS_TOKEN Browserless API token (default: "")
// BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
// BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
// KOKORO_VOICE Default TTS voice (default: af_bella)
// LOG_LEVEL debug | info | warn | error (default: info)
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
// KOKORO_VOICE Default TTS voice (default: af_bella)
// POCKETBASE_URL PocketBase API base URL (default: http://localhost:8090)
// POCKETBASE_EMAIL PocketBase admin email (default: admin@libnovel.local)
// POCKETBASE_PASSWORD PocketBase admin password (default: adminpassword)
// MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000)
// MINIO_ACCESS_KEY MinIO access key (default: minioadmin)
// MINIO_SECRET_KEY MinIO secret key (default: minioadmin)
// MINIO_USE_SSL Use TLS for MinIO (default: false)
// MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters)
// MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio)
// LOG_LEVEL debug | info | warn | error (default: info)
package main
import (
@@ -38,7 +46,7 @@ import (
"github.com/libnovel/scraper/internal/novelfire"
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/server"
"github.com/libnovel/scraper/internal/writer"
"github.com/libnovel/scraper/internal/storage"
)
func main() {
@@ -88,9 +96,30 @@ func run(log *slog.Logger) error {
bc := newBrowserClient(strategy, browserCfg)
urlClient := newBrowserClient(urlStrategy, browserCfg)
staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books")
w := writer.New(staticRoot)
nf := novelfire.New(bc, log, urlClient, w)
// ── Storage backends ────────────────────────────────────────────────────
minioCfg := storage.MinioConfig{
Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"),
AccessKey: envOr("MINIO_ACCESS_KEY", "minioadmin"),
SecretKey: envOr("MINIO_SECRET_KEY", "minioadmin"),
UseSSL: strings.ToLower(os.Getenv("MINIO_USE_SSL")) == "true",
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
}
pbCfg := storage.PocketBaseConfig{
BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"),
AdminEmail: envOr("POCKETBASE_EMAIL", "admin@libnovel.local"),
AdminPassword: envOr("POCKETBASE_PASSWORD", "adminpassword"),
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
store, err := storage.NewHybridStore(ctx, pbCfg, minioCfg)
if err != nil {
return fmt.Errorf("storage init failed: %w", err)
}
nf := novelfire.New(bc, log, urlClient, &rankingCacheAdapter{store: store})
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -104,13 +133,9 @@ func run(log *slog.Logger) error {
}
oCfg := orchestrator.Config{
Workers: workers,
StaticRoot: staticRoot,
Workers: workers,
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
switch cmd {
case "run":
// Optional --url flag.
@@ -121,10 +146,9 @@ func run(log *slog.Logger) error {
"strategy", strategy,
"workers", workers,
"max_concurrent", browserCfg.MaxConcurrent,
"static_root", oCfg.StaticRoot,
"single_book", oCfg.SingleBookURL,
)
o := orchestrator.New(oCfg, nf, log)
o := orchestrator.New(oCfg, nf, log, store)
return o.Run(ctx)
case "refresh":
@@ -133,13 +157,12 @@ func run(log *slog.Logger) error {
return fmt.Errorf("refresh command requires a book slug argument")
}
slug := args[1]
w := writer.New(oCfg.StaticRoot)
meta, ok, err := w.ReadMetadata(slug)
meta, ok, err := store.ReadMetadata(ctx, slug)
if err != nil {
return fmt.Errorf("failed to read metadata for %s: %w", slug, err)
}
if !ok {
return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot)
return fmt.Errorf("book %q not found in store", slug)
}
if meta.SourceURL == "" {
return fmt.Errorf("book %q has no source_url in metadata", slug)
@@ -149,7 +172,7 @@ func run(log *slog.Logger) error {
"slug", slug,
"source_url", meta.SourceURL,
)
o := orchestrator.New(oCfg, nf, log)
o := orchestrator.New(oCfg, nf, log, store)
return o.Run(ctx)
case "serve":
@@ -164,7 +187,7 @@ func run(log *slog.Logger) error {
"kokoro_url", kokoroURL,
"kokoro_voice", kokoroVoice,
)
srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice)
srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice)
return srv.ListenAndServe(ctx)
default:
@@ -192,6 +215,20 @@ func envOr(key, fallback string) string {
return fallback
}
// rankingCacheAdapter bridges storage.HybridStore (context-aware) to the
// context-free scraper.RankingPageCacher interface expected by novelfire.New.
type rankingCacheAdapter struct {
store *storage.HybridStore
}
func (a *rankingCacheAdapter) WriteRankingPageCache(page int, html string) error {
return a.store.WriteRankingPageCache(context.Background(), page, html)
}
func (a *rankingCacheAdapter) ReadRankingPageCache(page int) (string, error) {
return a.store.ReadRankingPageCache(context.Background(), page)
}
func printUsage() {
fmt.Fprintf(os.Stderr, `libnovel scraper