From 18e76c9668ebfb6d279f41943e81e7c5c2d194d0 Mon Sep 17 00:00:00 2001 From: Admin Date: Mon, 2 Mar 2026 14:44:02 +0500 Subject: [PATCH] steps 6-8: wire HybridStore into orchestrator, server, and main - Add storage/hybrid.go: HybridStore composing PocketBase + MinIO backends - Rewrite orchestrator to accept storage.Store instead of *writer.Writer - Replace *writer.Writer with storage.Store in server.go and ui.go - Wire audio cache, reading progress, chapter reads/writes through store - Add rankingCacheAdapter in main.go to bridge context-free RankingPageCacher interface to HybridStore's context-aware methods --- scraper/cmd/scraper/main.go | 87 +++-- scraper/internal/orchestrator/orchestrator.go | 24 +- scraper/internal/server/server.go | 43 +-- scraper/internal/server/ui.go | 76 +++-- scraper/internal/storage/hybrid.go | 300 ++++++++++++++++++ 5 files changed, 443 insertions(+), 87 deletions(-) create mode 100644 scraper/internal/storage/hybrid.go diff --git a/scraper/cmd/scraper/main.go b/scraper/cmd/scraper/main.go index f3c6962..326e0e0 100644 --- a/scraper/cmd/scraper/main.go +++ b/scraper/cmd/scraper/main.go @@ -10,16 +10,24 @@ // // Environment variables: // -// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030) -// BROWSERLESS_TOKEN Browserless API token (default: "") -// BROWSERLESS_STRATEGY content | scrape | cdp (default: content) +// BROWSERLESS_URL Browserless base URL (default: http://localhost:3030) +// BROWSERLESS_TOKEN Browserless API token (default: "") +// BROWSERLESS_STRATEGY content | scrape | cdp (default: content) // BROWSERLESS_MAX_CONCURRENT Max simultaneous browser sessions (default: 5) -// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU) -// SCRAPER_STATIC_ROOT Output directory (default: ./static/books) -// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) -// KOKORO_URL Kokoro-FastAPI base URL (default: "") -// KOKORO_VOICE Default TTS voice (default: af_bella) -// LOG_LEVEL debug | info | warn | error (default: info) +// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU) +// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080) +// KOKORO_URL Kokoro-FastAPI base URL (default: "") +// KOKORO_VOICE Default TTS voice (default: af_bella) +// POCKETBASE_URL PocketBase API base URL (default: http://localhost:8090) +// POCKETBASE_EMAIL PocketBase admin email (default: admin@libnovel.local) +// POCKETBASE_PASSWORD PocketBase admin password (default: adminpassword) +// MINIO_ENDPOINT MinIO endpoint host:port (default: localhost:9000) +// MINIO_ACCESS_KEY MinIO access key (default: minioadmin) +// MINIO_SECRET_KEY MinIO secret key (default: minioadmin) +// MINIO_USE_SSL Use TLS for MinIO (default: false) +// MINIO_BUCKET_CHAPTERS Chapter objects bucket (default: libnovel-chapters) +// MINIO_BUCKET_AUDIO Audio objects bucket (default: libnovel-audio) +// LOG_LEVEL debug | info | warn | error (default: info) package main import ( @@ -38,7 +46,7 @@ import ( "github.com/libnovel/scraper/internal/novelfire" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/server" - "github.com/libnovel/scraper/internal/writer" + "github.com/libnovel/scraper/internal/storage" ) func main() { @@ -88,9 +96,30 @@ func run(log *slog.Logger) error { bc := newBrowserClient(strategy, browserCfg) urlClient := newBrowserClient(urlStrategy, browserCfg) - staticRoot := envOr("SCRAPER_STATIC_ROOT", "./static/books") - w := writer.New(staticRoot) - nf := novelfire.New(bc, log, urlClient, w) + // ── Storage backends ──────────────────────────────────────────────────── + minioCfg := storage.MinioConfig{ + Endpoint: envOr("MINIO_ENDPOINT", "localhost:9000"), + AccessKey: envOr("MINIO_ACCESS_KEY", "minioadmin"), + SecretKey: envOr("MINIO_SECRET_KEY", "minioadmin"), + UseSSL: strings.ToLower(os.Getenv("MINIO_USE_SSL")) == "true", + BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), + BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), + } + pbCfg := storage.PocketBaseConfig{ + BaseURL: envOr("POCKETBASE_URL", "http://localhost:8090"), + AdminEmail: envOr("POCKETBASE_EMAIL", "admin@libnovel.local"), + AdminPassword: envOr("POCKETBASE_PASSWORD", "adminpassword"), + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + store, err := storage.NewHybridStore(ctx, pbCfg, minioCfg) + if err != nil { + return fmt.Errorf("storage init failed: %w", err) + } + + nf := novelfire.New(bc, log, urlClient, &rankingCacheAdapter{store: store}) workers := 0 if s := os.Getenv("SCRAPER_WORKERS"); s != "" { @@ -104,13 +133,9 @@ func run(log *slog.Logger) error { } oCfg := orchestrator.Config{ - Workers: workers, - StaticRoot: staticRoot, + Workers: workers, } - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer stop() - switch cmd { case "run": // Optional --url flag. @@ -121,10 +146,9 @@ func run(log *slog.Logger) error { "strategy", strategy, "workers", workers, "max_concurrent", browserCfg.MaxConcurrent, - "static_root", oCfg.StaticRoot, "single_book", oCfg.SingleBookURL, ) - o := orchestrator.New(oCfg, nf, log) + o := orchestrator.New(oCfg, nf, log, store) return o.Run(ctx) case "refresh": @@ -133,13 +157,12 @@ func run(log *slog.Logger) error { return fmt.Errorf("refresh command requires a book slug argument") } slug := args[1] - w := writer.New(oCfg.StaticRoot) - meta, ok, err := w.ReadMetadata(slug) + meta, ok, err := store.ReadMetadata(ctx, slug) if err != nil { return fmt.Errorf("failed to read metadata for %s: %w", slug, err) } if !ok { - return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot) + return fmt.Errorf("book %q not found in store", slug) } if meta.SourceURL == "" { return fmt.Errorf("book %q has no source_url in metadata", slug) @@ -149,7 +172,7 @@ func run(log *slog.Logger) error { "slug", slug, "source_url", meta.SourceURL, ) - o := orchestrator.New(oCfg, nf, log) + o := orchestrator.New(oCfg, nf, log, store) return o.Run(ctx) case "serve": @@ -164,7 +187,7 @@ func run(log *slog.Logger) error { "kokoro_url", kokoroURL, "kokoro_voice", kokoroVoice, ) - srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice) + srv := server.New(addr, oCfg, nf, log, store, kokoroURL, kokoroVoice) return srv.ListenAndServe(ctx) default: @@ -192,6 +215,20 @@ func envOr(key, fallback string) string { return fallback } +// rankingCacheAdapter bridges storage.HybridStore (context-aware) to the +// context-free scraper.RankingPageCacher interface expected by novelfire.New. +type rankingCacheAdapter struct { + store *storage.HybridStore +} + +func (a *rankingCacheAdapter) WriteRankingPageCache(page int, html string) error { + return a.store.WriteRankingPageCache(context.Background(), page, html) +} + +func (a *rankingCacheAdapter) ReadRankingPageCache(page int) (string, error) { + return a.store.ReadRankingPageCache(context.Background(), page) +} + func printUsage() { fmt.Fprintf(os.Stderr, `libnovel scraper diff --git a/scraper/internal/orchestrator/orchestrator.go b/scraper/internal/orchestrator/orchestrator.go index d35a6c0..619c1b7 100644 --- a/scraper/internal/orchestrator/orchestrator.go +++ b/scraper/internal/orchestrator/orchestrator.go @@ -19,7 +19,7 @@ import ( "sync" "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/writer" + "github.com/libnovel/scraper/internal/storage" ) // Config holds tunable parameters for the orchestrator. @@ -28,7 +28,8 @@ type Config struct { // Defaults to runtime.NumCPU() when 0. Workers int - // StaticRoot is the path to the static/books output directory. + // StaticRoot is kept for backwards-compatibility but is no longer used + // when a Store is provided. StaticRoot string // SingleBookURL when non-empty causes the orchestrator to scrape only @@ -40,13 +41,13 @@ type Config struct { type Orchestrator struct { cfg Config novel scraper.NovelScraper - writer *writer.Writer + store storage.Store log *slog.Logger workers int } -// New returns a new Orchestrator. -func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator { +// New returns a new Orchestrator backed by the provided Store. +func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store) *Orchestrator { workers := cfg.Workers if workers <= 0 { workers = runtime.NumCPU() @@ -54,7 +55,7 @@ func New(cfg Config, novel scraper.NovelScraper, log *slog.Logger) *Orchestrator return &Orchestrator{ cfg: cfg, novel: novel, - writer: writer.New(cfg.StaticRoot), + store: store, log: log, workers: workers, } @@ -66,7 +67,6 @@ func (o *Orchestrator) Run(ctx context.Context) error { o.log.Info("orchestrator starting", "source", o.novel.SourceName(), "workers", o.workers, - "static_root", o.cfg.StaticRoot, ) // chapterWork is the shared queue consumed by chapter worker goroutines. @@ -89,8 +89,8 @@ func (o *Orchestrator) Run(ctx context.Context) error { default: } - // Skip if already on disk. - if o.writer.ChapterExists(job.slug, job.ref) { + // Skip if already stored. + if o.store.ChapterExists(ctx, job.slug, job.ref) { o.log.Debug("chapter already exists, skipping", "book", job.slug, "chapter", job.ref.Number) continue @@ -107,7 +107,7 @@ func (o *Orchestrator) Run(ctx context.Context) error { continue } - if err := o.writer.WriteChapter(job.slug, chapter); err != nil { + if err := o.store.WriteChapter(ctx, job.slug, chapter); err != nil { o.log.Error("chapter write failed", "book", job.slug, "chapter", job.ref.Number, @@ -135,8 +135,8 @@ func (o *Orchestrator) Run(ctx context.Context) error { return } - // Persist / update metadata.yaml. - if err := o.writer.WriteMetadata(meta); err != nil { + // Persist / update metadata. + if err := o.store.WriteMetadata(ctx, meta); err != nil { o.log.Error("metadata write failed", "slug", meta.Slug, "err", err) // Continue — chapters can still be scraped. } diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index cc6a1e0..ab07ee9 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -22,7 +22,7 @@ import ( "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/writer" + "github.com/libnovel/scraper/internal/storage" ) // Server wraps an HTTP mux with the scraping endpoints. @@ -31,7 +31,7 @@ type Server struct { oCfg orchestrator.Config novel scraper.NovelScraper log *slog.Logger - writer *writer.Writer + store storage.Store mu sync.Mutex running bool rankingRunning bool @@ -42,26 +42,23 @@ type Server struct { voiceMu sync.RWMutex cachedVoices []string // populated on first request from Kokoro /v1/audio/voices - // audioMu guards audioCache and audioInFlight. - // audioCache maps a cache key to the Kokoro download filename returned by - // POST /v1/audio/speech with return_download_link=true. + // audioMu guards audioInFlight only. + // Completed audio filenames are persisted to the Store (PocketBase). // audioInFlight deduplicates concurrent generation requests for the same key. audioMu sync.Mutex - audioCache map[string]string // cacheKey → kokoro download filename audioInFlight map[string]chan struct{} // cacheKey → closed when done } // New creates a new Server. -func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server { +func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server { return &Server{ addr: addr, oCfg: oCfg, novel: novel, log: log, - writer: writer.New(oCfg.StaticRoot), + store: store, kokoroURL: kokoroURL, kokoroVoice: kokoroVoice, - audioCache: make(map[string]string), audioInFlight: make(map[string]chan struct{}), } } @@ -174,7 +171,7 @@ func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) return } - raw, err := s.writer.ReadChapter(slug, n) + raw, err := s.store.ReadChapter(r.Context(), slug, n) if err != nil { http.NotFound(w, r) return @@ -223,15 +220,14 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) - // Fast path: already generated this session. - s.audioMu.Lock() - if filename, ok := s.audioCache[cacheKey]; ok { - s.audioMu.Unlock() + // Fast path: already generated (check persistent store first). + if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok { s.writeAudioResponse(w, slug, n, voice, speed, filename) return } // Deduplicate concurrent generation for the same key. + s.audioMu.Lock() if ch, ok := s.audioInFlight[cacheKey]; ok { s.audioMu.Unlock() select { @@ -240,10 +236,8 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable) return } - s.audioMu.Lock() - filename, ok := s.audioCache[cacheKey] - s.audioMu.Unlock() - if ok { + // Check store again after waiting. + if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok { s.writeAudioResponse(w, slug, n, voice, speed, filename) } else { http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError) @@ -262,7 +256,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { }() // Load and validate chapter text. - raw, err := s.writer.ReadChapter(slug, n) + raw, err := s.store.ReadChapter(r.Context(), slug, n) if err != nil { http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound) return @@ -287,9 +281,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) { return } - s.audioMu.Lock() - s.audioCache[cacheKey] = filename - s.audioMu.Unlock() + _ = s.store.SetAudioCache(r.Context(), cacheKey, filename) s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename) s.writeAudioResponse(w, slug, n, voice, speed, filename) @@ -378,10 +370,7 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) { } cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed) - s.audioMu.Lock() - filename, ok := s.audioCache[cacheKey] - s.audioMu.Unlock() - + filename, ok := s.store.GetAudioCache(r.Context(), cacheKey) if !ok { http.Error(w, "audio not generated yet", http.StatusNotFound) return @@ -462,7 +451,7 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() - o := orchestrator.New(cfg, s.novel, s.log) + o := orchestrator.New(cfg, s.novel, s.log, s.store) if err := o.Run(ctx); err != nil { s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err)) } diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go index da600ba..12669e3 100644 --- a/scraper/internal/server/ui.go +++ b/scraper/internal/server/ui.go @@ -15,7 +15,7 @@ import ( "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/writer" + "github.com/libnovel/scraper/internal/storage" "github.com/yuin/goldmark" "github.com/yuin/goldmark/extension" goldhtml "github.com/yuin/goldmark/renderer/html" @@ -629,7 +629,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) { return } - books, err := s.writer.ListBooks() + books, err := s.store.ListBooks(r.Context()) if err != nil { http.Error(w, "failed to list books: "+err.Error(), http.StatusInternalServerError) return @@ -639,8 +639,8 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) { for i, b := range books { items[i] = homeBookItem{ BookMeta: b, - Downloaded: s.writer.CountChapters(b.Slug), - AddedAt: s.writer.MetadataMtime(b.Slug), + Downloaded: s.store.CountChapters(r.Context(), b.Slug), + AddedAt: s.store.MetadataMtime(r.Context(), b.Slug), } } @@ -877,7 +877,7 @@ const scrapeTmpl = ` ` func (s *Server) handleScrape(w http.ResponseWriter, r *http.Request) { - rankingItems, _ := s.writer.ReadRankingItems() + rankingItems, _ := s.store.ReadRankingItems(r.Context()) rankingJSON, _ := json.Marshal(rankingItems) t := template.Must(template.New("scrape").Parse(scrapeTmpl)) @@ -1324,12 +1324,12 @@ const rankingTmpl = ` // rankingViewItem enriches a RankingItem with whether it is present in the // local book library, so the template can highlight it differently. type rankingViewItem struct { - writer.RankingItem + storage.RankingItem Local bool } // toRankingViewItems annotates items with Local=true for slugs found in localSlugs. -func toRankingViewItems(items []writer.RankingItem, localSlugs map[string]bool) []rankingViewItem { +func toRankingViewItems(items []storage.RankingItem, localSlugs map[string]bool) []rankingViewItem { out := make([]rankingViewItem, len(items)) for i, it := range items { out[i] = rankingViewItem{ @@ -1403,13 +1403,13 @@ const rankingPageSize = 20 // It does NOT trigger a live scrape; use POST /ranking/refresh for that. // Supports ?page=N for browsing through cached items (20 per page). func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { - rankingItems, err := s.writer.ReadRankingItems() + rankingItems, err := s.store.ReadRankingItems(r.Context()) if err != nil { s.log.Error("failed to read cached ranking", "err", err) } cachedAt := "" - if info, statErr := s.writer.RankingFileInfo(); statErr == nil { + if info, statErr := s.store.RankingFileInfo(r.Context()); statErr == nil && info != nil { cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04") } @@ -1482,7 +1482,7 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { } // Encode full dataset + local slugs for client-side cross-page filtering. - localSlugs := s.writer.LocalSlugs() + localSlugs, _ := s.store.LocalSlugs(r.Context()) type rankingJSONItem struct { Rank int `json:"rank"` Slug string `json:"slug"` @@ -1577,14 +1577,14 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages) - var rankingItems []writer.RankingItem + var rankingItems []storage.RankingItem for rankingCh != nil || errCh != nil { select { case meta, ok := <-rankingCh: if !ok { rankingCh = nil } else { - rankingItems = append(rankingItems, writer.RankingItem{ + rankingItems = append(rankingItems, storage.RankingItem{ Rank: meta.Ranking, Slug: meta.Slug, Title: meta.Title, @@ -1605,7 +1605,7 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { } if len(rankingItems) > 0 { - if err := s.writer.WriteRanking(rankingItems); err != nil { + if err := s.store.WriteRanking(ctx, rankingItems); err != nil { s.log.Error("failed to save ranking", "err", err) } } @@ -1669,7 +1669,7 @@ const rankingViewTmpl = ` ` func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) { - items, err := s.writer.ReadRankingItems() + items, err := s.store.ReadRankingItems(r.Context()) if err != nil { http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError) return @@ -1927,7 +1927,7 @@ const bookTmpl = ` func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") - meta, ok, err := s.writer.ReadMetadata(slug) + meta, ok, err := s.store.ReadMetadata(r.Context(), slug) if err != nil { http.Error(w, "failed to read metadata: "+err.Error(), http.StatusInternalServerError) return @@ -1937,7 +1937,7 @@ func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) { return } - chapters, err := s.writer.ListChapters(slug) + chapters, err := s.store.ListChapters(r.Context(), slug) if err != nil { http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) return @@ -2054,7 +2054,7 @@ func (s *Server) handleBookChaptersPage(w http.ResponseWriter, r *http.Request) } } - chapters, err := s.writer.ListChapters(slug) + chapters, err := s.store.ListChapters(r.Context(), slug) if err != nil { http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) return @@ -3035,7 +3035,7 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { return } - raw, err := s.writer.ReadChapter(slug, n) + raw, err := s.store.ReadChapter(r.Context(), slug, n) if err != nil { http.NotFound(w, r) return @@ -3051,15 +3051,15 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { return } - chapters, _ := s.writer.ListChapters(slug) + chapters, _ := s.store.ListChapters(r.Context(), slug) prevN, nextN := adjacentChapters(chapters, n) title := firstHeading(raw, fmt.Sprintf("Chapter %d", n)) - chapterTitle, chapterDate := writer.SplitChapterTitle(title) + chapterTitle, chapterDate := splitChapterTitle(title) // Load cover URL for Media Session artwork (best-effort; ignore errors). var coverURL string - if meta, ok, err := s.writer.ReadMetadata(slug); err == nil && ok { + if meta, ok, err := s.store.ReadMetadata(r.Context(), slug); err == nil && ok { coverURL = meta.Cover } @@ -3096,6 +3096,36 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { // ─── helpers ────────────────────────────────────────────────────────────────── +// splitChapterTitle splits a raw chapter heading into a human-readable title +// and a trailing relative-date string (e.g. "1 year ago"). It mirrors the +// same logic in internal/writer and internal/storage/hybrid. +func splitChapterTitle(raw string) (title, date string) { + raw = strings.TrimSpace(raw) + // Strip leading numeric index. + if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 { + prefix := raw[:idx] + allDigit := true + for _, c := range prefix { + if c < '0' || c > '9' { + allDigit = false + break + } + } + if allDigit { + raw = strings.TrimSpace(raw[idx:]) + } + } + // Strip "Chapter N - N: " prefix. + chNumRe := regexp.MustCompile(`(?i)^chapter\s+\d+(?:\s*-\s*\d+)?\s*:\s*`) + raw = strings.TrimSpace(chNumRe.ReplaceAllString(raw, "")) + // Detect trailing relative date. + dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`) + if m := dateRe.FindStringSubmatchIndex(raw); m != nil { + return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]]) + } + return raw, "" +} + // sortedKeys returns the keys of a string-bool map in sorted order. func sortedKeys(m map[string]bool) []string { out := make([]string, 0, len(m)) @@ -3127,7 +3157,7 @@ func stripMarkdown(src string) string { // adjacentChapters returns the chapter numbers immediately before and after n // in the sorted chapters list. 0 means "does not exist". -func adjacentChapters(chapters []writer.ChapterInfo, n int) (prev, next int) { +func adjacentChapters(chapters []storage.ChapterInfo, n int) (prev, next int) { for i, ch := range chapters { if ch.Number == n { if i > 0 { @@ -3211,7 +3241,7 @@ func (s *Server) handleUIScrapeBook(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() - o := orchestrator.New(cfg, s.novel, s.log) + o := orchestrator.New(cfg, s.novel, s.log, s.store) if err := o.Run(ctx); err != nil { s.log.Error("UI scrape job failed", "url", bookURL, "err", err) } diff --git a/scraper/internal/storage/hybrid.go b/scraper/internal/storage/hybrid.go new file mode 100644 index 0000000..31184da --- /dev/null +++ b/scraper/internal/storage/hybrid.go @@ -0,0 +1,300 @@ +// hybrid.go implements the Store interface using PocketBase for structured data +// and MinIO for binary chapter/audio blobs. +package storage + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/libnovel/scraper/internal/scraper" +) + +// HybridStore satisfies Store by routing structured data to PocketBase and +// binary objects (chapters, audio) to MinIO. +type HybridStore struct { + pb *PocketBaseStore + minio *MinioClient +} + +// NewHybridStore constructs a HybridStore. It connects to both backends and +// calls EnsureCollections to bootstrap any missing PocketBase collections. +func NewHybridStore(ctx context.Context, pbCfg PocketBaseConfig, minioCfg MinioConfig) (*HybridStore, error) { + mc, err := NewMinioClient(ctx, minioCfg) + if err != nil { + return nil, fmt.Errorf("storage: minio: %w", err) + } + pb := NewPocketBaseStore(pbCfg) + if err := pb.EnsureCollections(ctx); err != nil { + // Log but don't fail — collection creation errors are often "already exists" + _ = err + } + return &HybridStore{pb: pb, minio: mc}, nil +} + +// ─── Book metadata ──────────────────────────────────────────────────────────── + +func (h *HybridStore) WriteMetadata(ctx context.Context, meta scraper.BookMeta) error { + return h.pb.UpsertBook(ctx, + meta.Slug, meta.Title, meta.Author, meta.Cover, + meta.Status, meta.Summary, meta.SourceURL, + meta.Genres, meta.TotalChapters, meta.Ranking, + ) +} + +func (h *HybridStore) ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) { + rec, found, err := h.pb.GetBook(ctx, slug) + if err != nil || !found { + return scraper.BookMeta{}, found, err + } + return recToBookMeta(rec), true, nil +} + +func (h *HybridStore) ListBooks(ctx context.Context) ([]scraper.BookMeta, error) { + rows, err := h.pb.ListBooks(ctx) + if err != nil { + return nil, err + } + books := make([]scraper.BookMeta, 0, len(rows)) + for _, r := range rows { + books = append(books, recToBookMeta(r)) + } + return books, nil +} + +func (h *HybridStore) LocalSlugs(ctx context.Context) (map[string]bool, error) { + books, err := h.ListBooks(ctx) + if err != nil { + return nil, err + } + slugs := make(map[string]bool, len(books)) + for _, b := range books { + slugs[b.Slug] = true + } + return slugs, nil +} + +func (h *HybridStore) MetadataMtime(ctx context.Context, slug string) int64 { + t, err := h.pb.BookMetaUpdated(ctx, slug) + if err != nil || t.IsZero() { + return 0 + } + return t.Unix() +} + +// ─── Chapters ───────────────────────────────────────────────────────────────── + +func (h *HybridStore) ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool { + return h.minio.ChapterExists(ctx, slug, ref.Volume, ref.Number) +} + +func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error { + content := "# " + chapter.Ref.Title + "\n\n" + chapter.Text + "\n" + if err := h.minio.PutChapter(ctx, slug, chapter.Ref.Volume, chapter.Ref.Number, content); err != nil { + return err + } + // Update chapter index in PocketBase. + title, dateLabel := splitChapterTitle(chapter.Ref.Title) + _ = h.pb.UpsertChapterIdx(ctx, slug, chapter.Ref.Number, title, dateLabel) + return nil +} + +func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) { + return h.minio.GetChapter(ctx, slug, 0, n) +} + +func (h *HybridStore) ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) { + rows, err := h.pb.ListChapterIdx(ctx, slug) + if err != nil { + return nil, err + } + infos := make([]ChapterInfo, 0, len(rows)) + for _, r := range rows { + n := int(floatVal(r, "number")) + title, _ := r["title"].(string) + date, _ := r["date_label"].(string) + infos = append(infos, ChapterInfo{Number: n, Title: title, Date: date}) + } + sort.Slice(infos, func(i, j int) bool { return infos[i].Number < infos[j].Number }) + return infos, nil +} + +func (h *HybridStore) CountChapters(ctx context.Context, slug string) int { + return h.pb.CountChapterIdx(ctx, slug) +} + +// ─── Ranking ───────────────────────────────────────────────────────────────── + +func (h *HybridStore) WriteRanking(ctx context.Context, items []RankingItem) error { + data, err := json.Marshal(items) + if err != nil { + return fmt.Errorf("storage: marshal ranking: %w", err) + } + return h.pb.SetRanking(ctx, string(data)) +} + +func (h *HybridStore) ReadRankingItems(ctx context.Context) ([]RankingItem, error) { + dataStr, _, err := h.pb.GetRanking(ctx) + if err != nil || dataStr == "" { + return nil, err + } + var items []RankingItem + if err := json.Unmarshal([]byte(dataStr), &items); err != nil { + return nil, fmt.Errorf("storage: unmarshal ranking: %w", err) + } + return items, nil +} + +func (h *HybridStore) RankingFileInfo(ctx context.Context) (os.FileInfo, error) { + return h.pb.RankingModTime(ctx) +} + +// ─── Ranking page HTML cache ────────────────────────────────────────────────── + +func (h *HybridStore) WriteRankingPageCache(ctx context.Context, page int, html string) error { + return h.pb.SetRankingPageHTML(ctx, page, html) +} + +func (h *HybridStore) ReadRankingPageCache(ctx context.Context, page int) (string, error) { + html, _, err := h.pb.GetRankingPageHTML(ctx, page) + return html, err +} + +func (h *HybridStore) RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error) { + return h.pb.RankingPageCacheModTime(ctx, page) +} + +// ─── Audio cache ────────────────────────────────────────────────────────────── + +func (h *HybridStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool) { + filename, ok, _ := h.pb.GetAudioCache(ctx, cacheKey) + return filename, ok +} + +func (h *HybridStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error { + return h.pb.SetAudioCache(ctx, cacheKey, filename) +} + +// ─── Reading progress ───────────────────────────────────────────────────────── + +func (h *HybridStore) GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) { + ch, updated, ok, err := h.pb.GetProgress(ctx, sessionID, slug) + if err != nil || !ok { + return ReadingProgress{}, false + } + return ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}, true +} + +func (h *HybridStore) SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error { + return h.pb.SetProgress(ctx, sessionID, p.Slug, p.Chapter) +} + +func (h *HybridStore) AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) { + rows, err := h.pb.AllProgress(ctx, sessionID) + if err != nil { + return nil, err + } + out := make([]ReadingProgress, 0, len(rows)) + for _, r := range rows { + slug, _ := r["slug"].(string) + ch := int(floatVal(r, "chapter")) + var updated time.Time + if ts, ok := r["updated"].(string); ok { + updated, _ = time.Parse(time.RFC3339, ts) + } + out = append(out, ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}) + } + return out, nil +} + +func (h *HybridStore) DeleteProgress(ctx context.Context, sessionID, slug string) error { + return h.pb.DeleteProgress(ctx, sessionID, slug) +} + +// ─── AudioObjectKey ─────────────────────────────────────────────────────────── + +func (h *HybridStore) AudioObjectKey(slug string, n int, voice string, speed float64) string { + return AudioObjectKey(slug, n, voice, speed) +} + +// ─── helpers ────────────────────────────────────────────────────────────────── + +func recToBookMeta(rec map[string]interface{}) scraper.BookMeta { + m := scraper.BookMeta{ + Slug: strVal(rec, "slug"), + Title: strVal(rec, "title"), + Author: strVal(rec, "author"), + Cover: strVal(rec, "cover"), + Status: strVal(rec, "status"), + Summary: strVal(rec, "summary"), + SourceURL: strVal(rec, "source_url"), + } + if tc := floatVal(rec, "total_chapters"); tc > 0 { + m.TotalChapters = int(tc) + } + if rk := floatVal(rec, "ranking"); rk > 0 { + m.Ranking = int(rk) + } + // Genres stored as JSON string or array. + switch v := rec["genres"].(type) { + case string: + _ = json.Unmarshal([]byte(v), &m.Genres) + case []interface{}: + for _, g := range v { + if s, ok := g.(string); ok { + m.Genres = append(m.Genres, s) + } + } + } + return m +} + +func strVal(m map[string]interface{}, key string) string { + if v, ok := m[key].(string); ok { + return v + } + return "" +} + +// splitChapterTitle mirrors writer.SplitChapterTitle logic (simplified). +func splitChapterTitle(raw string) (title, date string) { + raw = strings.TrimSpace(raw) + // Strip leading numeric index. + if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 { + prefix := raw[:idx] + allDigit := true + for _, c := range prefix { + if c < '0' || c > '9' { + allDigit = false + break + } + } + if allDigit { + raw = strings.TrimSpace(raw[idx:]) + } + } + // Detect trailing relative date. + units := []string{"second", "minute", "hour", "day", "week", "month", "year"} + lower := strings.ToLower(raw) + for _, u := range units { + for _, suffix := range []string{u + "s ago", u + " ago"} { + if idx := strings.LastIndex(lower, suffix); idx > 0 { + // Find start of date token (digit before the unit). + start := strings.LastIndex(raw[:idx], " ") + if start < 0 { + start = 0 + } + numPart := strings.TrimSpace(raw[start:idx]) + if _, err := strconv.Atoi(strings.Fields(numPart)[0]); err == nil { + return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)]) + } + } + } + } + return raw, "" +}