From b3358ac1d27a95c7ddea4dd52390a74a646035c4 Mon Sep 17 00:00:00 2001 From: Admin Date: Fri, 6 Mar 2026 17:09:57 +0500 Subject: [PATCH] feat: auto-persist metadata and chapter list on first book preview visit When a book is not in the local DB and the preview endpoint is hit, metadata and the chapter index are now saved to PocketBase in the background. Subsequent visits load from the local store instead of scraping live. Chapter text is not fetched until an explicit scrape job is triggered. --- .../orchestrator/orchestrator_test.go | 4 ++++ scraper/internal/server/handlers_preview.go | 21 +++++++++++++++++++ scraper/internal/storage/hybrid.go | 7 +++++++ scraper/internal/storage/pocketbase.go | 19 +++++++++++++++++ scraper/internal/storage/store.go | 5 +++++ 5 files changed, 56 insertions(+) diff --git a/scraper/internal/orchestrator/orchestrator_test.go b/scraper/internal/orchestrator/orchestrator_test.go index 9a1de11..5e08378 100644 --- a/scraper/internal/orchestrator/orchestrator_test.go +++ b/scraper/internal/orchestrator/orchestrator_test.go @@ -90,6 +90,10 @@ func (s *mockStore) WriteChapter(_ context.Context, slug string, ch scraper.Chap return nil } +func (s *mockStore) WriteChapterRefs(_ context.Context, _ string, _ []scraper.ChapterRef) error { + return nil +} + func (s *mockStore) WriteMetadata(_ context.Context, meta scraper.BookMeta) error { s.mu.Lock() defer s.mu.Unlock() diff --git a/scraper/internal/server/handlers_preview.go b/scraper/internal/server/handlers_preview.go index 983ff48..15505c4 100644 --- a/scraper/internal/server/handlers_preview.go +++ b/scraper/internal/server/handlers_preview.go @@ -13,6 +13,7 @@ package server // GET /api/chapter-text-preview/{slug}/{n} — scrape a single chapter text live import ( + "context" "encoding/json" "fmt" "net/http" @@ -75,6 +76,26 @@ func (s *Server) handleBookPreview(w http.ResponseWriter, r *http.Request) { chapters = []scraper.ChapterRef{} } + // If the book was not already in the library, persist the metadata and + // chapter list skeleton to PocketBase now so that subsequent visits load + // from the local store rather than scraping live again. Chapter text is + // NOT fetched here — that still requires an explicit scrape job. + if !inLib { + go func() { + bgCtx := context.Background() + if werr := s.store.WriteMetadata(bgCtx, meta); werr != nil { + s.log.Warn("book-preview: WriteMetadata failed (non-fatal)", "slug", slug, "err", werr) + } + if len(chapters) > 0 { + if werr := s.store.WriteChapterRefs(bgCtx, slug, chapters); werr != nil { + s.log.Warn("book-preview: WriteChapterRefs failed (non-fatal)", "slug", slug, "err", werr) + } + } + s.log.Info("book-preview: metadata+chapter list persisted", "slug", slug, "chapters", len(chapters)) + }() + inLib = true // will be true by the time the client navigates back + } + resp := BookPreviewResponse{ InLib: inLib, Meta: meta, diff --git a/scraper/internal/storage/hybrid.go b/scraper/internal/storage/hybrid.go index 63b0053..cd89cfb 100644 --- a/scraper/internal/storage/hybrid.go +++ b/scraper/internal/storage/hybrid.go @@ -119,6 +119,13 @@ func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scr return nil } +// WriteChapterRefs upserts chapter index rows (number + title) for all refs +// without writing any chapter text to MinIO. This pre-populates the chapter +// list when a book is first seen via a live preview. +func (h *HybridStore) WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error { + return h.pb.WriteChapterRefs(ctx, slug, refs) +} + func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) { return h.minio.GetChapter(ctx, slug, 0, n) } diff --git a/scraper/internal/storage/pocketbase.go b/scraper/internal/storage/pocketbase.go index c740fac..ea3dde9 100644 --- a/scraper/internal/storage/pocketbase.go +++ b/scraper/internal/storage/pocketbase.go @@ -28,6 +28,8 @@ import ( "strings" "sync" "time" + + "github.com/libnovel/scraper/internal/scraper" ) // PocketBaseConfig holds PocketBase connection settings. @@ -542,6 +544,23 @@ func (s *PocketBaseStore) UpsertChapterIdx(ctx context.Context, slug string, num }) } +// WriteChapterRefs upserts chapter index rows (number + title) for all refs +// without writing any chapter text. Errors are logged and skipped; the +// operation is best-effort. +func (s *PocketBaseStore) WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error { + var firstErr error + for _, ref := range refs { + if err := s.UpsertChapterIdx(ctx, slug, ref.Number, ref.Title, ""); err != nil { + s.log.Warn("pocketbase: WriteChapterRefs: upsert failed", + "slug", slug, "chapter", ref.Number, "err", err) + if firstErr == nil { + firstErr = err + } + } + } + return firstErr +} + func (s *PocketBaseStore) ListChapterIdx(ctx context.Context, slug string) ([]map[string]interface{}, error) { return s.pb.listAll(ctx, "chapters_idx", fmt.Sprintf(`slug="%s"`, pbEsc(slug)), "+number") diff --git a/scraper/internal/storage/store.go b/scraper/internal/storage/store.go index 3a04663..5b54bae 100644 --- a/scraper/internal/storage/store.go +++ b/scraper/internal/storage/store.go @@ -85,6 +85,11 @@ type Store interface { ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool // WriteChapter stores the chapter markdown. WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error + // WriteChapterRefs persists chapter metadata (number + title) into the + // chapters_idx table without fetching or storing any chapter text. + // It is used to pre-populate the chapter list when a book is first seen + // via a live preview, before its chapter text has been scraped. + WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error // ReadChapter returns the raw markdown for chapter number n. ReadChapter(ctx context.Context, slug string, n int) (string, error) // ListChapters returns all stored chapters for slug, sorted by number.