diff --git a/scraper/internal/orchestrator/orchestrator_test.go b/scraper/internal/orchestrator/orchestrator_test.go index 9a1de11..5e08378 100644 --- a/scraper/internal/orchestrator/orchestrator_test.go +++ b/scraper/internal/orchestrator/orchestrator_test.go @@ -90,6 +90,10 @@ func (s *mockStore) WriteChapter(_ context.Context, slug string, ch scraper.Chap return nil } +func (s *mockStore) WriteChapterRefs(_ context.Context, _ string, _ []scraper.ChapterRef) error { + return nil +} + func (s *mockStore) WriteMetadata(_ context.Context, meta scraper.BookMeta) error { s.mu.Lock() defer s.mu.Unlock() diff --git a/scraper/internal/server/handlers_preview.go b/scraper/internal/server/handlers_preview.go index 983ff48..15505c4 100644 --- a/scraper/internal/server/handlers_preview.go +++ b/scraper/internal/server/handlers_preview.go @@ -13,6 +13,7 @@ package server // GET /api/chapter-text-preview/{slug}/{n} — scrape a single chapter text live import ( + "context" "encoding/json" "fmt" "net/http" @@ -75,6 +76,26 @@ func (s *Server) handleBookPreview(w http.ResponseWriter, r *http.Request) { chapters = []scraper.ChapterRef{} } + // If the book was not already in the library, persist the metadata and + // chapter list skeleton to PocketBase now so that subsequent visits load + // from the local store rather than scraping live again. Chapter text is + // NOT fetched here — that still requires an explicit scrape job. + if !inLib { + go func() { + bgCtx := context.Background() + if werr := s.store.WriteMetadata(bgCtx, meta); werr != nil { + s.log.Warn("book-preview: WriteMetadata failed (non-fatal)", "slug", slug, "err", werr) + } + if len(chapters) > 0 { + if werr := s.store.WriteChapterRefs(bgCtx, slug, chapters); werr != nil { + s.log.Warn("book-preview: WriteChapterRefs failed (non-fatal)", "slug", slug, "err", werr) + } + } + s.log.Info("book-preview: metadata+chapter list persisted", "slug", slug, "chapters", len(chapters)) + }() + inLib = true // will be true by the time the client navigates back + } + resp := BookPreviewResponse{ InLib: inLib, Meta: meta, diff --git a/scraper/internal/storage/hybrid.go b/scraper/internal/storage/hybrid.go index 63b0053..cd89cfb 100644 --- a/scraper/internal/storage/hybrid.go +++ b/scraper/internal/storage/hybrid.go @@ -119,6 +119,13 @@ func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scr return nil } +// WriteChapterRefs upserts chapter index rows (number + title) for all refs +// without writing any chapter text to MinIO. This pre-populates the chapter +// list when a book is first seen via a live preview. +func (h *HybridStore) WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error { + return h.pb.WriteChapterRefs(ctx, slug, refs) +} + func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) { return h.minio.GetChapter(ctx, slug, 0, n) } diff --git a/scraper/internal/storage/pocketbase.go b/scraper/internal/storage/pocketbase.go index c740fac..ea3dde9 100644 --- a/scraper/internal/storage/pocketbase.go +++ b/scraper/internal/storage/pocketbase.go @@ -28,6 +28,8 @@ import ( "strings" "sync" "time" + + "github.com/libnovel/scraper/internal/scraper" ) // PocketBaseConfig holds PocketBase connection settings. @@ -542,6 +544,23 @@ func (s *PocketBaseStore) UpsertChapterIdx(ctx context.Context, slug string, num }) } +// WriteChapterRefs upserts chapter index rows (number + title) for all refs +// without writing any chapter text. Errors are logged and skipped; the +// operation is best-effort. +func (s *PocketBaseStore) WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error { + var firstErr error + for _, ref := range refs { + if err := s.UpsertChapterIdx(ctx, slug, ref.Number, ref.Title, ""); err != nil { + s.log.Warn("pocketbase: WriteChapterRefs: upsert failed", + "slug", slug, "chapter", ref.Number, "err", err) + if firstErr == nil { + firstErr = err + } + } + } + return firstErr +} + func (s *PocketBaseStore) ListChapterIdx(ctx context.Context, slug string) ([]map[string]interface{}, error) { return s.pb.listAll(ctx, "chapters_idx", fmt.Sprintf(`slug="%s"`, pbEsc(slug)), "+number") diff --git a/scraper/internal/storage/store.go b/scraper/internal/storage/store.go index 3a04663..5b54bae 100644 --- a/scraper/internal/storage/store.go +++ b/scraper/internal/storage/store.go @@ -85,6 +85,11 @@ type Store interface { ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool // WriteChapter stores the chapter markdown. WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error + // WriteChapterRefs persists chapter metadata (number + title) into the + // chapters_idx table without fetching or storing any chapter text. + // It is used to pre-populate the chapter list when a book is first seen + // via a live preview, before its chapter text has been scraped. + WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error // ReadChapter returns the raw markdown for chapter number n. ReadChapter(ctx context.Context, slug string, n int) (string, error) // ListChapters returns all stored chapters for slug, sorted by number.