feat: auto-persist metadata and chapter list on first book preview visit
Some checks failed
Deploy / Deploy Production (push) Has been skipped
Deploy / Cleanup Preview (push) Has been skipped
Deploy / Deploy Preview (push) Failing after 0s
CI / Scraper / Test (pull_request) Successful in 10s
CI / Scraper / Lint (pull_request) Successful in 19s
CI / UI / Build (pull_request) Successful in 20s
CI / Scraper / Build (pull_request) Successful in 10s
Some checks failed
Deploy / Deploy Production (push) Has been skipped
Deploy / Cleanup Preview (push) Has been skipped
Deploy / Deploy Preview (push) Failing after 0s
CI / Scraper / Test (pull_request) Successful in 10s
CI / Scraper / Lint (pull_request) Successful in 19s
CI / UI / Build (pull_request) Successful in 20s
CI / Scraper / Build (pull_request) Successful in 10s
When a book is not in the local DB and the preview endpoint is hit, metadata and the chapter index are now saved to PocketBase in the background. Subsequent visits load from the local store instead of scraping live. Chapter text is not fetched until an explicit scrape job is triggered.
This commit is contained in:
@@ -90,6 +90,10 @@ func (s *mockStore) WriteChapter(_ context.Context, slug string, ch scraper.Chap
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *mockStore) WriteChapterRefs(_ context.Context, _ string, _ []scraper.ChapterRef) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *mockStore) WriteMetadata(_ context.Context, meta scraper.BookMeta) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
@@ -13,6 +13,7 @@ package server
|
||||
// GET /api/chapter-text-preview/{slug}/{n} — scrape a single chapter text live
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -75,6 +76,26 @@ func (s *Server) handleBookPreview(w http.ResponseWriter, r *http.Request) {
|
||||
chapters = []scraper.ChapterRef{}
|
||||
}
|
||||
|
||||
// If the book was not already in the library, persist the metadata and
|
||||
// chapter list skeleton to PocketBase now so that subsequent visits load
|
||||
// from the local store rather than scraping live again. Chapter text is
|
||||
// NOT fetched here — that still requires an explicit scrape job.
|
||||
if !inLib {
|
||||
go func() {
|
||||
bgCtx := context.Background()
|
||||
if werr := s.store.WriteMetadata(bgCtx, meta); werr != nil {
|
||||
s.log.Warn("book-preview: WriteMetadata failed (non-fatal)", "slug", slug, "err", werr)
|
||||
}
|
||||
if len(chapters) > 0 {
|
||||
if werr := s.store.WriteChapterRefs(bgCtx, slug, chapters); werr != nil {
|
||||
s.log.Warn("book-preview: WriteChapterRefs failed (non-fatal)", "slug", slug, "err", werr)
|
||||
}
|
||||
}
|
||||
s.log.Info("book-preview: metadata+chapter list persisted", "slug", slug, "chapters", len(chapters))
|
||||
}()
|
||||
inLib = true // will be true by the time the client navigates back
|
||||
}
|
||||
|
||||
resp := BookPreviewResponse{
|
||||
InLib: inLib,
|
||||
Meta: meta,
|
||||
|
||||
@@ -119,6 +119,13 @@ func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scr
|
||||
return nil
|
||||
}
|
||||
|
||||
// WriteChapterRefs upserts chapter index rows (number + title) for all refs
|
||||
// without writing any chapter text to MinIO. This pre-populates the chapter
|
||||
// list when a book is first seen via a live preview.
|
||||
func (h *HybridStore) WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error {
|
||||
return h.pb.WriteChapterRefs(ctx, slug, refs)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
|
||||
return h.minio.GetChapter(ctx, slug, 0, n)
|
||||
}
|
||||
|
||||
@@ -28,6 +28,8 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// PocketBaseConfig holds PocketBase connection settings.
|
||||
@@ -542,6 +544,23 @@ func (s *PocketBaseStore) UpsertChapterIdx(ctx context.Context, slug string, num
|
||||
})
|
||||
}
|
||||
|
||||
// WriteChapterRefs upserts chapter index rows (number + title) for all refs
|
||||
// without writing any chapter text. Errors are logged and skipped; the
|
||||
// operation is best-effort.
|
||||
func (s *PocketBaseStore) WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error {
|
||||
var firstErr error
|
||||
for _, ref := range refs {
|
||||
if err := s.UpsertChapterIdx(ctx, slug, ref.Number, ref.Title, ""); err != nil {
|
||||
s.log.Warn("pocketbase: WriteChapterRefs: upsert failed",
|
||||
"slug", slug, "chapter", ref.Number, "err", err)
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func (s *PocketBaseStore) ListChapterIdx(ctx context.Context, slug string) ([]map[string]interface{}, error) {
|
||||
return s.pb.listAll(ctx, "chapters_idx",
|
||||
fmt.Sprintf(`slug="%s"`, pbEsc(slug)), "+number")
|
||||
|
||||
@@ -85,6 +85,11 @@ type Store interface {
|
||||
ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool
|
||||
// WriteChapter stores the chapter markdown.
|
||||
WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error
|
||||
// WriteChapterRefs persists chapter metadata (number + title) into the
|
||||
// chapters_idx table without fetching or storing any chapter text.
|
||||
// It is used to pre-populate the chapter list when a book is first seen
|
||||
// via a live preview, before its chapter text has been scraped.
|
||||
WriteChapterRefs(ctx context.Context, slug string, refs []scraper.ChapterRef) error
|
||||
// ReadChapter returns the raw markdown for chapter number n.
|
||||
ReadChapter(ctx context.Context, slug string, n int) (string, error)
|
||||
// ListChapters returns all stored chapters for slug, sorted by number.
|
||||
|
||||
Reference in New Issue
Block a user