// hybrid.go implements the Store interface using PocketBase for structured data // and MinIO for binary chapter/audio blobs. package storage import ( "context" "encoding/json" "fmt" "log/slog" "sort" "strconv" "strings" "time" "github.com/libnovel/scraper/internal/scraper" ) // HybridStore satisfies Store by routing structured data to PocketBase and // binary objects (chapters, audio) to MinIO. type HybridStore struct { pb *PocketBaseStore minio *MinioClient log *slog.Logger } // NewHybridStore constructs a HybridStore. It connects to both backends and // calls EnsureCollections to bootstrap any missing PocketBase collections. func NewHybridStore(ctx context.Context, pbCfg PocketBaseConfig, minioCfg MinioConfig, log *slog.Logger) (*HybridStore, error) { mc, err := NewMinioClient(ctx, minioCfg) if err != nil { return nil, fmt.Errorf("storage: minio: %w", err) } pb := NewPocketBaseStore(pbCfg, log) // Verify PocketBase credentials before proceeding. if err := pb.Ping(ctx); err != nil { return nil, fmt.Errorf("storage: pocketbase auth: %w", err) } if err := pb.EnsureCollections(ctx); err != nil { // Non-fatal: 400/422 means collections already exist. log.Warn("EnsureCollections returned an error (may be safe to ignore)", "err", err) } if err := pb.EnsureMigrations(ctx); err != nil { log.Warn("EnsureMigrations returned an error", "err", err) } return &HybridStore{pb: pb, minio: mc, log: log}, nil } // ─── Book metadata ──────────────────────────────────────────────────────────── func (h *HybridStore) WriteMetadata(ctx context.Context, meta scraper.BookMeta) error { return h.pb.UpsertBook(ctx, meta.Slug, meta.Title, meta.Author, meta.Cover, meta.Status, meta.Summary, meta.SourceURL, meta.Genres, meta.TotalChapters, meta.Ranking, ) } func (h *HybridStore) ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) { rec, found, err := h.pb.GetBook(ctx, slug) if err != nil || !found { return scraper.BookMeta{}, found, err } return recToBookMeta(rec), true, nil } func (h *HybridStore) ListBooks(ctx context.Context) ([]scraper.BookMeta, error) { rows, err := h.pb.ListBooks(ctx) if err != nil { return nil, err } books := make([]scraper.BookMeta, 0, len(rows)) for _, r := range rows { books = append(books, recToBookMeta(r)) } return books, nil } func (h *HybridStore) LocalSlugs(ctx context.Context) (map[string]bool, error) { books, err := h.ListBooks(ctx) if err != nil { return nil, err } slugs := make(map[string]bool, len(books)) for _, b := range books { slugs[b.Slug] = true } return slugs, nil } func (h *HybridStore) MetadataMtime(ctx context.Context, slug string) int64 { t, err := h.pb.BookMetaUpdated(ctx, slug) if err != nil { h.log.Warn("MetadataMtime: BookMetaUpdated failed", "slug", slug, "err", err) return 0 } if t.IsZero() { return 0 } return t.Unix() } // ─── Chapters ───────────────────────────────────────────────────────────────── func (h *HybridStore) ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool { return h.minio.ChapterExists(ctx, slug, ref.Volume, ref.Number) } func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error { content := "# " + chapter.Ref.Title + "\n\n" + chapter.Text + "\n" if err := h.minio.PutChapter(ctx, slug, chapter.Ref.Volume, chapter.Ref.Number, content); err != nil { return err } // Update chapter index in PocketBase. title, dateLabel := splitChapterTitle(chapter.Ref.Title) if err := h.pb.UpsertChapterIdx(ctx, slug, chapter.Ref.Number, title, dateLabel); err != nil { h.log.Warn("WriteChapter: failed to upsert chapter index in PocketBase", "slug", slug, "chapter", chapter.Ref.Number, "err", err) } return nil } func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) { return h.minio.GetChapter(ctx, slug, 0, n) } func (h *HybridStore) ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) { rows, err := h.pb.ListChapterIdx(ctx, slug) if err != nil { return nil, err } infos := make([]ChapterInfo, 0, len(rows)) for _, r := range rows { n := int(floatVal(r, "number")) title, _ := r["title"].(string) date, _ := r["date_label"].(string) infos = append(infos, ChapterInfo{Number: n, Title: title, Date: date}) } sort.Slice(infos, func(i, j int) bool { return infos[i].Number < infos[j].Number }) return infos, nil } func (h *HybridStore) CountChapters(ctx context.Context, slug string) int { return h.pb.CountChapterIdx(ctx, slug) } // ReindexChapters walks all MinIO objects for slug, reads the title from the // first line of each chapter markdown, and upserts them into chapters_idx. // This repairs the PocketBase index when it falls out of sync with MinIO. // Returns the number of chapters indexed and any non-fatal errors encountered. func (h *HybridStore) ReindexChapters(ctx context.Context, slug string) (int, error) { keys, err := h.minio.ListChapterKeys(ctx, slug) if err != nil { return 0, fmt.Errorf("reindex: list chapter keys: %w", err) } count := 0 var errs []string for _, key := range keys { // Parse chapter number from key: {slug}/vol-N/lo-hi/chapter-N.md n := chapterNumberFromKey(key) if n <= 0 { h.log.Warn("ReindexChapters: could not parse chapter number from key", "key", key) continue } raw, readErr := h.minio.GetChapter(ctx, slug, 0, n) if readErr != nil { errs = append(errs, fmt.Sprintf("ch%d: %v", n, readErr)) continue } // Extract title from first line ("# Title text") or fall back to empty. rawTitle := "" if line, _, found := strings.Cut(raw, "\n"); found || raw != "" { rawTitle = strings.TrimPrefix(strings.TrimSpace(line), "# ") } title, dateLabel := splitChapterTitle(rawTitle) if upsertErr := h.pb.UpsertChapterIdx(ctx, slug, n, title, dateLabel); upsertErr != nil { errs = append(errs, fmt.Sprintf("ch%d upsert: %v", n, upsertErr)) continue } count++ } if len(errs) > 0 { return count, fmt.Errorf("reindex: %d error(s): %s", len(errs), strings.Join(errs, "; ")) } return count, nil } // chapterNumberFromKey parses the chapter number from a MinIO object key of the // form "{slug}/vol-N/lo-hi/chapter-N.md". func chapterNumberFromKey(key string) int { // Grab the filename portion after the last '/'. parts := strings.Split(key, "/") if len(parts) == 0 { return 0 } filename := parts[len(parts)-1] // filename is "chapter-N.md" filename = strings.TrimSuffix(filename, ".md") filename = strings.TrimPrefix(filename, "chapter-") n, err := strconv.Atoi(filename) if err != nil || n <= 0 { return 0 } return n } // ─── Ranking ───────────────────────────────────────────────────────────────── func (h *HybridStore) WriteRankingItem(ctx context.Context, item RankingItem) error { return h.pb.UpsertRankingItem(ctx, item) } func (h *HybridStore) ReadRankingItems(ctx context.Context) ([]RankingItem, error) { return h.pb.ListRankingItems(ctx) } func (h *HybridStore) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) { last, err := h.pb.RankingLastUpdated(ctx) if err != nil { return false, err } if last.IsZero() { return false, nil } return time.Since(last) < maxAge, nil } // ─── Audio cache ────────────────────────────────────────────────────────────── func (h *HybridStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool) { filename, ok, err := h.pb.GetAudioCache(ctx, cacheKey) if err != nil { h.log.Warn("GetAudioCache: PocketBase lookup failed", "cache_key", cacheKey, "err", err) } return filename, ok } func (h *HybridStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error { return h.pb.SetAudioCache(ctx, cacheKey, filename) } // ─── Reading progress ───────────────────────────────────────────────────────── func (h *HybridStore) GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) { ch, updated, ok, err := h.pb.GetProgress(ctx, sessionID, slug) if err != nil { h.log.Warn("GetProgress: PocketBase lookup failed", "slug", slug, "err", err) return ReadingProgress{}, false } if !ok { return ReadingProgress{}, false } return ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}, true } func (h *HybridStore) SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error { return h.pb.SetProgress(ctx, sessionID, p.Slug, p.Chapter) } func (h *HybridStore) AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) { rows, err := h.pb.AllProgress(ctx, sessionID) if err != nil { return nil, err } out := make([]ReadingProgress, 0, len(rows)) for _, r := range rows { slug, _ := r["slug"].(string) ch := int(floatVal(r, "chapter")) var updated time.Time if ts, ok := r["updated"].(string); ok { updated, _ = time.Parse(time.RFC3339, ts) } out = append(out, ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}) } return out, nil } func (h *HybridStore) DeleteProgress(ctx context.Context, sessionID, slug string) error { return h.pb.DeleteProgress(ctx, sessionID, slug) } // ─── AudioObjectKey ─────────────────────────────────────────────────────────── func (h *HybridStore) AudioObjectKey(slug string, n int, voice string) string { return AudioObjectKey(slug, n, voice) } func (h *HybridStore) AudioExists(ctx context.Context, key string) bool { return h.minio.AudioExists(ctx, key) } // ─── PutAudio ───────────────────────────────────────────────────────────────── func (h *HybridStore) PutAudio(ctx context.Context, key string, data []byte) error { return h.minio.PutAudio(ctx, key, data) } // ─── Presigned URLs ─────────────────────────────────────────────────────────── func (h *HybridStore) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) { return h.minio.PresignChapter(ctx, slug, 0, n, expires) } func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) { return h.minio.PresignAudio(ctx, key, expires) } // ─── Browse page snapshots ──────────────────────────────────────────────────── func (h *HybridStore) SaveBrowsePage(ctx context.Context, key, html string) error { return h.minio.PutBrowsePage(ctx, key, html) } func (h *HybridStore) GetBrowsePage(ctx context.Context, key string) (string, bool, error) { return h.minio.GetBrowsePage(ctx, key) } func (h *HybridStore) BrowseHTMLKey(domain string, page int) string { return BrowseHTMLKey(domain, page) } func (h *HybridStore) BrowseCoverKey(domain, slug string) string { return BrowseCoverKey(domain, slug) } func (h *HybridStore) SaveBrowseAsset(ctx context.Context, key string, data []byte, contentType string) error { return h.minio.PutBrowseAsset(ctx, key, data, contentType) } func (h *HybridStore) GetBrowseAsset(ctx context.Context, key string) ([]byte, string, bool, error) { return h.minio.GetBrowseAsset(ctx, key) } // ─── Scraping tasks ─────────────────────────────────────────────────────────── func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) { return h.pb.CreateScrapingTask(ctx, kind, targetURL) } func (h *HybridStore) UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error { data := map[string]interface{}{ "status": u.Status, "books_found": u.BooksFound, "chapters_scraped": u.ChaptersScraped, "chapters_skipped": u.ChaptersSkipped, "errors": u.Errors, "error_message": u.ErrorMessage, } if !u.Finished.IsZero() { data["finished"] = u.Finished.UTC().Format(time.RFC3339) } return h.pb.UpdateScrapingTask(ctx, id, data) } func (h *HybridStore) ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error) { rows, err := h.pb.ListScrapingTasks(ctx) if err != nil { return nil, err } tasks := make([]ScrapeTask, 0, len(rows)) for _, r := range rows { t := ScrapeTask{ ID: strVal(r, "id"), Kind: strVal(r, "kind"), TargetURL: strVal(r, "target_url"), Status: strVal(r, "status"), BooksFound: int(floatVal(r, "books_found")), ChaptersScraped: int(floatVal(r, "chapters_scraped")), ChaptersSkipped: int(floatVal(r, "chapters_skipped")), Errors: int(floatVal(r, "errors")), ErrorMessage: strVal(r, "error_message"), } if ts, ok := r["started"].(string); ok { t.Started, _ = time.Parse(time.RFC3339, ts) } if ts, ok := r["finished"].(string); ok && ts != "" { t.Finished, _ = time.Parse(time.RFC3339, ts) } tasks = append(tasks, t) } return tasks, nil } // ─── helpers ────────────────────────────────────────────────────────────────── func recToBookMeta(rec map[string]interface{}) scraper.BookMeta { m := scraper.BookMeta{ Slug: strVal(rec, "slug"), Title: strVal(rec, "title"), Author: strVal(rec, "author"), Cover: strVal(rec, "cover"), Status: strVal(rec, "status"), Summary: strVal(rec, "summary"), SourceURL: strVal(rec, "source_url"), } if tc := floatVal(rec, "total_chapters"); tc > 0 { m.TotalChapters = int(tc) } if rk := floatVal(rec, "ranking"); rk > 0 { m.Ranking = int(rk) } // Genres stored as JSON string or array. switch v := rec["genres"].(type) { case string: _ = json.Unmarshal([]byte(v), &m.Genres) case []interface{}: for _, g := range v { if s, ok := g.(string); ok { m.Genres = append(m.Genres, s) } } } return m } func strVal(m map[string]interface{}, key string) string { if v, ok := m[key].(string); ok { return v } return "" } // splitChapterTitle mirrors writer.SplitChapterTitle logic (simplified). func splitChapterTitle(raw string) (title, date string) { raw = strings.TrimSpace(raw) // Strip leading numeric index. if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 { prefix := raw[:idx] allDigit := true for _, c := range prefix { if c < '0' || c > '9' { allDigit = false break } } if allDigit { raw = strings.TrimSpace(raw[idx:]) } } // Detect trailing relative date. Build a flat list of all suffixes once // to avoid a double-nested loop. units := []string{"second", "minute", "hour", "day", "week", "month", "year"} suffixes := make([]string, 0, len(units)*2) for _, u := range units { suffixes = append(suffixes, u+"s ago", u+" ago") } lower := strings.ToLower(raw) for _, suffix := range suffixes { idx := strings.LastIndex(lower, suffix) if idx <= 0 { continue } // Find start of the numeric token that precedes the unit. // Strip any whitespace that separates the number from the unit so // that LastIndex finds the space before the digit, not the one // between the digit and the unit word. before := strings.TrimRight(raw[:idx], " \t") start := strings.LastIndex(before, " ") if start < 0 { start = 0 } else { start++ // advance past the space to point at the digit } numPart := strings.TrimSpace(raw[start:idx]) fields := strings.Fields(numPart) if len(fields) > 0 { if _, err := strconv.Atoi(fields[0]); err == nil { return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)]) } } } return raw, "" }