steps 6-8: wire HybridStore into orchestrator, server, and main
- Add storage/hybrid.go: HybridStore composing PocketBase + MinIO backends - Rewrite orchestrator to accept storage.Store instead of *writer.Writer - Replace *writer.Writer with storage.Store in server.go and ui.go - Wire audio cache, reading progress, chapter reads/writes through store - Add rankingCacheAdapter in main.go to bridge context-free RankingPageCacher interface to HybridStore's context-aware methods
This commit is contained in:
Admin
300
scraper/internal/storage/hybrid.go
Normal file
300
scraper/internal/storage/hybrid.go
Normal file
@@ -0,0 +1,300 @@
|
||||
// hybrid.go implements the Store interface using PocketBase for structured data
|
||||
// and MinIO for binary chapter/audio blobs.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// HybridStore satisfies Store by routing structured data to PocketBase and
|
||||
// binary objects (chapters, audio) to MinIO.
|
||||
type HybridStore struct {
|
||||
pb *PocketBaseStore
|
||||
minio *MinioClient
|
||||
}
|
||||
|
||||
// NewHybridStore constructs a HybridStore. It connects to both backends and
|
||||
// calls EnsureCollections to bootstrap any missing PocketBase collections.
|
||||
func NewHybridStore(ctx context.Context, pbCfg PocketBaseConfig, minioCfg MinioConfig) (*HybridStore, error) {
|
||||
mc, err := NewMinioClient(ctx, minioCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: minio: %w", err)
|
||||
}
|
||||
pb := NewPocketBaseStore(pbCfg)
|
||||
if err := pb.EnsureCollections(ctx); err != nil {
|
||||
// Log but don't fail — collection creation errors are often "already exists"
|
||||
_ = err
|
||||
}
|
||||
return &HybridStore{pb: pb, minio: mc}, nil
|
||||
}
|
||||
|
||||
// ─── Book metadata ────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteMetadata(ctx context.Context, meta scraper.BookMeta) error {
|
||||
return h.pb.UpsertBook(ctx,
|
||||
meta.Slug, meta.Title, meta.Author, meta.Cover,
|
||||
meta.Status, meta.Summary, meta.SourceURL,
|
||||
meta.Genres, meta.TotalChapters, meta.Ranking,
|
||||
)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error) {
|
||||
rec, found, err := h.pb.GetBook(ctx, slug)
|
||||
if err != nil || !found {
|
||||
return scraper.BookMeta{}, found, err
|
||||
}
|
||||
return recToBookMeta(rec), true, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListBooks(ctx context.Context) ([]scraper.BookMeta, error) {
|
||||
rows, err := h.pb.ListBooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
books := make([]scraper.BookMeta, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
books = append(books, recToBookMeta(r))
|
||||
}
|
||||
return books, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) LocalSlugs(ctx context.Context) (map[string]bool, error) {
|
||||
books, err := h.ListBooks(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slugs := make(map[string]bool, len(books))
|
||||
for _, b := range books {
|
||||
slugs[b.Slug] = true
|
||||
}
|
||||
return slugs, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) MetadataMtime(ctx context.Context, slug string) int64 {
|
||||
t, err := h.pb.BookMetaUpdated(ctx, slug)
|
||||
if err != nil || t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
// ─── Chapters ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool {
|
||||
return h.minio.ChapterExists(ctx, slug, ref.Volume, ref.Number)
|
||||
}
|
||||
|
||||
func (h *HybridStore) WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error {
|
||||
content := "# " + chapter.Ref.Title + "\n\n" + chapter.Text + "\n"
|
||||
if err := h.minio.PutChapter(ctx, slug, chapter.Ref.Volume, chapter.Ref.Number, content); err != nil {
|
||||
return err
|
||||
}
|
||||
// Update chapter index in PocketBase.
|
||||
title, dateLabel := splitChapterTitle(chapter.Ref.Title)
|
||||
_ = h.pb.UpsertChapterIdx(ctx, slug, chapter.Ref.Number, title, dateLabel)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadChapter(ctx context.Context, slug string, n int) (string, error) {
|
||||
return h.minio.GetChapter(ctx, slug, 0, n)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error) {
|
||||
rows, err := h.pb.ListChapterIdx(ctx, slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
infos := make([]ChapterInfo, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
n := int(floatVal(r, "number"))
|
||||
title, _ := r["title"].(string)
|
||||
date, _ := r["date_label"].(string)
|
||||
infos = append(infos, ChapterInfo{Number: n, Title: title, Date: date})
|
||||
}
|
||||
sort.Slice(infos, func(i, j int) bool { return infos[i].Number < infos[j].Number })
|
||||
return infos, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) CountChapters(ctx context.Context, slug string) int {
|
||||
return h.pb.CountChapterIdx(ctx, slug)
|
||||
}
|
||||
|
||||
// ─── Ranking ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteRanking(ctx context.Context, items []RankingItem) error {
|
||||
data, err := json.Marshal(items)
|
||||
if err != nil {
|
||||
return fmt.Errorf("storage: marshal ranking: %w", err)
|
||||
}
|
||||
return h.pb.SetRanking(ctx, string(data))
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadRankingItems(ctx context.Context) ([]RankingItem, error) {
|
||||
dataStr, _, err := h.pb.GetRanking(ctx)
|
||||
if err != nil || dataStr == "" {
|
||||
return nil, err
|
||||
}
|
||||
var items []RankingItem
|
||||
if err := json.Unmarshal([]byte(dataStr), &items); err != nil {
|
||||
return nil, fmt.Errorf("storage: unmarshal ranking: %w", err)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) RankingFileInfo(ctx context.Context) (os.FileInfo, error) {
|
||||
return h.pb.RankingModTime(ctx)
|
||||
}
|
||||
|
||||
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) WriteRankingPageCache(ctx context.Context, page int, html string) error {
|
||||
return h.pb.SetRankingPageHTML(ctx, page, html)
|
||||
}
|
||||
|
||||
func (h *HybridStore) ReadRankingPageCache(ctx context.Context, page int) (string, error) {
|
||||
html, _, err := h.pb.GetRankingPageHTML(ctx, page)
|
||||
return html, err
|
||||
}
|
||||
|
||||
func (h *HybridStore) RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error) {
|
||||
return h.pb.RankingPageCacheModTime(ctx, page)
|
||||
}
|
||||
|
||||
// ─── Audio cache ──────────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) GetAudioCache(ctx context.Context, cacheKey string) (string, bool) {
|
||||
filename, ok, _ := h.pb.GetAudioCache(ctx, cacheKey)
|
||||
return filename, ok
|
||||
}
|
||||
|
||||
func (h *HybridStore) SetAudioCache(ctx context.Context, cacheKey, filename string) error {
|
||||
return h.pb.SetAudioCache(ctx, cacheKey, filename)
|
||||
}
|
||||
|
||||
// ─── Reading progress ─────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool) {
|
||||
ch, updated, ok, err := h.pb.GetProgress(ctx, sessionID, slug)
|
||||
if err != nil || !ok {
|
||||
return ReadingProgress{}, false
|
||||
}
|
||||
return ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated}, true
|
||||
}
|
||||
|
||||
func (h *HybridStore) SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error {
|
||||
return h.pb.SetProgress(ctx, sessionID, p.Slug, p.Chapter)
|
||||
}
|
||||
|
||||
func (h *HybridStore) AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error) {
|
||||
rows, err := h.pb.AllProgress(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]ReadingProgress, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
slug, _ := r["slug"].(string)
|
||||
ch := int(floatVal(r, "chapter"))
|
||||
var updated time.Time
|
||||
if ts, ok := r["updated"].(string); ok {
|
||||
updated, _ = time.Parse(time.RFC3339, ts)
|
||||
}
|
||||
out = append(out, ReadingProgress{Slug: slug, Chapter: ch, UpdatedAt: updated})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (h *HybridStore) DeleteProgress(ctx context.Context, sessionID, slug string) error {
|
||||
return h.pb.DeleteProgress(ctx, sessionID, slug)
|
||||
}
|
||||
|
||||
// ─── AudioObjectKey ───────────────────────────────────────────────────────────
|
||||
|
||||
func (h *HybridStore) AudioObjectKey(slug string, n int, voice string, speed float64) string {
|
||||
return AudioObjectKey(slug, n, voice, speed)
|
||||
}
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func recToBookMeta(rec map[string]interface{}) scraper.BookMeta {
|
||||
m := scraper.BookMeta{
|
||||
Slug: strVal(rec, "slug"),
|
||||
Title: strVal(rec, "title"),
|
||||
Author: strVal(rec, "author"),
|
||||
Cover: strVal(rec, "cover"),
|
||||
Status: strVal(rec, "status"),
|
||||
Summary: strVal(rec, "summary"),
|
||||
SourceURL: strVal(rec, "source_url"),
|
||||
}
|
||||
if tc := floatVal(rec, "total_chapters"); tc > 0 {
|
||||
m.TotalChapters = int(tc)
|
||||
}
|
||||
if rk := floatVal(rec, "ranking"); rk > 0 {
|
||||
m.Ranking = int(rk)
|
||||
}
|
||||
// Genres stored as JSON string or array.
|
||||
switch v := rec["genres"].(type) {
|
||||
case string:
|
||||
_ = json.Unmarshal([]byte(v), &m.Genres)
|
||||
case []interface{}:
|
||||
for _, g := range v {
|
||||
if s, ok := g.(string); ok {
|
||||
m.Genres = append(m.Genres, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func strVal(m map[string]interface{}, key string) string {
|
||||
if v, ok := m[key].(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// splitChapterTitle mirrors writer.SplitChapterTitle logic (simplified).
|
||||
func splitChapterTitle(raw string) (title, date string) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
// Strip leading numeric index.
|
||||
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
|
||||
prefix := raw[:idx]
|
||||
allDigit := true
|
||||
for _, c := range prefix {
|
||||
if c < '0' || c > '9' {
|
||||
allDigit = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allDigit {
|
||||
raw = strings.TrimSpace(raw[idx:])
|
||||
}
|
||||
}
|
||||
// Detect trailing relative date.
|
||||
units := []string{"second", "minute", "hour", "day", "week", "month", "year"}
|
||||
lower := strings.ToLower(raw)
|
||||
for _, u := range units {
|
||||
for _, suffix := range []string{u + "s ago", u + " ago"} {
|
||||
if idx := strings.LastIndex(lower, suffix); idx > 0 {
|
||||
// Find start of date token (digit before the unit).
|
||||
start := strings.LastIndex(raw[:idx], " ")
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
numPart := strings.TrimSpace(raw[start:idx])
|
||||
if _, err := strconv.Atoi(strings.Fields(numPart)[0]); err == nil {
|
||||
return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return raw, ""
|
||||
}
|
||||
Reference in New Issue
Block a user