Files
libnovel/scraper/internal/storage/store.go
Admin af3c487afb feat(e2e): use direct HTTP for chapter scraping, cap TTS at 200 chars
- e2e fixture: replace single contentClient with directClient (plain HTTP)
  for chapter/metadata/ranking + contentClient (Browserless) for urlClient
  only — matches production wiring and is significantly faster
- server: add max_chars field to audio request body; truncates stripped text
  to N runes before sending to Kokoro (used by e2e for quick TTS tests)
- fix: move RankingItem to scraper package to break novelfire→storage import
  cycle; storage.RankingItem is now a type alias for backward compat
- fix: update stale New() call in novelfire integration test (missing args)
- fix: replace removed blob-ranking methods in storage integration test with
  current per-item API (UpsertRankingItem/ListRankingItems/RankingLastUpdated)
- justfile: add test-e2e and e2e tasks
2026-03-03 20:02:15 +05:00

117 lines
6.2 KiB
Go

// Package storage defines the unified Store interface and helper types used by
// the server and orchestrator. Concrete implementations back the interface
// with PocketBase (structured data) and MinIO (binary objects).
package storage
import (
"context"
"time"
"github.com/libnovel/scraper/internal/scraper"
)
// ─── Shared types ─────────────────────────────────────────────────────────────
// ChapterInfo is a lightweight chapter descriptor (mirrors writer.ChapterInfo).
type ChapterInfo struct {
Number int
Title string
Date string
}
// RankingItem represents a single entry in the novel ranking list.
// Aliased from scraper.RankingItem for convenience within this package.
type RankingItem = scraper.RankingItem
// ReadingProgress holds a single user's reading position for one book.
type ReadingProgress struct {
Slug string `json:"slug"`
Chapter int `json:"chapter"`
UpdatedAt time.Time `json:"updated_at"`
}
// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro
// download filename so audio is not re-generated after a server restart.
type AudioCacheEntry struct {
CacheKey string `json:"cache_key"`
Filename string `json:"filename"`
}
// ─── Store interface ──────────────────────────────────────────────────────────
// Store is the single persistence abstraction consumed by the server and the
// orchestrator. Implementations may route calls to different backends
// (PocketBase for structured records, MinIO for binary blobs).
type Store interface {
// ── Book metadata ──────────────────────────────────────────────────────
// WriteMetadata upserts book metadata.
WriteMetadata(ctx context.Context, meta scraper.BookMeta) error
// ReadMetadata returns the metadata for slug. Returns (zero, false, nil)
// when the book is not found.
ReadMetadata(ctx context.Context, slug string) (scraper.BookMeta, bool, error)
// ListBooks returns all books, sorted alphabetically by title.
ListBooks(ctx context.Context) ([]scraper.BookMeta, error)
// LocalSlugs returns the set of slugs that have metadata stored.
LocalSlugs(ctx context.Context) (map[string]bool, error)
// MetadataMtime returns the Unix-second mtime of the metadata record, or 0.
MetadataMtime(ctx context.Context, slug string) int64
// ── Chapters (binary blobs in MinIO) ───────────────────────────────────
// ChapterExists returns true if the markdown file for the given ref exists.
ChapterExists(ctx context.Context, slug string, ref scraper.ChapterRef) bool
// WriteChapter stores the chapter markdown.
WriteChapter(ctx context.Context, slug string, chapter scraper.Chapter) error
// ReadChapter returns the raw markdown for chapter number n.
ReadChapter(ctx context.Context, slug string, n int) (string, error)
// ListChapters returns all stored chapters for slug, sorted by number.
ListChapters(ctx context.Context, slug string) ([]ChapterInfo, error)
// CountChapters returns the number of stored chapters for slug.
CountChapters(ctx context.Context, slug string) int
// ── Ranking ────────────────────────────────────────────────────────────
// WriteRankingItem upserts a single ranking entry (keyed on Slug).
WriteRankingItem(ctx context.Context, item RankingItem) error
// ReadRankingItems returns all ranking items sorted by rank ascending.
ReadRankingItems(ctx context.Context) ([]RankingItem, error)
// RankingFreshEnough returns true when ranking rows exist and the most
// recent Updated timestamp is within maxAge of now.
RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error)
// ── Audio cache ────────────────────────────────────────────────────────
// GetAudioCache returns the Kokoro filename for cacheKey, or ("", false).
GetAudioCache(ctx context.Context, cacheKey string) (string, bool)
// SetAudioCache persists a Kokoro filename for cacheKey.
SetAudioCache(ctx context.Context, cacheKey, filename string) error
// PutAudio stores raw audio bytes under the given MinIO object key.
PutAudio(ctx context.Context, key string, data []byte) error
// ── Reading progress ───────────────────────────────────────────────────
// GetProgress returns the reading progress for the given session ID and slug.
// Returns (zero, false) if no progress is recorded.
GetProgress(ctx context.Context, sessionID, slug string) (ReadingProgress, bool)
// SetProgress saves or updates reading progress.
SetProgress(ctx context.Context, sessionID string, p ReadingProgress) error
// AllProgress returns all progress entries for a session.
AllProgress(ctx context.Context, sessionID string) ([]ReadingProgress, error)
// DeleteProgress removes progress for a specific slug.
DeleteProgress(ctx context.Context, sessionID, slug string) error
// ── Audio object paths (MinIO) ─────────────────────────────────────────
// AudioObjectKey returns the MinIO object key for a cached audio file.
AudioObjectKey(slug string, n int, voice string, speed float64) string
// ── Presigned URLs ─────────────────────────────────────────────────────
// PresignChapter returns a presigned GET URL for a chapter markdown object.
PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error)
// PresignAudio returns a presigned GET URL for an audio object.
PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
}