refactor(ranking): replace blob cache with per-item PocketBase storage

- Replace SetRanking/GetRanking/SetRankingPageHTML/GetRankingPageHTML blob methods
  with WriteRankingItem/ReadRankingItems/RankingFreshEnough per-item operations
- Add 24h staleness gate in ScrapeRanking to skip re-scraping fresh data
- Add GET /api/ranking endpoint returning []RankingItem sorted by rank
- Remove RankingPageCacher interface and rankingCacheAdapter adapter
- Update integration tests to use new per-item upsert semantics
- Include e2e test suite (scraper/internal/e2e/)
This commit is contained in:
Admin
2026-03-03 19:37:49 +05:00
parent 56bf4dde22
commit b8d4d94b18
10 changed files with 1173 additions and 351 deletions

View File

@@ -6,7 +6,6 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"strconv"
"strings"
@@ -130,43 +129,23 @@ func (h *HybridStore) CountChapters(ctx context.Context, slug string) int {
// ─── Ranking ─────────────────────────────────────────────────────────────────
func (h *HybridStore) WriteRanking(ctx context.Context, items []RankingItem) error {
data, err := json.Marshal(items)
if err != nil {
return fmt.Errorf("storage: marshal ranking: %w", err)
}
return h.pb.SetRanking(ctx, string(data))
func (h *HybridStore) WriteRankingItem(ctx context.Context, item RankingItem) error {
return h.pb.UpsertRankingItem(ctx, item)
}
func (h *HybridStore) ReadRankingItems(ctx context.Context) ([]RankingItem, error) {
dataStr, _, err := h.pb.GetRanking(ctx)
if err != nil || dataStr == "" {
return nil, err
return h.pb.ListRankingItems(ctx)
}
func (h *HybridStore) RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error) {
last, err := h.pb.RankingLastUpdated(ctx)
if err != nil {
return false, err
}
var items []RankingItem
if err := json.Unmarshal([]byte(dataStr), &items); err != nil {
return nil, fmt.Errorf("storage: unmarshal ranking: %w", err)
if last.IsZero() {
return false, nil
}
return items, nil
}
func (h *HybridStore) RankingFileInfo(ctx context.Context) (os.FileInfo, error) {
return h.pb.RankingModTime(ctx)
}
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
func (h *HybridStore) WriteRankingPageCache(ctx context.Context, page int, html string) error {
return h.pb.SetRankingPageHTML(ctx, page, html)
}
func (h *HybridStore) ReadRankingPageCache(ctx context.Context, page int) (string, error) {
html, _, err := h.pb.GetRankingPageHTML(ctx, page)
return html, err
}
func (h *HybridStore) RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error) {
return h.pb.RankingPageCacheModTime(ctx, page)
return time.Since(last) < maxAge, nil
}
// ─── Audio cache ──────────────────────────────────────────────────────────────
@@ -222,6 +201,12 @@ func (h *HybridStore) AudioObjectKey(slug string, n int, voice string, speed flo
return AudioObjectKey(slug, n, voice, speed)
}
// ─── PutAudio ─────────────────────────────────────────────────────────────────
func (h *HybridStore) PutAudio(ctx context.Context, key string, data []byte) error {
return h.minio.PutAudio(ctx, key, data)
}
// ─── Presigned URLs ───────────────────────────────────────────────────────────
func (h *HybridStore) PresignChapter(ctx context.Context, slug string, n int, expires time.Duration) (string, error) {
@@ -300,8 +285,11 @@ func splitChapterTitle(raw string) (title, date string) {
start = 0
}
numPart := strings.TrimSpace(raw[start:idx])
if _, err := strconv.Atoi(strings.Fields(numPart)[0]); err == nil {
return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)])
fields := strings.Fields(numPart)
if len(fields) > 0 {
if _, err := strconv.Atoi(fields[0]); err == nil {
return strings.TrimSpace(raw[:start]), strings.TrimSpace(raw[start : idx+len(suffix)])
}
}
}
}

View File

@@ -233,23 +233,36 @@ func TestHybridStore_WriteReadChapter(t *testing.T) {
})
}
// TestHybridStore_WriteReadRanking exercises WriteRanking → ReadRankingItems
// round-trip.
// TestHybridStore_WriteReadRanking exercises WriteRankingItem → ReadRankingItems
// round-trip and RankingFreshEnough.
func TestHybridStore_WriteReadRanking(t *testing.T) {
hs := newTestHybridStore(t)
slug1 := "integ-rank-1-" + fmt.Sprintf("%d", time.Now().UnixMilli())
slug2 := "integ-rank-2-" + fmt.Sprintf("%d", time.Now().UnixMilli())
slug3 := "integ-rank-3-" + fmt.Sprintf("%d", time.Now().UnixMilli())
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
t.Cleanup(func() {
cleanCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
for _, sl := range []string{slug1, slug2, slug3} {
_ = hs.pb.pb.deleteWhere(cleanCtx, "ranking", fmt.Sprintf(`slug="%s"`, sl))
}
})
items := []RankingItem{
{Rank: 1, Slug: "top-novel", Title: "Top Novel", Author: "Author A", Status: "Ongoing", SourceURL: "https://example.com/book/top-novel"},
{Rank: 2, Slug: "second-novel", Title: "Second Novel", Author: "Author B", Genres: []string{"Action"}, Status: "Completed"},
{Rank: 3, Slug: "third-novel", Title: "Third Novel"},
{Rank: 1, Slug: slug1, Title: "Top Novel", Author: "Author A", Status: "Ongoing", SourceURL: "https://example.com/book/top"},
{Rank: 2, Slug: slug2, Title: "Second Novel", Author: "Author B", Genres: []string{"Action"}, Status: "Completed"},
{Rank: 3, Slug: slug3, Title: "Third Novel"},
}
t.Run("WriteRanking", func(t *testing.T) {
if err := hs.WriteRanking(ctx, items); err != nil {
t.Fatalf("WriteRanking: %v", err)
t.Run("WriteRankingItem", func(t *testing.T) {
for _, item := range items {
if err := hs.WriteRankingItem(ctx, item); err != nil {
t.Fatalf("WriteRankingItem(%s): %v", item.Slug, err)
}
}
t.Logf("wrote %d ranking items", len(items))
})
@@ -259,34 +272,43 @@ func TestHybridStore_WriteReadRanking(t *testing.T) {
if err != nil {
t.Fatalf("ReadRankingItems: %v", err)
}
if len(got) != len(items) {
t.Errorf("ReadRankingItems returned %d items, want %d", len(got), len(items))
}
for i, item := range got {
t.Logf("items[%d]: rank=%d slug=%q title=%q", i, item.Rank, item.Slug, item.Title)
}
if len(got) > 0 {
if got[0].Rank != 1 {
t.Errorf("items[0].Rank = %d, want 1", got[0].Rank)
}
if got[0].Title != "Top Novel" {
t.Errorf("items[0].Title = %q, want %q", got[0].Title, "Top Novel")
// Filter to just our test slugs (other tests may leave rows).
var ours []RankingItem
slugSet := map[string]bool{slug1: true, slug2: true, slug3: true}
for _, g := range got {
if slugSet[g.Slug] {
ours = append(ours, g)
}
}
if len(ours) != 3 {
t.Fatalf("ReadRankingItems returned %d test items, want 3", len(ours))
}
// Verify order by rank.
for i := 1; i < len(ours); i++ {
if ours[i].Rank <= ours[i-1].Rank {
t.Errorf("items not sorted by rank: ours[%d].Rank=%d, ours[%d].Rank=%d",
i, ours[i].Rank, i-1, ours[i-1].Rank)
}
}
// Verify fields.
if ours[0].Title != "Top Novel" {
t.Errorf("ours[0].Title = %q, want %q", ours[0].Title, "Top Novel")
}
if ours[0].Author != "Author A" {
t.Errorf("ours[0].Author = %q, want %q", ours[0].Author, "Author A")
}
t.Logf("ranking items: %+v", ours)
})
t.Run("RankingFileInfo", func(t *testing.T) {
fi, err := hs.RankingFileInfo(ctx)
t.Run("RankingFreshEnough", func(t *testing.T) {
fresh, err := hs.RankingFreshEnough(ctx, 24*time.Hour)
if err != nil {
t.Fatalf("RankingFileInfo: %v", err)
t.Fatalf("RankingFreshEnough: %v", err)
}
if fi == nil {
t.Fatal("RankingFileInfo returned nil")
if !fresh {
t.Error("RankingFreshEnough(24h) returned false immediately after writing items")
}
if fi.ModTime().IsZero() {
t.Error("RankingFileInfo.ModTime() is zero")
}
t.Logf("ranking file info: modtime=%s", fi.ModTime())
t.Logf("ranking fresh=true")
})
}

View File

@@ -5,8 +5,8 @@
// books — slug(text,unique), title, author, cover, status, genres(json),
// summary, total_chapters(number), source_url, ranking(number), updated(date)
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
// ranking — data(json), updated(date) [single row, upserted by slug="_ranking_"]
// ranking_html — page(number,unique), html(text), updated(date)
// ranking — rank(number), slug(text,unique), title, author, cover, status,
// genres(json), source_url, updated(date)
// progress — session_id(text), slug(text), chapter(number), updated(date)
// audio_cache — cache_key(text,unique), filename(text), updated(date)
// app_users — username(text,unique), password_hash(text), role(text), created(date)
@@ -20,7 +20,6 @@ import (
"io"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
@@ -248,20 +247,22 @@ func (s *PocketBaseStore) Ping(ctx context.Context) error {
func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
// We just attempt to create each collection; 400/422 errors for "already
// exists" are silently ignored.
// PocketBase v0.22+ uses "fields"; older versions used "schema".
// We use "fields" which is the current API.
collections := []map[string]interface{}{
{
"name": "books",
"type": "base",
"schema": []map[string]interface{}{
{"name": "slug", "type": "text", "required": true, "options": map[string]interface{}{"min": 1}},
"fields": []map[string]interface{}{
{"name": "slug", "type": "text", "required": true},
{"name": "title", "type": "text", "required": true},
{"name": "author", "type": "text"},
{"name": "cover", "type": "url"},
{"name": "cover", "type": "text"},
{"name": "status", "type": "text"},
{"name": "genres", "type": "json"},
{"name": "summary", "type": "text"},
{"name": "total_chapters", "type": "number"},
{"name": "source_url", "type": "url"},
{"name": "source_url", "type": "text"},
{"name": "ranking", "type": "number"},
{"name": "meta_updated", "type": "date"},
},
@@ -269,7 +270,7 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
{
"name": "chapters_idx",
"type": "base",
"schema": []map[string]interface{}{
"fields": []map[string]interface{}{
{"name": "slug", "type": "text", "required": true},
{"name": "number", "type": "number", "required": true},
{"name": "title", "type": "text"},
@@ -279,25 +280,22 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
{
"name": "ranking",
"type": "base",
"schema": []map[string]interface{}{
{"name": "key", "type": "text", "required": true},
{"name": "data", "type": "json"},
{"name": "updated", "type": "date"},
},
},
{
"name": "ranking_html",
"type": "base",
"schema": []map[string]interface{}{
{"name": "page", "type": "number", "required": true},
{"name": "html", "type": "text"},
"fields": []map[string]interface{}{
{"name": "rank", "type": "number", "required": true},
{"name": "slug", "type": "text", "required": true},
{"name": "title", "type": "text"},
{"name": "author", "type": "text"},
{"name": "cover", "type": "text"},
{"name": "status", "type": "text"},
{"name": "genres", "type": "json"},
{"name": "source_url", "type": "text"},
{"name": "updated", "type": "date"},
},
},
{
"name": "progress",
"type": "base",
"schema": []map[string]interface{}{
"fields": []map[string]interface{}{
{"name": "session_id", "type": "text", "required": true},
{"name": "slug", "type": "text", "required": true},
{"name": "chapter", "type": "number"},
@@ -307,7 +305,7 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
{
"name": "audio_cache",
"type": "base",
"schema": []map[string]interface{}{
"fields": []map[string]interface{}{
{"name": "cache_key", "type": "text", "required": true},
{"name": "filename", "type": "text"},
{"name": "updated", "type": "date"},
@@ -316,8 +314,8 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
{
"name": "app_users",
"type": "base",
"schema": []map[string]interface{}{
{"name": "username", "type": "text", "required": true, "options": map[string]interface{}{"min": 3, "max": 32}},
"fields": []map[string]interface{}{
{"name": "username", "type": "text", "required": true},
{"name": "password_hash", "type": "text", "required": true},
{"name": "role", "type": "text"},
{"name": "created", "type": "date"},
@@ -406,52 +404,82 @@ func (s *PocketBaseStore) CountChapterIdx(ctx context.Context, slug string) int
return len(rows)
}
// ─── Ranking ──────────────────────────────────────────────────────────────────
// ─── Ranking (per-item) ───────────────────────────────────────────────────────
func (s *PocketBaseStore) SetRanking(ctx context.Context, dataJSON string) error {
return s.pb.upsert(ctx, "ranking", `key="_ranking_"`, map[string]interface{}{
"key": "_ranking_",
"data": dataJSON,
"updated": time.Now().UTC().Format(time.RFC3339),
func (s *PocketBaseStore) UpsertRankingItem(ctx context.Context, item RankingItem) error {
genresJSON, _ := json.Marshal(item.Genres)
return s.pb.upsert(ctx, "ranking", fmt.Sprintf(`slug="%s"`, pbEsc(item.Slug)), map[string]interface{}{
"rank": item.Rank,
"slug": item.Slug,
"title": item.Title,
"author": item.Author,
"cover": item.Cover,
"status": item.Status,
"genres": string(genresJSON),
"source_url": item.SourceURL,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetRanking(ctx context.Context) (string, time.Time, error) {
rec, err := s.pb.listOne(ctx, "ranking", `key="_ranking_"`)
if err != nil || rec == nil {
return "", time.Time{}, err
func (s *PocketBaseStore) ListRankingItems(ctx context.Context) ([]RankingItem, error) {
rows, err := s.pb.listAll(ctx, "ranking", "", "+rank")
if err != nil {
return nil, err
}
data, _ := rec["data"].(string)
var updated time.Time
if ts, ok := rec["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
items := make([]RankingItem, 0, len(rows))
for _, r := range rows {
item := RankingItem{
Rank: int(floatVal(r, "rank")),
Slug: strVal(r, "slug"),
Title: strVal(r, "title"),
Author: strVal(r, "author"),
Cover: strVal(r, "cover"),
Status: strVal(r, "status"),
SourceURL: strVal(r, "source_url"),
}
if ts, ok := r["updated"].(string); ok {
item.Updated, _ = time.Parse(time.RFC3339, ts)
}
switch v := r["genres"].(type) {
case string:
_ = json.Unmarshal([]byte(v), &item.Genres)
case []interface{}:
for _, g := range v {
if s, ok := g.(string); ok {
item.Genres = append(item.Genres, s)
}
}
}
items = append(items, item)
}
return data, updated, nil
return items, nil
}
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
func (s *PocketBaseStore) SetRankingPageHTML(ctx context.Context, page int, html string) error {
return s.pb.upsert(ctx, "ranking_html",
fmt.Sprintf(`page=%d`, page),
map[string]interface{}{
"page": page,
"html": html,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetRankingPageHTML(ctx context.Context, page int) (string, time.Time, error) {
rec, err := s.pb.listOne(ctx, "ranking_html", fmt.Sprintf(`page=%d`, page))
if err != nil || rec == nil {
return "", time.Time{}, err
// RankingLastUpdated returns the most recent Updated time across all ranking rows,
// or the zero time if no rows exist.
func (s *PocketBaseStore) RankingLastUpdated(ctx context.Context) (time.Time, error) {
// listAll with sort "-updated" and perPage=1 is the cheapest approach.
q := url.Values{}
q.Set("sort", "-updated")
q.Set("perPage", "1")
path := fmt.Sprintf("/api/collections/ranking/records?%s", q.Encode())
resp, err := s.pb.do(ctx, http.MethodGet, path, nil)
if err != nil {
return time.Time{}, err
}
html, _ := rec["html"].(string)
var updated time.Time
if ts, ok := rec["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
defer resp.Body.Close()
var result struct {
Items []map[string]interface{} `json:"items"`
}
return html, updated, nil
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return time.Time{}, err
}
if len(result.Items) == 0 {
return time.Time{}, nil
}
ts, _ := result.Items[0]["updated"].(string)
t, _ := time.Parse(time.RFC3339, ts)
return t, nil
}
// ─── Reading progress ─────────────────────────────────────────────────────────
@@ -516,45 +544,6 @@ func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (s
return filename, filename != "", nil
}
// ─── rankingFileInfo is a minimal os.FileInfo implementation ─────────────────
type rankingFileInfo struct {
modTime time.Time
}
func (r rankingFileInfo) Name() string { return "ranking" }
func (r rankingFileInfo) Size() int64 { return 0 }
func (r rankingFileInfo) Mode() os.FileMode { return 0o444 }
func (r rankingFileInfo) ModTime() time.Time { return r.modTime }
func (r rankingFileInfo) IsDir() bool { return false }
func (r rankingFileInfo) Sys() interface{} { return nil }
var _ os.FileInfo = rankingFileInfo{}
// RankingModTime returns file-info-compatible data for the ranking record.
func (s *PocketBaseStore) RankingModTime(ctx context.Context) (os.FileInfo, error) {
_, updated, err := s.GetRanking(ctx)
if err != nil {
return nil, err
}
if updated.IsZero() {
return nil, nil
}
return rankingFileInfo{modTime: updated}, nil
}
// RankingPageCacheModTime returns file-info for a cached ranking page.
func (s *PocketBaseStore) RankingPageCacheModTime(ctx context.Context, page int) (os.FileInfo, error) {
_, updated, err := s.GetRankingPageHTML(ctx, page)
if err != nil {
return nil, err
}
if updated.IsZero() {
return nil, nil
}
return rankingFileInfo{modTime: updated}, nil
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// pbEsc escapes a string for use in a PocketBase filter expression.

View File

@@ -5,7 +5,6 @@ package storage
import (
"context"
"os"
"time"
"github.com/libnovel/scraper/internal/scraper"
@@ -22,14 +21,15 @@ type ChapterInfo struct {
// RankingItem represents a single entry in the novel ranking list.
type RankingItem struct {
Rank int `json:"rank"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author,omitempty"`
Cover string `json:"cover,omitempty"`
Status string `json:"status,omitempty"`
Genres []string `json:"genres,omitempty"`
SourceURL string `json:"source_url,omitempty"`
Rank int `json:"rank"`
Slug string `json:"slug"`
Title string `json:"title"`
Author string `json:"author,omitempty"`
Cover string `json:"cover,omitempty"`
Status string `json:"status,omitempty"`
Genres []string `json:"genres,omitempty"`
SourceURL string `json:"source_url,omitempty"`
Updated time.Time `json:"updated,omitempty"`
}
// ReadingProgress holds a single user's reading position for one book.
@@ -81,22 +81,13 @@ type Store interface {
// ── Ranking ────────────────────────────────────────────────────────────
// WriteRanking persists the ranking list.
WriteRanking(ctx context.Context, items []RankingItem) error
// ReadRankingItems returns the stored ranking items.
// WriteRankingItem upserts a single ranking entry (keyed on Slug).
WriteRankingItem(ctx context.Context, item RankingItem) error
// ReadRankingItems returns all ranking items sorted by rank ascending.
ReadRankingItems(ctx context.Context) ([]RankingItem, error)
// RankingFileInfo returns os.FileInfo-like data for the ranking record.
// Returns (nil, nil) when no ranking has been stored yet.
RankingFileInfo(ctx context.Context) (os.FileInfo, error)
// ── Ranking page HTML cache ────────────────────────────────────────────
// WriteRankingPageCache stores raw HTML for a ranking page.
WriteRankingPageCache(ctx context.Context, page int, html string) error
// ReadRankingPageCache returns cached HTML for a ranking page, or "" on miss.
ReadRankingPageCache(ctx context.Context, page int) (string, error)
// RankingPageCacheInfo returns file-like info for a cached ranking page.
RankingPageCacheInfo(ctx context.Context, page int) (os.FileInfo, error)
// RankingFreshEnough returns true when ranking rows exist and the most
// recent Updated timestamp is within maxAge of now.
RankingFreshEnough(ctx context.Context, maxAge time.Duration) (bool, error)
// ── Audio cache ────────────────────────────────────────────────────────
@@ -104,6 +95,8 @@ type Store interface {
GetAudioCache(ctx context.Context, cacheKey string) (string, bool)
// SetAudioCache persists a Kokoro filename for cacheKey.
SetAudioCache(ctx context.Context, cacheKey, filename string) error
// PutAudio stores raw audio bytes under the given MinIO object key.
PutAudio(ctx context.Context, key string, data []byte) error
// ── Reading progress ───────────────────────────────────────────────────