kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

refactor(ranking): replace blob cache with per-item PocketBase storage

Browse Source 
- Replace SetRanking/GetRanking/SetRankingPageHTML/GetRankingPageHTML blob methods
  with WriteRankingItem/ReadRankingItems/RankingFreshEnough per-item operations
- Add 24h staleness gate in ScrapeRanking to skip re-scraping fresh data
- Add GET /api/ranking endpoint returning []RankingItem sorted by rank
- Remove RankingPageCacher interface and rankingCacheAdapter adapter
- Update integration tests to use new per-item upsert semantics
- Include e2e test suite (scraper/internal/e2e/)
This commit is contained in:
Admin
2026-03-03 19:37:49 +05:00
parent 56bf4dde22
commit b8d4d94b18
10 changed files with 1173 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
scraper/internal/storage/pocketbase.go
View File

@@ -5,8 +5,8 @@
// books — slug(text,unique), title, author, cover, status, genres(json),
// summary, total_chapters(number), source_url, ranking(number), updated(date)
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
// ranking — data(json), updated(date) [single row, upserted by slug="_ranking_"]
// ranking_html — page(number,unique), html(text), updated(date)
// ranking — rank(number), slug(text,unique), title, author, cover, status,
// genres(json), source_url, updated(date)
// progress — session_id(text), slug(text), chapter(number), updated(date)
// audio_cache — cache_key(text,unique), filename(text), updated(date)
// app_users — username(text,unique), password_hash(text), role(text), created(date)
@@ -20,7 +20,6 @@ import (
"io"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
@@ -248,20 +247,22 @@ func (s *PocketBaseStore) Ping(ctx context.Context) error {
func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
// We just attempt to create each collection; 400/422 errors for "already
// exists" are silently ignored.
// PocketBase v0.22+ uses "fields"; older versions used "schema".
// We use "fields" which is the current API.
collections := []map[string]interface{}{
{
"name": "books",
"type": "base",
"schema": []map[string]interface{}{
{"name": "slug", "type": "text", "required": true, "options": map[string]interface{}{"min": 1}},
"fields": []map[string]interface{}{
{"name": "slug", "type": "text", "required": true},
{"name": "title", "type": "text", "required": true},
{"name": "author", "type": "text"},
{"name": "cover", "type": "url"},
{"name": "cover", "type": "text"},
{"name": "status", "type": "text"},
{"name": "genres", "type": "json"},
{"name": "summary", "type": "text"},
{"name": "total_chapters", "type": "number"},
{"name": "source_url", "type": "url"},
{"name": "source_url", "type": "text"},
{"name": "ranking", "type": "number"},
{"name": "meta_updated", "type": "date"},
},
@@ -269,7 +270,7 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
{
"name": "chapters_idx",
"type": "base",
"schema": []map[string]interface{}{
"fields": []map[string]interface{}{
{"name": "slug", "type": "text", "required": true},
{"name": "number", "type": "number", "required": true},
{"name": "title", "type": "text"},
@@ -279,25 +280,22 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
{
"name": "ranking",
"type": "base",
"schema": []map[string]interface{}{
{"name": "key", "type": "text", "required": true},
{"name": "data", "type": "json"},
{"name": "updated", "type": "date"},
},
},
{
"name": "ranking_html",
"type": "base",
"schema": []map[string]interface{}{
{"name": "page", "type": "number", "required": true},
{"name": "html", "type": "text"},
"fields": []map[string]interface{}{
{"name": "rank", "type": "number", "required": true},
{"name": "slug", "type": "text", "required": true},
{"name": "title", "type": "text"},
{"name": "author", "type": "text"},
{"name": "cover", "type": "text"},
{"name": "status", "type": "text"},
{"name": "genres", "type": "json"},
{"name": "source_url", "type": "text"},
{"name": "updated", "type": "date"},
},
},
{
"name": "progress",
"type": "base",
"schema": []map[string]interface{}{
"fields": []map[string]interface{}{
{"name": "session_id", "type": "text", "required": true},
{"name": "slug", "type": "text", "required": true},
{"name": "chapter", "type": "number"},
@@ -307,7 +305,7 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
{
"name": "audio_cache",
"type": "base",
"schema": []map[string]interface{}{
"fields": []map[string]interface{}{
{"name": "cache_key", "type": "text", "required": true},
{"name": "filename", "type": "text"},
{"name": "updated", "type": "date"},
@@ -316,8 +314,8 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
{
"name": "app_users",
"type": "base",
"schema": []map[string]interface{}{
{"name": "username", "type": "text", "required": true, "options": map[string]interface{}{"min": 3, "max": 32}},
"fields": []map[string]interface{}{
{"name": "username", "type": "text", "required": true},
{"name": "password_hash", "type": "text", "required": true},
{"name": "role", "type": "text"},
{"name": "created", "type": "date"},
@@ -406,52 +404,82 @@ func (s *PocketBaseStore) CountChapterIdx(ctx context.Context, slug string) int
return len(rows)
}
// ─── Ranking ──────────────────────────────────────────────────────────────────
// ─── Ranking (per-item) ───────────────────────────────────────────────────────
func (s *PocketBaseStore) SetRanking(ctx context.Context, dataJSON string) error {
return s.pb.upsert(ctx, "ranking", `key="_ranking_"`, map[string]interface{}{
"key": "_ranking_",
"data": dataJSON,
"updated": time.Now().UTC().Format(time.RFC3339),
func (s *PocketBaseStore) UpsertRankingItem(ctx context.Context, item RankingItem) error {
genresJSON, _ := json.Marshal(item.Genres)
return s.pb.upsert(ctx, "ranking", fmt.Sprintf(`slug="%s"`, pbEsc(item.Slug)), map[string]interface{}{
"rank": item.Rank,
"slug": item.Slug,
"title": item.Title,
"author": item.Author,
"cover": item.Cover,
"status": item.Status,
"genres": string(genresJSON),
"source_url": item.SourceURL,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetRanking(ctx context.Context) (string, time.Time, error) {
rec, err := s.pb.listOne(ctx, "ranking", `key="_ranking_"`)
if err != nil || rec == nil {
return "", time.Time{}, err
func (s *PocketBaseStore) ListRankingItems(ctx context.Context) ([]RankingItem, error) {
rows, err := s.pb.listAll(ctx, "ranking", "", "+rank")
if err != nil {
return nil, err
}
data, _ := rec["data"].(string)
var updated time.Time
if ts, ok := rec["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
items := make([]RankingItem, 0, len(rows))
for _, r := range rows {
item := RankingItem{
Rank: int(floatVal(r, "rank")),
Slug: strVal(r, "slug"),
Title: strVal(r, "title"),
Author: strVal(r, "author"),
Cover: strVal(r, "cover"),
Status: strVal(r, "status"),
SourceURL: strVal(r, "source_url"),
}
if ts, ok := r["updated"].(string); ok {
item.Updated, _ = time.Parse(time.RFC3339, ts)
}
switch v := r["genres"].(type) {
case string:
_ = json.Unmarshal([]byte(v), &item.Genres)
case []interface{}:
for _, g := range v {
if s, ok := g.(string); ok {
item.Genres = append(item.Genres, s)
}
}
}
items = append(items, item)
}
return data, updated, nil
return items, nil
}
// ─── Ranking page HTML cache ──────────────────────────────────────────────────
func (s *PocketBaseStore) SetRankingPageHTML(ctx context.Context, page int, html string) error {
return s.pb.upsert(ctx, "ranking_html",
fmt.Sprintf(`page=%d`, page),
map[string]interface{}{
"page": page,
"html": html,
"updated": time.Now().UTC().Format(time.RFC3339),
})
}
func (s *PocketBaseStore) GetRankingPageHTML(ctx context.Context, page int) (string, time.Time, error) {
rec, err := s.pb.listOne(ctx, "ranking_html", fmt.Sprintf(`page=%d`, page))
if err != nil || rec == nil {
return "", time.Time{}, err
// RankingLastUpdated returns the most recent Updated time across all ranking rows,
// or the zero time if no rows exist.
func (s *PocketBaseStore) RankingLastUpdated(ctx context.Context) (time.Time, error) {
// listAll with sort "-updated" and perPage=1 is the cheapest approach.
q := url.Values{}
q.Set("sort", "-updated")
q.Set("perPage", "1")
path := fmt.Sprintf("/api/collections/ranking/records?%s", q.Encode())
resp, err := s.pb.do(ctx, http.MethodGet, path, nil)
if err != nil {
return time.Time{}, err
}
html, _ := rec["html"].(string)
var updated time.Time
if ts, ok := rec["updated"].(string); ok {
updated, _ = time.Parse(time.RFC3339, ts)
defer resp.Body.Close()
var result struct {
Items []map[string]interface{} `json:"items"`
}
return html, updated, nil
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return time.Time{}, err
}
if len(result.Items) == 0 {
return time.Time{}, nil
}
ts, _ := result.Items[0]["updated"].(string)
t, _ := time.Parse(time.RFC3339, ts)
return t, nil
}
// ─── Reading progress ─────────────────────────────────────────────────────────
@@ -516,45 +544,6 @@ func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (s
return filename, filename != "", nil
}
// ─── rankingFileInfo is a minimal os.FileInfo implementation ─────────────────
type rankingFileInfo struct {
modTime time.Time
}
func (r rankingFileInfo) Name() string { return "ranking" }
func (r rankingFileInfo) Size() int64 { return 0 }
func (r rankingFileInfo) Mode() os.FileMode { return 0o444 }
func (r rankingFileInfo) ModTime() time.Time { return r.modTime }
func (r rankingFileInfo) IsDir() bool { return false }
func (r rankingFileInfo) Sys() interface{} { return nil }
var _ os.FileInfo = rankingFileInfo{}
// RankingModTime returns file-info-compatible data for the ranking record.
func (s *PocketBaseStore) RankingModTime(ctx context.Context) (os.FileInfo, error) {
_, updated, err := s.GetRanking(ctx)
if err != nil {
return nil, err
}
if updated.IsZero() {
return nil, nil
}
return rankingFileInfo{modTime: updated}, nil
}
// RankingPageCacheModTime returns file-info for a cached ranking page.
func (s *PocketBaseStore) RankingPageCacheModTime(ctx context.Context, page int) (os.FileInfo, error) {
_, updated, err := s.GetRankingPageHTML(ctx, page)
if err != nil {
return nil, err
}
if updated.IsZero() {
return nil, nil
}
return rankingFileInfo{modTime: updated}, nil
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// pbEsc escapes a string for use in a PocketBase filter expression.