Compare commits

...

1 Commits

Author SHA1 Message Date
Admin
69818089a6 perf(ui): eliminate full listBooks() scan on every page load
Some checks failed
Release / Test backend (push) Failing after 10s
Release / Docker / backend (push) Has been skipped
Release / Docker / runner (push) Has been skipped
Release / Docker / caddy (push) Failing after 10s
Release / Check ui (push) Successful in 32s
CI / Test backend (pull_request) Successful in 39s
CI / Check ui (pull_request) Successful in 48s
Release / Upload source maps (push) Successful in 2m17s
CI / Docker / caddy (pull_request) Successful in 2m44s
Release / Docker / ui (push) Successful in 2m31s
Release / Gitea Release (push) Has been skipped
CI / Docker / runner (pull_request) Successful in 1m38s
CI / Docker / ui (pull_request) Successful in 1m28s
CI / Docker / backend (pull_request) Successful in 2m11s
The home, library, and /books routes were fetching all 15k books from
PocketBase on every SSR request (31 sequential HTTP calls per request).

Changes:
- Add src/lib/server/cache.ts: generic Valkey JSON cache
- Add getBooksBySlugs(): single PB query fetching only requested slugs,
  with fallback to the 5-min Valkey cache populated by listBooks()
- listBooks(): now caches results in Valkey for 5 min (safety net for
  admin routes that still need the full list)
- Home + /api/home: replaced listBooks()+filter with getBooksBySlugs()
  on progress slugs only — typically 1 PB request instead of 31
- /books + /api/library: same pattern using progress+saved slug union
2026-03-26 16:14:47 +05:00
6 changed files with 172 additions and 45 deletions

View File

@@ -0,0 +1,72 @@
/**
* Generic Valkey (Redis-compatible) cache.
*
* Reuses the same ioredis singleton from presignCache.ts but exposes a
* simple typed get/set/invalidate API for arbitrary JSON values.
*
* Usage:
* const books = await cache.get<Book[]>('books:all');
* await cache.set('books:all', books, 5 * 60);
* await cache.invalidate('books:all');
*/
import Redis from 'ioredis';
let _client: Redis | null = null;
function client(): Redis {
if (!_client) {
const url = process.env.VALKEY_URL ?? 'redis://valkey:6379';
_client = new Redis(url, {
lazyConnect: false,
enableOfflineQueue: true,
maxRetriesPerRequest: 2
});
_client.on('error', (err: Error) => {
console.error('[cache] Valkey error:', err.message);
});
}
return _client;
}
/** Return the cached value for key, or null if absent / expired / error. */
export async function get<T>(key: string): Promise<T | null> {
try {
const raw = await client().get(key);
if (!raw) return null;
return JSON.parse(raw) as T;
} catch {
return null;
}
}
/**
* Store a value under key for ttlSeconds seconds.
* Silently no-ops on Valkey errors so callers never crash.
*/
export async function set<T>(key: string, value: T, ttlSeconds: number): Promise<void> {
try {
await client().set(key, JSON.stringify(value), 'EX', ttlSeconds);
} catch {
// non-fatal
}
}
/** Delete a key immediately (e.g. after a write that invalidates it). */
export async function invalidate(key: string): Promise<void> {
try {
await client().del(key);
} catch {
// non-fatal
}
}
/** Invalidate all keys matching a glob pattern (e.g. 'books:*'). */
export async function invalidatePattern(pattern: string): Promise<void> {
try {
const keys = await client().keys(pattern);
if (keys.length > 0) await client().del(...keys);
} catch {
// non-fatal
}
}

View File

@@ -6,6 +6,7 @@
import { env } from '$env/dynamic/private';
import { log } from '$lib/server/logger';
import * as cache from '$lib/server/cache';
const PB_URL = env.POCKETBASE_URL ?? 'http://localhost:8090';
const PB_EMAIL = env.POCKETBASE_ADMIN_EMAIL ?? 'admin@libnovel.local';
@@ -200,16 +201,65 @@ async function listOne<T>(collection: string, filter: string): Promise<T | null>
// ─── Books ────────────────────────────────────────────────────────────────────
const BOOKS_CACHE_KEY = 'books:all';
const BOOKS_CACHE_TTL = 5 * 60; // 5 minutes
export async function listBooks(): Promise<Book[]> {
const cached = await cache.get<Book[]>(BOOKS_CACHE_KEY);
if (cached) {
log.debug('pocketbase', 'listBooks cache hit', { total: cached.length });
return cached;
}
const books = await listAll<Book>('books', '', '+title');
const nullTitles = books.filter((b) => b.title == null).length;
if (nullTitles > 0) {
log.warn('pocketbase', 'listBooks: books with null title', { count: nullTitles, total: books.length });
}
log.debug('pocketbase', 'listBooks', { total: books.length, nullTitles });
log.debug('pocketbase', 'listBooks cache miss', { total: books.length, nullTitles });
await cache.set(BOOKS_CACHE_KEY, books, BOOKS_CACHE_TTL);
return books;
}
/**
* Fetch only the books whose slugs are in the given set.
* Uses PocketBase filter `slug IN (...)` — a single request regardless of how
* many slugs are requested. Falls back to empty array on error.
*
* Use this instead of listBooks() whenever you only need a small subset of
* books (e.g. the user's reading list or saved shelf).
*
* PocketBase filter syntax for IN: slug='a' || slug='b' || ...
* Limited to 200 slugs to keep the filter URL sane; callers with larger sets
* should fall back to listBooks().
*/
export async function getBooksBySlugs(slugs: Iterable<string>): Promise<Book[]> {
const slugArr = [...new Set(slugs)].slice(0, 200);
if (slugArr.length === 0) return [];
// Check cache for each slug individually (populated by prior listBooks calls).
// If all slugs hit, skip the network round-trip entirely.
const cached = await cache.get<Book[]>(BOOKS_CACHE_KEY);
if (cached) {
const slugSet = new Set(slugArr);
const found = cached.filter((b) => slugSet.has(b.slug));
if (found.length === slugArr.length) {
log.debug('pocketbase', 'getBooksBySlugs cache hit', { count: found.length });
return found;
}
}
// Build filter: slug='a' || slug='b' || ...
const filter = slugArr.map((s) => `slug='${s.replace(/'/g, "\\'")}'`).join(' || ');
const books = await listAll<Book>('books', filter, '+title');
log.debug('pocketbase', 'getBooksBySlugs', { requested: slugArr.length, found: books.length });
return books;
}
/** Invalidate the books cache (call after a book is created/updated/deleted). */
export async function invalidateBooksCache(): Promise<void> {
await cache.invalidate(BOOKS_CACHE_KEY);
}
export async function getBook(slug: string): Promise<Book | null> {
return listOne<Book>('books', `slug="${slug}"`);
}

View File

@@ -1,6 +1,6 @@
import type { PageServerLoad } from './$types';
import {
listBooks,
getBooksBySlugs,
recentlyAddedBooks,
allProgress,
getHomeStats,
@@ -10,14 +10,15 @@ import { log } from '$lib/server/logger';
import type { Book, Progress } from '$lib/server/pocketbase';
export const load: PageServerLoad = async ({ locals }) => {
let allBooks: Book[] = [];
// Step 1: fetch progress + recent books + stats in parallel.
// We intentionally do NOT call listBooks() here — we only need books that
// appear in the user's progress list, which is a tiny subset of 15k books.
let recentBooks: Book[] = [];
let progressList: Progress[] = [];
let stats = { totalBooks: 0, totalChapters: 0 };
try {
[allBooks, recentBooks, progressList, stats] = await Promise.all([
listBooks(),
[recentBooks, progressList, stats] = await Promise.all([
recentlyAddedBooks(8),
allProgress(locals.sessionId, locals.user?.id),
getHomeStats()
@@ -26,8 +27,14 @@ export const load: PageServerLoad = async ({ locals }) => {
log.error('home', 'failed to load home data', { err: String(e) });
}
// Build slug → book lookup
const bookMap = new Map<string, Book>(allBooks.map((b) => [b.slug, b]));
// Step 2: fetch only the books we actually need for continue-reading.
// This is O(progress entries) instead of O(15k books).
const progressSlugs = progressList.map((p) => p.slug);
const progressBooks = progressSlugs.length > 0
? await getBooksBySlugs(progressSlugs).catch(() => [] as Book[])
: [];
const bookMap = new Map<string, Book>(progressBooks.map((b) => [b.slug, b]));
// Continue reading: progress entries joined with book data, most recent first
const continueReading = progressList

View File

@@ -1,7 +1,7 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import {
listBooks,
getBooksBySlugs,
recentlyAddedBooks,
allProgress,
getHomeStats,
@@ -17,14 +17,12 @@ import type { Book, Progress } from '$lib/server/pocketbase';
* Requires authentication (enforced by layout guard).
*/
export const GET: RequestHandler = async ({ locals }) => {
let allBooks: Book[] = [];
let recentBooks: Book[] = [];
let progressList: Progress[] = [];
let stats = { totalBooks: 0, totalChapters: 0 };
try {
[allBooks, recentBooks, progressList, stats] = await Promise.all([
listBooks(),
[recentBooks, progressList, stats] = await Promise.all([
recentlyAddedBooks(8),
allProgress(locals.sessionId, locals.user?.id),
getHomeStats()
@@ -33,7 +31,13 @@ export const GET: RequestHandler = async ({ locals }) => {
log.error('api/home', 'failed to load home data', { err: String(e) });
}
const bookMap = new Map<string, Book>(allBooks.map((b) => [b.slug, b]));
// Fetch only the books we actually need for continue-reading.
const progressSlugs = progressList.map((p) => p.slug);
const progressBooks = progressSlugs.length > 0
? await getBooksBySlugs(progressSlugs).catch(() => [] as Book[])
: [];
const bookMap = new Map<string, Book>(progressBooks.map((b) => [b.slug, b]));
const continueReading = progressList
.filter((p) => bookMap.has(p.slug))

View File

@@ -1,7 +1,8 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { listBooks, allProgress, getSavedSlugs } from '$lib/server/pocketbase';
import { getBooksBySlugs, allProgress, getSavedSlugs } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
import type { Book } from '$lib/server/pocketbase';
/**
* GET /api/library
@@ -11,23 +12,25 @@ import { log } from '$lib/server/logger';
* Response shape mirrors LibraryItem in the iOS APIClient.
*/
export const GET: RequestHandler = async ({ locals }) => {
let allBooks: Awaited<ReturnType<typeof listBooks>>;
let progressList: Awaited<ReturnType<typeof allProgress>>;
let savedSlugs: Set<string>;
let progressList: Awaited<ReturnType<typeof allProgress>> = [];
let savedSlugs: Set<string> = new Set();
try {
[allBooks, progressList, savedSlugs] = await Promise.all([
listBooks(),
[progressList, savedSlugs] = await Promise.all([
allProgress(locals.sessionId, locals.user?.id),
getSavedSlugs(locals.sessionId, locals.user?.id)
]);
} catch (e) {
log.error('api/library', 'failed to load library data', { err: String(e) });
allBooks = [];
progressList = [];
savedSlugs = new Set();
}
// Fetch only the books the user actually has in their library.
const progressSlugs = new Set(progressList.map((p) => p.slug));
const allNeededSlugs = new Set([...progressSlugs, ...savedSlugs]);
const books = allNeededSlugs.size > 0
? await getBooksBySlugs(allNeededSlugs).catch(() => [] as Book[])
: [];
const progressMap: Record<string, number> = {};
const progressUpdatedMap: Record<string, string> = {};
for (const p of progressList) {
@@ -35,9 +38,6 @@ export const GET: RequestHandler = async ({ locals }) => {
progressUpdatedMap[p.slug] = p.updated;
}
const progressSlugs = new Set(progressList.map((p) => p.slug));
const books = allBooks.filter((b) => progressSlugs.has(b.slug) || savedSlugs.has(b.slug));
const withProgress = books.filter((b) => progressSlugs.has(b.slug));
const savedOnly = books
.filter((b) => !progressSlugs.has(b.slug))

View File

@@ -1,48 +1,42 @@
import { error } from '@sveltejs/kit';
import type { PageServerLoad } from './$types';
import { listBooks, allProgress, getSavedSlugs } from '$lib/server/pocketbase';
import { getBooksBySlugs, allProgress, getSavedSlugs } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
import type { Book } from '$lib/server/pocketbase';
export const load: PageServerLoad = async ({ locals }) => {
let allBooks: Awaited<ReturnType<typeof listBooks>>;
let progressList: Awaited<ReturnType<typeof allProgress>>;
let savedSlugs: Set<string>;
let progressList: Awaited<ReturnType<typeof allProgress>> = [];
let savedSlugs: Set<string> = new Set();
try {
[allBooks, progressList, savedSlugs] = await Promise.all([
listBooks(),
[progressList, savedSlugs] = await Promise.all([
allProgress(locals.sessionId, locals.user?.id),
getSavedSlugs(locals.sessionId, locals.user?.id)
]);
} catch (e) {
log.error('books', 'failed to load library data', { err: String(e) });
allBooks = [];
progressList = [];
savedSlugs = new Set();
}
// Fetch only the books the user actually has in their library.
const progressSlugs = new Set(progressList.map((p) => p.slug));
const allNeededSlugs = new Set([...progressSlugs, ...savedSlugs]);
const books = allNeededSlugs.size > 0
? await getBooksBySlugs(allNeededSlugs).catch(() => [] as Book[])
: [];
// Build a quick lookup: slug → last chapter read
const progressMap: Record<string, number> = {};
const progressUpdatedMap: Record<string, string> = {};
for (const p of progressList) {
progressMap[p.slug] = p.chapter;
progressUpdatedMap[p.slug] = p.updated;
}
// Library = books the user has started reading OR explicitly saved
const progressSlugs = new Set(progressList.map((p) => p.slug));
const books = allBooks.filter((b) => progressSlugs.has(b.slug) || savedSlugs.has(b.slug));
// Sort: books with progress first (most-recently-read order is implicit via progressList),
// then saved-only books alphabetically.
// Sort: books with progress first (most-recently-read), then saved-only alphabetically.
const withProgress = books.filter((b) => progressSlugs.has(b.slug));
const savedOnly = books
.filter((b) => !progressSlugs.has(b.slug))
.sort((a, b) => (a.title ?? '').localeCompare(b.title ?? ''));
// Re-sort withProgress by most recent progress update
const progressUpdatedMap: Record<string, string> = {};
for (const p of progressList) {
progressUpdatedMap[p.slug] = p.updated;
}
withProgress.sort((a, b) => {
const ta = progressUpdatedMap[a.slug] ?? '';
const tb = progressUpdatedMap[b.slug] ?? '';