perf: cache admin job lists + targeted polling for audio/translation pages
Some checks failed
Release / Test backend (push) Successful in 41s
Release / Check ui (push) Failing after 34s
Release / Upload source maps (push) Has been skipped
Release / Docker / ui (push) Has been skipped
Release / Docker / caddy (push) Successful in 40s
Release / Docker / runner (push) Has been cancelled
Release / Gitea Release (push) Has been cancelled
Release / Docker / backend (push) Has been cancelled
Some checks failed
Release / Test backend (push) Successful in 41s
Release / Check ui (push) Failing after 34s
Release / Upload source maps (push) Has been skipped
Release / Docker / ui (push) Has been skipped
Release / Docker / caddy (push) Successful in 40s
Release / Docker / runner (push) Has been cancelled
Release / Gitea Release (push) Has been cancelled
Release / Docker / backend (push) Has been cancelled
- Add 30s Valkey cache to listScrapingTasks, listAudioJobs, listTranslationJobs (use listN(500) instead of unbounded listAll to cap at one request) - Delete listAudioCache() — derive AudioCacheEntry[] from jobs in server load - Add listBookSlugs() with 10min cache — replaces full listBooks() in translation load - Add GET /api/admin/audio-jobs, /api/admin/translation-jobs, /api/admin/scrape-tasks (lightweight polling endpoints backed by the Valkey cache) - Replace invalidateAll() interval polling in audio+translation pages with targeted fetch to the new endpoints (avoids re-running full server load)
This commit is contained in:
@@ -248,6 +248,14 @@ const RATINGS_CACHE_TTL = 5 * 60; // 5 minutes
|
|||||||
const HOME_STATS_CACHE_KEY = 'home:stats';
|
const HOME_STATS_CACHE_KEY = 'home:stats';
|
||||||
const HOME_STATS_CACHE_TTL = 10 * 60; // 10 minutes — counts don't need to be exact
|
const HOME_STATS_CACHE_TTL = 10 * 60; // 10 minutes — counts don't need to be exact
|
||||||
|
|
||||||
|
const SCRAPING_TASKS_CACHE_KEY = 'admin:scraping_tasks';
|
||||||
|
const AUDIO_JOBS_CACHE_KEY = 'admin:audio_jobs';
|
||||||
|
const TRANSLATION_JOBS_CACHE_KEY = 'admin:translation_jobs';
|
||||||
|
const ADMIN_JOBS_CACHE_TTL = 30; // 30 seconds — admin views poll frequently
|
||||||
|
|
||||||
|
const BOOK_SLUGS_CACHE_KEY = 'books:slugs';
|
||||||
|
const BOOK_SLUGS_CACHE_TTL = 10 * 60; // 10 minutes — slugs change rarely
|
||||||
|
|
||||||
async function getAllRatings(): Promise<BookRating[]> {
|
async function getAllRatings(): Promise<BookRating[]> {
|
||||||
const cached = await cache.get<BookRating[]>(RATINGS_CACHE_KEY);
|
const cached = await cache.get<BookRating[]>(RATINGS_CACHE_KEY);
|
||||||
if (cached) return cached;
|
if (cached) return cached;
|
||||||
@@ -272,6 +280,31 @@ export async function listBooks(): Promise<Book[]> {
|
|||||||
return books;
|
return books;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface BookSlug {
|
||||||
|
slug: string;
|
||||||
|
title: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns only the slug and title of every book. Cheaper than listBooks() —
|
||||||
|
* used for datalist autocomplete in admin forms. Cached for 10 minutes.
|
||||||
|
*/
|
||||||
|
export async function listBookSlugs(): Promise<BookSlug[]> {
|
||||||
|
const cached = await cache.get<BookSlug[]>(BOOK_SLUGS_CACHE_KEY);
|
||||||
|
if (cached) return cached;
|
||||||
|
// Re-use full books cache if already warm — avoids a second PocketBase call.
|
||||||
|
const fullCached = await cache.get<Book[]>(BOOKS_CACHE_KEY);
|
||||||
|
if (fullCached) {
|
||||||
|
const slugs = fullCached.map((b) => ({ slug: b.slug, title: b.title }));
|
||||||
|
await cache.set(BOOK_SLUGS_CACHE_KEY, slugs, BOOK_SLUGS_CACHE_TTL);
|
||||||
|
return slugs;
|
||||||
|
}
|
||||||
|
const items = await listAll<BookSlug>('books', '', '+title').catch(() => [] as BookSlug[]);
|
||||||
|
const slugs = items.map((b) => ({ slug: b.slug, title: b.title }));
|
||||||
|
await cache.set(BOOK_SLUGS_CACHE_KEY, slugs, BOOK_SLUGS_CACHE_TTL);
|
||||||
|
return slugs;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetch only the books whose slugs are in the given set.
|
* Fetch only the books whose slugs are in the given set.
|
||||||
* Uses PocketBase filter `slug IN (...)` — a single request regardless of how
|
* Uses PocketBase filter `slug IN (...)` — a single request regardless of how
|
||||||
@@ -1052,16 +1085,6 @@ export interface AudioCacheEntry {
|
|||||||
updated: string;
|
updated: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function listAudioCache(): Promise<AudioCacheEntry[]> {
|
|
||||||
const jobs = await listAll<AudioJob>('audio_jobs', 'status="done"', '-finished');
|
|
||||||
return jobs.map((j) => ({
|
|
||||||
id: j.id,
|
|
||||||
cache_key: j.cache_key,
|
|
||||||
filename: `${j.cache_key}.mp3`,
|
|
||||||
updated: j.finished
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
export interface ScrapingTask {
|
export interface ScrapingTask {
|
||||||
@@ -1081,7 +1104,11 @@ export interface ScrapingTask {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function listScrapingTasks(): Promise<ScrapingTask[]> {
|
export async function listScrapingTasks(): Promise<ScrapingTask[]> {
|
||||||
return listAll<ScrapingTask>('scraping_tasks', '', '-started');
|
const cached = await cache.get<ScrapingTask[]>(SCRAPING_TASKS_CACHE_KEY);
|
||||||
|
if (cached) return cached;
|
||||||
|
const tasks = await listN<ScrapingTask>('scraping_tasks', 500, '', '-started');
|
||||||
|
await cache.set(SCRAPING_TASKS_CACHE_KEY, tasks, ADMIN_JOBS_CACHE_TTL);
|
||||||
|
return tasks;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getScrapingTask(id: string): Promise<ScrapingTask | null> {
|
export async function getScrapingTask(id: string): Promise<ScrapingTask | null> {
|
||||||
@@ -1103,7 +1130,11 @@ export interface AudioJob {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function listAudioJobs(): Promise<AudioJob[]> {
|
export async function listAudioJobs(): Promise<AudioJob[]> {
|
||||||
return listAll<AudioJob>('audio_jobs', '', '-started');
|
const cached = await cache.get<AudioJob[]>(AUDIO_JOBS_CACHE_KEY);
|
||||||
|
if (cached) return cached;
|
||||||
|
const jobs = await listN<AudioJob>('audio_jobs', 500, '', '-started');
|
||||||
|
await cache.set(AUDIO_JOBS_CACHE_KEY, jobs, ADMIN_JOBS_CACHE_TTL);
|
||||||
|
return jobs;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1130,7 +1161,11 @@ export interface TranslationJob {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function listTranslationJobs(): Promise<TranslationJob[]> {
|
export async function listTranslationJobs(): Promise<TranslationJob[]> {
|
||||||
return listAll<TranslationJob>('translation_jobs', '', '-started');
|
const cached = await cache.get<TranslationJob[]>(TRANSLATION_JOBS_CACHE_KEY);
|
||||||
|
if (cached) return cached;
|
||||||
|
const jobs = await listN<TranslationJob>('translation_jobs', 500, '', '-started');
|
||||||
|
await cache.set(TRANSLATION_JOBS_CACHE_KEY, jobs, ADMIN_JOBS_CACHE_TTL);
|
||||||
|
return jobs;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getAudioTime(
|
export async function getAudioTime(
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { redirect } from '@sveltejs/kit';
|
import { redirect } from '@sveltejs/kit';
|
||||||
import type { PageServerLoad } from './$types';
|
import type { PageServerLoad } from './$types';
|
||||||
import { listAudioCache, listAudioJobs, type AudioCacheEntry, type AudioJob } from '$lib/server/pocketbase';
|
import { listAudioJobs, type AudioCacheEntry, type AudioJob } from '$lib/server/pocketbase';
|
||||||
import { log } from '$lib/server/logger';
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
export const load: PageServerLoad = async ({ locals }) => {
|
export const load: PageServerLoad = async ({ locals }) => {
|
||||||
@@ -8,16 +8,20 @@ export const load: PageServerLoad = async ({ locals }) => {
|
|||||||
redirect(302, '/');
|
redirect(302, '/');
|
||||||
}
|
}
|
||||||
|
|
||||||
const [entries, jobs] = await Promise.all([
|
const jobs = await listAudioJobs().catch((e): AudioJob[] => {
|
||||||
listAudioCache().catch((e): AudioCacheEntry[] => {
|
|
||||||
log.warn('admin/audio', 'failed to load audio cache', { err: String(e) });
|
|
||||||
return [];
|
|
||||||
}),
|
|
||||||
listAudioJobs().catch((e): AudioJob[] => {
|
|
||||||
log.warn('admin/audio', 'failed to load audio jobs', { err: String(e) });
|
log.warn('admin/audio', 'failed to load audio jobs', { err: String(e) });
|
||||||
return [];
|
return [];
|
||||||
})
|
});
|
||||||
]);
|
|
||||||
|
// Derive cache entries from done jobs — no second query needed.
|
||||||
|
const entries: AudioCacheEntry[] = jobs
|
||||||
|
.filter((j) => j.status === 'done')
|
||||||
|
.map((j) => ({
|
||||||
|
id: j.id,
|
||||||
|
cache_key: j.cache_key,
|
||||||
|
filename: `${j.cache_key}.mp3`,
|
||||||
|
updated: j.finished
|
||||||
|
}));
|
||||||
|
|
||||||
return { entries, jobs };
|
return { entries, jobs };
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { untrack } from 'svelte';
|
import { untrack } from 'svelte';
|
||||||
import { invalidateAll } from '$app/navigation';
|
|
||||||
import type { PageData } from './$types';
|
import type { PageData } from './$types';
|
||||||
import type { AudioJob, AudioCacheEntry } from '$lib/server/pocketbase';
|
import type { AudioJob, AudioCacheEntry } from '$lib/server/pocketbase';
|
||||||
import * as m from '$lib/paraglide/messages.js';
|
import * as m from '$lib/paraglide/messages.js';
|
||||||
@@ -21,8 +20,17 @@
|
|||||||
|
|
||||||
$effect(() => {
|
$effect(() => {
|
||||||
if (!hasInFlight) return;
|
if (!hasInFlight) return;
|
||||||
const id = setInterval(() => {
|
const id = setInterval(async () => {
|
||||||
invalidateAll();
|
const res = await fetch('/api/admin/audio-jobs').catch(() => null);
|
||||||
|
if (res?.ok) {
|
||||||
|
const body = await res.json().catch(() => null);
|
||||||
|
if (body?.jobs) {
|
||||||
|
jobs = body.jobs;
|
||||||
|
entries = (body.jobs as AudioJob[])
|
||||||
|
.filter((j) => j.status === 'done')
|
||||||
|
.map((j) => ({ id: j.id, cache_key: j.cache_key, filename: `${j.cache_key}.mp3`, updated: j.finished }));
|
||||||
|
}
|
||||||
|
}
|
||||||
}, 3000);
|
}, 3000);
|
||||||
return () => clearInterval(id);
|
return () => clearInterval(id);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { redirect } from '@sveltejs/kit';
|
import { redirect } from '@sveltejs/kit';
|
||||||
import type { Actions, PageServerLoad } from './$types';
|
import type { Actions, PageServerLoad } from './$types';
|
||||||
import { listBooks, listTranslationJobs, type TranslationJob } from '$lib/server/pocketbase';
|
import { listBookSlugs, listTranslationJobs, type TranslationJob } from '$lib/server/pocketbase';
|
||||||
import { backendFetch } from '$lib/server/scraper';
|
import { backendFetch } from '$lib/server/scraper';
|
||||||
import { log } from '$lib/server/logger';
|
import { log } from '$lib/server/logger';
|
||||||
|
|
||||||
@@ -10,8 +10,8 @@ export const load: PageServerLoad = async ({ locals }) => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const [books, jobs] = await Promise.all([
|
const [books, jobs] = await Promise.all([
|
||||||
listBooks().catch((e): Awaited<ReturnType<typeof listBooks>> => {
|
listBookSlugs().catch((e): Awaited<ReturnType<typeof listBookSlugs>> => {
|
||||||
log.warn('admin/translation', 'failed to load books', { err: String(e) });
|
log.warn('admin/translation', 'failed to load book slugs', { err: String(e) });
|
||||||
return [];
|
return [];
|
||||||
}),
|
}),
|
||||||
listTranslationJobs().catch((e): TranslationJob[] => {
|
listTranslationJobs().catch((e): TranslationJob[] => {
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import { untrack } from 'svelte';
|
import { untrack } from 'svelte';
|
||||||
import { invalidateAll } from '$app/navigation';
|
|
||||||
import { enhance } from '$app/forms';
|
import { enhance } from '$app/forms';
|
||||||
import type { PageData, ActionData } from './$types';
|
import type { PageData, ActionData } from './$types';
|
||||||
import type { TranslationJob } from '$lib/server/pocketbase';
|
import type { TranslationJob } from '$lib/server/pocketbase';
|
||||||
@@ -19,8 +18,12 @@
|
|||||||
|
|
||||||
$effect(() => {
|
$effect(() => {
|
||||||
if (!hasInFlight) return;
|
if (!hasInFlight) return;
|
||||||
const id = setInterval(() => {
|
const id = setInterval(async () => {
|
||||||
invalidateAll();
|
const res = await fetch('/api/admin/translation-jobs').catch(() => null);
|
||||||
|
if (res?.ok) {
|
||||||
|
const body = await res.json().catch(() => null);
|
||||||
|
if (body?.jobs) jobs = body.jobs;
|
||||||
|
}
|
||||||
}, 3000);
|
}, 3000);
|
||||||
return () => clearInterval(id);
|
return () => clearInterval(id);
|
||||||
});
|
});
|
||||||
|
|||||||
16
ui/src/routes/api/admin/audio-jobs/+server.ts
Normal file
16
ui/src/routes/api/admin/audio-jobs/+server.ts
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { listAudioJobs } from '$lib/server/pocketbase';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/admin/audio-jobs
|
||||||
|
* Returns the current audio jobs list (served from 30 s Valkey cache).
|
||||||
|
* Used by the admin audio page for lightweight polling instead of invalidateAll().
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ locals }) => {
|
||||||
|
if (!locals.user || locals.user.role !== 'admin') {
|
||||||
|
throw error(403, 'Forbidden');
|
||||||
|
}
|
||||||
|
const jobs = await listAudioJobs().catch(() => []);
|
||||||
|
return json({ jobs });
|
||||||
|
};
|
||||||
16
ui/src/routes/api/admin/scrape-tasks/+server.ts
Normal file
16
ui/src/routes/api/admin/scrape-tasks/+server.ts
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { listScrapingTasks } from '$lib/server/pocketbase';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/admin/scrape-tasks
|
||||||
|
* Returns the current scraping task list (served from 30 s Valkey cache).
|
||||||
|
* Used by the admin scrape page for lightweight polling instead of invalidateAll().
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ locals }) => {
|
||||||
|
if (!locals.user || locals.user.role !== 'admin') {
|
||||||
|
throw error(403, 'Forbidden');
|
||||||
|
}
|
||||||
|
const tasks = await listScrapingTasks().catch(() => []);
|
||||||
|
return json({ tasks });
|
||||||
|
};
|
||||||
16
ui/src/routes/api/admin/translation-jobs/+server.ts
Normal file
16
ui/src/routes/api/admin/translation-jobs/+server.ts
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
import { json, error } from '@sveltejs/kit';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
import { listTranslationJobs } from '$lib/server/pocketbase';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/admin/translation-jobs
|
||||||
|
* Returns the current translation jobs list (served from 30 s Valkey cache).
|
||||||
|
* Used by the admin translation page for lightweight polling instead of invalidateAll().
|
||||||
|
*/
|
||||||
|
export const GET: RequestHandler = async ({ locals }) => {
|
||||||
|
if (!locals.user || locals.user.role !== 'admin') {
|
||||||
|
throw error(403, 'Forbidden');
|
||||||
|
}
|
||||||
|
const jobs = await listTranslationJobs().catch(() => []);
|
||||||
|
return json({ jobs });
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user