From ad2d1a2603806c62b065273c76f3083ce2031701 Mon Sep 17 00:00:00 2001 From: Admin Date: Sun, 5 Apr 2026 00:32:18 +0500 Subject: [PATCH] feat: stream chapter-name generation via SSE batching Split chapter-name LLM requests into 100-chapter batches and stream results back as SSE so large books (e.g. Shadow Slave: 2916 chapters) never time out or truncate. Frontend shows live batch progress inline and accumulates proposals as they arrive. --- backend/internal/backend/handlers_textgen.go | 194 ++++++++---- ui/src/routes/admin/image-gen/+page.server.ts | 41 ++- ui/src/routes/admin/image-gen/+page.svelte | 258 +++++++++++++--- ui/src/routes/admin/text-gen/+page.server.ts | 37 ++- ui/src/routes/admin/text-gen/+page.svelte | 283 +++++++++++++++--- .../admin/text-gen/chapter-names/+server.ts | 25 +- 6 files changed, 660 insertions(+), 178 deletions(-) diff --git a/backend/internal/backend/handlers_textgen.go b/backend/internal/backend/handlers_textgen.go index 2c21536..c42dcd3 100644 --- a/backend/internal/backend/handlers_textgen.go +++ b/backend/internal/backend/handlers_textgen.go @@ -10,6 +10,10 @@ import ( "github.com/libnovel/backend/internal/domain" ) +// chapterNamesBatchSize is the number of chapters sent per LLM request. +// Keeps output well within the 4096-token response limit (~30 tokens/title). +const chapterNamesBatchSize = 100 + // handleAdminTextGenModels handles GET /api/admin/text-gen/models. // Returns the list of supported Cloudflare AI text generation models. func (s *Server) handleAdminTextGenModels(w http.ResponseWriter, r *http.Request) { @@ -36,16 +40,6 @@ type textGenChapterNamesRequest struct { MaxTokens int `json:"max_tokens"` } -// textGenChapterNamesResponse is the JSON body returned by POST /api/admin/text-gen/chapter-names. -type textGenChapterNamesResponse struct { - // Chapters is the list of proposed chapter titles, indexed by chapter number. - Chapters []proposedChapterTitle `json:"chapters"` - // Model is the model that was used. - Model string `json:"model"` - // RawResponse is the raw model output for debugging / manual editing. - RawResponse string `json:"raw_response"` -} - // proposedChapterTitle is a single chapter with its AI-proposed title. type proposedChapterTitle struct { Number int `json:"number"` @@ -55,12 +49,35 @@ type proposedChapterTitle struct { NewTitle string `json:"new_title"` } +// chapterNamesBatchEvent is one SSE event emitted per processed batch. +type chapterNamesBatchEvent struct { + // Batch is the 1-based batch index. + Batch int `json:"batch"` + // TotalBatches is the total number of batches. + TotalBatches int `json:"total_batches"` + // ChaptersDone is the cumulative count of chapters processed so far. + ChaptersDone int `json:"chapters_done"` + // TotalChapters is the total chapter count for this book. + TotalChapters int `json:"total_chapters"` + // Model is the CF AI model used. + Model string `json:"model"` + // Chapters contains the proposed titles for this batch. + Chapters []proposedChapterTitle `json:"chapters"` + // Error is non-empty if this batch failed. + Error string `json:"error,omitempty"` + // Done is true on the final sentinel event (no Chapters). + Done bool `json:"done,omitempty"` +} + // handleAdminTextGenChapterNames handles POST /api/admin/text-gen/chapter-names. // -// Reads all chapter titles for the given slug, sends them to the LLM with the -// requested naming pattern, and returns proposed replacements. Does NOT persist -// anything — the frontend shows a diff and the user must confirm via -// POST /api/admin/text-gen/chapter-names/apply. +// Splits all chapters into batches of chapterNamesBatchSize, sends each batch +// to the LLM sequentially, and streams results back as Server-Sent Events so +// the frontend can show live progress. Each SSE data line is a JSON-encoded +// chapterNamesBatchEvent. The final event has Done=true. +// +// Does NOT persist anything — the frontend shows a diff and the user must +// confirm via POST /api/admin/text-gen/chapter-names/apply. func (s *Server) handleAdminTextGenChapterNames(w http.ResponseWriter, r *http.Request) { if s.deps.TextGen == nil { jsonError(w, http.StatusServiceUnavailable, "text generation not configured (CFAI_ACCOUNT_ID/CFAI_API_TOKEN missing)") @@ -92,11 +109,29 @@ func (s *Server) handleAdminTextGenChapterNames(w http.ResponseWriter, r *http.R return } - // Build the prompt. - var chapterListSB strings.Builder - for _, ch := range chapters { - chapterListSB.WriteString(fmt.Sprintf("%d: %s\n", ch.Number, ch.Title)) + model := cfai.TextModel(req.Model) + if model == "" { + model = cfai.DefaultTextModel } + // 4096 tokens comfortably fits 100 chapter titles (~30 tokens each). + maxTokens := req.MaxTokens + if maxTokens <= 0 { + maxTokens = 4096 + } + + // Index existing titles for old/new diff. + existing := make(map[int]string, len(chapters)) + for _, ch := range chapters { + existing[ch.Number] = ch.Title + } + + // Partition chapters into batches. + batches := chunkChapters(chapters, chapterNamesBatchSize) + totalBatches := len(batches) + + s.deps.Log.Info("admin: text-gen chapter-names requested", + "slug", req.Slug, "chapters", len(chapters), + "batches", totalBatches, "model", model, "max_tokens", maxTokens) systemPrompt := `You are a chapter title editor for a web novel platform. ` + `The user provides a list of chapter numbers with their current titles, ` + @@ -111,64 +146,91 @@ func (s *Server) handleAdminTextGenChapterNames(w http.ResponseWriter, r *http.R `5. Each element: {"number": , "title": }. ` + `6. Output every chapter in the input list, in order. Do not skip any.` - userPrompt := fmt.Sprintf( - "Naming pattern: %s\n\nChapters:\n%s", - req.Pattern, - chapterListSB.String(), - ) + // Switch to SSE before writing anything. + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("X-Accel-Buffering", "no") // disable nginx/caddy buffering + flusher, canFlush := w.(http.Flusher) - model := cfai.TextModel(req.Model) - if model == "" { - model = cfai.DefaultTextModel + sseWrite := func(evt chapterNamesBatchEvent) { + b, _ := json.Marshal(evt) + fmt.Fprintf(w, "data: %s\n\n", b) + if canFlush { + flusher.Flush() + } } - // Default to 4096 tokens so large chapter lists are not truncated. - maxTokens := req.MaxTokens - if maxTokens <= 0 { - maxTokens = 4096 - } + chaptersDone := 0 + for i, batch := range batches { + if r.Context().Err() != nil { + return // client disconnected + } - s.deps.Log.Info("admin: text-gen chapter-names requested", - "slug", req.Slug, "chapters", len(chapters), "model", model, "max_tokens", maxTokens) + var chapterListSB strings.Builder + for _, ch := range batch { + chapterListSB.WriteString(fmt.Sprintf("%d: %s\n", ch.Number, ch.Title)) + } + userPrompt := fmt.Sprintf("Naming pattern: %s\n\nChapters:\n%s", req.Pattern, chapterListSB.String()) - raw, genErr := s.deps.TextGen.Generate(r.Context(), cfai.TextRequest{ - Model: model, - Messages: []cfai.TextMessage{ - {Role: "system", Content: systemPrompt}, - {Role: "user", Content: userPrompt}, - }, - MaxTokens: maxTokens, - }) - if genErr != nil { - s.deps.Log.Error("admin: text-gen chapter-names failed", "err", genErr) - jsonError(w, http.StatusBadGateway, "text generation failed: "+genErr.Error()) - return - } + raw, genErr := s.deps.TextGen.Generate(r.Context(), cfai.TextRequest{ + Model: model, + Messages: []cfai.TextMessage{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + }, + MaxTokens: maxTokens, + }) + if genErr != nil { + s.deps.Log.Error("admin: text-gen chapter-names batch failed", + "batch", i+1, "err", genErr) + sseWrite(chapterNamesBatchEvent{ + Batch: i + 1, + TotalBatches: totalBatches, + ChaptersDone: chaptersDone, + TotalChapters: len(chapters), + Model: string(model), + Error: genErr.Error(), + }) + continue + } - // Parse the JSON array from the model response. - proposed := parseChapterTitlesJSON(raw) + proposed := parseChapterTitlesJSON(raw) + result := make([]proposedChapterTitle, 0, len(proposed)) + for _, p := range proposed { + result = append(result, proposedChapterTitle{ + Number: p.Number, + OldTitle: existing[p.Number], + NewTitle: p.Title, + }) + } + chaptersDone += len(batch) - // Build the response: merge proposed titles with old titles. - // Index existing chapters by number for O(1) lookup. - existing := make(map[int]string, len(chapters)) - for _, ch := range chapters { - existing[ch.Number] = ch.Title - } - - result := make([]proposedChapterTitle, 0, len(proposed)) - for _, p := range proposed { - result = append(result, proposedChapterTitle{ - Number: p.Number, - OldTitle: existing[p.Number], - NewTitle: p.Title, + sseWrite(chapterNamesBatchEvent{ + Batch: i + 1, + TotalBatches: totalBatches, + ChaptersDone: chaptersDone, + TotalChapters: len(chapters), + Model: string(model), + Chapters: result, }) } - writeJSON(w, 0, textGenChapterNamesResponse{ - Chapters: result, - Model: string(model), - RawResponse: raw, - }) + // Final sentinel event. + sseWrite(chapterNamesBatchEvent{Done: true, TotalChapters: len(chapters), Model: string(model)}) +} + +// chunkChapters splits a chapter slice into batches of at most size n. +func chunkChapters(chapters []domain.ChapterInfo, n int) [][]domain.ChapterInfo { + var batches [][]domain.ChapterInfo + for len(chapters) > 0 { + end := n + if end > len(chapters) { + end = len(chapters) + } + batches = append(batches, chapters[:end]) + chapters = chapters[end:] + } + return batches } // parseChapterTitlesJSON extracts the JSON array from a model response. diff --git a/ui/src/routes/admin/image-gen/+page.server.ts b/ui/src/routes/admin/image-gen/+page.server.ts index b44793d..da09808 100644 --- a/ui/src/routes/admin/image-gen/+page.server.ts +++ b/ui/src/routes/admin/image-gen/+page.server.ts @@ -1,6 +1,7 @@ import type { PageServerLoad } from './$types'; import { backendFetch } from '$lib/server/scraper'; import { log } from '$lib/server/logger'; +import { listBooks } from '$lib/server/pocketbase'; export interface ImageModelInfo { id: string; @@ -11,18 +12,36 @@ export interface ImageModelInfo { description: string; } +export interface BookSummary { + slug: string; + title: string; + summary: string; + cover: string; +} + export const load: PageServerLoad = async () => { // parent layout already guards admin role - try { - const res = await backendFetch('/api/admin/image-gen/models'); - if (!res.ok) { - log.warn('admin/image-gen', 'failed to load models', { status: res.status }); - return { models: [] as ImageModelInfo[] }; - } - const data = await res.json(); - return { models: (data.models ?? []) as ImageModelInfo[] }; - } catch (e) { - log.warn('admin/image-gen', 'backend unreachable', { err: String(e) }); - return { models: [] as ImageModelInfo[] }; + const [modelsResult, books] = await Promise.allSettled([ + (async () => { + const res = await backendFetch('/api/admin/image-gen/models'); + if (!res.ok) throw new Error(`status ${res.status}`); + const data = await res.json(); + return (data.models ?? []) as ImageModelInfo[]; + })(), + listBooks() + ]); + + if (modelsResult.status === 'rejected') { + log.warn('admin/image-gen', 'failed to load models', { err: String(modelsResult.reason) }); } + + return { + models: modelsResult.status === 'fulfilled' ? modelsResult.value : ([] as ImageModelInfo[]), + books: (books.status === 'fulfilled' ? books.value : []).map((b) => ({ + slug: b.slug, + title: b.title, + summary: b.summary ?? '', + cover: b.cover ?? '' + })) as BookSummary[] + }; }; diff --git a/ui/src/routes/admin/image-gen/+page.svelte b/ui/src/routes/admin/image-gen/+page.svelte index 1ab9b9d..49d1089 100644 --- a/ui/src/routes/admin/image-gen/+page.svelte +++ b/ui/src/routes/admin/image-gen/+page.svelte @@ -1,26 +1,120 @@