c8e0cf2813
Add warmVoiceSamples() goroutine launched from ListenAndServe.
It waits up to 30s for Kokoro to become reachable, then generates
missing voice sample clips for all available voices and uploads them
to MinIO (_voice-samples/{voice}.mp3). Already-existing samples are
skipped, so the operation is idempotent on restarts.
1674 lines
54 KiB
Go
1674 lines
54 KiB
Go
// Package server exposes the scraper as an HTTP API service.
|
|
//
|
|
// Endpoints:
|
|
//
|
|
// POST /scrape — enqueue a full catalogue scrape
|
|
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
|
|
// GET /health — liveness probe
|
|
// GET /api/progress — get reading progress map (session-scoped)
|
|
// POST /api/progress/{slug} — set reading progress
|
|
// DELETE /api/progress/{slug} — delete reading progress
|
|
// GET /api/presign/chapter/{slug}/{n} — presigned MinIO URL for chapter markdown
|
|
// GET /api/presign/audio/{slug}/{n} — presigned MinIO URL for chapter audio
|
|
// GET /api/chapter-text/{slug}/{n} — plain text of chapter (markdown stripped)
|
|
// POST /api/audio/{slug}/{n} — trigger Kokoro audio generation
|
|
// GET /api/audio-proxy/{slug}/{n} — proxy generated audio from Kokoro
|
|
package server
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/libnovel/scraper/internal/orchestrator"
|
|
"github.com/libnovel/scraper/internal/scraper"
|
|
"github.com/libnovel/scraper/internal/storage"
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
// Server wraps an HTTP mux with the scraping endpoints.
|
|
type Server struct {
|
|
addr string
|
|
oCfg orchestrator.Config
|
|
novel scraper.NovelScraper
|
|
log *slog.Logger
|
|
store storage.Store
|
|
mu sync.Mutex
|
|
running bool
|
|
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
|
kokoroVoice string // default voice, e.g. af_bella
|
|
|
|
// SingleFile CLI settings for browse-page snapshots.
|
|
singleFilePath string // path to single-file binary, e.g. /usr/local/bin/single-file
|
|
browserlessURL string // Browserless base URL, e.g. http://browserless:3000
|
|
|
|
// voiceMu guards cachedVoices.
|
|
voiceMu sync.RWMutex
|
|
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
|
|
|
|
// audioMu guards audioInFlight only.
|
|
// Completed audio filenames are persisted to the Store (PocketBase).
|
|
// audioInFlight deduplicates concurrent generation requests for the same key.
|
|
audioMu sync.Mutex
|
|
audioInFlight map[string]chan struct{} // cacheKey → closed when done
|
|
|
|
// browseMu guards browseInFlight — keys of MinIO objects currently being
|
|
// captured by a background SingleFile goroutine.
|
|
browseMu sync.Mutex
|
|
browseInFlight map[string]struct{}
|
|
}
|
|
|
|
// New creates a new Server.
|
|
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
|
|
return &Server{
|
|
addr: addr,
|
|
oCfg: oCfg,
|
|
novel: novel,
|
|
log: log,
|
|
store: store,
|
|
kokoroURL: kokoroURL,
|
|
kokoroVoice: kokoroVoice,
|
|
singleFilePath: os.Getenv("SINGLEFILE_PATH"),
|
|
browserlessURL: os.Getenv("BROWSERLESS_URL"),
|
|
audioInFlight: make(map[string]chan struct{}),
|
|
browseInFlight: make(map[string]struct{}),
|
|
}
|
|
}
|
|
|
|
// voices returns the list of available Kokoro voices. On the first call it
|
|
// fetches GET /v1/audio/voices from the Kokoro service and caches the result.
|
|
// If the fetch fails (Kokoro not up yet, network error, etc.) it falls back to
|
|
// the hardcoded kokoroVoices list so the UI is never empty.
|
|
func (s *Server) voices() []string {
|
|
s.voiceMu.RLock()
|
|
cached := s.cachedVoices
|
|
s.voiceMu.RUnlock()
|
|
if len(cached) > 0 {
|
|
return cached
|
|
}
|
|
|
|
if s.kokoroURL != "" {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
|
|
if err == nil {
|
|
req.Header.Set("Accept", "application/json")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err == nil {
|
|
defer resp.Body.Close()
|
|
var payload struct {
|
|
Voices []string `json:"voices"`
|
|
}
|
|
if resp.StatusCode == http.StatusOK && json.NewDecoder(resp.Body).Decode(&payload) == nil && len(payload.Voices) > 0 {
|
|
s.voiceMu.Lock()
|
|
s.cachedVoices = payload.Voices
|
|
s.voiceMu.Unlock()
|
|
s.log.Info("fetched kokoro voices", "count", len(payload.Voices))
|
|
return payload.Voices
|
|
}
|
|
}
|
|
}
|
|
s.log.Warn("could not fetch kokoro voices, using built-in list")
|
|
}
|
|
|
|
return kokoroVoices
|
|
}
|
|
|
|
// ListenAndServe starts the HTTP server and blocks until the provided context
|
|
// is cancelled.
|
|
func (s *Server) ListenAndServe(ctx context.Context) error {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /health", s.handleHealth)
|
|
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
|
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
|
// Browse API — fetches and parses novelfire catalogue page
|
|
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
|
// Ranking API
|
|
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
|
|
// Cover image proxy (serves images stored in browse MinIO bucket)
|
|
mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
|
|
// Scrape status
|
|
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
|
|
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
|
|
// Re-index chapters for a book from MinIO into PocketBase chapters_idx
|
|
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
|
|
// Progress API
|
|
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
|
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
|
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
|
|
// Presigned URL API (for SvelteKit UI)
|
|
mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter)
|
|
mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio)
|
|
mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample)
|
|
// Plain-text chapter content (used server-side for audio generation)
|
|
mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText)
|
|
// Voices list (proxied from Kokoro)
|
|
mux.HandleFunc("GET /api/voices", s.handleVoices)
|
|
// Voice sample generation — generates a short audio clip for each voice
|
|
// and stores it in MinIO for UI preview playback.
|
|
voiceSampleHandler := http.TimeoutHandler(
|
|
http.HandlerFunc(s.handleGenerateVoiceSamples),
|
|
15*time.Minute,
|
|
`{"error":"voice sample generation timed out"}`,
|
|
)
|
|
mux.Handle("POST /api/audio/voice-samples", voiceSampleHandler)
|
|
// Server-side audio generation via Kokoro /v1/audio/speech.
|
|
// Generation can take several minutes, so wrap in its own timeout handler.
|
|
audioGenHandler := http.TimeoutHandler(
|
|
http.HandlerFunc(s.handleAudioGenerate),
|
|
10*time.Minute,
|
|
`{"error":"audio generation timed out"}`,
|
|
)
|
|
mux.Handle("POST /api/audio/{slug}/{n}", audioGenHandler)
|
|
// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
|
|
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
|
|
|
srv := &http.Server{
|
|
Addr: s.addr,
|
|
Handler: mux,
|
|
ReadTimeout: 15 * time.Second,
|
|
WriteTimeout: 60 * time.Second,
|
|
IdleTimeout: 60 * time.Second,
|
|
}
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() { errCh <- srv.ListenAndServe() }()
|
|
|
|
s.log.Info("HTTP server listening", "addr", s.addr)
|
|
|
|
// Pre-populate voice samples in the background so the UI voice selector
|
|
// has playable previews without requiring a manual trigger.
|
|
go s.warmVoiceSamples(ctx)
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return srv.Shutdown(shutCtx)
|
|
case err := <-errCh:
|
|
return err
|
|
}
|
|
}
|
|
|
|
func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
|
}
|
|
|
|
// handleGetRanking returns all ranking items sorted by rank ascending.
|
|
// Cover fields that hold a MinIO object key (e.g. "novelfire.net/assets/book-covers/slug.jpg")
|
|
// are rewritten to a /api/cover/{key} proxy URL so the UI can fetch them
|
|
// without knowing about the internal MinIO topology.
|
|
func (s *Server) handleGetRanking(w http.ResponseWriter, r *http.Request) {
|
|
items, err := s.store.ReadRankingItems(r.Context())
|
|
if err != nil {
|
|
s.log.Error("ranking read failed", "err", err)
|
|
http.Error(w, `{"error":"failed to read ranking"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
if items == nil {
|
|
items = []storage.RankingItem{}
|
|
}
|
|
// Rewrite cover keys to proxy URLs.
|
|
// Keys stored by triggerBrowseSnapshot look like:
|
|
// "novelfire.net/assets/book-covers/shadow-slave.jpg"
|
|
// We expose them as:
|
|
// "/api/cover/novelfire.net/shadow-slave"
|
|
// (the handler strips the domain and slug from the path, reconstructs the key)
|
|
for i := range items {
|
|
cover := items[i].Cover
|
|
if cover != "" && !strings.HasPrefix(cover, "http") {
|
|
// cover is a MinIO key; extract domain + slug for the proxy path.
|
|
// Key format: {domain}/assets/book-covers/{slug}.jpg
|
|
parts := strings.SplitN(cover, "/assets/book-covers/", 2)
|
|
if len(parts) == 2 {
|
|
domain := parts[0]
|
|
slug := strings.TrimSuffix(parts[1], ".jpg")
|
|
items[i].Cover = "/api/cover/" + domain + "/" + slug
|
|
}
|
|
}
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(items)
|
|
}
|
|
|
|
// handleGetCover proxies a cover image stored in the MinIO browse bucket.
|
|
// Route: GET /api/cover/{domain}/{slug}
|
|
// It reconstructs the MinIO key as {domain}/assets/book-covers/{slug}.jpg,
|
|
// fetches the object, and streams it to the client.
|
|
// Returns 404 if not yet downloaded, allowing the UI to fall back to the
|
|
// original source URL.
|
|
func (s *Server) handleGetCover(w http.ResponseWriter, r *http.Request) {
|
|
domain := r.PathValue("domain")
|
|
slug := r.PathValue("slug")
|
|
if domain == "" || slug == "" {
|
|
http.Error(w, "missing domain or slug", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
key := s.store.BrowseCoverKey(domain, slug)
|
|
|
|
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
data, contentType, ok, err := s.store.GetBrowseAsset(ctx, key)
|
|
if err != nil {
|
|
s.log.Warn("handleGetCover: GetBrowseAsset error", "key", key, "err", err)
|
|
http.Error(w, "storage error", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
if !ok {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
if contentType == "" {
|
|
contentType = "image/jpeg"
|
|
}
|
|
w.Header().Set("Content-Type", contentType)
|
|
w.Header().Set("Cache-Control", "public, max-age=86400")
|
|
_, _ = w.Write(data)
|
|
}
|
|
|
|
// ─── Session cookie helpers ───────────────────────────────────────────────────
|
|
|
|
const sessionCookieName = "libnovel_session"
|
|
|
|
// sessionID returns the session ID from the request cookie, or "" if absent.
|
|
func sessionID(r *http.Request) string {
|
|
c, err := r.Cookie(sessionCookieName)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return c.Value
|
|
}
|
|
|
|
// newSessionID generates a random 16-byte hex session ID.
|
|
func newSessionID() (string, error) {
|
|
b := make([]byte, 16)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(b), nil
|
|
}
|
|
|
|
// ensureSession issues a new session cookie if the request does not already
|
|
// carry one, and returns the session ID (either existing or newly issued).
|
|
func ensureSession(w http.ResponseWriter, r *http.Request) string {
|
|
if id := sessionID(r); id != "" {
|
|
return id
|
|
}
|
|
id, err := newSessionID()
|
|
if err != nil {
|
|
// Very unlikely, but fall back to a timestamp-based ID.
|
|
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
|
|
}
|
|
http.SetCookie(w, &http.Cookie{
|
|
Name: sessionCookieName,
|
|
Value: id,
|
|
Path: "/",
|
|
HttpOnly: true,
|
|
SameSite: http.SameSiteLaxMode,
|
|
MaxAge: 365 * 24 * 60 * 60, // 1 year
|
|
})
|
|
return id
|
|
}
|
|
|
|
// ─── Reading progress API ─────────────────────────────────────────────────────
|
|
|
|
// handleGetProgress handles GET /api/progress.
|
|
// Returns JSON: {"slug": chapterNum, ...} merged with {"slug_ts": timestampMs, ...}
|
|
func (s *Server) handleGetProgress(w http.ResponseWriter, r *http.Request) {
|
|
sid := ensureSession(w, r)
|
|
entries, err := s.store.AllProgress(r.Context(), sid)
|
|
if err != nil {
|
|
s.log.Error("AllProgress failed", "err", err)
|
|
entries = nil
|
|
}
|
|
|
|
progress := make(map[string]interface{}, len(entries)*2)
|
|
for _, p := range entries {
|
|
progress[p.Slug] = p.Chapter
|
|
progress[p.Slug+"_ts"] = p.UpdatedAt.UnixMilli()
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(progress)
|
|
}
|
|
|
|
// handleSetProgress handles POST /api/progress/{slug}.
|
|
// Body: {"chapter": N}
|
|
func (s *Server) handleSetProgress(w http.ResponseWriter, r *http.Request) {
|
|
sid := ensureSession(w, r)
|
|
slug := r.PathValue("slug")
|
|
if slug == "" {
|
|
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
var body struct {
|
|
Chapter int `json:"chapter"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Chapter < 1 {
|
|
http.Error(w, `{"error":"invalid body"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
p := storage.ReadingProgress{
|
|
Slug: slug,
|
|
Chapter: body.Chapter,
|
|
UpdatedAt: time.Now(),
|
|
}
|
|
if err := s.store.SetProgress(r.Context(), sid, p); err != nil {
|
|
s.log.Error("SetProgress failed", "slug", slug, "err", err)
|
|
http.Error(w, `{"error":"store error"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{})
|
|
}
|
|
|
|
// handleDeleteProgress handles DELETE /api/progress/{slug}.
|
|
func (s *Server) handleDeleteProgress(w http.ResponseWriter, r *http.Request) {
|
|
sid := ensureSession(w, r)
|
|
slug := r.PathValue("slug")
|
|
if slug == "" {
|
|
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if err := s.store.DeleteProgress(r.Context(), sid, slug); err != nil {
|
|
s.log.Error("DeleteProgress failed", "slug", slug, "err", err)
|
|
// Non-fatal — treat as success.
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{})
|
|
}
|
|
|
|
// handleChapterText returns the plain text of a chapter (markdown stripped)
|
|
// for server-side audio generation. Called by handleAudioGenerate internally.
|
|
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
|
if err != nil {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
fmt.Fprint(w, stripMarkdown(raw))
|
|
}
|
|
|
|
// ─── Audio generation via Kokoro /v1/audio/speech ────────────────────────────
|
|
//
|
|
// handleAudioGenerate handles POST /api/audio/{slug}/{n}.
|
|
//
|
|
// It calls Kokoro's POST /v1/audio/speech with return_download_link=true.
|
|
// Kokoro generates the audio, saves it to its own temp storage, and returns
|
|
// the download filename in the X-Download-Path response header.
|
|
// We cache that filename (in memory, keyed by slug/chapter/voice/speed) and
|
|
// return a proxy URL that the browser sets as audio.src.
|
|
//
|
|
// On a cache hit the proxy URL is returned immediately without re-generating.
|
|
// Concurrent requests for the same key are deduplicated.
|
|
func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.Error(w, `{"error":"invalid chapter"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Parse optional voice/speed from JSON body.
|
|
voice := s.kokoroVoice
|
|
speed := 1.0
|
|
var body struct {
|
|
Voice string `json:"voice"`
|
|
Speed float64 `json:"speed"`
|
|
MaxChars int `json:"max_chars"`
|
|
}
|
|
if r.Body != nil {
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
}
|
|
if body.Voice != "" {
|
|
voice = body.Voice
|
|
}
|
|
if body.Speed > 0 {
|
|
speed = body.Speed
|
|
}
|
|
|
|
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
|
|
|
// Fast path: already generated (check persistent store first).
|
|
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
|
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
|
return
|
|
}
|
|
|
|
// Deduplicate concurrent generation for the same key.
|
|
s.audioMu.Lock()
|
|
if ch, ok := s.audioInFlight[cacheKey]; ok {
|
|
s.audioMu.Unlock()
|
|
select {
|
|
case <-ch:
|
|
case <-r.Context().Done():
|
|
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
// Check store again after waiting.
|
|
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
|
|
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
|
} else {
|
|
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
|
|
}
|
|
return
|
|
}
|
|
ch := make(chan struct{})
|
|
s.audioInFlight[cacheKey] = ch
|
|
s.audioMu.Unlock()
|
|
|
|
defer func() {
|
|
s.audioMu.Lock()
|
|
delete(s.audioInFlight, cacheKey)
|
|
s.audioMu.Unlock()
|
|
close(ch)
|
|
}()
|
|
|
|
// Load and validate chapter text.
|
|
raw, err := s.store.ReadChapter(r.Context(), slug, n)
|
|
if err != nil {
|
|
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
|
|
return
|
|
}
|
|
text := stripMarkdown(raw)
|
|
if text == "" {
|
|
http.Error(w, `{"error":"chapter text is empty"}`, http.StatusUnprocessableEntity)
|
|
return
|
|
}
|
|
if body.MaxChars > 0 && len([]rune(text)) > body.MaxChars {
|
|
text = string([]rune(text)[:body.MaxChars])
|
|
}
|
|
if s.kokoroURL == "" {
|
|
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
|
|
// Call Kokoro POST /v1/audio/speech with return_download_link=true.
|
|
// Kokoro saves the generated audio to its own temp storage and returns the
|
|
// download path in the X-Download-Path response header.
|
|
filename, err := s.generateSpeech(r.Context(), text, voice, speed)
|
|
if err != nil {
|
|
s.log.Error("kokoro speech generation failed", "slug", slug, "chapter", n, "err", err)
|
|
http.Error(w, `{"error":"speech generation failed"}`, http.StatusBadGateway)
|
|
return
|
|
}
|
|
|
|
if err := s.store.SetAudioCache(r.Context(), cacheKey, filename); err != nil {
|
|
s.log.Warn("audio cache write failed", "slug", slug, "chapter", n, "cache_key", cacheKey, "err", err)
|
|
}
|
|
|
|
// Download generated audio from Kokoro and persist to MinIO synchronously
|
|
// so that the presigned URL returned to the client is immediately valid.
|
|
minioKey := s.store.AudioObjectKey(slug, n, voice, speed)
|
|
audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
|
|
if dlErr != nil {
|
|
s.log.Warn("audio MinIO upload skipped: kokoro download failed",
|
|
"slug", slug, "chapter", n, "filename", filename, "err", dlErr)
|
|
} else if putErr := s.store.PutAudio(r.Context(), minioKey, audioData); putErr != nil {
|
|
s.log.Warn("audio MinIO upload failed",
|
|
"slug", slug, "chapter", n, "key", minioKey, "err", putErr)
|
|
} else {
|
|
s.log.Info("audio uploaded to MinIO", "slug", slug, "chapter", n, "key", minioKey)
|
|
}
|
|
|
|
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
|
|
s.writeAudioResponse(w, slug, n, voice, speed, filename)
|
|
}
|
|
|
|
// generateSpeech calls POST /v1/audio/speech on Kokoro with return_download_link=true
|
|
// and returns the filename from the X-Download-Path response header.
|
|
func (s *Server) generateSpeech(ctx context.Context, text, voice string, speed float64) (string, error) {
|
|
reqBody, _ := json.Marshal(map[string]interface{}{
|
|
"model": "kokoro",
|
|
"input": text,
|
|
"voice": voice,
|
|
"response_format": "mp3",
|
|
"speed": speed,
|
|
"stream": false,
|
|
"return_download_link": true,
|
|
})
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
|
s.kokoroURL+"/v1/audio/speech", bytes.NewReader(reqBody))
|
|
if err != nil {
|
|
return "", fmt.Errorf("build request: %w", err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("kokoro request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
// Drain body so the connection can be reused.
|
|
_, _ = io.Copy(io.Discard, resp.Body)
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", fmt.Errorf("kokoro status %d", resp.StatusCode)
|
|
}
|
|
|
|
// X-Download-Path is e.g. "/download/speech_abc123.mp3"
|
|
dlPath := resp.Header.Get("X-Download-Path")
|
|
if dlPath == "" {
|
|
return "", fmt.Errorf("kokoro did not return X-Download-Path header")
|
|
}
|
|
|
|
// Extract just the filename from the path.
|
|
filename := dlPath
|
|
if idx := strings.LastIndex(dlPath, "/"); idx >= 0 {
|
|
filename = dlPath[idx+1:]
|
|
}
|
|
if filename == "" {
|
|
return "", fmt.Errorf("empty filename in X-Download-Path: %q", dlPath)
|
|
}
|
|
return filename, nil
|
|
}
|
|
|
|
// downloadFromKokoro downloads a generated audio file from Kokoro's temp storage
|
|
// using GET /v1/download/{filename} and returns the raw bytes.
|
|
func (s *Server) downloadFromKokoro(ctx context.Context, filename string) ([]byte, error) {
|
|
url := s.kokoroURL + "/v1/download/" + filename
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("build download request: %w", err)
|
|
}
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("kokoro download request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("kokoro download status %d", resp.StatusCode)
|
|
}
|
|
data, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read kokoro download body: %w", err)
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
// writeAudioResponse writes the JSON response for a generated audio chapter.
|
|
// The URL points to our proxy handler GET /api/audio-proxy/{slug}/{n}.
|
|
func (s *Server) writeAudioResponse(w http.ResponseWriter, slug string, n int, voice string, speed float64, filename string) {
|
|
proxyURL := fmt.Sprintf("/api/audio-proxy/%s/%d?voice=%s&speed=%.1f", slug, n, voice, speed)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"url": proxyURL,
|
|
"filename": filename,
|
|
})
|
|
}
|
|
|
|
// handleAudioProxy handles GET /api/audio-proxy/{slug}/{n}.
|
|
// It looks up the Kokoro download filename for this chapter (voice/speed) and
|
|
// proxies GET /v1/download/{filename} from the Kokoro server back to the browser.
|
|
func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
voice := r.URL.Query().Get("voice")
|
|
if voice == "" {
|
|
voice = s.kokoroVoice
|
|
}
|
|
speedStr := r.URL.Query().Get("speed")
|
|
speed := 1.0
|
|
if speedStr != "" {
|
|
if v, err := strconv.ParseFloat(speedStr, 64); err == nil && v > 0 {
|
|
speed = v
|
|
}
|
|
}
|
|
|
|
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
|
|
filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
|
|
if !ok {
|
|
http.Error(w, "audio not generated yet", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
kokoroURL := s.kokoroURL + "/v1/download/" + filename
|
|
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, kokoroURL, nil)
|
|
if err != nil {
|
|
http.Error(w, "failed to build proxy request", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
http.Error(w, "kokoro download failed", http.StatusBadGateway)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
http.Error(w, fmt.Sprintf("kokoro returned %d", resp.StatusCode), http.StatusBadGateway)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "audio/mpeg")
|
|
w.Header().Set("Cache-Control", "public, max-age=3600")
|
|
if cl := resp.Header.Get("Content-Length"); cl != "" {
|
|
w.Header().Set("Content-Length", cl)
|
|
}
|
|
_, _ = io.Copy(w, resp.Body)
|
|
}
|
|
|
|
// ─── Presigned URL handlers ───────────────────────────────────────────────────
|
|
|
|
// handlePresignChapter handles GET /api/presign/chapter/{slug}/{n}.
|
|
// Returns a short-lived presigned MinIO URL for the chapter markdown object.
|
|
// The SvelteKit server uses this to fetch chapter content server-side.
|
|
func (s *Server) handlePresignChapter(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 || slug == "" {
|
|
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
url, err := s.store.PresignChapter(r.Context(), slug, n, 15*time.Minute)
|
|
if err != nil {
|
|
s.log.Error("presign chapter failed", "slug", slug, "n", n, "err", err)
|
|
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
|
}
|
|
|
|
// handlePresignAudio handles GET /api/presign/audio/{slug}/{n}.
|
|
// Returns a presigned MinIO URL for the audio object (if it has been generated).
|
|
// Query params: voice, speed (optional, defaults to server defaults).
|
|
func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
n, err := strconv.Atoi(r.PathValue("n"))
|
|
if err != nil || n < 1 || slug == "" {
|
|
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
voice := r.URL.Query().Get("voice")
|
|
if voice == "" {
|
|
voice = s.kokoroVoice
|
|
}
|
|
speed := 1.0
|
|
if sv := r.URL.Query().Get("speed"); sv != "" {
|
|
if v, err := strconv.ParseFloat(sv, 64); err == nil && v > 0 {
|
|
speed = v
|
|
}
|
|
}
|
|
|
|
key := s.store.AudioObjectKey(slug, n, voice, speed)
|
|
|
|
// Return 404 when the object hasn't been uploaded yet — the client treats
|
|
// this as "audio not ready" and will either poll or trigger generation.
|
|
if !s.store.AudioExists(r.Context(), key) {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
|
|
if err != nil {
|
|
s.log.Error("presign audio failed", "slug", slug, "n", n, "err", err)
|
|
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
|
}
|
|
|
|
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
|
|
cfg := s.oCfg
|
|
cfg.SingleBookURL = "" // full catalogue
|
|
|
|
s.runAsync(w, cfg)
|
|
}
|
|
|
|
func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
|
var body struct {
|
|
URL string `json:"url"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
|
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
cfg := s.oCfg
|
|
cfg.SingleBookURL = body.URL
|
|
|
|
s.runAsync(w, cfg)
|
|
}
|
|
|
|
// runAsync launches an orchestrator in the background and returns 202 Accepted.
|
|
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
|
|
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
|
s.mu.Lock()
|
|
if s.running {
|
|
s.mu.Unlock()
|
|
http.Error(w, `{"error":"a scrape job is already running"}`, http.StatusConflict)
|
|
return
|
|
}
|
|
s.running = true
|
|
s.mu.Unlock()
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusAccepted)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"status": "accepted"})
|
|
|
|
go func() {
|
|
defer func() {
|
|
s.mu.Lock()
|
|
s.running = false
|
|
s.mu.Unlock()
|
|
}()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
|
|
defer cancel()
|
|
|
|
// Determine task kind and target.
|
|
kind := "catalogue"
|
|
targetURL := ""
|
|
if cfg.SingleBookURL != "" {
|
|
kind = "book"
|
|
targetURL = cfg.SingleBookURL
|
|
}
|
|
|
|
// Create the task record in PocketBase.
|
|
taskID, err := s.store.CreateScrapeTask(ctx, kind, targetURL)
|
|
if err != nil {
|
|
s.log.Warn("could not create scraping_tasks record", "err", err)
|
|
// Non-fatal: continue without task tracking.
|
|
}
|
|
|
|
// flush pushes the latest counters to PocketBase (best-effort).
|
|
flush := func(p orchestrator.Progress, status, errMsg string, finished bool) {
|
|
if taskID == "" {
|
|
return
|
|
}
|
|
u := storage.ScrapeTaskUpdate{
|
|
Status: status,
|
|
BooksFound: p.BooksFound,
|
|
ChaptersScraped: p.ChaptersScraped,
|
|
ChaptersSkipped: p.ChaptersSkipped,
|
|
Errors: p.Errors,
|
|
ErrorMessage: errMsg,
|
|
}
|
|
if finished {
|
|
u.Finished = time.Now().UTC()
|
|
}
|
|
if updateErr := s.store.UpdateScrapeTask(ctx, taskID, u); updateErr != nil {
|
|
s.log.Warn("could not update scraping_tasks record", "task_id", taskID, "err", updateErr)
|
|
}
|
|
}
|
|
|
|
cfg.OnProgress = func(p orchestrator.Progress) {
|
|
flush(p, "running", "", false)
|
|
}
|
|
|
|
o := orchestrator.New(cfg, s.novel, s.log, s.store)
|
|
runErr := o.Run(ctx)
|
|
|
|
// After a successful full-catalogue run, refresh the ranking list.
|
|
if runErr == nil && cfg.SingleBookURL == "" {
|
|
s.log.Info("runAsync: starting ScrapeRanking after catalogue run")
|
|
rankCtx, rankCancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
|
defer rankCancel()
|
|
rankEntries, rankErrs := s.novel.ScrapeRanking(rankCtx, 0)
|
|
rank := 1
|
|
for meta := range rankEntries {
|
|
item := storage.RankingItem{
|
|
Rank: rank,
|
|
Slug: meta.Slug,
|
|
Title: meta.Title,
|
|
Author: meta.Author,
|
|
Cover: meta.Cover,
|
|
Status: meta.Status,
|
|
Genres: meta.Genres,
|
|
SourceURL: meta.SourceURL,
|
|
}
|
|
if werr := s.store.WriteRankingItem(rankCtx, item); werr != nil {
|
|
s.log.Warn("runAsync: WriteRankingItem failed", "slug", meta.Slug, "err", werr)
|
|
}
|
|
rank++
|
|
}
|
|
if rerr := <-rankErrs; rerr != nil {
|
|
s.log.Warn("runAsync: ScrapeRanking finished with error", "err", rerr)
|
|
} else {
|
|
s.log.Info("runAsync: ScrapeRanking complete", "count", rank-1)
|
|
}
|
|
}
|
|
|
|
// Determine final status.
|
|
finalStatus := "done"
|
|
errMsg := ""
|
|
if runErr != nil {
|
|
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", runErr))
|
|
if ctx.Err() != nil {
|
|
finalStatus = "cancelled"
|
|
} else {
|
|
finalStatus = "failed"
|
|
}
|
|
errMsg = runErr.Error()
|
|
}
|
|
|
|
// Best-effort: read last known progress counters via a zero-value
|
|
// OnProgress — we don't have a snapshot here, so re-use whatever the
|
|
// last OnProgress call delivered (the orchestrator calls notify() at
|
|
// the very end, so this is always accurate after Run returns).
|
|
// We issue one final flush with the terminal status and finished time.
|
|
if taskID != "" {
|
|
// Re-fetch current counters by listing the task (cheapest path).
|
|
tasks, listErr := s.store.ListScrapeTasks(ctx)
|
|
var last storage.ScrapeTaskUpdate
|
|
if listErr == nil {
|
|
for _, t := range tasks {
|
|
if t.ID == taskID {
|
|
last = storage.ScrapeTaskUpdate{
|
|
BooksFound: t.BooksFound,
|
|
ChaptersScraped: t.ChaptersScraped,
|
|
ChaptersSkipped: t.ChaptersSkipped,
|
|
Errors: t.Errors,
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
last.Status = finalStatus
|
|
last.ErrorMessage = errMsg
|
|
last.Finished = time.Now().UTC()
|
|
if updateErr := s.store.UpdateScrapeTask(ctx, taskID, last); updateErr != nil {
|
|
s.log.Warn("could not finalize scraping_tasks record", "task_id", taskID, "err", updateErr)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// ─── Browse API ───────────────────────────────────────────────────────────────
|
|
|
|
// NovelListing represents a single novel entry from the novelfire browse page.
|
|
type NovelListing struct {
|
|
Slug string `json:"slug"`
|
|
Title string `json:"title"`
|
|
Cover string `json:"cover"`
|
|
Rank string `json:"rank"`
|
|
Rating string `json:"rating"`
|
|
Chapters string `json:"chapters"`
|
|
URL string `json:"url"`
|
|
}
|
|
|
|
const novelFireBase = "https://novelfire.net"
|
|
const novelFireDomain = "novelfire.net"
|
|
|
|
// handleBrowse handles GET /api/browse.
|
|
// Query params:
|
|
//
|
|
// page (default 1)
|
|
// genre (default "all")
|
|
// sort (default "popular")
|
|
// status (default "all")
|
|
// type (default "all-novel")
|
|
//
|
|
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
|
|
//
|
|
// Cache strategy: check MinIO browse bucket first (key: {domain}/html/page-N.html);
|
|
// if a snapshot exists, parse it and return structured JSON.
|
|
// On a cache miss, fetch live from novelfire.net, return the result, and
|
|
// trigger a background SingleFile snapshot + ranking population.
|
|
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
|
q := r.URL.Query()
|
|
page := q.Get("page")
|
|
if page == "" {
|
|
page = "1"
|
|
}
|
|
genre := q.Get("genre")
|
|
if genre == "" {
|
|
genre = "all"
|
|
}
|
|
sortBy := q.Get("sort")
|
|
if sortBy == "" {
|
|
sortBy = "popular"
|
|
}
|
|
status := q.Get("status")
|
|
if status == "" {
|
|
status = "all"
|
|
}
|
|
novelType := q.Get("type")
|
|
if novelType == "" {
|
|
novelType = "all-novel"
|
|
}
|
|
|
|
pageNum, _ := strconv.Atoi(page)
|
|
if pageNum <= 0 {
|
|
pageNum = 1
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
|
|
defer cancel()
|
|
|
|
// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
|
|
cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
|
|
if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 {
|
|
novels, hasNext := parseBrowsePage(strings.NewReader(html))
|
|
s.log.Debug("browse: served from cache", "key", cacheKey)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Header().Set("Cache-Control", "public, max-age=300")
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"novels": novels,
|
|
"page": pageNum,
|
|
"hasNext": hasNext,
|
|
})
|
|
return
|
|
}
|
|
|
|
// ── Live fallback: direct fetch from novelfire.net ───────────────────
|
|
// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
|
|
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
|
|
novelFireBase, genre, sortBy, status, novelType, page)
|
|
|
|
var novels []NovelListing
|
|
var hasNext bool
|
|
var fetchErr error
|
|
for attempt := 1; attempt <= 3; attempt++ {
|
|
if attempt > 1 {
|
|
select {
|
|
case <-ctx.Done():
|
|
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
|
|
return
|
|
case <-time.After(time.Duration(attempt) * time.Second):
|
|
}
|
|
}
|
|
|
|
var req *http.Request
|
|
req, fetchErr = http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
|
|
if fetchErr != nil {
|
|
http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
|
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
|
req.Header.Set("Accept-Encoding", "gzip, deflate, br")
|
|
req.Header.Set("Cache-Control", "no-cache")
|
|
req.Header.Set("Pragma", "no-cache")
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
fetchErr = err
|
|
s.log.Warn("browse fetch failed, retrying", "url", targetURL, "attempt", attempt, "err", err)
|
|
continue
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
_, _ = io.Copy(io.Discard, resp.Body)
|
|
resp.Body.Close()
|
|
fetchErr = fmt.Errorf("upstream returned %d", resp.StatusCode)
|
|
s.log.Warn("browse upstream error, retrying", "url", targetURL, "attempt", attempt, "status", resp.StatusCode)
|
|
continue
|
|
}
|
|
|
|
novels, hasNext = parseBrowsePage(resp.Body)
|
|
resp.Body.Close()
|
|
fetchErr = nil
|
|
break
|
|
}
|
|
if fetchErr != nil {
|
|
s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr)
|
|
http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway)
|
|
return
|
|
}
|
|
|
|
// ── Background: populate MinIO cache via SingleFile ───────────────────
|
|
// Fire-and-forget: capture the JS-rendered page with SingleFile, store
|
|
// it in MinIO, then parse it to populate the ranking collection.
|
|
s.triggerBrowseSnapshot(cacheKey, targetURL)
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Header().Set("Cache-Control", "public, max-age=300")
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"novels": novels,
|
|
"page": pageNum,
|
|
"hasNext": hasNext,
|
|
})
|
|
}
|
|
|
|
// triggerBrowseSnapshot fires a background goroutine that:
|
|
// 1. Runs SingleFile CLI to capture the fully-rendered novelfire browse page
|
|
// and stores the self-contained HTML at {domain}/html/page-N.html in MinIO.
|
|
// 2. Parses the stored HTML to extract novel listings.
|
|
// 3. For each listing, upserts a ranking record in PocketBase (rank, slug,
|
|
// title, cover key, source_url).
|
|
// 4. Fires a separate goroutine per cover image to download and store it at
|
|
// {domain}/assets/book-covers/{slug}.jpg in MinIO.
|
|
//
|
|
// It is a no-op when:
|
|
// - SINGLEFILE_PATH is not set (SingleFile not installed)
|
|
// - a capture for this cache key is already in progress
|
|
//
|
|
// The goroutine uses a fresh context so it outlives the HTTP request.
|
|
func (s *Server) triggerBrowseSnapshot(cacheKey, pageURL string) {
|
|
if s.singleFilePath == "" {
|
|
return
|
|
}
|
|
|
|
s.browseMu.Lock()
|
|
if _, inflight := s.browseInFlight[cacheKey]; inflight {
|
|
s.browseMu.Unlock()
|
|
return
|
|
}
|
|
s.browseInFlight[cacheKey] = struct{}{}
|
|
s.browseMu.Unlock()
|
|
|
|
go func() {
|
|
defer func() {
|
|
s.browseMu.Lock()
|
|
delete(s.browseInFlight, cacheKey)
|
|
s.browseMu.Unlock()
|
|
}()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
|
|
// Convert http(s) → ws(s) for the SingleFile --browser-server flag.
|
|
wsEndpoint := s.browserlessURL
|
|
wsEndpoint = strings.Replace(wsEndpoint, "http://", "ws://", 1)
|
|
wsEndpoint = strings.Replace(wsEndpoint, "https://", "wss://", 1)
|
|
|
|
tmpFile, err := os.CreateTemp("", "libnovel-browse-*.html")
|
|
if err != nil {
|
|
s.log.Warn("triggerBrowseSnapshot: create temp file failed", "key", cacheKey, "err", err)
|
|
return
|
|
}
|
|
tmpPath := tmpFile.Name()
|
|
tmpFile.Close()
|
|
defer os.Remove(tmpPath)
|
|
|
|
//nolint:gosec
|
|
cmd := exec.CommandContext(ctx, s.singleFilePath,
|
|
pageURL,
|
|
"--browser-server="+wsEndpoint,
|
|
"--output="+tmpPath,
|
|
)
|
|
if out, runErr := cmd.CombinedOutput(); runErr != nil {
|
|
s.log.Warn("triggerBrowseSnapshot: SingleFile failed",
|
|
"key", cacheKey, "err", runErr, "output", string(out))
|
|
return
|
|
}
|
|
|
|
htmlBytes, readErr := os.ReadFile(tmpPath)
|
|
if readErr != nil {
|
|
s.log.Warn("triggerBrowseSnapshot: read output failed",
|
|
"key", cacheKey, "err", readErr)
|
|
return
|
|
}
|
|
if len(htmlBytes) == 0 {
|
|
s.log.Warn("triggerBrowseSnapshot: SingleFile produced empty output, skipping cache",
|
|
"key", cacheKey)
|
|
return
|
|
}
|
|
|
|
// Store the HTML snapshot.
|
|
if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
|
|
s.log.Warn("triggerBrowseSnapshot: SaveBrowsePage failed",
|
|
"key", cacheKey, "err", putErr)
|
|
return
|
|
}
|
|
s.log.Info("triggerBrowseSnapshot: cached browse page",
|
|
"key", cacheKey, "bytes", len(htmlBytes))
|
|
|
|
// Parse the stored HTML to extract novel listings.
|
|
novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
|
|
if len(novels) == 0 {
|
|
return
|
|
}
|
|
|
|
// Upsert each novel into the ranking PocketBase collection and
|
|
// kick off a background cover download.
|
|
for i, novel := range novels {
|
|
rank := i + 1
|
|
coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)
|
|
|
|
item := storage.RankingItem{
|
|
Rank: rank,
|
|
Slug: novel.Slug,
|
|
Title: novel.Title,
|
|
Cover: coverKey, // stored as MinIO key; UI fetches via /api/cover/...
|
|
SourceURL: novel.URL,
|
|
}
|
|
if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
|
|
s.log.Warn("triggerBrowseSnapshot: WriteRankingItem failed",
|
|
"slug", novel.Slug, "err", werr)
|
|
}
|
|
|
|
// Download and store the cover image in a separate goroutine.
|
|
coverURL := novel.Cover
|
|
if coverURL != "" {
|
|
go s.downloadAndStoreCover(coverKey, coverURL)
|
|
}
|
|
}
|
|
|
|
s.log.Info("triggerBrowseSnapshot: ranking populated", "count", len(novels), "key", cacheKey)
|
|
}()
|
|
}
|
|
|
|
// downloadAndStoreCover fetches a cover image URL and stores it in MinIO under
|
|
// the given key. Errors are logged but not propagated — this is best-effort.
|
|
func (s *Server) downloadAndStoreCover(key, imageURL string) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
// Skip if already stored.
|
|
if _, _, ok, _ := s.store.GetBrowseAsset(ctx, key); ok {
|
|
return
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, imageURL, nil)
|
|
if err != nil {
|
|
s.log.Warn("downloadAndStoreCover: build request failed", "url", imageURL, "err", err)
|
|
return
|
|
}
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
s.log.Warn("downloadAndStoreCover: fetch failed", "url", imageURL, "err", err)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
s.log.Warn("downloadAndStoreCover: non-200 response", "url", imageURL, "status", resp.StatusCode)
|
|
return
|
|
}
|
|
|
|
data, readErr := io.ReadAll(resp.Body)
|
|
if readErr != nil {
|
|
s.log.Warn("downloadAndStoreCover: read body failed", "url", imageURL, "err", readErr)
|
|
return
|
|
}
|
|
|
|
contentType := resp.Header.Get("Content-Type")
|
|
if contentType == "" {
|
|
contentType = "image/jpeg"
|
|
}
|
|
|
|
if putErr := s.store.SaveBrowseAsset(ctx, key, data, contentType); putErr != nil {
|
|
s.log.Warn("downloadAndStoreCover: SaveBrowseAsset failed", "key", key, "err", putErr)
|
|
return
|
|
}
|
|
s.log.Debug("downloadAndStoreCover: stored cover", "key", key, "bytes", len(data))
|
|
}
|
|
|
|
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
|
|
// Returns novels and whether a "next page" link was found.
|
|
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
|
|
doc, err := html.Parse(r)
|
|
if err != nil {
|
|
return nil, false
|
|
}
|
|
|
|
var novels []NovelListing
|
|
hasNext := false
|
|
|
|
var walk func(*html.Node)
|
|
walk = func(n *html.Node) {
|
|
if n.Type == html.ElementNode {
|
|
switch n.Data {
|
|
case "li":
|
|
if hasClass(n, "novel-item") {
|
|
if novel, ok := parseNovelItem(n); ok {
|
|
novels = append(novels, novel)
|
|
}
|
|
}
|
|
// pagination li with class "next"
|
|
if hasClass(n, "next") {
|
|
hasNext = true
|
|
}
|
|
case "a":
|
|
// Detect "next" pagination link
|
|
if hasClass(n, "next") || attrVal(n, "rel") == "next" {
|
|
hasNext = true
|
|
}
|
|
// Also check aria-label="Next"
|
|
if attrVal(n, "aria-label") == "Next" {
|
|
hasNext = true
|
|
}
|
|
}
|
|
}
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
walk(c)
|
|
}
|
|
}
|
|
walk(doc)
|
|
return novels, hasNext
|
|
}
|
|
|
|
// parseNovelItem extracts a NovelListing from a <li class="novel-item"> node.
|
|
func parseNovelItem(li *html.Node) (NovelListing, bool) {
|
|
var novel NovelListing
|
|
|
|
var walk func(*html.Node)
|
|
walk = func(n *html.Node) {
|
|
if n.Type == html.ElementNode {
|
|
switch n.Data {
|
|
case "a":
|
|
href := attrVal(n, "href")
|
|
if strings.HasPrefix(href, "/book/") {
|
|
slug := strings.TrimPrefix(href, "/book/")
|
|
slug = strings.TrimSuffix(slug, "/")
|
|
if novel.Slug == "" {
|
|
novel.Slug = slug
|
|
novel.URL = novelFireBase + href
|
|
}
|
|
}
|
|
case "img":
|
|
// lazy-loaded covers use data-src
|
|
src := attrVal(n, "data-src")
|
|
if src == "" {
|
|
src = attrVal(n, "src")
|
|
}
|
|
if src != "" && novel.Cover == "" {
|
|
if !strings.HasPrefix(src, "http") {
|
|
src = novelFireBase + src
|
|
}
|
|
novel.Cover = src
|
|
}
|
|
case "h4":
|
|
if hasClass(n, "novel-title") && novel.Title == "" {
|
|
novel.Title = strings.TrimSpace(textContent(n))
|
|
}
|
|
case "span":
|
|
cls := attrVal(n, "class")
|
|
if strings.Contains(cls, "_bl") && novel.Rank == "" {
|
|
novel.Rank = strings.TrimSpace(textContent(n))
|
|
}
|
|
if strings.Contains(cls, "_br") && novel.Rating == "" {
|
|
novel.Rating = strings.TrimSpace(textContent(n))
|
|
}
|
|
}
|
|
}
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
walk(c)
|
|
}
|
|
}
|
|
walk(li)
|
|
|
|
// Extract chapter count from the novel stats text (contains "N Chapters")
|
|
novel.Chapters = extractChapters(li)
|
|
|
|
if novel.Slug == "" || novel.Title == "" {
|
|
return novel, false
|
|
}
|
|
return novel, true
|
|
}
|
|
|
|
// extractChapters finds the chapter count text within a novel-item node.
|
|
func extractChapters(n *html.Node) string {
|
|
var result string
|
|
var walk func(*html.Node)
|
|
walk = func(node *html.Node) {
|
|
if node.Type == html.ElementNode {
|
|
cls := attrVal(node, "class")
|
|
if strings.Contains(cls, "novel-stats") || strings.Contains(cls, "chapter") {
|
|
txt := strings.TrimSpace(textContent(node))
|
|
if strings.Contains(txt, "Chapter") || strings.Contains(txt, "chapter") {
|
|
// Extract just the numeric part if possible
|
|
result = txt
|
|
return
|
|
}
|
|
}
|
|
}
|
|
for c := node.FirstChild; c != nil; c = c.NextSibling {
|
|
walk(c)
|
|
}
|
|
}
|
|
walk(n)
|
|
return result
|
|
}
|
|
|
|
// hasClass reports whether an HTML node has the given CSS class.
|
|
func hasClass(n *html.Node, cls string) bool {
|
|
for _, a := range n.Attr {
|
|
if a.Key == "class" {
|
|
for _, c := range strings.Fields(a.Val) {
|
|
if c == cls {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// attrVal returns the value of an attribute on an HTML node, or "".
|
|
func attrVal(n *html.Node, key string) string {
|
|
for _, a := range n.Attr {
|
|
if a.Key == key {
|
|
return a.Val
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// textContent returns the concatenated text content of a node and its descendants.
|
|
func textContent(n *html.Node) string {
|
|
if n.Type == html.TextNode {
|
|
return n.Data
|
|
}
|
|
var sb strings.Builder
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
sb.WriteString(textContent(c))
|
|
}
|
|
return sb.String()
|
|
}
|
|
|
|
// ─── Scrape status API ────────────────────────────────────────────────────────
|
|
|
|
// handleScrapeStatus handles GET /api/scrape/status.
|
|
// Returns JSON: {"running": bool}
|
|
func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) {
|
|
s.mu.Lock()
|
|
running := s.running
|
|
s.mu.Unlock()
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]bool{"running": running})
|
|
}
|
|
|
|
// handleScrapeTasks handles GET /api/scrape/tasks.
|
|
// Returns JSON array of all scraping_tasks records, newest first.
|
|
func (s *Server) handleScrapeTasks(w http.ResponseWriter, r *http.Request) {
|
|
tasks, err := s.store.ListScrapeTasks(r.Context())
|
|
if err != nil {
|
|
s.log.Error("handleScrapeTasks: list failed", "err", err)
|
|
http.Error(w, `{"error":"failed to list tasks"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
if tasks == nil {
|
|
tasks = []storage.ScrapeTask{}
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(tasks)
|
|
}
|
|
|
|
// handleReindex handles POST /api/reindex/{slug}.
|
|
// It rebuilds the chapters_idx PocketBase collection for the given book by
|
|
// walking its MinIO objects. Use this when chapters were scraped but the index
|
|
// is out of sync (e.g. after a failed UpsertChapterIdx during scraping).
|
|
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
|
|
slug := r.PathValue("slug")
|
|
if slug == "" {
|
|
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
type reindexer interface {
|
|
ReindexChapters(ctx context.Context, slug string) (int, error)
|
|
}
|
|
ri, ok := s.store.(reindexer)
|
|
if !ok {
|
|
http.Error(w, `{"error":"store does not support reindex"}`, http.StatusNotImplemented)
|
|
return
|
|
}
|
|
|
|
count, err := ri.ReindexChapters(r.Context(), slug)
|
|
if err != nil {
|
|
s.log.Error("reindex failed", "slug", slug, "indexed", count, "err", err)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"error": err.Error(),
|
|
"indexed": count,
|
|
})
|
|
return
|
|
}
|
|
|
|
s.log.Info("reindex complete", "slug", slug, "indexed", count)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"slug": slug,
|
|
"indexed": count,
|
|
})
|
|
}
|
|
|
|
// ─── Voices API ───────────────────────────────────────────────────────────────
|
|
|
|
// handleVoices handles GET /api/voices.
|
|
// Returns the list of available Kokoro voices as JSON: {"voices": [...]}
|
|
func (s *Server) handleVoices(w http.ResponseWriter, _ *http.Request) {
|
|
voices := s.voices()
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{"voices": voices})
|
|
}
|
|
|
|
// ─── Voice sample generation ──────────────────────────────────────────────────
|
|
|
|
// voiceSampleText is the short passage used for voice sample previews.
|
|
const voiceSampleText = "The ancient library held secrets older than memory itself, its dust-laden shelves stretching upward into shadow. She reached for the worn leather spine, fingers trembling with anticipation."
|
|
|
|
// voiceSampleKey returns the MinIO object key for a voice sample.
|
|
// Key: _voice-samples/{voice}.mp3
|
|
func voiceSampleKey(voice string) string {
|
|
safe := strings.Map(func(r rune) rune {
|
|
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
|
|
(r >= '0' && r <= '9') || r == '_' || r == '-' {
|
|
return r
|
|
}
|
|
return '_'
|
|
}, voice)
|
|
return fmt.Sprintf("_voice-samples/%s.mp3", safe)
|
|
}
|
|
|
|
// warmVoiceSamples runs at startup in a background goroutine.
|
|
// It generates a short audio sample for every available Kokoro voice that
|
|
// doesn't already have one in MinIO, so the UI voice selector has playable
|
|
// previews without requiring a manual trigger.
|
|
// It respects ctx cancellation and waits up to 30 s for Kokoro to become
|
|
// reachable before giving up.
|
|
func (s *Server) warmVoiceSamples(ctx context.Context) {
|
|
if s.kokoroURL == "" {
|
|
return
|
|
}
|
|
|
|
// Wait for Kokoro to be reachable (it may still be starting up).
|
|
deadline := time.Now().Add(30 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err == nil {
|
|
resp.Body.Close()
|
|
if resp.StatusCode == http.StatusOK {
|
|
break
|
|
}
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(3 * time.Second):
|
|
}
|
|
}
|
|
|
|
voices := s.voices()
|
|
s.log.Info("warming voice samples", "voices", len(voices))
|
|
|
|
generated, skipped, failed := 0, 0, 0
|
|
for _, voice := range voices {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
|
|
key := voiceSampleKey(voice)
|
|
if s.store.AudioExists(ctx, key) {
|
|
skipped++
|
|
continue
|
|
}
|
|
|
|
filename, err := s.generateSpeech(ctx, voiceSampleText, voice, 1.0)
|
|
if err != nil {
|
|
s.log.Warn("voice sample warmup: generation failed", "voice", voice, "err", err)
|
|
failed++
|
|
continue
|
|
}
|
|
|
|
audioData, err := s.downloadFromKokoro(ctx, filename)
|
|
if err != nil {
|
|
s.log.Warn("voice sample warmup: download failed", "voice", voice, "err", err)
|
|
failed++
|
|
continue
|
|
}
|
|
|
|
if err := s.store.PutAudio(ctx, key, audioData); err != nil {
|
|
s.log.Warn("voice sample warmup: upload failed", "voice", voice, "key", key, "err", err)
|
|
failed++
|
|
continue
|
|
}
|
|
|
|
s.log.Debug("voice sample warmed", "voice", voice)
|
|
generated++
|
|
}
|
|
|
|
s.log.Info("voice sample warmup complete",
|
|
"generated", generated, "skipped", skipped, "failed", failed)
|
|
}
|
|
|
|
// handleGenerateVoiceSamples handles POST /api/audio/voice-samples.
|
|
// It generates short audio samples for each available voice and stores them
|
|
// in the audio MinIO bucket so the UI can play them during voice selection.
|
|
// Already-generated samples are skipped (idempotent).
|
|
// Optional JSON body: {"voices": ["af_bella", ...]} to generate a subset.
|
|
// Returns: {"generated": [...], "skipped": [...], "failed": [...]}
|
|
func (s *Server) handleGenerateVoiceSamples(w http.ResponseWriter, r *http.Request) {
|
|
if s.kokoroURL == "" {
|
|
http.Error(w, `{"error":"kokoro not configured"}`, http.StatusServiceUnavailable)
|
|
return
|
|
}
|
|
|
|
// Parse optional voice list from body.
|
|
var body struct {
|
|
Voices []string `json:"voices"`
|
|
}
|
|
if r.Body != nil {
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
}
|
|
|
|
targetVoices := body.Voices
|
|
if len(targetVoices) == 0 {
|
|
targetVoices = s.voices()
|
|
}
|
|
|
|
type result struct {
|
|
Generated []string `json:"generated"`
|
|
Skipped []string `json:"skipped"`
|
|
Failed []string `json:"failed"`
|
|
}
|
|
var res result
|
|
|
|
for _, voice := range targetVoices {
|
|
key := voiceSampleKey(voice)
|
|
|
|
// Skip if already uploaded.
|
|
if s.store.AudioExists(r.Context(), key) {
|
|
res.Skipped = append(res.Skipped, voice)
|
|
s.log.Debug("voice sample already exists, skipping", "voice", voice)
|
|
continue
|
|
}
|
|
|
|
// Generate via Kokoro (speed 1.0 for samples).
|
|
filename, err := s.generateSpeech(r.Context(), voiceSampleText, voice, 1.0)
|
|
if err != nil {
|
|
s.log.Warn("voice sample generation failed", "voice", voice, "err", err)
|
|
res.Failed = append(res.Failed, voice)
|
|
continue
|
|
}
|
|
|
|
// Download from Kokoro and upload to MinIO.
|
|
audioData, dlErr := s.downloadFromKokoro(r.Context(), filename)
|
|
if dlErr != nil {
|
|
s.log.Warn("voice sample kokoro download failed", "voice", voice, "err", dlErr)
|
|
res.Failed = append(res.Failed, voice)
|
|
continue
|
|
}
|
|
|
|
if putErr := s.store.PutAudio(r.Context(), key, audioData); putErr != nil {
|
|
s.log.Warn("voice sample MinIO upload failed", "voice", voice, "key", key, "err", putErr)
|
|
res.Failed = append(res.Failed, voice)
|
|
continue
|
|
}
|
|
|
|
s.log.Info("voice sample generated", "voice", voice, "key", key)
|
|
res.Generated = append(res.Generated, voice)
|
|
}
|
|
|
|
if res.Generated == nil {
|
|
res.Generated = []string{}
|
|
}
|
|
if res.Skipped == nil {
|
|
res.Skipped = []string{}
|
|
}
|
|
if res.Failed == nil {
|
|
res.Failed = []string{}
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(res)
|
|
}
|
|
|
|
// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}.
|
|
// Returns a presigned URL for the voice sample audio file stored in MinIO.
|
|
// Returns 404 if the sample has not been generated yet.
|
|
func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) {
|
|
voice := r.PathValue("voice")
|
|
if voice == "" {
|
|
http.Error(w, `{"error":"missing voice"}`, http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
key := voiceSampleKey(voice)
|
|
|
|
if !s.store.AudioExists(r.Context(), key) {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
|
|
url, err := s.store.PresignAudio(r.Context(), key, 1*time.Hour)
|
|
if err != nil {
|
|
s.log.Error("presign voice sample failed", "voice", voice, "err", err)
|
|
http.Error(w, `{"error":"presign failed"}`, http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"url": url})
|
|
}
|