Compare commits

...

4 Commits

Author SHA1 Message Date
Admin
1118392811 chore: add .gitignore, CrowdSec acquis config, update architecture docs
- .gitignore: exclude crowdsec/.crowdsec.env (bouncer API key), .env.local,
  and compiled Go binaries (backend/bin/, backend/healthcheck)
- crowdsec/acquis.yaml: log acquisition config for CrowdSec agent
- docs: update d2 and mermaid diagrams to reflect MinIO bucket renames
  (libnovel-chapters→chapters, libnovel-audio→audio, libnovel-browse→catalogue),
  remove /api/browse, update presigned URL paths
2026-03-23 16:36:23 +05:00
Admin
baa403efa2 feat(caddy): add CrowdSec bouncer module
Build caddy with caddy-crowdsec-bouncer/http so the Caddyfile can
reference the crowdsec layer directive for IP-level blocking.
2026-03-23 16:36:09 +05:00
Admin
0ed1112b20 feat(ui): add Sentry/GlitchTip, Umami analytics, presign auto-trigger TTS, feedback link
- Add @sentry/sveltekit: server-side init in hooks.server.ts, client-side in
  hooks.client.ts; opt-in via PUBLIC_GLITCHTIP_DSN env var; handleError wired up
- Add Umami analytics: script tag in +layout.svelte (opt-in via PUBLIC_UMAMI_WEBSITE_ID);
  track book_view, chapter_read, audio_played events via window.umami?.track
- Presign audio endpoint now auto-triggers Kokoro TTS generation when audio is
  missing (404 → POST /api/audio → return 202); AudioPlayer handles 202 status
- Fix listAudioCache: was querying non-existent audio_cache collection; now projects
  from audio_jobs where status='done'
- Add Feedback nav link (desktop + mobile + footer) pointing to feedback.libnovel.cc
- Admin scrape page: add full catalogue scrape button; fix retryTask for catalogue
  kind; show retry button for all failed/cancelled tasks regardless of kind
2026-03-23 16:35:58 +05:00
Admin
16a12ede4d feat(backend): remove browse-page cache, add Sentry/GlitchTip, rename MinIO buckets
- Remove BrowseStore interface + all browse MinIO ops (PutBrowsePage, GetBrowsePage,
  BrowseObjectKey, putBrowse, getBrowse); the /api/browse endpoint was replaced by
  the Meilisearch-backed /api/catalogue route
- Remove browse_refresh.go and the 6h browse refresh ticker in runner
- Add Sentry/GlitchTip error tracking to both backend and runner binaries via
  github.com/getsentry/sentry-go; opt-in via GLITCHTIP_DSN env var
- Wrap http.ServeMux with sentryhttp middleware for automatic panic recovery
- Rename MinIO bucket defaults: libnovel-chapters→chapters, libnovel-audio→audio,
  libnovel-browse→catalogue (env vars still override)
- Remove staged compiled binaries (backend/bin/runner, backend/healthcheck)
2026-03-23 16:35:39 +05:00
36 changed files with 2668 additions and 712 deletions

5
v3/.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
# CrowdSec — generated bouncer API key; never commit
crowdsec/.crowdsec.env
# Local env overrides
.env.local

Binary file not shown.

View File

@@ -2,7 +2,7 @@
//
// It exposes all endpoints consumed by the SvelteKit UI: book/chapter reads,
// scrape-task creation, presigned MinIO URLs, audio-task creation, reading
// progress, live novelfire.net browse/search, and Kokoro voice list.
// progress, live novelfire.net search, and Kokoro voice list.
//
// All heavy lifting (scraping, TTS generation) is delegated to the runner
// binary via PocketBase task records. The backend never scrapes directly.
@@ -19,7 +19,9 @@ import (
"os"
"os/signal"
"syscall"
"time"
"github.com/getsentry/sentry-go"
"github.com/libnovel/backend/internal/backend"
"github.com/libnovel/backend/internal/config"
"github.com/libnovel/backend/internal/kokoro"
@@ -43,6 +45,19 @@ func main() {
func run() error {
cfg := config.Load()
// ── Sentry / GlitchTip error tracking ────────────────────────────────────
if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" {
if err := sentry.Init(sentry.ClientOptions{
Dsn: dsn,
Release: version + "@" + commit,
TracesSampleRate: 0.1,
}); err != nil {
fmt.Fprintf(os.Stderr, "backend: sentry init warning: %v\n", err)
} else {
defer sentry.Flush(2 * time.Second)
}
}
// ── Logger ───────────────────────────────────────────────────────────────
log := buildLogger(cfg.LogLevel)
log.Info("backend starting",
@@ -95,7 +110,6 @@ func run() error {
AudioStore: store,
PresignStore: store,
ProgressStore: store,
BrowseStore: store,
CoverStore: store,
Producer: store,
TaskReader: store,

View File

@@ -19,6 +19,7 @@ import (
"syscall"
"time"
"github.com/getsentry/sentry-go"
"github.com/libnovel/backend/internal/browser"
"github.com/libnovel/backend/internal/config"
"github.com/libnovel/backend/internal/kokoro"
@@ -44,6 +45,19 @@ func main() {
func run() error {
cfg := config.Load()
// ── Sentry / GlitchTip error tracking ────────────────────────────────────
if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" {
if err := sentry.Init(sentry.ClientOptions{
Dsn: dsn,
Release: version + "@" + commit,
TracesSampleRate: 0.1,
}); err != nil {
fmt.Fprintf(os.Stderr, "runner: sentry init warning: %v\n", err)
} else {
defer sentry.Flush(2 * time.Second)
}
}
// ── Logger ──────────────────────────────────────────────────────────────
log := buildLogger(cfg.LogLevel)
log.Info("runner starting",
@@ -105,21 +119,20 @@ func run() error {
// ── Runner ──────────────────────────────────────────────────────────────
rCfg := runner.Config{
WorkerID: cfg.Runner.WorkerID,
PollInterval: cfg.Runner.PollInterval,
MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape,
MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio,
OrchestratorWorkers: workers,
MetricsAddr: cfg.Runner.MetricsAddr,
CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval,
SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh,
WorkerID: cfg.Runner.WorkerID,
PollInterval: cfg.Runner.PollInterval,
MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape,
MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio,
OrchestratorWorkers: workers,
MetricsAddr: cfg.Runner.MetricsAddr,
CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval,
SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh,
}
deps := runner.Dependencies{
Consumer: store,
BookWriter: store,
BookReader: store,
AudioStore: store,
BrowseStore: store,
CoverStore: store,
SearchIndex: searchIndex,
Novel: novel,

View File

@@ -13,6 +13,7 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/getsentry/sentry-go v0.43.0 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/google/uuid v1.6.0 // indirect

View File

@@ -8,6 +8,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
@@ -57,5 +59,6 @@ golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

Binary file not shown.

View File

@@ -176,80 +176,6 @@ type NovelListing struct {
URL string `json:"url"`
}
// handleBrowse handles GET /api/browse.
// Fetches novelfire.net live (no MinIO cache in the new backend).
// Query params: page (default 1), genre (default "all"), sort (default "popular"),
// status (default "all"), type (default "all-novel")
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
page := q.Get("page")
if page == "" {
page = "1"
}
genre := q.Get("genre")
if genre == "" {
genre = "all"
}
sortBy := q.Get("sort")
if sortBy == "" {
sortBy = "popular"
}
status := q.Get("status")
if status == "" {
status = "all"
}
novelType := q.Get("type")
if novelType == "" {
novelType = "all-novel"
}
pageNum, _ := strconv.Atoi(page)
if pageNum <= 0 {
pageNum = 1
}
// ── Try MinIO cache first ─────────────────────────────────────────────
// Only page 1 is cached; higher pages fall through to live fetch.
if pageNum == 1 && s.deps.BrowseStore != nil {
if data, ok, err := s.deps.BrowseStore.GetBrowsePage(r.Context(), genre, sortBy, status, novelType, 1); err == nil && ok {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "public, max-age=300")
_, _ = w.Write(data)
return
}
}
// ── Fall back to live novelfire.net fetch ──────────────────────────────
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
defer cancel()
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
novelFireBase, genre, sortBy, status, novelType, pageNum)
novels, hasNext, err := s.fetchBrowsePage(ctx, targetURL)
if err != nil {
// Live fetch also failed — return empty list with cached=false flag so
// the UI can show a "not ready yet" state instead of a hard error.
s.deps.Log.Error("handleBrowse: fetch failed (no cache)", "url", targetURL, "err", err)
w.Header().Set("Cache-Control", "no-store")
writeJSON(w, 0, map[string]any{
"novels": []any{},
"page": pageNum,
"hasNext": false,
"cached": false,
})
return
}
w.Header().Set("Cache-Control", "public, max-age=300")
writeJSON(w, 0, map[string]any{
"novels": novels,
"page": pageNum,
"hasNext": hasNext,
"cached": false,
})
}
// handleSearch handles GET /api/search.
// Query params: q (min 2 chars), source ("local"|"remote"|"all", default "all")
//
@@ -832,7 +758,13 @@ func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
writeJSON(w, 0, map[string]string{"url": u})
}
// voiceSampleText is the phrase synthesised for every voice sample.
const voiceSampleText = "Hello! This is a preview of what I sound like. I hope you enjoy listening to your stories with my voice."
// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}.
// If the sample has not been generated yet it synthesises it on the fly via
// Kokoro, stores the result in MinIO, and returns the presigned URL — so the
// caller always gets a playable URL in a single request.
func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) {
voice := r.PathValue("voice")
if voice == "" {
@@ -841,9 +773,21 @@ func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request
}
key := kokoro.VoiceSampleKey(voice)
// Generate sample on demand when it is not in MinIO yet.
if !s.deps.AudioStore.AudioExists(r.Context(), key) {
http.NotFound(w, r)
return
s.deps.Log.Info("generating voice sample on demand", "voice", voice)
mp3, err := s.deps.Kokoro.GenerateAudio(r.Context(), voiceSampleText, voice)
if err != nil {
s.deps.Log.Error("voice sample generation failed", "voice", voice, "err", err)
jsonError(w, http.StatusInternalServerError, "voice sample generation failed")
return
}
if err := s.deps.AudioStore.PutAudio(r.Context(), key, mp3); err != nil {
s.deps.Log.Error("voice sample upload failed", "voice", voice, "err", err)
jsonError(w, http.StatusInternalServerError, "voice sample upload failed")
return
}
}
u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour)

View File

@@ -6,7 +6,7 @@
// picks up and executes those tasks asynchronously
// - Presigned MinIO URLs for media playback/upload
// - Session-scoped reading progress
// - Live novelfire.net browse/search (no scraper interface needed; direct HTTP)
// - Live novelfire.net search (no scraper interface needed; direct HTTP)
// - Kokoro voice list
//
// The backend never scrapes directly. All scraping (metadata, chapter list,
@@ -28,6 +28,7 @@ import (
"sync"
"time"
sentryhttp "github.com/getsentry/sentry-go/http"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/meili"
@@ -47,8 +48,6 @@ type Dependencies struct {
PresignStore bookstore.PresignStore
// ProgressStore reads/writes per-session reading progress.
ProgressStore bookstore.ProgressStore
// BrowseStore reads cached browse page snapshots from MinIO.
BrowseStore bookstore.BrowseStore
// CoverStore reads and writes book cover images from MinIO.
// If nil, the cover endpoint falls back to a CDN redirect.
CoverStore bookstore.CoverStore
@@ -122,8 +121,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
// Cancel a pending task (scrape or audio)
mux.HandleFunc("POST /api/cancel-task/{id}", s.handleCancelTask)
// Browse & search (live novelfire.net)
mux.HandleFunc("GET /api/browse", s.handleBrowse)
// Browse & search
mux.HandleFunc("GET /api/search", s.handleSearch)
// Catalogue (Meilisearch-backed browse + search — preferred path for UI)
@@ -174,7 +172,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
srv := &http.Server{
Addr: s.cfg.Addr,
Handler: mux,
Handler: sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle(mux),
ReadTimeout: 15 * time.Second,
WriteTimeout: 60 * time.Second,
IdleTimeout: 60 * time.Second,

View File

@@ -128,18 +128,6 @@ type ProgressStore interface {
DeleteProgress(ctx context.Context, sessionID, slug string) error
}
// BrowseStore covers browse page snapshot storage.
// The runner writes snapshots; the backend reads them.
type BrowseStore interface {
// PutBrowsePage stores a raw JSON snapshot for a browse page.
// genre, sort, status, novelType and page identify the page.
PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error
// GetBrowsePage retrieves a raw JSON snapshot. Returns (nil, false, nil)
// when no snapshot exists for the given parameters.
GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error)
}
// CoverStore covers book cover image storage in MinIO.
// The runner writes covers during catalogue refresh; the backend reads them.
type CoverStore interface {

View File

@@ -145,10 +145,10 @@ func Load() Config {
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
UseSSL: envBool("MINIO_USE_SSL", false),
PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true),
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "chapters"),
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "audio"),
BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "avatars"),
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "catalogue"),
},
Kokoro: Kokoro{

View File

@@ -33,8 +33,8 @@ func TestLoad_Defaults(t *testing.T) {
if cfg.PocketBase.URL != "http://localhost:8090" {
t.Errorf("PocketBase.URL: want http://localhost:8090, got %q", cfg.PocketBase.URL)
}
if cfg.MinIO.BucketChapters != "libnovel-chapters" {
t.Errorf("MinIO.BucketChapters: want libnovel-chapters, got %q", cfg.MinIO.BucketChapters)
if cfg.MinIO.BucketChapters != "chapters" {
t.Errorf("MinIO.BucketChapters: want chapters, got %q", cfg.MinIO.BucketChapters)
}
if cfg.MinIO.UseSSL != false {
t.Errorf("MinIO.UseSSL: want false, got %v", cfg.MinIO.UseSSL)

View File

@@ -1,176 +0,0 @@
package runner
// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
// browse page snapshots and stores them in MinIO.
//
// Design:
// - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
// - Fetches page 1 for each combination of the standard genre/sort/status
// filter values and stores the parsed JSON blob in MinIO via BrowseStore.
// - The backend's handleBrowse then serves from MinIO instead of calling
// novelfire.net live, which avoids IP-based rate-limiting on the server.
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"time"
)
// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
type browseNovelListing struct {
Slug string `json:"slug"`
Title string `json:"title"`
Cover string `json:"cover"`
URL string `json:"url"`
}
// browseSnapshot is the JSON structure stored in MinIO.
type browseSnapshot struct {
Novels []browseNovelListing `json:"novels"`
Page int `json:"page"`
HasNext bool `json:"hasNext"`
// CachedAt is the UTC time the snapshot was written (ISO 8601).
CachedAt string `json:"cachedAt"`
}
// browseCombos lists the filter combinations to pre-fetch.
// Each entry is (genre, sort, status, novelType).
var browseCombos = []struct{ genre, sort, status, novelType string }{
{"all", "popular", "all", "all-novel"},
{"all", "popular", "ongoing", "all-novel"},
{"all", "popular", "completed", "all-novel"},
{"all", "new", "all", "all-novel"},
{"all", "new", "ongoing", "all-novel"},
{"all", "new", "completed", "all-novel"},
{"all", "top-rated", "all", "all-novel"},
{"all", "top-rated", "ongoing", "all-novel"},
{"all", "top-rated", "completed", "all-novel"},
}
const novelFireBrowseBase = "https://novelfire.net"
// runBrowseRefresh fetches all browse combos from novelfire.net and stores
// the results in MinIO. Errors per-combo are logged but do not abort the
// whole refresh cycle.
func (r *Runner) runBrowseRefresh(ctx context.Context) {
if r.deps.BrowseStore == nil {
r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
return
}
log := r.deps.Log.With("op", "browse_refresh")
log.Info("runner: browse refresh starting", "combos", len(browseCombos))
ok, fail := 0, 0
for _, c := range browseCombos {
if ctx.Err() != nil {
break
}
novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
if err != nil {
log.Warn("runner: browse fetch failed",
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
fail++
continue
}
snap := browseSnapshot{
Novels: novels,
Page: 1,
HasNext: hasNext,
CachedAt: time.Now().UTC().Format(time.RFC3339),
}
data, _ := json.Marshal(snap)
if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
log.Warn("runner: browse put failed",
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
fail++
continue
}
ok++
}
log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
}
// fetchBrowsePage calls novelfire.net and returns a list of novel listings
// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
novelFireBrowseBase, genre, sort, status, novelType)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
if err != nil {
return nil, false, fmt.Errorf("build request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
httpClient := &http.Client{Timeout: 45 * time.Second}
resp, err := httpClient.Do(req)
if err != nil {
return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, resp.Body)
return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
}
return parseBrowseHTML(resp.Body)
}
// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
// in backend/handlers.go — kept separate to avoid coupling packages.
func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
data, err := io.ReadAll(r)
if err != nil {
return nil, false, err
}
body := string(data)
hasNext := strings.Contains(body, `rel="next"`) ||
strings.Contains(body, `aria-label="Next"`) ||
strings.Contains(body, `class="next"`)
slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
slugMatches := slugRe.FindAllStringSubmatch(body, -1)
titleMatches := titleRe.FindAllStringSubmatch(body, -1)
coverMatches := coverRe.FindAllStringSubmatch(body, -1)
var novels []browseNovelListing
seen := make(map[string]bool)
for i, sm := range slugMatches {
slug := sm[1]
if seen[slug] {
continue
}
seen[slug] = true
item := browseNovelListing{
Slug: slug,
URL: novelFireBrowseBase + "/book/" + slug,
}
if i < len(titleMatches) {
item.Title = strings.TrimSpace(titleMatches[i][1])
}
if i < len(coverMatches) {
item.Cover = coverMatches[i][1]
}
if item.Title != "" {
novels = append(novels, item)
}
}
return novels, hasNext, nil
}

View File

@@ -49,9 +49,6 @@ type Config struct {
// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
// task is considered orphaned and reset to pending. Defaults to 2m when 0.
StaleTaskThreshold time.Duration
// BrowseRefreshInterval is how often the runner pre-fetches browse page
// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
BrowseRefreshInterval time.Duration
// CatalogueRefreshInterval is how often the runner walks the full catalogue,
// scrapes per-book metadata, downloads covers, and re-indexes everything in
// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
@@ -76,10 +73,7 @@ type Dependencies struct {
BookReader bookstore.BookReader
// AudioStore persists generated audio and checks key existence.
AudioStore bookstore.AudioStore
// BrowseStore stores browse page snapshots in MinIO.
BrowseStore bookstore.BrowseStore
// CoverStore stores book cover images in MinIO.
// If nil, cover downloads are skipped during catalogue refresh.
CoverStore bookstore.CoverStore
// SearchIndex indexes books in Meilisearch after scraping.
// If nil a no-op is used.
@@ -125,9 +119,6 @@ func New(cfg Config, deps Dependencies) *Runner {
if cfg.StaleTaskThreshold <= 0 {
cfg.StaleTaskThreshold = 2 * time.Minute
}
if cfg.BrowseRefreshInterval <= 0 {
cfg.BrowseRefreshInterval = 6 * time.Hour
}
if cfg.CatalogueRefreshInterval <= 0 {
cfg.CatalogueRefreshInterval = 24 * time.Hour
}
@@ -151,7 +142,6 @@ func (r *Runner) Run(ctx context.Context) error {
"poll_interval", r.cfg.PollInterval,
"max_scrape", r.cfg.MaxConcurrentScrape,
"max_audio", r.cfg.MaxConcurrentAudio,
"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
"catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval,
"metrics_addr", r.cfg.MetricsAddr,
)
@@ -173,14 +163,9 @@ func (r *Runner) Run(ctx context.Context) error {
tick := time.NewTicker(r.cfg.PollInterval)
defer tick.Stop()
browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
defer browseTick.Stop()
catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval)
defer catalogueTick.Stop()
// Run one browse refresh immediately on startup.
go r.runBrowseRefresh(ctx)
// Run one catalogue refresh immediately on startup (unless skipped by flag).
if !r.cfg.SkipInitialCatalogueRefresh {
go r.runCatalogueRefresh(ctx)
@@ -207,8 +192,6 @@ func (r *Runner) Run(ctx context.Context) error {
r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
}
return nil
case <-browseTick.C:
go r.runBrowseRefresh(ctx)
case <-catalogueTick.C:
go r.runCatalogueRefresh(ctx)
case <-tick.C:

View File

@@ -119,12 +119,6 @@ func AvatarObjectKey(userID, ext string) string {
return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
}
// BrowseObjectKey returns the MinIO object key for a cached browse page snapshot.
// Format: browse/{genre}/{sort}/{status}/{type}/page-{n}.json
func BrowseObjectKey(genre, sort, status, novelType string, page int) string {
return fmt.Sprintf("browse/%s/%s/%s/%s/page-%d.json", genre, sort, status, novelType, page)
}
// CoverObjectKey returns the MinIO object key for a book cover image.
// Format: covers/{slug}.jpg
func CoverObjectKey(slug string) string {
@@ -207,26 +201,6 @@ func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string)
return keys, nil
}
// ── Browse operations ─────────────────────────────────────────────────────────
// putBrowse stores raw JSON bytes for a browse page snapshot.
func (m *minioClient) putBrowse(ctx context.Context, key string, data []byte) error {
return m.putObject(ctx, m.bucketBrowse, key, "application/json", data)
}
// getBrowse retrieves a browse page snapshot. Returns (nil, false, nil) when
// the object does not exist.
func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool, error) {
if !m.objectExists(ctx, m.bucketBrowse, key) {
return nil, false, nil
}
data, err := m.getObject(ctx, m.bucketBrowse, key)
if err != nil {
return nil, false, err
}
return data, true, nil
}
// ── Cover operations ──────────────────────────────────────────────────────────
// putCover stores a raw cover image in the browse bucket under covers/{slug}.jpg.

View File

@@ -50,7 +50,6 @@ var _ bookstore.RankingStore = (*Store)(nil)
var _ bookstore.AudioStore = (*Store)(nil)
var _ bookstore.PresignStore = (*Store)(nil)
var _ bookstore.ProgressStore = (*Store)(nil)
var _ bookstore.BrowseStore = (*Store)(nil)
var _ bookstore.CoverStore = (*Store)(nil)
var _ taskqueue.Producer = (*Store)(nil)
var _ taskqueue.Consumer = (*Store)(nil)
@@ -790,25 +789,6 @@ func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
}, nil
}
// ── BrowseStore ────────────────────────────────────────────────────────────────
func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error {
key := BrowseObjectKey(genre, sort, status, novelType, page)
if err := s.mc.putBrowse(ctx, key, data); err != nil {
return fmt.Errorf("PutBrowsePage: %w", err)
}
return nil
}
func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) {
key := BrowseObjectKey(genre, sort, status, novelType, page)
data, ok, err := s.mc.getBrowse(ctx, key)
if err != nil {
return nil, false, fmt.Errorf("GetBrowsePage: %w", err)
}
return data, ok, nil
}
// ── CoverStore ─────────────────────────────────────────────────────────────────
func (s *Store) PutCover(ctx context.Context, slug string, data []byte, contentType string) error {

View File

@@ -1,7 +1,8 @@
FROM caddy:2-builder AS builder
RUN xcaddy build \
--with github.com/mholt/caddy-ratelimit
--with github.com/mholt/caddy-ratelimit \
--with github.com/hslatman/caddy-crowdsec-bouncer/http
FROM caddy:2-alpine
COPY --from=builder /usr/bin/caddy /usr/bin/caddy

12
v3/crowdsec/acquis.yaml Normal file
View File

@@ -0,0 +1,12 @@
# CrowdSec log acquisition — tells the CrowdSec agent which logs to parse.
#
# Caddy writes JSON access logs to /var/log/caddy/access.log (mounted from the
# caddy_logs Docker volume). CrowdSec reads the same volume at the same path.
#
# The `crowdsecurity/caddy` collection (installed via COLLECTIONS env var)
# provides the parser that understands Caddy's JSON log format.
filenames:
- /var/log/caddy/access.log
labels:
type: caddy

View File

@@ -171,7 +171,7 @@ client -> caddy: HTTPS :443
caddy -> sk: "/* (catch-all)\n→ SvelteKit handles auth"
caddy -> be: "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/* /api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/* /api/catalogue /api/ranking"
caddy -> storage.mn: "/avatars/*\n(presigned GETs)"
caddy -> storage.mn: "/avatars/*\n/audio/*\n/chapters/*\n(presigned MinIO GETs)"
# ─── SvelteKit → Backend (server-side proxy) ──────────────────────────────────

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 57 KiB

After

Width:  |  Height:  |  Size: 57 KiB

View File

@@ -46,7 +46,7 @@ storage: Storage {
minio: MinIO {
shape: cylinder
label: "MinIO :9000\n\nbuckets:\n libnovel-chapters\n libnovel-audio\n avatars\n libnovel-browse"
label: "MinIO :9000\n\nbuckets:\n chapters\n audio\n avatars\n catalogue"
}
pocketbase: PocketBase {
@@ -135,7 +135,7 @@ app.ui -> app.backend: REST API calls (server-side)\n/api/catalogue /api/book-pr
app.caddy -> app.ui: "/* (catch-all)\n/api/scrape/*\n/api/chapter-text-preview/*\n→ SvelteKit (auth enforced)"
app.caddy -> app.backend: "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*"
app.caddy -> storage.minio: "/avatars/*\n(presigned avatar GETs)"
app.caddy -> storage.minio: "/avatars/*\n/audio/*\n/chapters/*\n(presigned MinIO GETs)"
# ─── External → App ───────────────────────────────────────────────────────────

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 58 KiB

After

Width:  |  Height:  |  Size: 58 KiB

View File

@@ -22,7 +22,7 @@ flowchart TD
H --> I[Worker pool — N goroutines\nRUNNER_MAX_CONCURRENT_SCRAPE]
I --> J[For each missing chapter:\nGET chapter HTML from novelfire.net]
J --> K[Parse HTML → Markdown\nhtmlutil.NodeToMarkdown]
K --> L[PUT object to MinIO\nlibnovel-chapters/{slug}/{n}.md]
K --> L[PUT object to MinIO\nchapters/{slug}/{n}.md]
L --> M[Upsert book doc\nto Meilisearch index: books]
M --> Z
F --> M
@@ -87,7 +87,7 @@ flowchart TD
H --> I["GET /api/chapter-text/{slug}/{n}\nfrom backend — plain text"]
I --> J[POST /v1/audio/speech\nto Kokoro-FastAPI\nbody: text + voice]
J --> K[Stream MP3 response]
K --> L[PUT object to MinIO\nlibnovel-audio/{slug}/{n}/{voice}.mp3]
K --> L[PUT object to MinIO\naudio/{slug}/{n}/{voice}.mp3]
L --> M[UPDATE audio_task\nstatus=done]
N(["Client polls\nGET /api/audio/status/{slug}/{n}"]) --> O{status?}

2245
v3/ui/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -29,6 +29,7 @@
"dependencies": {
"@aws-sdk/client-s3": "^3.1005.0",
"@aws-sdk/s3-request-presigner": "^3.1005.0",
"@sentry/sveltekit": "^10.45.0",
"cropperjs": "^1.6.2",
"ioredis": "^5.3.2",
"marked": "^17.0.3",

8
v3/ui/src/app.d.ts vendored
View File

@@ -13,6 +13,14 @@ declare global {
// interface PageState {}
// interface Platform {}
}
// Umami analytics — injected by the script tag in +layout.svelte when
// PUBLIC_UMAMI_WEBSITE_ID is set. Optional so calls are no-ops in dev.
interface Window {
umami?: {
track(event: string, data?: Record<string, unknown>): void;
};
}
}
export {};

13
v3/ui/src/hooks.client.ts Normal file
View File

@@ -0,0 +1,13 @@
import * as Sentry from '@sentry/sveltekit';
import { env } from '$env/dynamic/public';
// Sentry / GlitchTip client-side error tracking.
// No-op when PUBLIC_GLITCHTIP_DSN is unset (e.g. local dev).
if (env.PUBLIC_GLITCHTIP_DSN) {
Sentry.init({
dsn: env.PUBLIC_GLITCHTIP_DSN,
tracesSampleRate: 0.1
});
}
export const handleError = Sentry.handleErrorWithSentry();

View File

@@ -1,10 +1,24 @@
import type { Handle } from '@sveltejs/kit';
import { handleErrorWithSentry } from '@sentry/sveltekit';
import * as Sentry from '@sentry/sveltekit';
import { randomBytes, createHmac } from 'node:crypto';
import { env } from '$env/dynamic/private';
import { env as pubEnv } from '$env/dynamic/public';
import { log } from '$lib/server/logger';
import { createUserSession, touchUserSession, isSessionRevoked } from '$lib/server/pocketbase';
import { drain as drainPresignCache } from '$lib/server/presignCache';
// ─── Sentry / GlitchTip server-side error tracking ────────────────────────────
// No-op when PUBLIC_GLITCHTIP_DSN is unset (e.g. local dev).
if (pubEnv.PUBLIC_GLITCHTIP_DSN) {
Sentry.init({
dsn: pubEnv.PUBLIC_GLITCHTIP_DSN,
tracesSampleRate: 0.1
});
}
export const handleError = handleErrorWithSentry();
// ─── Graceful shutdown ────────────────────────────────────────────────────────
//
// When Docker/Kubernetes sends SIGTERM (or the user sends SIGINT), we:

View File

@@ -327,7 +327,9 @@
voice: targetVoice
});
const res = await fetch(`/api/presign/audio?${params}`);
if (res.status === 404) return null;
// 202: TTS was just enqueued by the presign endpoint — audio not ready yet.
// 404: legacy fallback (should no longer occur after endpoint change).
if (res.status === 202 || res.status === 404) return null;
if (!res.ok) throw new Error(`presign HTTP ${res.status}`);
const data = (await res.json()) as { url: string };
return data.url;
@@ -608,6 +610,11 @@
// Not yet loaded — start the full flow.
await startPlayback();
// Track audio play after successful load.
if (audioStore.status === 'ready') {
window.umami?.track('audio_played', { slug, chapter });
}
}
function formatTime(s: number): string {

View File

@@ -662,6 +662,8 @@ export async function setAudioTime(
}
// ─── Audio cache ──────────────────────────────────────────────────────────────
// There is no separate audio_cache collection — completed audio jobs in the
// audio_jobs collection serve as the cache record. We project them here.
export interface AudioCacheEntry {
id: string;
@@ -671,7 +673,13 @@ export interface AudioCacheEntry {
}
export async function listAudioCache(): Promise<AudioCacheEntry[]> {
return listAll<AudioCacheEntry>('audio_cache', '', '-updated');
const jobs = await listAll<AudioJob>('audio_jobs', 'status="done"', '-finished');
return jobs.map((j) => ({
id: j.id,
cache_key: j.cache_key,
filename: `${j.cache_key}.mp3`,
updated: j.finished
}));
}
// ─── Scraping tasks ───────────────────────────────────────────────────────────

View File

@@ -170,6 +170,14 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>libnovel</title>
<!-- Umami analytics — no-op when PUBLIC_UMAMI_WEBSITE_ID is unset -->
{#if env.PUBLIC_UMAMI_WEBSITE_ID}
<script
defer
src="https://analytics.libnovel.cc/script.js"
data-website-id={env.PUBLIC_UMAMI_WEBSITE_ID}
></script>
{/if}
</svelte:head>
<!-- Persistent audio element — always in the DOM, never conditionally unmounted.
@@ -232,12 +240,20 @@
>
Library
</a>
<a
href="/catalogue"
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/catalogue') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
>
Discover
</a>
<a
href="/catalogue"
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/catalogue') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
>
Discover
</a>
<a
href="https://feedback.libnovel.cc"
target="_blank"
rel="noopener noreferrer"
class="hidden sm:block text-sm transition-colors text-zinc-400 hover:text-zinc-100"
>
Feedback
</a>
<div class="ml-auto flex items-center gap-4">
<!-- Desktop: admin + profile + sign out (hidden on mobile) -->
@@ -311,13 +327,22 @@
>
Library
</a>
<a
href="/catalogue"
onclick={() => (menuOpen = false)}
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/catalogue') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
>
Discover
</a>
<a
href="/catalogue"
onclick={() => (menuOpen = false)}
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/catalogue') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
>
Discover
</a>
<a
href="https://feedback.libnovel.cc"
target="_blank"
rel="noopener noreferrer"
onclick={() => (menuOpen = false)}
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100"
>
Feedback ↗
</a>
<a
href="/profile"
onclick={() => (menuOpen = false)}
@@ -374,10 +399,22 @@
<div class="max-w-6xl mx-auto px-4 py-6 flex flex-col items-center gap-4 text-xs text-zinc-600">
<!-- Top row: site links -->
<nav class="flex flex-wrap items-center justify-center gap-x-5 gap-y-2">
<a href="/books" class="hover:text-zinc-400 transition-colors">Library</a>
<a href="/catalogue" class="hover:text-zinc-400 transition-colors">Discover</a>
<a
href="https://novelfire.net"
<a href="/books" class="hover:text-zinc-400 transition-colors">Library</a>
<a href="/catalogue" class="hover:text-zinc-400 transition-colors">Discover</a>
<a
href="https://feedback.libnovel.cc"
target="_blank"
rel="noopener noreferrer"
class="hover:text-zinc-400 transition-colors flex items-center gap-1"
>
Feedback
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14" />
</svg>
</a>
<a
href="https://novelfire.net"
target="_blank"
rel="noopener noreferrer"
class="hover:text-zinc-400 transition-colors flex items-center gap-1"

View File

@@ -32,6 +32,33 @@
tasks = data.tasks;
});
// ── Full catalogue scrape ───────────────────────────────────────────────────
let catalogueError = $state('');
let cataloguing = $state(false);
async function triggerCatalogueScrape() {
if (running || cataloguing) return;
cataloguing = true;
catalogueError = '';
try {
const res = await fetch('/api/scrape', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: '{}'
});
if (!res.ok) {
const d = await res.json().catch(() => ({}));
catalogueError = d.error ?? d.message ?? `Error ${res.status}`;
} else {
running = true;
}
} catch {
catalogueError = 'Network error.';
} finally {
cataloguing = false;
}
}
// ── Single book scrape ──────────────────────────────────────────────────────
let scrapeUrl = $state('');
let scrapeError = $state('');
@@ -114,13 +141,14 @@
}
function retryTask(task: ScrapingTask) {
if (task.kind === 'book_range') {
if (task.kind === 'catalogue') {
triggerCatalogueScrape();
} else if (task.kind === 'book_range') {
rangeUrl = task.target_url ?? '';
rangeFrom = task.from_chapter ?? 1;
rangeTo = task.to_chapter > 0 ? task.to_chapter : null;
scrollToRangeForm();
} else {
// book / catalogue — just pre-fill the single-book form
scrapeUrl = task.target_url ?? '';
scrollToBookForm();
}
@@ -220,7 +248,25 @@
</div>
<!-- Scrape controls -->
<div class="grid gap-4 sm:grid-cols-2">
<div class="grid gap-4 sm:grid-cols-2 lg:grid-cols-3">
<!-- Full catalogue -->
<div class="bg-zinc-800 rounded-xl border border-zinc-700 p-5 space-y-3">
<div>
<h2 class="text-sm font-semibold text-zinc-300">Scrape full catalogue</h2>
<p class="text-xs text-zinc-500 mt-1">Re-crawls all novelfire.net pages and picks up new books.</p>
</div>
<button
onclick={triggerCatalogueScrape}
disabled={running || cataloguing}
class="w-full px-4 py-2 rounded-lg bg-amber-600 text-zinc-900 font-semibold text-sm hover:bg-amber-500 transition-colors disabled:opacity-50"
>
{cataloguing ? 'Queuing…' : running ? 'Already running…' : 'Start catalogue scrape'}
</button>
{#if catalogueError}
<p class="text-sm text-red-400">{catalogueError}</p>
{/if}
</div>
<!-- Single book -->
<div id="book-form" class="bg-zinc-800 rounded-xl border border-zinc-700 p-5 space-y-3">
<h2 class="text-sm font-semibold text-zinc-300">Scrape a single book</h2>
@@ -379,7 +425,7 @@
Continue ▶
</button>
{/if}
{#if (task.status === 'failed' || task.status === 'cancelled') && task.kind !== 'catalogue'}
{#if task.status === 'failed' || task.status === 'cancelled'}
<button
onclick={() => retryTask(task)}
class="px-2 py-1 rounded text-xs font-medium bg-sky-900/60 text-sky-300 hover:bg-sky-800/60 transition-colors"
@@ -447,7 +493,7 @@
Continue ▶
</button>
{/if}
{#if (task.status === 'failed' || task.status === 'cancelled') && task.kind !== 'catalogue'}
{#if task.status === 'failed' || task.status === 'cancelled'}
<button
onclick={() => retryTask(task)}
class="flex-1 px-3 py-1.5 rounded-lg text-xs font-medium bg-sky-900/60 text-sky-300 hover:bg-sky-800/60 transition-colors"

View File

@@ -3,12 +3,23 @@ import type { RequestHandler } from './$types';
import { presignAudio } from '$lib/server/minio';
import { log } from '$lib/server/logger';
import * as cache from '$lib/server/presignCache';
import { backendFetch } from '$lib/server/scraper';
/**
* GET /api/presign/audio?slug=...&n=...&voice=...
* Returns a presigned MinIO URL for the audio file so the browser
* can stream it directly without going through the server.
* Returns 404 when the audio has not been generated yet.
*
* Returns a presigned MinIO URL for the audio file so the client can stream
* it directly without going through the server.
*
* When the audio has not been generated yet, this endpoint automatically
* enqueues a TTS generation job and returns 202 Accepted with the job status,
* so callers can poll GET /api/audio/status/{slug}/{n}?voice=... until done,
* then call this endpoint again to get the URL.
*
* Responses:
* 200 { url: string } — audio ready, stream from MinIO
* 202 { task_id: string, status: string } — TTS enqueued, poll for completion
* 202 { status: "pending"|"generating" } — TTS already in progress
*
* Results are cached in-process for 50 minutes (MinIO URLs are valid 1 hour)
* to avoid a backend + MinIO round-trip on every "Play" click.
@@ -38,10 +49,52 @@ export const GET: RequestHandler = async ({ url }) => {
return json({ url: presignedUrl });
} catch (e) {
const status = (e as { status?: number }).status;
if (status === 404) {
error(404, 'Audio not found');
if (status !== 404) {
log.error('presign', 'presign audio failed', { slug, n, err: String(e) });
error(500, `Could not get presigned URL: ${e}`);
}
log.error('presign', 'presign audio failed', { slug, n, err: String(e) });
error(500, `Could not get presigned URL: ${e}`);
}
// Audio not found — automatically trigger TTS generation so the caller
// doesn't need a separate POST step. Return 202 with the job status.
log.info('presign', 'audio not found, triggering TTS generation', { slug, n, voice });
try {
const triggerRes = await backendFetch(`/api/audio/${slug}/${n}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(voice ? { voice } : {})
});
if (!triggerRes.ok) {
const text = await triggerRes.text().catch(() => '');
log.error('presign', 'audio trigger failed', { slug, n, status: triggerRes.status, body: text });
error(triggerRes.status as Parameters<typeof error>[0], text || 'Audio generation failed');
}
const data = (await triggerRes.json()) as
| { url: string; status: 'done' }
| { task_id: string; status: string };
// If the backend says it's already done (race: generated between presign
// check and the POST), try to presign once more and return 200.
if (triggerRes.status === 200 || ('status' in data && data.status === 'done')) {
try {
const presignedUrl = await presignAudio(slug, n, voice || undefined);
await cache.set(cacheKey, presignedUrl);
return json({ url: presignedUrl });
} catch {
// Ignore — fall through to 202 below.
}
}
// Generation is in progress — return 202 so the caller can poll.
return new Response(JSON.stringify(data), {
status: 202,
headers: { 'Content-Type': 'application/json' }
});
} catch (e) {
if ((e as { status?: number }).status) throw e; // re-throw SvelteKit errors
log.error('presign', 'audio trigger error', { slug, n, err: String(e) });
error(500, `Could not trigger audio generation: ${e}`);
}
};

View File

@@ -1,12 +1,15 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { presignVoiceSample } from '$lib/server/minio';
import { log } from '$lib/server/logger';
import * as cache from '$lib/server/presignCache';
/**
* GET /api/presign/voice-sample?voice=af_bella
* Returns a presigned URL for the voice sample audio file.
* Returns 404 if the sample has not been generated yet.
*
* The backend generates the sample on demand via Kokoro TTS if it does not
* exist yet, so this endpoint always returns 200 { url } (or 5xx on failure).
*
* Results are cached in-process for 50 minutes to avoid a backend + MinIO
* round-trip on every voice-selection preview play.
@@ -25,16 +28,13 @@ export const GET: RequestHandler = async ({ url }) => {
return json({ url: cached });
}
// Slow path: call backend → MinIO presign.
// Slow path: call backend → generate if needed → MinIO presign.
try {
const presignedUrl = await presignVoiceSample(voice);
await cache.set(cacheKey, presignedUrl);
return json({ url: presignedUrl });
} catch (e) {
const status = (e as { status?: number }).status;
if (status === 404) {
error(404, 'Voice sample not found');
}
log.error('presign', 'presign voice sample failed', { voice, err: String(e) });
error(502, `Failed to presign voice sample: ${e}`);
}
};

View File

@@ -1,11 +1,17 @@
<script lang="ts">
import { untrack } from 'svelte';
import { onMount, untrack } from 'svelte';
import { invalidateAll } from '$app/navigation';
import type { PageData } from './$types';
import CommentsSection from '$lib/components/CommentsSection.svelte';
let { data }: { data: PageData } = $props();
onMount(() => {
if (data.book?.slug) {
window.umami?.track('book_view', { slug: data.book.slug });
}
});
// ── Save / unsave ─────────────────────────────────────────────────────────
let saved = $state(untrack(() => data.saved));
let saving = $state(false);

View File

@@ -19,6 +19,12 @@
const wordCount = $derived(countWords(html));
onMount(async () => {
// Umami analytics: track chapter reads
window.umami?.track('chapter_read', {
slug: data.book.slug,
chapter: data.chapter.number
});
// Record reading progress (skip for preview chapters)
if (!data.isPreview) {
try {