Compare commits
4 Commits
b9b69cee44
...
1118392811
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1118392811 | ||
|
|
baa403efa2 | ||
|
|
0ed1112b20 | ||
|
|
16a12ede4d |
5
v3/.gitignore
vendored
Normal file
5
v3/.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
# CrowdSec — generated bouncer API key; never commit
|
||||
crowdsec/.crowdsec.env
|
||||
|
||||
# Local env overrides
|
||||
.env.local
|
||||
Binary file not shown.
@@ -2,7 +2,7 @@
|
||||
//
|
||||
// It exposes all endpoints consumed by the SvelteKit UI: book/chapter reads,
|
||||
// scrape-task creation, presigned MinIO URLs, audio-task creation, reading
|
||||
// progress, live novelfire.net browse/search, and Kokoro voice list.
|
||||
// progress, live novelfire.net search, and Kokoro voice list.
|
||||
//
|
||||
// All heavy lifting (scraping, TTS generation) is delegated to the runner
|
||||
// binary via PocketBase task records. The backend never scrapes directly.
|
||||
@@ -19,7 +19,9 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/libnovel/backend/internal/backend"
|
||||
"github.com/libnovel/backend/internal/config"
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
@@ -43,6 +45,19 @@ func main() {
|
||||
func run() error {
|
||||
cfg := config.Load()
|
||||
|
||||
// ── Sentry / GlitchTip error tracking ────────────────────────────────────
|
||||
if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" {
|
||||
if err := sentry.Init(sentry.ClientOptions{
|
||||
Dsn: dsn,
|
||||
Release: version + "@" + commit,
|
||||
TracesSampleRate: 0.1,
|
||||
}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "backend: sentry init warning: %v\n", err)
|
||||
} else {
|
||||
defer sentry.Flush(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Logger ───────────────────────────────────────────────────────────────
|
||||
log := buildLogger(cfg.LogLevel)
|
||||
log.Info("backend starting",
|
||||
@@ -95,7 +110,6 @@ func run() error {
|
||||
AudioStore: store,
|
||||
PresignStore: store,
|
||||
ProgressStore: store,
|
||||
BrowseStore: store,
|
||||
CoverStore: store,
|
||||
Producer: store,
|
||||
TaskReader: store,
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/libnovel/backend/internal/browser"
|
||||
"github.com/libnovel/backend/internal/config"
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
@@ -44,6 +45,19 @@ func main() {
|
||||
func run() error {
|
||||
cfg := config.Load()
|
||||
|
||||
// ── Sentry / GlitchTip error tracking ────────────────────────────────────
|
||||
if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" {
|
||||
if err := sentry.Init(sentry.ClientOptions{
|
||||
Dsn: dsn,
|
||||
Release: version + "@" + commit,
|
||||
TracesSampleRate: 0.1,
|
||||
}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "runner: sentry init warning: %v\n", err)
|
||||
} else {
|
||||
defer sentry.Flush(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Logger ──────────────────────────────────────────────────────────────
|
||||
log := buildLogger(cfg.LogLevel)
|
||||
log.Info("runner starting",
|
||||
@@ -105,21 +119,20 @@ func run() error {
|
||||
|
||||
// ── Runner ──────────────────────────────────────────────────────────────
|
||||
rCfg := runner.Config{
|
||||
WorkerID: cfg.Runner.WorkerID,
|
||||
PollInterval: cfg.Runner.PollInterval,
|
||||
MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape,
|
||||
MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio,
|
||||
OrchestratorWorkers: workers,
|
||||
MetricsAddr: cfg.Runner.MetricsAddr,
|
||||
CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval,
|
||||
SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh,
|
||||
WorkerID: cfg.Runner.WorkerID,
|
||||
PollInterval: cfg.Runner.PollInterval,
|
||||
MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape,
|
||||
MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio,
|
||||
OrchestratorWorkers: workers,
|
||||
MetricsAddr: cfg.Runner.MetricsAddr,
|
||||
CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval,
|
||||
SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh,
|
||||
}
|
||||
deps := runner.Dependencies{
|
||||
Consumer: store,
|
||||
BookWriter: store,
|
||||
BookReader: store,
|
||||
AudioStore: store,
|
||||
BrowseStore: store,
|
||||
CoverStore: store,
|
||||
SearchIndex: searchIndex,
|
||||
Novel: novel,
|
||||
|
||||
@@ -13,6 +13,7 @@ require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/getsentry/sentry-go v0.43.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
|
||||
@@ -8,6 +8,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
|
||||
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
@@ -57,5 +59,6 @@ golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
Binary file not shown.
@@ -176,80 +176,6 @@ type NovelListing struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// handleBrowse handles GET /api/browse.
|
||||
// Fetches novelfire.net live (no MinIO cache in the new backend).
|
||||
// Query params: page (default 1), genre (default "all"), sort (default "popular"),
|
||||
// status (default "all"), type (default "all-novel")
|
||||
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page := q.Get("page")
|
||||
if page == "" {
|
||||
page = "1"
|
||||
}
|
||||
genre := q.Get("genre")
|
||||
if genre == "" {
|
||||
genre = "all"
|
||||
}
|
||||
sortBy := q.Get("sort")
|
||||
if sortBy == "" {
|
||||
sortBy = "popular"
|
||||
}
|
||||
status := q.Get("status")
|
||||
if status == "" {
|
||||
status = "all"
|
||||
}
|
||||
novelType := q.Get("type")
|
||||
if novelType == "" {
|
||||
novelType = "all-novel"
|
||||
}
|
||||
|
||||
pageNum, _ := strconv.Atoi(page)
|
||||
if pageNum <= 0 {
|
||||
pageNum = 1
|
||||
}
|
||||
|
||||
// ── Try MinIO cache first ─────────────────────────────────────────────
|
||||
// Only page 1 is cached; higher pages fall through to live fetch.
|
||||
if pageNum == 1 && s.deps.BrowseStore != nil {
|
||||
if data, ok, err := s.deps.BrowseStore.GetBrowsePage(r.Context(), genre, sortBy, status, novelType, 1); err == nil && ok {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
_, _ = w.Write(data)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ── Fall back to live novelfire.net fetch ──────────────────────────────
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
|
||||
defer cancel()
|
||||
|
||||
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d",
|
||||
novelFireBase, genre, sortBy, status, novelType, pageNum)
|
||||
|
||||
novels, hasNext, err := s.fetchBrowsePage(ctx, targetURL)
|
||||
if err != nil {
|
||||
// Live fetch also failed — return empty list with cached=false flag so
|
||||
// the UI can show a "not ready yet" state instead of a hard error.
|
||||
s.deps.Log.Error("handleBrowse: fetch failed (no cache)", "url", targetURL, "err", err)
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, 0, map[string]any{
|
||||
"novels": []any{},
|
||||
"page": pageNum,
|
||||
"hasNext": false,
|
||||
"cached": false,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
writeJSON(w, 0, map[string]any{
|
||||
"novels": novels,
|
||||
"page": pageNum,
|
||||
"hasNext": hasNext,
|
||||
"cached": false,
|
||||
})
|
||||
}
|
||||
|
||||
// handleSearch handles GET /api/search.
|
||||
// Query params: q (min 2 chars), source ("local"|"remote"|"all", default "all")
|
||||
//
|
||||
@@ -832,7 +758,13 @@ func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 0, map[string]string{"url": u})
|
||||
}
|
||||
|
||||
// voiceSampleText is the phrase synthesised for every voice sample.
|
||||
const voiceSampleText = "Hello! This is a preview of what I sound like. I hope you enjoy listening to your stories with my voice."
|
||||
|
||||
// handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}.
|
||||
// If the sample has not been generated yet it synthesises it on the fly via
|
||||
// Kokoro, stores the result in MinIO, and returns the presigned URL — so the
|
||||
// caller always gets a playable URL in a single request.
|
||||
func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) {
|
||||
voice := r.PathValue("voice")
|
||||
if voice == "" {
|
||||
@@ -841,9 +773,21 @@ func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
|
||||
key := kokoro.VoiceSampleKey(voice)
|
||||
|
||||
// Generate sample on demand when it is not in MinIO yet.
|
||||
if !s.deps.AudioStore.AudioExists(r.Context(), key) {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
s.deps.Log.Info("generating voice sample on demand", "voice", voice)
|
||||
mp3, err := s.deps.Kokoro.GenerateAudio(r.Context(), voiceSampleText, voice)
|
||||
if err != nil {
|
||||
s.deps.Log.Error("voice sample generation failed", "voice", voice, "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "voice sample generation failed")
|
||||
return
|
||||
}
|
||||
if err := s.deps.AudioStore.PutAudio(r.Context(), key, mp3); err != nil {
|
||||
s.deps.Log.Error("voice sample upload failed", "voice", voice, "err", err)
|
||||
jsonError(w, http.StatusInternalServerError, "voice sample upload failed")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour)
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
// picks up and executes those tasks asynchronously
|
||||
// - Presigned MinIO URLs for media playback/upload
|
||||
// - Session-scoped reading progress
|
||||
// - Live novelfire.net browse/search (no scraper interface needed; direct HTTP)
|
||||
// - Live novelfire.net search (no scraper interface needed; direct HTTP)
|
||||
// - Kokoro voice list
|
||||
//
|
||||
// The backend never scrapes directly. All scraping (metadata, chapter list,
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sentryhttp "github.com/getsentry/sentry-go/http"
|
||||
"github.com/libnovel/backend/internal/bookstore"
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
"github.com/libnovel/backend/internal/meili"
|
||||
@@ -47,8 +48,6 @@ type Dependencies struct {
|
||||
PresignStore bookstore.PresignStore
|
||||
// ProgressStore reads/writes per-session reading progress.
|
||||
ProgressStore bookstore.ProgressStore
|
||||
// BrowseStore reads cached browse page snapshots from MinIO.
|
||||
BrowseStore bookstore.BrowseStore
|
||||
// CoverStore reads and writes book cover images from MinIO.
|
||||
// If nil, the cover endpoint falls back to a CDN redirect.
|
||||
CoverStore bookstore.CoverStore
|
||||
@@ -122,8 +121,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
// Cancel a pending task (scrape or audio)
|
||||
mux.HandleFunc("POST /api/cancel-task/{id}", s.handleCancelTask)
|
||||
|
||||
// Browse & search (live novelfire.net)
|
||||
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
||||
// Browse & search
|
||||
mux.HandleFunc("GET /api/search", s.handleSearch)
|
||||
|
||||
// Catalogue (Meilisearch-backed browse + search — preferred path for UI)
|
||||
@@ -174,7 +172,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: s.cfg.Addr,
|
||||
Handler: mux,
|
||||
Handler: sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle(mux),
|
||||
ReadTimeout: 15 * time.Second,
|
||||
WriteTimeout: 60 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
|
||||
@@ -128,18 +128,6 @@ type ProgressStore interface {
|
||||
DeleteProgress(ctx context.Context, sessionID, slug string) error
|
||||
}
|
||||
|
||||
// BrowseStore covers browse page snapshot storage.
|
||||
// The runner writes snapshots; the backend reads them.
|
||||
type BrowseStore interface {
|
||||
// PutBrowsePage stores a raw JSON snapshot for a browse page.
|
||||
// genre, sort, status, novelType and page identify the page.
|
||||
PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error
|
||||
|
||||
// GetBrowsePage retrieves a raw JSON snapshot. Returns (nil, false, nil)
|
||||
// when no snapshot exists for the given parameters.
|
||||
GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error)
|
||||
}
|
||||
|
||||
// CoverStore covers book cover image storage in MinIO.
|
||||
// The runner writes covers during catalogue refresh; the backend reads them.
|
||||
type CoverStore interface {
|
||||
|
||||
@@ -145,10 +145,10 @@ func Load() Config {
|
||||
SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"),
|
||||
UseSSL: envBool("MINIO_USE_SSL", false),
|
||||
PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true),
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
|
||||
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "chapters"),
|
||||
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "audio"),
|
||||
BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "avatars"),
|
||||
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
|
||||
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "catalogue"),
|
||||
},
|
||||
|
||||
Kokoro: Kokoro{
|
||||
|
||||
@@ -33,8 +33,8 @@ func TestLoad_Defaults(t *testing.T) {
|
||||
if cfg.PocketBase.URL != "http://localhost:8090" {
|
||||
t.Errorf("PocketBase.URL: want http://localhost:8090, got %q", cfg.PocketBase.URL)
|
||||
}
|
||||
if cfg.MinIO.BucketChapters != "libnovel-chapters" {
|
||||
t.Errorf("MinIO.BucketChapters: want libnovel-chapters, got %q", cfg.MinIO.BucketChapters)
|
||||
if cfg.MinIO.BucketChapters != "chapters" {
|
||||
t.Errorf("MinIO.BucketChapters: want chapters, got %q", cfg.MinIO.BucketChapters)
|
||||
}
|
||||
if cfg.MinIO.UseSSL != false {
|
||||
t.Errorf("MinIO.UseSSL: want false, got %v", cfg.MinIO.UseSSL)
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
package runner
|
||||
|
||||
// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
|
||||
// browse page snapshots and stores them in MinIO.
|
||||
//
|
||||
// Design:
|
||||
// - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
|
||||
// - Fetches page 1 for each combination of the standard genre/sort/status
|
||||
// filter values and stores the parsed JSON blob in MinIO via BrowseStore.
|
||||
// - The backend's handleBrowse then serves from MinIO instead of calling
|
||||
// novelfire.net live, which avoids IP-based rate-limiting on the server.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
|
||||
type browseNovelListing struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Cover string `json:"cover"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// browseSnapshot is the JSON structure stored in MinIO.
|
||||
type browseSnapshot struct {
|
||||
Novels []browseNovelListing `json:"novels"`
|
||||
Page int `json:"page"`
|
||||
HasNext bool `json:"hasNext"`
|
||||
// CachedAt is the UTC time the snapshot was written (ISO 8601).
|
||||
CachedAt string `json:"cachedAt"`
|
||||
}
|
||||
|
||||
// browseCombos lists the filter combinations to pre-fetch.
|
||||
// Each entry is (genre, sort, status, novelType).
|
||||
var browseCombos = []struct{ genre, sort, status, novelType string }{
|
||||
{"all", "popular", "all", "all-novel"},
|
||||
{"all", "popular", "ongoing", "all-novel"},
|
||||
{"all", "popular", "completed", "all-novel"},
|
||||
{"all", "new", "all", "all-novel"},
|
||||
{"all", "new", "ongoing", "all-novel"},
|
||||
{"all", "new", "completed", "all-novel"},
|
||||
{"all", "top-rated", "all", "all-novel"},
|
||||
{"all", "top-rated", "ongoing", "all-novel"},
|
||||
{"all", "top-rated", "completed", "all-novel"},
|
||||
}
|
||||
|
||||
const novelFireBrowseBase = "https://novelfire.net"
|
||||
|
||||
// runBrowseRefresh fetches all browse combos from novelfire.net and stores
|
||||
// the results in MinIO. Errors per-combo are logged but do not abort the
|
||||
// whole refresh cycle.
|
||||
func (r *Runner) runBrowseRefresh(ctx context.Context) {
|
||||
if r.deps.BrowseStore == nil {
|
||||
r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
|
||||
return
|
||||
}
|
||||
|
||||
log := r.deps.Log.With("op", "browse_refresh")
|
||||
log.Info("runner: browse refresh starting", "combos", len(browseCombos))
|
||||
|
||||
ok, fail := 0, 0
|
||||
for _, c := range browseCombos {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
|
||||
if err != nil {
|
||||
log.Warn("runner: browse fetch failed",
|
||||
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
|
||||
fail++
|
||||
continue
|
||||
}
|
||||
|
||||
snap := browseSnapshot{
|
||||
Novels: novels,
|
||||
Page: 1,
|
||||
HasNext: hasNext,
|
||||
CachedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
data, _ := json.Marshal(snap)
|
||||
if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
|
||||
log.Warn("runner: browse put failed",
|
||||
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
|
||||
fail++
|
||||
continue
|
||||
}
|
||||
ok++
|
||||
}
|
||||
|
||||
log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
|
||||
}
|
||||
|
||||
// fetchBrowsePage calls novelfire.net and returns a list of novel listings
|
||||
// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
|
||||
func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
|
||||
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
|
||||
novelFireBrowseBase, genre, sort, status, novelType)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
|
||||
httpClient := &http.Client{Timeout: 45 * time.Second}
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
|
||||
}
|
||||
|
||||
return parseBrowseHTML(resp.Body)
|
||||
}
|
||||
|
||||
// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
|
||||
// in backend/handlers.go — kept separate to avoid coupling packages.
|
||||
func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
body := string(data)
|
||||
|
||||
hasNext := strings.Contains(body, `rel="next"`) ||
|
||||
strings.Contains(body, `aria-label="Next"`) ||
|
||||
strings.Contains(body, `class="next"`)
|
||||
|
||||
slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
|
||||
titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
|
||||
coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
|
||||
|
||||
slugMatches := slugRe.FindAllStringSubmatch(body, -1)
|
||||
titleMatches := titleRe.FindAllStringSubmatch(body, -1)
|
||||
coverMatches := coverRe.FindAllStringSubmatch(body, -1)
|
||||
|
||||
var novels []browseNovelListing
|
||||
seen := make(map[string]bool)
|
||||
for i, sm := range slugMatches {
|
||||
slug := sm[1]
|
||||
if seen[slug] {
|
||||
continue
|
||||
}
|
||||
seen[slug] = true
|
||||
|
||||
item := browseNovelListing{
|
||||
Slug: slug,
|
||||
URL: novelFireBrowseBase + "/book/" + slug,
|
||||
}
|
||||
if i < len(titleMatches) {
|
||||
item.Title = strings.TrimSpace(titleMatches[i][1])
|
||||
}
|
||||
if i < len(coverMatches) {
|
||||
item.Cover = coverMatches[i][1]
|
||||
}
|
||||
if item.Title != "" {
|
||||
novels = append(novels, item)
|
||||
}
|
||||
}
|
||||
|
||||
return novels, hasNext, nil
|
||||
}
|
||||
@@ -49,9 +49,6 @@ type Config struct {
|
||||
// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
|
||||
// task is considered orphaned and reset to pending. Defaults to 2m when 0.
|
||||
StaleTaskThreshold time.Duration
|
||||
// BrowseRefreshInterval is how often the runner pre-fetches browse page
|
||||
// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
|
||||
BrowseRefreshInterval time.Duration
|
||||
// CatalogueRefreshInterval is how often the runner walks the full catalogue,
|
||||
// scrapes per-book metadata, downloads covers, and re-indexes everything in
|
||||
// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
|
||||
@@ -76,10 +73,7 @@ type Dependencies struct {
|
||||
BookReader bookstore.BookReader
|
||||
// AudioStore persists generated audio and checks key existence.
|
||||
AudioStore bookstore.AudioStore
|
||||
// BrowseStore stores browse page snapshots in MinIO.
|
||||
BrowseStore bookstore.BrowseStore
|
||||
// CoverStore stores book cover images in MinIO.
|
||||
// If nil, cover downloads are skipped during catalogue refresh.
|
||||
CoverStore bookstore.CoverStore
|
||||
// SearchIndex indexes books in Meilisearch after scraping.
|
||||
// If nil a no-op is used.
|
||||
@@ -125,9 +119,6 @@ func New(cfg Config, deps Dependencies) *Runner {
|
||||
if cfg.StaleTaskThreshold <= 0 {
|
||||
cfg.StaleTaskThreshold = 2 * time.Minute
|
||||
}
|
||||
if cfg.BrowseRefreshInterval <= 0 {
|
||||
cfg.BrowseRefreshInterval = 6 * time.Hour
|
||||
}
|
||||
if cfg.CatalogueRefreshInterval <= 0 {
|
||||
cfg.CatalogueRefreshInterval = 24 * time.Hour
|
||||
}
|
||||
@@ -151,7 +142,6 @@ func (r *Runner) Run(ctx context.Context) error {
|
||||
"poll_interval", r.cfg.PollInterval,
|
||||
"max_scrape", r.cfg.MaxConcurrentScrape,
|
||||
"max_audio", r.cfg.MaxConcurrentAudio,
|
||||
"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
|
||||
"catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval,
|
||||
"metrics_addr", r.cfg.MetricsAddr,
|
||||
)
|
||||
@@ -173,14 +163,9 @@ func (r *Runner) Run(ctx context.Context) error {
|
||||
tick := time.NewTicker(r.cfg.PollInterval)
|
||||
defer tick.Stop()
|
||||
|
||||
browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
|
||||
defer browseTick.Stop()
|
||||
|
||||
catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval)
|
||||
defer catalogueTick.Stop()
|
||||
|
||||
// Run one browse refresh immediately on startup.
|
||||
go r.runBrowseRefresh(ctx)
|
||||
// Run one catalogue refresh immediately on startup (unless skipped by flag).
|
||||
if !r.cfg.SkipInitialCatalogueRefresh {
|
||||
go r.runCatalogueRefresh(ctx)
|
||||
@@ -207,8 +192,6 @@ func (r *Runner) Run(ctx context.Context) error {
|
||||
r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
|
||||
}
|
||||
return nil
|
||||
case <-browseTick.C:
|
||||
go r.runBrowseRefresh(ctx)
|
||||
case <-catalogueTick.C:
|
||||
go r.runCatalogueRefresh(ctx)
|
||||
case <-tick.C:
|
||||
|
||||
@@ -119,12 +119,6 @@ func AvatarObjectKey(userID, ext string) string {
|
||||
return fmt.Sprintf("%s/%s.%s", userID, ext, ext)
|
||||
}
|
||||
|
||||
// BrowseObjectKey returns the MinIO object key for a cached browse page snapshot.
|
||||
// Format: browse/{genre}/{sort}/{status}/{type}/page-{n}.json
|
||||
func BrowseObjectKey(genre, sort, status, novelType string, page int) string {
|
||||
return fmt.Sprintf("browse/%s/%s/%s/%s/page-%d.json", genre, sort, status, novelType, page)
|
||||
}
|
||||
|
||||
// CoverObjectKey returns the MinIO object key for a book cover image.
|
||||
// Format: covers/{slug}.jpg
|
||||
func CoverObjectKey(slug string) string {
|
||||
@@ -207,26 +201,6 @@ func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string)
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// ── Browse operations ─────────────────────────────────────────────────────────
|
||||
|
||||
// putBrowse stores raw JSON bytes for a browse page snapshot.
|
||||
func (m *minioClient) putBrowse(ctx context.Context, key string, data []byte) error {
|
||||
return m.putObject(ctx, m.bucketBrowse, key, "application/json", data)
|
||||
}
|
||||
|
||||
// getBrowse retrieves a browse page snapshot. Returns (nil, false, nil) when
|
||||
// the object does not exist.
|
||||
func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool, error) {
|
||||
if !m.objectExists(ctx, m.bucketBrowse, key) {
|
||||
return nil, false, nil
|
||||
}
|
||||
data, err := m.getObject(ctx, m.bucketBrowse, key)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return data, true, nil
|
||||
}
|
||||
|
||||
// ── Cover operations ──────────────────────────────────────────────────────────
|
||||
|
||||
// putCover stores a raw cover image in the browse bucket under covers/{slug}.jpg.
|
||||
|
||||
@@ -50,7 +50,6 @@ var _ bookstore.RankingStore = (*Store)(nil)
|
||||
var _ bookstore.AudioStore = (*Store)(nil)
|
||||
var _ bookstore.PresignStore = (*Store)(nil)
|
||||
var _ bookstore.ProgressStore = (*Store)(nil)
|
||||
var _ bookstore.BrowseStore = (*Store)(nil)
|
||||
var _ bookstore.CoverStore = (*Store)(nil)
|
||||
var _ taskqueue.Producer = (*Store)(nil)
|
||||
var _ taskqueue.Consumer = (*Store)(nil)
|
||||
@@ -790,25 +789,6 @@ func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ── BrowseStore ────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error {
|
||||
key := BrowseObjectKey(genre, sort, status, novelType, page)
|
||||
if err := s.mc.putBrowse(ctx, key, data); err != nil {
|
||||
return fmt.Errorf("PutBrowsePage: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) {
|
||||
key := BrowseObjectKey(genre, sort, status, novelType, page)
|
||||
data, ok, err := s.mc.getBrowse(ctx, key)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("GetBrowsePage: %w", err)
|
||||
}
|
||||
return data, ok, nil
|
||||
}
|
||||
|
||||
// ── CoverStore ─────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *Store) PutCover(ctx context.Context, slug string, data []byte, contentType string) error {
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
FROM caddy:2-builder AS builder
|
||||
|
||||
RUN xcaddy build \
|
||||
--with github.com/mholt/caddy-ratelimit
|
||||
--with github.com/mholt/caddy-ratelimit \
|
||||
--with github.com/hslatman/caddy-crowdsec-bouncer/http
|
||||
|
||||
FROM caddy:2-alpine
|
||||
COPY --from=builder /usr/bin/caddy /usr/bin/caddy
|
||||
|
||||
12
v3/crowdsec/acquis.yaml
Normal file
12
v3/crowdsec/acquis.yaml
Normal file
@@ -0,0 +1,12 @@
|
||||
# CrowdSec log acquisition — tells the CrowdSec agent which logs to parse.
|
||||
#
|
||||
# Caddy writes JSON access logs to /var/log/caddy/access.log (mounted from the
|
||||
# caddy_logs Docker volume). CrowdSec reads the same volume at the same path.
|
||||
#
|
||||
# The `crowdsecurity/caddy` collection (installed via COLLECTIONS env var)
|
||||
# provides the parser that understands Caddy's JSON log format.
|
||||
|
||||
filenames:
|
||||
- /var/log/caddy/access.log
|
||||
labels:
|
||||
type: caddy
|
||||
@@ -171,7 +171,7 @@ client -> caddy: HTTPS :443
|
||||
|
||||
caddy -> sk: "/* (catch-all)\n→ SvelteKit handles auth"
|
||||
caddy -> be: "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/* /api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/* /api/catalogue /api/ranking"
|
||||
caddy -> storage.mn: "/avatars/*\n(presigned GETs)"
|
||||
caddy -> storage.mn: "/avatars/*\n/audio/*\n/chapters/*\n(presigned MinIO GETs)"
|
||||
|
||||
# ─── SvelteKit → Backend (server-side proxy) ──────────────────────────────────
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 57 KiB After Width: | Height: | Size: 57 KiB |
@@ -46,7 +46,7 @@ storage: Storage {
|
||||
|
||||
minio: MinIO {
|
||||
shape: cylinder
|
||||
label: "MinIO :9000\n\nbuckets:\n libnovel-chapters\n libnovel-audio\n avatars\n libnovel-browse"
|
||||
label: "MinIO :9000\n\nbuckets:\n chapters\n audio\n avatars\n catalogue"
|
||||
}
|
||||
|
||||
pocketbase: PocketBase {
|
||||
@@ -135,7 +135,7 @@ app.ui -> app.backend: REST API calls (server-side)\n/api/catalogue /api/book-pr
|
||||
|
||||
app.caddy -> app.ui: "/* (catch-all)\n/api/scrape/*\n/api/chapter-text-preview/*\n→ SvelteKit (auth enforced)"
|
||||
app.caddy -> app.backend: "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*"
|
||||
app.caddy -> storage.minio: "/avatars/*\n(presigned avatar GETs)"
|
||||
app.caddy -> storage.minio: "/avatars/*\n/audio/*\n/chapters/*\n(presigned MinIO GETs)"
|
||||
|
||||
# ─── External → App ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 58 KiB After Width: | Height: | Size: 58 KiB |
@@ -22,7 +22,7 @@ flowchart TD
|
||||
H --> I[Worker pool — N goroutines\nRUNNER_MAX_CONCURRENT_SCRAPE]
|
||||
I --> J[For each missing chapter:\nGET chapter HTML from novelfire.net]
|
||||
J --> K[Parse HTML → Markdown\nhtmlutil.NodeToMarkdown]
|
||||
K --> L[PUT object to MinIO\nlibnovel-chapters/{slug}/{n}.md]
|
||||
K --> L[PUT object to MinIO\nchapters/{slug}/{n}.md]
|
||||
L --> M[Upsert book doc\nto Meilisearch index: books]
|
||||
M --> Z
|
||||
F --> M
|
||||
@@ -87,7 +87,7 @@ flowchart TD
|
||||
H --> I["GET /api/chapter-text/{slug}/{n}\nfrom backend — plain text"]
|
||||
I --> J[POST /v1/audio/speech\nto Kokoro-FastAPI\nbody: text + voice]
|
||||
J --> K[Stream MP3 response]
|
||||
K --> L[PUT object to MinIO\nlibnovel-audio/{slug}/{n}/{voice}.mp3]
|
||||
K --> L[PUT object to MinIO\naudio/{slug}/{n}/{voice}.mp3]
|
||||
L --> M[UPDATE audio_task\nstatus=done]
|
||||
|
||||
N(["Client polls\nGET /api/audio/status/{slug}/{n}"]) --> O{status?}
|
||||
|
||||
2245
v3/ui/package-lock.json
generated
2245
v3/ui/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,7 @@
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1005.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.1005.0",
|
||||
"@sentry/sveltekit": "^10.45.0",
|
||||
"cropperjs": "^1.6.2",
|
||||
"ioredis": "^5.3.2",
|
||||
"marked": "^17.0.3",
|
||||
|
||||
8
v3/ui/src/app.d.ts
vendored
8
v3/ui/src/app.d.ts
vendored
@@ -13,6 +13,14 @@ declare global {
|
||||
// interface PageState {}
|
||||
// interface Platform {}
|
||||
}
|
||||
|
||||
// Umami analytics — injected by the script tag in +layout.svelte when
|
||||
// PUBLIC_UMAMI_WEBSITE_ID is set. Optional so calls are no-ops in dev.
|
||||
interface Window {
|
||||
umami?: {
|
||||
track(event: string, data?: Record<string, unknown>): void;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export {};
|
||||
|
||||
13
v3/ui/src/hooks.client.ts
Normal file
13
v3/ui/src/hooks.client.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import * as Sentry from '@sentry/sveltekit';
|
||||
import { env } from '$env/dynamic/public';
|
||||
|
||||
// Sentry / GlitchTip client-side error tracking.
|
||||
// No-op when PUBLIC_GLITCHTIP_DSN is unset (e.g. local dev).
|
||||
if (env.PUBLIC_GLITCHTIP_DSN) {
|
||||
Sentry.init({
|
||||
dsn: env.PUBLIC_GLITCHTIP_DSN,
|
||||
tracesSampleRate: 0.1
|
||||
});
|
||||
}
|
||||
|
||||
export const handleError = Sentry.handleErrorWithSentry();
|
||||
@@ -1,10 +1,24 @@
|
||||
import type { Handle } from '@sveltejs/kit';
|
||||
import { handleErrorWithSentry } from '@sentry/sveltekit';
|
||||
import * as Sentry from '@sentry/sveltekit';
|
||||
import { randomBytes, createHmac } from 'node:crypto';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { env as pubEnv } from '$env/dynamic/public';
|
||||
import { log } from '$lib/server/logger';
|
||||
import { createUserSession, touchUserSession, isSessionRevoked } from '$lib/server/pocketbase';
|
||||
import { drain as drainPresignCache } from '$lib/server/presignCache';
|
||||
|
||||
// ─── Sentry / GlitchTip server-side error tracking ────────────────────────────
|
||||
// No-op when PUBLIC_GLITCHTIP_DSN is unset (e.g. local dev).
|
||||
if (pubEnv.PUBLIC_GLITCHTIP_DSN) {
|
||||
Sentry.init({
|
||||
dsn: pubEnv.PUBLIC_GLITCHTIP_DSN,
|
||||
tracesSampleRate: 0.1
|
||||
});
|
||||
}
|
||||
|
||||
export const handleError = handleErrorWithSentry();
|
||||
|
||||
// ─── Graceful shutdown ────────────────────────────────────────────────────────
|
||||
//
|
||||
// When Docker/Kubernetes sends SIGTERM (or the user sends SIGINT), we:
|
||||
|
||||
@@ -327,7 +327,9 @@
|
||||
voice: targetVoice
|
||||
});
|
||||
const res = await fetch(`/api/presign/audio?${params}`);
|
||||
if (res.status === 404) return null;
|
||||
// 202: TTS was just enqueued by the presign endpoint — audio not ready yet.
|
||||
// 404: legacy fallback (should no longer occur after endpoint change).
|
||||
if (res.status === 202 || res.status === 404) return null;
|
||||
if (!res.ok) throw new Error(`presign HTTP ${res.status}`);
|
||||
const data = (await res.json()) as { url: string };
|
||||
return data.url;
|
||||
@@ -608,6 +610,11 @@
|
||||
|
||||
// Not yet loaded — start the full flow.
|
||||
await startPlayback();
|
||||
|
||||
// Track audio play after successful load.
|
||||
if (audioStore.status === 'ready') {
|
||||
window.umami?.track('audio_played', { slug, chapter });
|
||||
}
|
||||
}
|
||||
|
||||
function formatTime(s: number): string {
|
||||
|
||||
@@ -662,6 +662,8 @@ export async function setAudioTime(
|
||||
}
|
||||
|
||||
// ─── Audio cache ──────────────────────────────────────────────────────────────
|
||||
// There is no separate audio_cache collection — completed audio jobs in the
|
||||
// audio_jobs collection serve as the cache record. We project them here.
|
||||
|
||||
export interface AudioCacheEntry {
|
||||
id: string;
|
||||
@@ -671,7 +673,13 @@ export interface AudioCacheEntry {
|
||||
}
|
||||
|
||||
export async function listAudioCache(): Promise<AudioCacheEntry[]> {
|
||||
return listAll<AudioCacheEntry>('audio_cache', '', '-updated');
|
||||
const jobs = await listAll<AudioJob>('audio_jobs', 'status="done"', '-finished');
|
||||
return jobs.map((j) => ({
|
||||
id: j.id,
|
||||
cache_key: j.cache_key,
|
||||
filename: `${j.cache_key}.mp3`,
|
||||
updated: j.finished
|
||||
}));
|
||||
}
|
||||
|
||||
// ─── Scraping tasks ───────────────────────────────────────────────────────────
|
||||
|
||||
@@ -170,6 +170,14 @@
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>libnovel</title>
|
||||
<!-- Umami analytics — no-op when PUBLIC_UMAMI_WEBSITE_ID is unset -->
|
||||
{#if env.PUBLIC_UMAMI_WEBSITE_ID}
|
||||
<script
|
||||
defer
|
||||
src="https://analytics.libnovel.cc/script.js"
|
||||
data-website-id={env.PUBLIC_UMAMI_WEBSITE_ID}
|
||||
></script>
|
||||
{/if}
|
||||
</svelte:head>
|
||||
|
||||
<!-- Persistent audio element — always in the DOM, never conditionally unmounted.
|
||||
@@ -232,12 +240,20 @@
|
||||
>
|
||||
Library
|
||||
</a>
|
||||
<a
|
||||
href="/catalogue"
|
||||
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/catalogue') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
|
||||
>
|
||||
Discover
|
||||
</a>
|
||||
<a
|
||||
href="/catalogue"
|
||||
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/catalogue') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
|
||||
>
|
||||
Discover
|
||||
</a>
|
||||
<a
|
||||
href="https://feedback.libnovel.cc"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="hidden sm:block text-sm transition-colors text-zinc-400 hover:text-zinc-100"
|
||||
>
|
||||
Feedback
|
||||
</a>
|
||||
|
||||
<div class="ml-auto flex items-center gap-4">
|
||||
<!-- Desktop: admin + profile + sign out (hidden on mobile) -->
|
||||
@@ -311,13 +327,22 @@
|
||||
>
|
||||
Library
|
||||
</a>
|
||||
<a
|
||||
href="/catalogue"
|
||||
onclick={() => (menuOpen = false)}
|
||||
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/catalogue') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
|
||||
>
|
||||
Discover
|
||||
</a>
|
||||
<a
|
||||
href="/catalogue"
|
||||
onclick={() => (menuOpen = false)}
|
||||
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/catalogue') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
|
||||
>
|
||||
Discover
|
||||
</a>
|
||||
<a
|
||||
href="https://feedback.libnovel.cc"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
onclick={() => (menuOpen = false)}
|
||||
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100"
|
||||
>
|
||||
Feedback ↗
|
||||
</a>
|
||||
<a
|
||||
href="/profile"
|
||||
onclick={() => (menuOpen = false)}
|
||||
@@ -374,10 +399,22 @@
|
||||
<div class="max-w-6xl mx-auto px-4 py-6 flex flex-col items-center gap-4 text-xs text-zinc-600">
|
||||
<!-- Top row: site links -->
|
||||
<nav class="flex flex-wrap items-center justify-center gap-x-5 gap-y-2">
|
||||
<a href="/books" class="hover:text-zinc-400 transition-colors">Library</a>
|
||||
<a href="/catalogue" class="hover:text-zinc-400 transition-colors">Discover</a>
|
||||
<a
|
||||
href="https://novelfire.net"
|
||||
<a href="/books" class="hover:text-zinc-400 transition-colors">Library</a>
|
||||
<a href="/catalogue" class="hover:text-zinc-400 transition-colors">Discover</a>
|
||||
<a
|
||||
href="https://feedback.libnovel.cc"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="hover:text-zinc-400 transition-colors flex items-center gap-1"
|
||||
>
|
||||
Feedback
|
||||
<svg class="w-3 h-3" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M10 6H6a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2v-4M14 4h6m0 0v6m0-6L10 14" />
|
||||
</svg>
|
||||
</a>
|
||||
<a
|
||||
href="https://novelfire.net"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="hover:text-zinc-400 transition-colors flex items-center gap-1"
|
||||
|
||||
@@ -32,6 +32,33 @@
|
||||
tasks = data.tasks;
|
||||
});
|
||||
|
||||
// ── Full catalogue scrape ───────────────────────────────────────────────────
|
||||
let catalogueError = $state('');
|
||||
let cataloguing = $state(false);
|
||||
|
||||
async function triggerCatalogueScrape() {
|
||||
if (running || cataloguing) return;
|
||||
cataloguing = true;
|
||||
catalogueError = '';
|
||||
try {
|
||||
const res = await fetch('/api/scrape', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: '{}'
|
||||
});
|
||||
if (!res.ok) {
|
||||
const d = await res.json().catch(() => ({}));
|
||||
catalogueError = d.error ?? d.message ?? `Error ${res.status}`;
|
||||
} else {
|
||||
running = true;
|
||||
}
|
||||
} catch {
|
||||
catalogueError = 'Network error.';
|
||||
} finally {
|
||||
cataloguing = false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Single book scrape ──────────────────────────────────────────────────────
|
||||
let scrapeUrl = $state('');
|
||||
let scrapeError = $state('');
|
||||
@@ -114,13 +141,14 @@
|
||||
}
|
||||
|
||||
function retryTask(task: ScrapingTask) {
|
||||
if (task.kind === 'book_range') {
|
||||
if (task.kind === 'catalogue') {
|
||||
triggerCatalogueScrape();
|
||||
} else if (task.kind === 'book_range') {
|
||||
rangeUrl = task.target_url ?? '';
|
||||
rangeFrom = task.from_chapter ?? 1;
|
||||
rangeTo = task.to_chapter > 0 ? task.to_chapter : null;
|
||||
scrollToRangeForm();
|
||||
} else {
|
||||
// book / catalogue — just pre-fill the single-book form
|
||||
scrapeUrl = task.target_url ?? '';
|
||||
scrollToBookForm();
|
||||
}
|
||||
@@ -220,7 +248,25 @@
|
||||
</div>
|
||||
|
||||
<!-- Scrape controls -->
|
||||
<div class="grid gap-4 sm:grid-cols-2">
|
||||
<div class="grid gap-4 sm:grid-cols-2 lg:grid-cols-3">
|
||||
<!-- Full catalogue -->
|
||||
<div class="bg-zinc-800 rounded-xl border border-zinc-700 p-5 space-y-3">
|
||||
<div>
|
||||
<h2 class="text-sm font-semibold text-zinc-300">Scrape full catalogue</h2>
|
||||
<p class="text-xs text-zinc-500 mt-1">Re-crawls all novelfire.net pages and picks up new books.</p>
|
||||
</div>
|
||||
<button
|
||||
onclick={triggerCatalogueScrape}
|
||||
disabled={running || cataloguing}
|
||||
class="w-full px-4 py-2 rounded-lg bg-amber-600 text-zinc-900 font-semibold text-sm hover:bg-amber-500 transition-colors disabled:opacity-50"
|
||||
>
|
||||
{cataloguing ? 'Queuing…' : running ? 'Already running…' : 'Start catalogue scrape'}
|
||||
</button>
|
||||
{#if catalogueError}
|
||||
<p class="text-sm text-red-400">{catalogueError}</p>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- Single book -->
|
||||
<div id="book-form" class="bg-zinc-800 rounded-xl border border-zinc-700 p-5 space-y-3">
|
||||
<h2 class="text-sm font-semibold text-zinc-300">Scrape a single book</h2>
|
||||
@@ -379,7 +425,7 @@
|
||||
Continue ▶
|
||||
</button>
|
||||
{/if}
|
||||
{#if (task.status === 'failed' || task.status === 'cancelled') && task.kind !== 'catalogue'}
|
||||
{#if task.status === 'failed' || task.status === 'cancelled'}
|
||||
<button
|
||||
onclick={() => retryTask(task)}
|
||||
class="px-2 py-1 rounded text-xs font-medium bg-sky-900/60 text-sky-300 hover:bg-sky-800/60 transition-colors"
|
||||
@@ -447,7 +493,7 @@
|
||||
Continue ▶
|
||||
</button>
|
||||
{/if}
|
||||
{#if (task.status === 'failed' || task.status === 'cancelled') && task.kind !== 'catalogue'}
|
||||
{#if task.status === 'failed' || task.status === 'cancelled'}
|
||||
<button
|
||||
onclick={() => retryTask(task)}
|
||||
class="flex-1 px-3 py-1.5 rounded-lg text-xs font-medium bg-sky-900/60 text-sky-300 hover:bg-sky-800/60 transition-colors"
|
||||
|
||||
@@ -3,12 +3,23 @@ import type { RequestHandler } from './$types';
|
||||
import { presignAudio } from '$lib/server/minio';
|
||||
import { log } from '$lib/server/logger';
|
||||
import * as cache from '$lib/server/presignCache';
|
||||
import { backendFetch } from '$lib/server/scraper';
|
||||
|
||||
/**
|
||||
* GET /api/presign/audio?slug=...&n=...&voice=...
|
||||
* Returns a presigned MinIO URL for the audio file so the browser
|
||||
* can stream it directly without going through the server.
|
||||
* Returns 404 when the audio has not been generated yet.
|
||||
*
|
||||
* Returns a presigned MinIO URL for the audio file so the client can stream
|
||||
* it directly without going through the server.
|
||||
*
|
||||
* When the audio has not been generated yet, this endpoint automatically
|
||||
* enqueues a TTS generation job and returns 202 Accepted with the job status,
|
||||
* so callers can poll GET /api/audio/status/{slug}/{n}?voice=... until done,
|
||||
* then call this endpoint again to get the URL.
|
||||
*
|
||||
* Responses:
|
||||
* 200 { url: string } — audio ready, stream from MinIO
|
||||
* 202 { task_id: string, status: string } — TTS enqueued, poll for completion
|
||||
* 202 { status: "pending"|"generating" } — TTS already in progress
|
||||
*
|
||||
* Results are cached in-process for 50 minutes (MinIO URLs are valid 1 hour)
|
||||
* to avoid a backend + MinIO round-trip on every "Play" click.
|
||||
@@ -38,10 +49,52 @@ export const GET: RequestHandler = async ({ url }) => {
|
||||
return json({ url: presignedUrl });
|
||||
} catch (e) {
|
||||
const status = (e as { status?: number }).status;
|
||||
if (status === 404) {
|
||||
error(404, 'Audio not found');
|
||||
if (status !== 404) {
|
||||
log.error('presign', 'presign audio failed', { slug, n, err: String(e) });
|
||||
error(500, `Could not get presigned URL: ${e}`);
|
||||
}
|
||||
log.error('presign', 'presign audio failed', { slug, n, err: String(e) });
|
||||
error(500, `Could not get presigned URL: ${e}`);
|
||||
}
|
||||
|
||||
// Audio not found — automatically trigger TTS generation so the caller
|
||||
// doesn't need a separate POST step. Return 202 with the job status.
|
||||
log.info('presign', 'audio not found, triggering TTS generation', { slug, n, voice });
|
||||
try {
|
||||
const triggerRes = await backendFetch(`/api/audio/${slug}/${n}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(voice ? { voice } : {})
|
||||
});
|
||||
|
||||
if (!triggerRes.ok) {
|
||||
const text = await triggerRes.text().catch(() => '');
|
||||
log.error('presign', 'audio trigger failed', { slug, n, status: triggerRes.status, body: text });
|
||||
error(triggerRes.status as Parameters<typeof error>[0], text || 'Audio generation failed');
|
||||
}
|
||||
|
||||
const data = (await triggerRes.json()) as
|
||||
| { url: string; status: 'done' }
|
||||
| { task_id: string; status: string };
|
||||
|
||||
// If the backend says it's already done (race: generated between presign
|
||||
// check and the POST), try to presign once more and return 200.
|
||||
if (triggerRes.status === 200 || ('status' in data && data.status === 'done')) {
|
||||
try {
|
||||
const presignedUrl = await presignAudio(slug, n, voice || undefined);
|
||||
await cache.set(cacheKey, presignedUrl);
|
||||
return json({ url: presignedUrl });
|
||||
} catch {
|
||||
// Ignore — fall through to 202 below.
|
||||
}
|
||||
}
|
||||
|
||||
// Generation is in progress — return 202 so the caller can poll.
|
||||
return new Response(JSON.stringify(data), {
|
||||
status: 202,
|
||||
headers: { 'Content-Type': 'application/json' }
|
||||
});
|
||||
} catch (e) {
|
||||
if ((e as { status?: number }).status) throw e; // re-throw SvelteKit errors
|
||||
log.error('presign', 'audio trigger error', { slug, n, err: String(e) });
|
||||
error(500, `Could not trigger audio generation: ${e}`);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import { json, error } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { presignVoiceSample } from '$lib/server/minio';
|
||||
import { log } from '$lib/server/logger';
|
||||
import * as cache from '$lib/server/presignCache';
|
||||
|
||||
/**
|
||||
* GET /api/presign/voice-sample?voice=af_bella
|
||||
* Returns a presigned URL for the voice sample audio file.
|
||||
* Returns 404 if the sample has not been generated yet.
|
||||
*
|
||||
* The backend generates the sample on demand via Kokoro TTS if it does not
|
||||
* exist yet, so this endpoint always returns 200 { url } (or 5xx on failure).
|
||||
*
|
||||
* Results are cached in-process for 50 minutes to avoid a backend + MinIO
|
||||
* round-trip on every voice-selection preview play.
|
||||
@@ -25,16 +28,13 @@ export const GET: RequestHandler = async ({ url }) => {
|
||||
return json({ url: cached });
|
||||
}
|
||||
|
||||
// Slow path: call backend → MinIO presign.
|
||||
// Slow path: call backend → generate if needed → MinIO presign.
|
||||
try {
|
||||
const presignedUrl = await presignVoiceSample(voice);
|
||||
await cache.set(cacheKey, presignedUrl);
|
||||
return json({ url: presignedUrl });
|
||||
} catch (e) {
|
||||
const status = (e as { status?: number }).status;
|
||||
if (status === 404) {
|
||||
error(404, 'Voice sample not found');
|
||||
}
|
||||
log.error('presign', 'presign voice sample failed', { voice, err: String(e) });
|
||||
error(502, `Failed to presign voice sample: ${e}`);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
<script lang="ts">
|
||||
import { untrack } from 'svelte';
|
||||
import { onMount, untrack } from 'svelte';
|
||||
import { invalidateAll } from '$app/navigation';
|
||||
import type { PageData } from './$types';
|
||||
import CommentsSection from '$lib/components/CommentsSection.svelte';
|
||||
|
||||
let { data }: { data: PageData } = $props();
|
||||
|
||||
onMount(() => {
|
||||
if (data.book?.slug) {
|
||||
window.umami?.track('book_view', { slug: data.book.slug });
|
||||
}
|
||||
});
|
||||
|
||||
// ── Save / unsave ─────────────────────────────────────────────────────────
|
||||
let saved = $state(untrack(() => data.saved));
|
||||
let saving = $state(false);
|
||||
|
||||
@@ -19,6 +19,12 @@
|
||||
const wordCount = $derived(countWords(html));
|
||||
|
||||
onMount(async () => {
|
||||
// Umami analytics: track chapter reads
|
||||
window.umami?.track('chapter_read', {
|
||||
slug: data.book.slug,
|
||||
chapter: data.chapter.number
|
||||
});
|
||||
|
||||
// Record reading progress (skip for preview chapters)
|
||||
if (!data.isPreview) {
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user