diff --git a/v3/backend/bin/runner b/v3/backend/bin/runner deleted file mode 100755 index 6de49f4..0000000 Binary files a/v3/backend/bin/runner and /dev/null differ diff --git a/v3/backend/cmd/backend/main.go b/v3/backend/cmd/backend/main.go index 1a53e5a..4643122 100644 --- a/v3/backend/cmd/backend/main.go +++ b/v3/backend/cmd/backend/main.go @@ -2,7 +2,7 @@ // // It exposes all endpoints consumed by the SvelteKit UI: book/chapter reads, // scrape-task creation, presigned MinIO URLs, audio-task creation, reading -// progress, live novelfire.net browse/search, and Kokoro voice list. +// progress, live novelfire.net search, and Kokoro voice list. // // All heavy lifting (scraping, TTS generation) is delegated to the runner // binary via PocketBase task records. The backend never scrapes directly. @@ -19,7 +19,9 @@ import ( "os" "os/signal" "syscall" + "time" + "github.com/getsentry/sentry-go" "github.com/libnovel/backend/internal/backend" "github.com/libnovel/backend/internal/config" "github.com/libnovel/backend/internal/kokoro" @@ -43,6 +45,19 @@ func main() { func run() error { cfg := config.Load() + // ── Sentry / GlitchTip error tracking ──────────────────────────────────── + if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" { + if err := sentry.Init(sentry.ClientOptions{ + Dsn: dsn, + Release: version + "@" + commit, + TracesSampleRate: 0.1, + }); err != nil { + fmt.Fprintf(os.Stderr, "backend: sentry init warning: %v\n", err) + } else { + defer sentry.Flush(2 * time.Second) + } + } + // ── Logger ─────────────────────────────────────────────────────────────── log := buildLogger(cfg.LogLevel) log.Info("backend starting", @@ -95,7 +110,6 @@ func run() error { AudioStore: store, PresignStore: store, ProgressStore: store, - BrowseStore: store, CoverStore: store, Producer: store, TaskReader: store, diff --git a/v3/backend/cmd/runner/main.go b/v3/backend/cmd/runner/main.go index f534d68..45ebef4 100644 --- a/v3/backend/cmd/runner/main.go +++ b/v3/backend/cmd/runner/main.go @@ -19,6 +19,7 @@ import ( "syscall" "time" + "github.com/getsentry/sentry-go" "github.com/libnovel/backend/internal/browser" "github.com/libnovel/backend/internal/config" "github.com/libnovel/backend/internal/kokoro" @@ -44,6 +45,19 @@ func main() { func run() error { cfg := config.Load() + // ── Sentry / GlitchTip error tracking ──────────────────────────────────── + if dsn := os.Getenv("GLITCHTIP_DSN"); dsn != "" { + if err := sentry.Init(sentry.ClientOptions{ + Dsn: dsn, + Release: version + "@" + commit, + TracesSampleRate: 0.1, + }); err != nil { + fmt.Fprintf(os.Stderr, "runner: sentry init warning: %v\n", err) + } else { + defer sentry.Flush(2 * time.Second) + } + } + // ── Logger ────────────────────────────────────────────────────────────── log := buildLogger(cfg.LogLevel) log.Info("runner starting", @@ -105,21 +119,20 @@ func run() error { // ── Runner ────────────────────────────────────────────────────────────── rCfg := runner.Config{ - WorkerID: cfg.Runner.WorkerID, - PollInterval: cfg.Runner.PollInterval, - MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape, - MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio, - OrchestratorWorkers: workers, - MetricsAddr: cfg.Runner.MetricsAddr, - CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval, - SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh, + WorkerID: cfg.Runner.WorkerID, + PollInterval: cfg.Runner.PollInterval, + MaxConcurrentScrape: cfg.Runner.MaxConcurrentScrape, + MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio, + OrchestratorWorkers: workers, + MetricsAddr: cfg.Runner.MetricsAddr, + CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval, + SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh, } deps := runner.Dependencies{ Consumer: store, BookWriter: store, BookReader: store, AudioStore: store, - BrowseStore: store, CoverStore: store, SearchIndex: searchIndex, Novel: novel, diff --git a/v3/backend/go.mod b/v3/backend/go.mod index c6e2633..f108996 100644 --- a/v3/backend/go.mod +++ b/v3/backend/go.mod @@ -13,6 +13,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/getsentry/sentry-go v0.43.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/uuid v1.6.0 // indirect diff --git a/v3/backend/go.sum b/v3/backend/go.sum index db607e6..6026879 100644 --- a/v3/backend/go.sum +++ b/v3/backend/go.sum @@ -8,6 +8,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4= +github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= @@ -57,5 +59,6 @@ golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/v3/backend/healthcheck b/v3/backend/healthcheck deleted file mode 100755 index 9d0e8ec..0000000 Binary files a/v3/backend/healthcheck and /dev/null differ diff --git a/v3/backend/internal/backend/handlers.go b/v3/backend/internal/backend/handlers.go index eda8eec..5c8a4d2 100644 --- a/v3/backend/internal/backend/handlers.go +++ b/v3/backend/internal/backend/handlers.go @@ -176,80 +176,6 @@ type NovelListing struct { URL string `json:"url"` } -// handleBrowse handles GET /api/browse. -// Fetches novelfire.net live (no MinIO cache in the new backend). -// Query params: page (default 1), genre (default "all"), sort (default "popular"), -// status (default "all"), type (default "all-novel") -func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query() - page := q.Get("page") - if page == "" { - page = "1" - } - genre := q.Get("genre") - if genre == "" { - genre = "all" - } - sortBy := q.Get("sort") - if sortBy == "" { - sortBy = "popular" - } - status := q.Get("status") - if status == "" { - status = "all" - } - novelType := q.Get("type") - if novelType == "" { - novelType = "all-novel" - } - - pageNum, _ := strconv.Atoi(page) - if pageNum <= 0 { - pageNum = 1 - } - - // ── Try MinIO cache first ───────────────────────────────────────────── - // Only page 1 is cached; higher pages fall through to live fetch. - if pageNum == 1 && s.deps.BrowseStore != nil { - if data, ok, err := s.deps.BrowseStore.GetBrowsePage(r.Context(), genre, sortBy, status, novelType, 1); err == nil && ok { - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Cache-Control", "public, max-age=300") - _, _ = w.Write(data) - return - } - } - - // ── Fall back to live novelfire.net fetch ────────────────────────────── - ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second) - defer cancel() - - targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%d", - novelFireBase, genre, sortBy, status, novelType, pageNum) - - novels, hasNext, err := s.fetchBrowsePage(ctx, targetURL) - if err != nil { - // Live fetch also failed — return empty list with cached=false flag so - // the UI can show a "not ready yet" state instead of a hard error. - s.deps.Log.Error("handleBrowse: fetch failed (no cache)", "url", targetURL, "err", err) - w.Header().Set("Cache-Control", "no-store") - writeJSON(w, 0, map[string]any{ - "novels": []any{}, - "page": pageNum, - "hasNext": false, - "cached": false, - }) - return - } - - w.Header().Set("Cache-Control", "public, max-age=300") - writeJSON(w, 0, map[string]any{ - "novels": novels, - "page": pageNum, - "hasNext": hasNext, - "cached": false, - }) -} - // handleSearch handles GET /api/search. // Query params: q (min 2 chars), source ("local"|"remote"|"all", default "all") // @@ -832,7 +758,13 @@ func (s *Server) handlePresignAudio(w http.ResponseWriter, r *http.Request) { writeJSON(w, 0, map[string]string{"url": u}) } +// voiceSampleText is the phrase synthesised for every voice sample. +const voiceSampleText = "Hello! This is a preview of what I sound like. I hope you enjoy listening to your stories with my voice." + // handlePresignVoiceSample handles GET /api/presign/voice-sample/{voice}. +// If the sample has not been generated yet it synthesises it on the fly via +// Kokoro, stores the result in MinIO, and returns the presigned URL — so the +// caller always gets a playable URL in a single request. func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request) { voice := r.PathValue("voice") if voice == "" { @@ -841,9 +773,21 @@ func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request } key := kokoro.VoiceSampleKey(voice) + + // Generate sample on demand when it is not in MinIO yet. if !s.deps.AudioStore.AudioExists(r.Context(), key) { - http.NotFound(w, r) - return + s.deps.Log.Info("generating voice sample on demand", "voice", voice) + mp3, err := s.deps.Kokoro.GenerateAudio(r.Context(), voiceSampleText, voice) + if err != nil { + s.deps.Log.Error("voice sample generation failed", "voice", voice, "err", err) + jsonError(w, http.StatusInternalServerError, "voice sample generation failed") + return + } + if err := s.deps.AudioStore.PutAudio(r.Context(), key, mp3); err != nil { + s.deps.Log.Error("voice sample upload failed", "voice", voice, "err", err) + jsonError(w, http.StatusInternalServerError, "voice sample upload failed") + return + } } u, err := s.deps.PresignStore.PresignAudio(r.Context(), key, 1*time.Hour) diff --git a/v3/backend/internal/backend/server.go b/v3/backend/internal/backend/server.go index 148213a..21e7084 100644 --- a/v3/backend/internal/backend/server.go +++ b/v3/backend/internal/backend/server.go @@ -6,7 +6,7 @@ // picks up and executes those tasks asynchronously // - Presigned MinIO URLs for media playback/upload // - Session-scoped reading progress -// - Live novelfire.net browse/search (no scraper interface needed; direct HTTP) +// - Live novelfire.net search (no scraper interface needed; direct HTTP) // - Kokoro voice list // // The backend never scrapes directly. All scraping (metadata, chapter list, @@ -28,6 +28,7 @@ import ( "sync" "time" + sentryhttp "github.com/getsentry/sentry-go/http" "github.com/libnovel/backend/internal/bookstore" "github.com/libnovel/backend/internal/kokoro" "github.com/libnovel/backend/internal/meili" @@ -47,8 +48,6 @@ type Dependencies struct { PresignStore bookstore.PresignStore // ProgressStore reads/writes per-session reading progress. ProgressStore bookstore.ProgressStore - // BrowseStore reads cached browse page snapshots from MinIO. - BrowseStore bookstore.BrowseStore // CoverStore reads and writes book cover images from MinIO. // If nil, the cover endpoint falls back to a CDN redirect. CoverStore bookstore.CoverStore @@ -122,8 +121,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error { // Cancel a pending task (scrape or audio) mux.HandleFunc("POST /api/cancel-task/{id}", s.handleCancelTask) - // Browse & search (live novelfire.net) - mux.HandleFunc("GET /api/browse", s.handleBrowse) + // Browse & search mux.HandleFunc("GET /api/search", s.handleSearch) // Catalogue (Meilisearch-backed browse + search — preferred path for UI) @@ -174,7 +172,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error { srv := &http.Server{ Addr: s.cfg.Addr, - Handler: mux, + Handler: sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle(mux), ReadTimeout: 15 * time.Second, WriteTimeout: 60 * time.Second, IdleTimeout: 60 * time.Second, diff --git a/v3/backend/internal/bookstore/bookstore.go b/v3/backend/internal/bookstore/bookstore.go index 65ab4f9..d638d36 100644 --- a/v3/backend/internal/bookstore/bookstore.go +++ b/v3/backend/internal/bookstore/bookstore.go @@ -128,18 +128,6 @@ type ProgressStore interface { DeleteProgress(ctx context.Context, sessionID, slug string) error } -// BrowseStore covers browse page snapshot storage. -// The runner writes snapshots; the backend reads them. -type BrowseStore interface { - // PutBrowsePage stores a raw JSON snapshot for a browse page. - // genre, sort, status, novelType and page identify the page. - PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error - - // GetBrowsePage retrieves a raw JSON snapshot. Returns (nil, false, nil) - // when no snapshot exists for the given parameters. - GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) -} - // CoverStore covers book cover image storage in MinIO. // The runner writes covers during catalogue refresh; the backend reads them. type CoverStore interface { diff --git a/v3/backend/internal/config/config.go b/v3/backend/internal/config/config.go index f7fb68b..2d05f6d 100644 --- a/v3/backend/internal/config/config.go +++ b/v3/backend/internal/config/config.go @@ -145,10 +145,10 @@ func Load() Config { SecretKey: envOr("MINIO_SECRET_KEY", "changeme123"), UseSSL: envBool("MINIO_USE_SSL", false), PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true), - BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), - BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), + BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "chapters"), + BucketAudio: envOr("MINIO_BUCKET_AUDIO", "audio"), BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "avatars"), - BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"), + BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "catalogue"), }, Kokoro: Kokoro{ diff --git a/v3/backend/internal/config/config_test.go b/v3/backend/internal/config/config_test.go index e424d75..d281b44 100644 --- a/v3/backend/internal/config/config_test.go +++ b/v3/backend/internal/config/config_test.go @@ -33,8 +33,8 @@ func TestLoad_Defaults(t *testing.T) { if cfg.PocketBase.URL != "http://localhost:8090" { t.Errorf("PocketBase.URL: want http://localhost:8090, got %q", cfg.PocketBase.URL) } - if cfg.MinIO.BucketChapters != "libnovel-chapters" { - t.Errorf("MinIO.BucketChapters: want libnovel-chapters, got %q", cfg.MinIO.BucketChapters) + if cfg.MinIO.BucketChapters != "chapters" { + t.Errorf("MinIO.BucketChapters: want chapters, got %q", cfg.MinIO.BucketChapters) } if cfg.MinIO.UseSSL != false { t.Errorf("MinIO.UseSSL: want false, got %v", cfg.MinIO.UseSSL) diff --git a/v3/backend/internal/runner/browse_refresh.go b/v3/backend/internal/runner/browse_refresh.go deleted file mode 100644 index c742005..0000000 --- a/v3/backend/internal/runner/browse_refresh.go +++ /dev/null @@ -1,176 +0,0 @@ -package runner - -// browse_refresh.go — independent 6-hour loop that fetches novelfire.net -// browse page snapshots and stores them in MinIO. -// -// Design: -// - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run(). -// - Fetches page 1 for each combination of the standard genre/sort/status -// filter values and stores the parsed JSON blob in MinIO via BrowseStore. -// - The backend's handleBrowse then serves from MinIO instead of calling -// novelfire.net live, which avoids IP-based rate-limiting on the server. - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "regexp" - "strings" - "time" -) - -// browseNovelListing mirrors backend.NovelListing for JSON serialisation. -type browseNovelListing struct { - Slug string `json:"slug"` - Title string `json:"title"` - Cover string `json:"cover"` - URL string `json:"url"` -} - -// browseSnapshot is the JSON structure stored in MinIO. -type browseSnapshot struct { - Novels []browseNovelListing `json:"novels"` - Page int `json:"page"` - HasNext bool `json:"hasNext"` - // CachedAt is the UTC time the snapshot was written (ISO 8601). - CachedAt string `json:"cachedAt"` -} - -// browseCombos lists the filter combinations to pre-fetch. -// Each entry is (genre, sort, status, novelType). -var browseCombos = []struct{ genre, sort, status, novelType string }{ - {"all", "popular", "all", "all-novel"}, - {"all", "popular", "ongoing", "all-novel"}, - {"all", "popular", "completed", "all-novel"}, - {"all", "new", "all", "all-novel"}, - {"all", "new", "ongoing", "all-novel"}, - {"all", "new", "completed", "all-novel"}, - {"all", "top-rated", "all", "all-novel"}, - {"all", "top-rated", "ongoing", "all-novel"}, - {"all", "top-rated", "completed", "all-novel"}, -} - -const novelFireBrowseBase = "https://novelfire.net" - -// runBrowseRefresh fetches all browse combos from novelfire.net and stores -// the results in MinIO. Errors per-combo are logged but do not abort the -// whole refresh cycle. -func (r *Runner) runBrowseRefresh(ctx context.Context) { - if r.deps.BrowseStore == nil { - r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured") - return - } - - log := r.deps.Log.With("op", "browse_refresh") - log.Info("runner: browse refresh starting", "combos", len(browseCombos)) - - ok, fail := 0, 0 - for _, c := range browseCombos { - if ctx.Err() != nil { - break - } - novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType) - if err != nil { - log.Warn("runner: browse fetch failed", - "genre", c.genre, "sort", c.sort, "status", c.status, "err", err) - fail++ - continue - } - - snap := browseSnapshot{ - Novels: novels, - Page: 1, - HasNext: hasNext, - CachedAt: time.Now().UTC().Format(time.RFC3339), - } - data, _ := json.Marshal(snap) - if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil { - log.Warn("runner: browse put failed", - "genre", c.genre, "sort", c.sort, "status", c.status, "err", err) - fail++ - continue - } - ok++ - } - - log.Info("runner: browse refresh finished", "ok", ok, "failed", fail) -} - -// fetchBrowsePage calls novelfire.net and returns a list of novel listings -// plus a hasNext flag. Mirrors the logic in backend/handlers.go. -func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) { - pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1", - novelFireBrowseBase, genre, sort, status, novelType) - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) - if err != nil { - return nil, false, fmt.Errorf("build request: %w", err) - } - req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)") - req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") - req.Header.Set("Accept-Language", "en-US,en;q=0.9") - - httpClient := &http.Client{Timeout: 45 * time.Second} - resp, err := httpClient.Do(req) - if err != nil { - return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - _, _ = io.Copy(io.Discard, resp.Body) - return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL) - } - - return parseBrowseHTML(resp.Body) -} - -// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage -// in backend/handlers.go — kept separate to avoid coupling packages. -func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) { - data, err := io.ReadAll(r) - if err != nil { - return nil, false, err - } - body := string(data) - - hasNext := strings.Contains(body, `rel="next"`) || - strings.Contains(body, `aria-label="Next"`) || - strings.Contains(body, `class="next"`) - - slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`) - titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`) - coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`) - - slugMatches := slugRe.FindAllStringSubmatch(body, -1) - titleMatches := titleRe.FindAllStringSubmatch(body, -1) - coverMatches := coverRe.FindAllStringSubmatch(body, -1) - - var novels []browseNovelListing - seen := make(map[string]bool) - for i, sm := range slugMatches { - slug := sm[1] - if seen[slug] { - continue - } - seen[slug] = true - - item := browseNovelListing{ - Slug: slug, - URL: novelFireBrowseBase + "/book/" + slug, - } - if i < len(titleMatches) { - item.Title = strings.TrimSpace(titleMatches[i][1]) - } - if i < len(coverMatches) { - item.Cover = coverMatches[i][1] - } - if item.Title != "" { - novels = append(novels, item) - } - } - - return novels, hasNext, nil -} diff --git a/v3/backend/internal/runner/runner.go b/v3/backend/internal/runner/runner.go index 3c7f3a6..24f4757 100644 --- a/v3/backend/internal/runner/runner.go +++ b/v3/backend/internal/runner/runner.go @@ -49,9 +49,6 @@ type Config struct { // StaleTaskThreshold is how old a heartbeat must be (or absent) before the // task is considered orphaned and reset to pending. Defaults to 2m when 0. StaleTaskThreshold time.Duration - // BrowseRefreshInterval is how often the runner pre-fetches browse page - // snapshots from novelfire.net and stores them in MinIO. Defaults to 6h. - BrowseRefreshInterval time.Duration // CatalogueRefreshInterval is how often the runner walks the full catalogue, // scrapes per-book metadata, downloads covers, and re-indexes everything in // Meilisearch. Defaults to 24h (expensive — full catalogue walk). @@ -76,10 +73,7 @@ type Dependencies struct { BookReader bookstore.BookReader // AudioStore persists generated audio and checks key existence. AudioStore bookstore.AudioStore - // BrowseStore stores browse page snapshots in MinIO. - BrowseStore bookstore.BrowseStore // CoverStore stores book cover images in MinIO. - // If nil, cover downloads are skipped during catalogue refresh. CoverStore bookstore.CoverStore // SearchIndex indexes books in Meilisearch after scraping. // If nil a no-op is used. @@ -125,9 +119,6 @@ func New(cfg Config, deps Dependencies) *Runner { if cfg.StaleTaskThreshold <= 0 { cfg.StaleTaskThreshold = 2 * time.Minute } - if cfg.BrowseRefreshInterval <= 0 { - cfg.BrowseRefreshInterval = 6 * time.Hour - } if cfg.CatalogueRefreshInterval <= 0 { cfg.CatalogueRefreshInterval = 24 * time.Hour } @@ -151,7 +142,6 @@ func (r *Runner) Run(ctx context.Context) error { "poll_interval", r.cfg.PollInterval, "max_scrape", r.cfg.MaxConcurrentScrape, "max_audio", r.cfg.MaxConcurrentAudio, - "browse_refresh_interval", r.cfg.BrowseRefreshInterval, "catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval, "metrics_addr", r.cfg.MetricsAddr, ) @@ -173,14 +163,9 @@ func (r *Runner) Run(ctx context.Context) error { tick := time.NewTicker(r.cfg.PollInterval) defer tick.Stop() - browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval) - defer browseTick.Stop() - catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval) defer catalogueTick.Stop() - // Run one browse refresh immediately on startup. - go r.runBrowseRefresh(ctx) // Run one catalogue refresh immediately on startup (unless skipped by flag). if !r.cfg.SkipInitialCatalogueRefresh { go r.runCatalogueRefresh(ctx) @@ -207,8 +192,6 @@ func (r *Runner) Run(ctx context.Context) error { r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit") } return nil - case <-browseTick.C: - go r.runBrowseRefresh(ctx) case <-catalogueTick.C: go r.runCatalogueRefresh(ctx) case <-tick.C: diff --git a/v3/backend/internal/storage/minio.go b/v3/backend/internal/storage/minio.go index 3dee9e7..3f5217a 100644 --- a/v3/backend/internal/storage/minio.go +++ b/v3/backend/internal/storage/minio.go @@ -119,12 +119,6 @@ func AvatarObjectKey(userID, ext string) string { return fmt.Sprintf("%s/%s.%s", userID, ext, ext) } -// BrowseObjectKey returns the MinIO object key for a cached browse page snapshot. -// Format: browse/{genre}/{sort}/{status}/{type}/page-{n}.json -func BrowseObjectKey(genre, sort, status, novelType string, page int) string { - return fmt.Sprintf("browse/%s/%s/%s/%s/page-%d.json", genre, sort, status, novelType, page) -} - // CoverObjectKey returns the MinIO object key for a book cover image. // Format: covers/{slug}.jpg func CoverObjectKey(slug string) string { @@ -207,26 +201,6 @@ func (m *minioClient) listObjectKeys(ctx context.Context, bucket, prefix string) return keys, nil } -// ── Browse operations ───────────────────────────────────────────────────────── - -// putBrowse stores raw JSON bytes for a browse page snapshot. -func (m *minioClient) putBrowse(ctx context.Context, key string, data []byte) error { - return m.putObject(ctx, m.bucketBrowse, key, "application/json", data) -} - -// getBrowse retrieves a browse page snapshot. Returns (nil, false, nil) when -// the object does not exist. -func (m *minioClient) getBrowse(ctx context.Context, key string) ([]byte, bool, error) { - if !m.objectExists(ctx, m.bucketBrowse, key) { - return nil, false, nil - } - data, err := m.getObject(ctx, m.bucketBrowse, key) - if err != nil { - return nil, false, err - } - return data, true, nil -} - // ── Cover operations ────────────────────────────────────────────────────────── // putCover stores a raw cover image in the browse bucket under covers/{slug}.jpg. diff --git a/v3/backend/internal/storage/store.go b/v3/backend/internal/storage/store.go index a0159b2..c24c4b5 100644 --- a/v3/backend/internal/storage/store.go +++ b/v3/backend/internal/storage/store.go @@ -50,7 +50,6 @@ var _ bookstore.RankingStore = (*Store)(nil) var _ bookstore.AudioStore = (*Store)(nil) var _ bookstore.PresignStore = (*Store)(nil) var _ bookstore.ProgressStore = (*Store)(nil) -var _ bookstore.BrowseStore = (*Store)(nil) var _ bookstore.CoverStore = (*Store)(nil) var _ taskqueue.Producer = (*Store)(nil) var _ taskqueue.Consumer = (*Store)(nil) @@ -790,25 +789,6 @@ func parseAudioTask(raw json.RawMessage) (domain.AudioTask, error) { }, nil } -// ── BrowseStore ──────────────────────────────────────────────────────────────── - -func (s *Store) PutBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int, data []byte) error { - key := BrowseObjectKey(genre, sort, status, novelType, page) - if err := s.mc.putBrowse(ctx, key, data); err != nil { - return fmt.Errorf("PutBrowsePage: %w", err) - } - return nil -} - -func (s *Store) GetBrowsePage(ctx context.Context, genre, sort, status, novelType string, page int) ([]byte, bool, error) { - key := BrowseObjectKey(genre, sort, status, novelType, page) - data, ok, err := s.mc.getBrowse(ctx, key) - if err != nil { - return nil, false, fmt.Errorf("GetBrowsePage: %w", err) - } - return data, ok, nil -} - // ── CoverStore ───────────────────────────────────────────────────────────────── func (s *Store) PutCover(ctx context.Context, slug string, data []byte, contentType string) error {