All checks were successful
Release / Scraper / Test (push) Successful in 20s
Release / UI / Build (push) Successful in 25s
CI / Scraper / Lint (pull_request) Successful in 10s
Release / v2 / Build ui-v2 (push) Successful in 28s
CI / Scraper / Test (pull_request) Successful in 15s
CI / UI / Build (pull_request) Successful in 25s
Release / Scraper / Docker (push) Successful in 55s
Release / UI / Docker (push) Successful in 43s
CI / Scraper / Docker Push (pull_request) Has been skipped
CI / UI / Docker Push (pull_request) Has been skipped
Release / v2 / Docker / ui-v2 (push) Successful in 36s
Release / v2 / Test backend (push) Successful in 3m51s
Release / v2 / Docker / runner (push) Successful in 36s
Release / v2 / Docker / backend (push) Successful in 1m34s
iOS CI / Build (pull_request) Successful in 5m33s
iOS CI / Test (pull_request) Successful in 11m25s
- Runner fetches 9 browse combos (genre×sort×status) every 6h and stores JSON snapshots in MinIO libnovel-browse bucket (browse_refresh.go) - Backend handleBrowse reads page-1 results from MinIO first; falls back to live novelfire.net fetch; returns empty+cached:false on total failure instead of 502 - Add BrowseStore interface (bookstore.go), MinIO put/get helpers (minio.go), Store methods + compile-time assertion (store.go), BucketBrowse config, wiring in cmd/backend and cmd/runner, docker-compose-new bucket init - Fix ReapStaleTasks: PocketBase datetime fields require heartbeat_at=null (not heartbeat_at="") in filter expressions, and nil (not "") in patch payload — was causing 400 errors on every reap cycle
288 lines
9.5 KiB
Go
288 lines
9.5 KiB
Go
// Package backend implements the HTTP API server for the LibNovel backend.
|
|
//
|
|
// The server exposes all endpoints consumed by the SvelteKit UI:
|
|
// - Book/chapter reads from PocketBase/MinIO via bookstore interfaces
|
|
// - Task creation (scrape + audio) via taskqueue.Producer — the runner binary
|
|
// picks up and executes those tasks asynchronously
|
|
// - Presigned MinIO URLs for media playback/upload
|
|
// - Session-scoped reading progress
|
|
// - Live novelfire.net browse/search (no scraper interface needed; direct HTTP)
|
|
// - Kokoro voice list
|
|
//
|
|
// The backend never scrapes directly. All scraping (metadata, chapter list,
|
|
// chapter text, audio TTS) is delegated to the runner binary via PocketBase
|
|
// task records. GET /api/book-preview enqueues a task when the book is absent.
|
|
//
|
|
// All external dependencies are injected as interfaces; concrete types live in
|
|
// internal/storage and are wired by cmd/backend/main.go.
|
|
package backend
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/libnovel/backend/internal/bookstore"
|
|
"github.com/libnovel/backend/internal/kokoro"
|
|
"github.com/libnovel/backend/internal/taskqueue"
|
|
)
|
|
|
|
// Dependencies holds all external services the backend server depends on.
|
|
// Every field is an interface so test doubles can be injected freely.
|
|
type Dependencies struct {
|
|
// BookReader reads book metadata and chapter text from PocketBase/MinIO.
|
|
BookReader bookstore.BookReader
|
|
// RankingStore reads ranking data from PocketBase.
|
|
RankingStore bookstore.RankingStore
|
|
// AudioStore checks audio object existence and computes MinIO keys.
|
|
AudioStore bookstore.AudioStore
|
|
// PresignStore generates short-lived MinIO URLs.
|
|
PresignStore bookstore.PresignStore
|
|
// ProgressStore reads/writes per-session reading progress.
|
|
ProgressStore bookstore.ProgressStore
|
|
// BrowseStore reads cached browse page snapshots from MinIO.
|
|
BrowseStore bookstore.BrowseStore
|
|
// Producer creates scrape/audio tasks in PocketBase.
|
|
Producer taskqueue.Producer
|
|
// TaskReader reads scrape/audio task records from PocketBase.
|
|
TaskReader taskqueue.Reader
|
|
// Kokoro is the TTS client (used for voice list only in the backend;
|
|
// audio generation is done by the runner).
|
|
Kokoro kokoro.Client
|
|
// Log is the structured logger.
|
|
Log *slog.Logger
|
|
}
|
|
|
|
// Config holds HTTP server tuning parameters.
|
|
type Config struct {
|
|
// Addr is the listen address, e.g. ":8080".
|
|
Addr string
|
|
// DefaultVoice is used when no voice is specified in audio requests.
|
|
DefaultVoice string
|
|
// Version and Commit are embedded in /health and /api/version responses.
|
|
Version string
|
|
Commit string
|
|
}
|
|
|
|
// Server is the HTTP API server.
|
|
type Server struct {
|
|
cfg Config
|
|
deps Dependencies
|
|
|
|
// voiceMu guards cachedVoices. Populated lazily on first GET /api/voices.
|
|
voiceMu sync.RWMutex
|
|
cachedVoices []string
|
|
}
|
|
|
|
// New creates a Server from cfg and deps.
|
|
func New(cfg Config, deps Dependencies) *Server {
|
|
if cfg.DefaultVoice == "" {
|
|
cfg.DefaultVoice = "af_bella"
|
|
}
|
|
if deps.Log == nil {
|
|
deps.Log = slog.Default()
|
|
}
|
|
return &Server{cfg: cfg, deps: deps}
|
|
}
|
|
|
|
// ListenAndServe registers all routes and starts the HTTP server.
|
|
// It blocks until ctx is cancelled, then performs a graceful shutdown.
|
|
func (s *Server) ListenAndServe(ctx context.Context) error {
|
|
mux := http.NewServeMux()
|
|
|
|
// Health / version
|
|
mux.HandleFunc("GET /health", s.handleHealth)
|
|
mux.HandleFunc("GET /api/version", s.handleVersion)
|
|
|
|
// Scrape task creation (202 Accepted — runner executes asynchronously)
|
|
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
|
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
|
mux.HandleFunc("POST /scrape/book/range", s.handleScrapeBookRange)
|
|
|
|
// Scrape task status / history
|
|
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
|
|
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
|
|
|
|
// Cancel a pending task (scrape or audio)
|
|
mux.HandleFunc("POST /api/cancel-task/{id}", s.handleCancelTask)
|
|
|
|
// Browse & search (live novelfire.net)
|
|
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
|
mux.HandleFunc("GET /api/search", s.handleSearch)
|
|
|
|
// Ranking (from PocketBase)
|
|
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
|
|
|
|
// Cover proxy (live URL redirect)
|
|
mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
|
|
|
|
// Book preview (enqueues scrape task if not in library; returns stored data if already scraped)
|
|
mux.HandleFunc("GET /api/book-preview/{slug}", s.handleBookPreview)
|
|
|
|
// Chapter text (served from MinIO via PocketBase index)
|
|
mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText)
|
|
// Raw markdown chapter content — served directly from MinIO by the backend.
|
|
// Use this instead of presign+fetch to avoid SvelteKit→MinIO network path.
|
|
mux.HandleFunc("GET /api/chapter-markdown/{slug}/{n}", s.handleChapterMarkdown)
|
|
|
|
// Reindex chapters_idx from MinIO
|
|
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
|
|
|
|
// Audio task creation (backend creates task; runner executes)
|
|
mux.HandleFunc("POST /api/audio/{slug}/{n}", s.handleAudioGenerate)
|
|
mux.HandleFunc("GET /api/audio/status/{slug}/{n}", s.handleAudioStatus)
|
|
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
|
|
|
// Voices list
|
|
mux.HandleFunc("GET /api/voices", s.handleVoices)
|
|
|
|
// Presigned URLs
|
|
mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter)
|
|
mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio)
|
|
mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample)
|
|
mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload)
|
|
mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar)
|
|
|
|
// Reading progress
|
|
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
|
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
|
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
|
|
|
|
srv := &http.Server{
|
|
Addr: s.cfg.Addr,
|
|
Handler: mux,
|
|
ReadTimeout: 15 * time.Second,
|
|
WriteTimeout: 60 * time.Second,
|
|
IdleTimeout: 60 * time.Second,
|
|
}
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() { errCh <- srv.ListenAndServe() }()
|
|
s.deps.Log.Info("backend: HTTP server listening", "addr", s.cfg.Addr)
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
s.deps.Log.Info("backend: context cancelled, starting graceful shutdown")
|
|
shutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
if err := srv.Shutdown(shutCtx); err != nil {
|
|
s.deps.Log.Error("backend: graceful shutdown failed", "err", err)
|
|
return err
|
|
}
|
|
s.deps.Log.Info("backend: shutdown complete")
|
|
return nil
|
|
case err := <-errCh:
|
|
return err
|
|
}
|
|
}
|
|
|
|
// ── Session cookie helpers ─────────────────────────────────────────────────────
|
|
|
|
const sessionCookieName = "libnovel_session"
|
|
|
|
func sessionID(r *http.Request) string {
|
|
c, err := r.Cookie(sessionCookieName)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return c.Value
|
|
}
|
|
|
|
func newSessionID() (string, error) {
|
|
b := make([]byte, 16)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(b), nil
|
|
}
|
|
|
|
func ensureSession(w http.ResponseWriter, r *http.Request) string {
|
|
if id := sessionID(r); id != "" {
|
|
return id
|
|
}
|
|
id, err := newSessionID()
|
|
if err != nil {
|
|
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
|
|
}
|
|
http.SetCookie(w, &http.Cookie{
|
|
Name: sessionCookieName,
|
|
Value: id,
|
|
Path: "/",
|
|
HttpOnly: true,
|
|
SameSite: http.SameSiteLaxMode,
|
|
MaxAge: 365 * 24 * 60 * 60,
|
|
})
|
|
return id
|
|
}
|
|
|
|
// ── Utility helpers ────────────────────────────────────────────────────────────
|
|
|
|
// writeJSON writes v as a JSON response with status code. Status 0 → 200.
|
|
func writeJSON(w http.ResponseWriter, status int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if status != 0 {
|
|
w.WriteHeader(status)
|
|
}
|
|
_ = json.NewEncoder(w).Encode(v)
|
|
}
|
|
|
|
// jsonError writes a JSON error body and the given status code.
|
|
func jsonError(w http.ResponseWriter, status int, msg string) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
|
|
}
|
|
|
|
// voices returns the list of available Kokoro voices. On the first call it
|
|
// fetches from the Kokoro service and caches the result. Falls back to the
|
|
// hardcoded list on error.
|
|
func (s *Server) voices(ctx context.Context) []string {
|
|
s.voiceMu.RLock()
|
|
cached := s.cachedVoices
|
|
s.voiceMu.RUnlock()
|
|
if len(cached) > 0 {
|
|
return cached
|
|
}
|
|
|
|
if s.deps.Kokoro == nil {
|
|
return kokoroVoices
|
|
}
|
|
|
|
fetchCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
list, err := s.deps.Kokoro.ListVoices(fetchCtx)
|
|
if err != nil || len(list) == 0 {
|
|
s.deps.Log.Warn("backend: could not fetch kokoro voices, using built-in list", "err", err)
|
|
return kokoroVoices
|
|
}
|
|
|
|
s.voiceMu.Lock()
|
|
s.cachedVoices = list
|
|
s.voiceMu.Unlock()
|
|
s.deps.Log.Info("backend: fetched kokoro voices", "count", len(list))
|
|
return list
|
|
}
|
|
|
|
// handleHealth handles GET /health.
|
|
func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
|
writeJSON(w, 0, map[string]string{
|
|
"status": "ok",
|
|
"version": s.cfg.Version,
|
|
"commit": s.cfg.Commit,
|
|
})
|
|
}
|
|
|
|
// handleVersion handles GET /api/version.
|
|
func (s *Server) handleVersion(w http.ResponseWriter, _ *http.Request) {
|
|
writeJSON(w, 0, map[string]string{
|
|
"version": s.cfg.Version,
|
|
"commit": s.cfg.Commit,
|
|
})
|
|
}
|