1d00fd4e2e
Adds GET /api/search Go endpoint that queries PocketBase books by title/author substring and fetches novelfire.net /search results via parseBrowsePage(), merging results with local-first de-duplication. The browse page gains a search input that navigates to ?q=; results show local vs remote counts and hide pagination/filter controls while in search mode.
270 lines
9.8 KiB
Go
270 lines
9.8 KiB
Go
// Package server exposes the scraper as an HTTP API service.
|
|
//
|
|
// Endpoints:
|
|
//
|
|
// POST /scrape — enqueue a full catalogue scrape
|
|
// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."})
|
|
// GET /health — liveness probe
|
|
// GET /api/progress — get reading progress map (session-scoped)
|
|
// POST /api/progress/{slug} — set reading progress
|
|
// DELETE /api/progress/{slug} — delete reading progress
|
|
// GET /api/presign/chapter/{slug}/{n} — presigned MinIO URL for chapter markdown
|
|
// GET /api/presign/audio/{slug}/{n} — presigned MinIO URL for chapter audio
|
|
// GET /api/chapter-text/{slug}/{n} — plain text of chapter (markdown stripped)
|
|
// POST /api/audio/{slug}/{n} — trigger Kokoro audio generation
|
|
// GET /api/audio-proxy/{slug}/{n} — proxy generated audio from Kokoro
|
|
package server
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/libnovel/scraper/internal/orchestrator"
|
|
"github.com/libnovel/scraper/internal/scraper"
|
|
"github.com/libnovel/scraper/internal/storage"
|
|
)
|
|
|
|
// Server wraps an HTTP mux with the scraping endpoints.
|
|
type Server struct {
|
|
addr string
|
|
oCfg orchestrator.Config
|
|
novel scraper.NovelScraper
|
|
log *slog.Logger
|
|
store storage.Store
|
|
mu sync.Mutex
|
|
running bool
|
|
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
|
|
kokoroVoice string // default voice, e.g. af_bella
|
|
|
|
// voiceMu guards cachedVoices.
|
|
voiceMu sync.RWMutex
|
|
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
|
|
|
|
// audioMu guards audioInFlight only.
|
|
// Completed audio filenames are persisted to the Store (PocketBase).
|
|
// audioInFlight deduplicates concurrent generation requests for the same key.
|
|
audioMu sync.Mutex
|
|
audioInFlight map[string]chan struct{} // cacheKey → closed when done
|
|
|
|
// browseMu guards browseInFlight — keys currently being refreshed
|
|
// in the background.
|
|
browseMu sync.Mutex
|
|
browseInFlight map[string]struct{}
|
|
|
|
// browseMemCache is a short-lived in-process cache for browse results.
|
|
// It is populated whenever a live upstream fetch succeeds and used as a
|
|
// last-resort fallback when both MinIO and the upstream are unavailable.
|
|
// Key: the MinIO cache key (same as used for BrowseHTMLKey).
|
|
browseMemCacheMu sync.RWMutex
|
|
browseMemCache map[string]browseCacheEntry
|
|
}
|
|
|
|
type browseCacheEntry struct {
|
|
novels []NovelListing
|
|
hasNext bool
|
|
cachedAt time.Time
|
|
}
|
|
|
|
// New creates a new Server.
|
|
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
|
|
return &Server{
|
|
addr: addr,
|
|
oCfg: oCfg,
|
|
novel: novel,
|
|
log: log,
|
|
store: store,
|
|
kokoroURL: kokoroURL,
|
|
kokoroVoice: kokoroVoice,
|
|
audioInFlight: make(map[string]chan struct{}),
|
|
browseInFlight: make(map[string]struct{}),
|
|
browseMemCache: make(map[string]browseCacheEntry),
|
|
}
|
|
}
|
|
|
|
// voices returns the list of available Kokoro voices. On the first call it
|
|
// fetches GET /v1/audio/voices from the Kokoro service and caches the result.
|
|
// If the fetch fails (Kokoro not up yet, network error, etc.) it falls back to
|
|
// the hardcoded kokoroVoices list so the UI is never empty.
|
|
func (s *Server) voices() []string {
|
|
s.voiceMu.RLock()
|
|
cached := s.cachedVoices
|
|
s.voiceMu.RUnlock()
|
|
if len(cached) > 0 {
|
|
return cached
|
|
}
|
|
|
|
if s.kokoroURL == "" {
|
|
return kokoroVoices
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.kokoroURL+"/v1/audio/voices", nil)
|
|
if err != nil {
|
|
s.log.Warn("could not fetch kokoro voices, using built-in list", "err", err)
|
|
return kokoroVoices
|
|
}
|
|
req.Header.Set("Accept", "application/json")
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
s.log.Warn("could not fetch kokoro voices, using built-in list", "err", err)
|
|
return kokoroVoices
|
|
}
|
|
defer resp.Body.Close()
|
|
var payload struct {
|
|
Voices []string `json:"voices"`
|
|
}
|
|
if resp.StatusCode != http.StatusOK || json.NewDecoder(resp.Body).Decode(&payload) != nil || len(payload.Voices) == 0 {
|
|
s.log.Warn("could not fetch kokoro voices, using built-in list")
|
|
return kokoroVoices
|
|
}
|
|
s.voiceMu.Lock()
|
|
s.cachedVoices = payload.Voices
|
|
s.voiceMu.Unlock()
|
|
s.log.Info("fetched kokoro voices", "count", len(payload.Voices))
|
|
return payload.Voices
|
|
}
|
|
|
|
// ListenAndServe starts the HTTP server and blocks until the provided context
|
|
// is cancelled.
|
|
func (s *Server) ListenAndServe(ctx context.Context) error {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /health", s.handleHealth)
|
|
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
|
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
|
mux.HandleFunc("POST /scrape/book/range", s.handleScrapeBookRange)
|
|
// Browse API — fetches and parses novelfire catalogue page
|
|
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
|
// Ranking API
|
|
mux.HandleFunc("GET /api/ranking", s.handleGetRanking)
|
|
// Cover image proxy (serves images stored in browse MinIO bucket)
|
|
mux.HandleFunc("GET /api/cover/{domain}/{slug}", s.handleGetCover)
|
|
// Scrape status
|
|
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
|
|
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
|
|
// Re-index chapters for a book from MinIO into PocketBase chapters_idx
|
|
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
|
|
// On-demand preview (no store writes) — for books not yet in the library
|
|
mux.HandleFunc("GET /api/book-preview/{slug}", s.handleBookPreview)
|
|
mux.HandleFunc("GET /api/chapter-text-preview/{slug}/{n}", s.handleChapterTextPreview)
|
|
// Search: local PocketBase + remote novelfire.net
|
|
mux.HandleFunc("GET /api/search", s.handleSearch)
|
|
// Progress API
|
|
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
|
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
|
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
|
|
// Presigned URL API (for SvelteKit UI)
|
|
mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter)
|
|
mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio)
|
|
mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample)
|
|
// Plain-text chapter content (used server-side for audio generation)
|
|
mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText)
|
|
// Voices list (proxied from Kokoro)
|
|
mux.HandleFunc("GET /api/voices", s.handleVoices)
|
|
// Voice sample generation — generates a short audio clip for each voice
|
|
// and stores it in MinIO for UI preview playback.
|
|
voiceSampleHandler := http.TimeoutHandler(
|
|
http.HandlerFunc(s.handleGenerateVoiceSamples),
|
|
15*time.Minute,
|
|
`{"error":"voice sample generation timed out"}`,
|
|
)
|
|
mux.Handle("POST /api/audio/voice-samples", voiceSampleHandler)
|
|
// Server-side audio generation via Kokoro /v1/audio/speech.
|
|
// Generation can take several minutes, so wrap in its own timeout handler.
|
|
audioGenHandler := http.TimeoutHandler(
|
|
http.HandlerFunc(s.handleAudioGenerate),
|
|
10*time.Minute,
|
|
`{"error":"audio generation timed out"}`,
|
|
)
|
|
mux.Handle("POST /api/audio/{slug}/{n}", audioGenHandler)
|
|
// Proxy route: fetches the generated file from Kokoro /v1/download/{filename}.
|
|
mux.HandleFunc("GET /api/audio-proxy/{slug}/{n}", s.handleAudioProxy)
|
|
|
|
srv := &http.Server{
|
|
Addr: s.addr,
|
|
Handler: mux,
|
|
ReadTimeout: 15 * time.Second,
|
|
WriteTimeout: 60 * time.Second,
|
|
IdleTimeout: 60 * time.Second,
|
|
}
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() { errCh <- srv.ListenAndServe() }()
|
|
|
|
s.log.Info("HTTP server listening", "addr", s.addr)
|
|
|
|
// Pre-populate voice samples in the background so the UI voice selector
|
|
// has playable previews without requiring a manual trigger.
|
|
go s.warmVoiceSamples(ctx)
|
|
|
|
// Warm the browse cache on startup: if page 1 is not cached in MinIO yet,
|
|
// trigger a background SingleFile snapshot immediately so the first user
|
|
// request is served from cache rather than hitting novelfire.net live.
|
|
go s.warmBrowseCache()
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
return srv.Shutdown(shutCtx)
|
|
case err := <-errCh:
|
|
return err
|
|
}
|
|
}
|
|
|
|
func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
|
}
|
|
|
|
// ─── Session cookie helpers ───────────────────────────────────────────────────
|
|
|
|
const sessionCookieName = "libnovel_session"
|
|
|
|
// sessionID returns the session ID from the request cookie, or "" if absent.
|
|
func sessionID(r *http.Request) string {
|
|
c, err := r.Cookie(sessionCookieName)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return c.Value
|
|
}
|
|
|
|
// newSessionID generates a random 16-byte hex session ID.
|
|
func newSessionID() (string, error) {
|
|
b := make([]byte, 16)
|
|
if _, err := rand.Read(b); err != nil {
|
|
return "", err
|
|
}
|
|
return hex.EncodeToString(b), nil
|
|
}
|
|
|
|
// ensureSession issues a new session cookie if the request does not already
|
|
// carry one, and returns the session ID (either existing or newly issued).
|
|
func ensureSession(w http.ResponseWriter, r *http.Request) string {
|
|
if id := sessionID(r); id != "" {
|
|
return id
|
|
}
|
|
id, err := newSessionID()
|
|
if err != nil {
|
|
// Very unlikely, but fall back to a timestamp-based ID.
|
|
id = fmt.Sprintf("fallback-%d", time.Now().UnixNano())
|
|
}
|
|
http.SetCookie(w, &http.Cookie{
|
|
Name: sessionCookieName,
|
|
Value: id,
|
|
Path: "/",
|
|
HttpOnly: true,
|
|
SameSite: http.SameSiteLaxMode,
|
|
MaxAge: 365 * 24 * 60 * 60, // 1 year
|
|
})
|
|
return id
|
|
}
|