feat(v3): add v3 stack — backend rewrite, renamed env vars, docs

- New Go backend binary (backend + runner) replacing old scraper/
- Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose
- Rename scraperFetch → backendFetch across all 19 UI server files
- Remove SCRAPER_PROXY env var and proxy transport from browser.Config
- Add Meilisearch, Valkey, Caddy to docker-compose
- Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
This commit is contained in:
Admin
2026-03-22 17:27:32 +05:00
parent 29d0eeb7e8
commit a85636d5db
178 changed files with 25951 additions and 0 deletions

View File

@@ -0,0 +1,176 @@
package runner
// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
// browse page snapshots and stores them in MinIO.
//
// Design:
// - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
// - Fetches page 1 for each combination of the standard genre/sort/status
// filter values and stores the parsed JSON blob in MinIO via BrowseStore.
// - The backend's handleBrowse then serves from MinIO instead of calling
// novelfire.net live, which avoids IP-based rate-limiting on the server.
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"time"
)
// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
type browseNovelListing struct {
Slug string `json:"slug"`
Title string `json:"title"`
Cover string `json:"cover"`
URL string `json:"url"`
}
// browseSnapshot is the JSON structure stored in MinIO.
type browseSnapshot struct {
Novels []browseNovelListing `json:"novels"`
Page int `json:"page"`
HasNext bool `json:"hasNext"`
// CachedAt is the UTC time the snapshot was written (ISO 8601).
CachedAt string `json:"cachedAt"`
}
// browseCombos lists the filter combinations to pre-fetch.
// Each entry is (genre, sort, status, novelType).
var browseCombos = []struct{ genre, sort, status, novelType string }{
{"all", "popular", "all", "all-novel"},
{"all", "popular", "ongoing", "all-novel"},
{"all", "popular", "completed", "all-novel"},
{"all", "new", "all", "all-novel"},
{"all", "new", "ongoing", "all-novel"},
{"all", "new", "completed", "all-novel"},
{"all", "top-rated", "all", "all-novel"},
{"all", "top-rated", "ongoing", "all-novel"},
{"all", "top-rated", "completed", "all-novel"},
}
const novelFireBrowseBase = "https://novelfire.net"
// runBrowseRefresh fetches all browse combos from novelfire.net and stores
// the results in MinIO. Errors per-combo are logged but do not abort the
// whole refresh cycle.
func (r *Runner) runBrowseRefresh(ctx context.Context) {
if r.deps.BrowseStore == nil {
r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
return
}
log := r.deps.Log.With("op", "browse_refresh")
log.Info("runner: browse refresh starting", "combos", len(browseCombos))
ok, fail := 0, 0
for _, c := range browseCombos {
if ctx.Err() != nil {
break
}
novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
if err != nil {
log.Warn("runner: browse fetch failed",
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
fail++
continue
}
snap := browseSnapshot{
Novels: novels,
Page: 1,
HasNext: hasNext,
CachedAt: time.Now().UTC().Format(time.RFC3339),
}
data, _ := json.Marshal(snap)
if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
log.Warn("runner: browse put failed",
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
fail++
continue
}
ok++
}
log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
}
// fetchBrowsePage calls novelfire.net and returns a list of novel listings
// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
novelFireBrowseBase, genre, sort, status, novelType)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
if err != nil {
return nil, false, fmt.Errorf("build request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
httpClient := &http.Client{Timeout: 45 * time.Second}
resp, err := httpClient.Do(req)
if err != nil {
return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, resp.Body)
return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
}
return parseBrowseHTML(resp.Body)
}
// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
// in backend/handlers.go — kept separate to avoid coupling packages.
func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
data, err := io.ReadAll(r)
if err != nil {
return nil, false, err
}
body := string(data)
hasNext := strings.Contains(body, `rel="next"`) ||
strings.Contains(body, `aria-label="Next"`) ||
strings.Contains(body, `class="next"`)
slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
slugMatches := slugRe.FindAllStringSubmatch(body, -1)
titleMatches := titleRe.FindAllStringSubmatch(body, -1)
coverMatches := coverRe.FindAllStringSubmatch(body, -1)
var novels []browseNovelListing
seen := make(map[string]bool)
for i, sm := range slugMatches {
slug := sm[1]
if seen[slug] {
continue
}
seen[slug] = true
item := browseNovelListing{
Slug: slug,
URL: novelFireBrowseBase + "/book/" + slug,
}
if i < len(titleMatches) {
item.Title = strings.TrimSpace(titleMatches[i][1])
}
if i < len(coverMatches) {
item.Cover = coverMatches[i][1]
}
if item.Title != "" {
novels = append(novels, item)
}
}
return novels, hasNext, nil
}

View File

@@ -0,0 +1,177 @@
package runner
// catalogue_refresh.go — independent loop that walks the full novelfire.net
// catalogue, scrapes per-book metadata, downloads cover images to MinIO, and
// indexes every book in Meilisearch.
//
// Design:
// - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run().
// - Also fires once on startup.
// - ScrapeCatalogue streams CatalogueEntry values over a channel — we iterate
// and call ScrapeMetadata for each entry.
// - Per-request random jitter (13s) prevents hammering novelfire.net.
// - Cover images are fetched from the URL embedded in BookMeta.Cover and
// stored in MinIO (browse bucket, key: covers/{slug}.jpg).
// - WriteMetadata + UpsertBook are called for every successfully scraped book.
// - Errors for individual books are logged and skipped; the loop continues.
// - The cover URL stored in BookMeta.Cover is rewritten to the internal proxy
// path (/api/cover/novelfire.net/{slug}) so the UI always fetches via the
// backend, which will serve from MinIO.
import (
"context"
"fmt"
"io"
"math/rand"
"net/http"
"time"
)
// runCatalogueRefresh performs one full catalogue walk: scrapes metadata for
// every book on novelfire.net, downloads covers to MinIO, and upserts to
// Meilisearch. Errors for individual books are logged and skipped.
func (r *Runner) runCatalogueRefresh(ctx context.Context) {
if r.deps.Novel == nil {
r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured")
return
}
if r.deps.BookWriter == nil {
r.deps.Log.Warn("runner: catalogue refresh skipped — BookWriter not configured")
return
}
log := r.deps.Log.With("op", "catalogue_refresh")
log.Info("runner: catalogue refresh starting")
entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
ok, skipped, errCount := 0, 0, 0
for entry := range entries {
if ctx.Err() != nil {
break
}
// Random jitter between books to avoid rate-limiting.
jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond
select {
case <-ctx.Done():
break
case <-time.After(jitter):
}
meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL)
if err != nil {
log.Warn("runner: catalogue refresh: metadata scrape failed",
"url", entry.URL, "err", err)
errCount++
continue
}
// Rewrite cover URL to backend proxy path so UI never hits CDN directly.
originalCover := meta.Cover
meta.Cover = fmt.Sprintf("/api/cover/novelfire.net/%s", meta.Slug)
// Persist to PocketBase.
if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil {
log.Warn("runner: catalogue refresh: WriteMetadata failed",
"slug", meta.Slug, "err", err)
errCount++
continue
}
// Index in Meilisearch.
if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
log.Warn("runner: catalogue refresh: UpsertBook failed",
"slug", meta.Slug, "err", err)
// non-fatal — continue
}
// Download and store cover image in MinIO if we have a cover URL
// and a CoverStore is wired in.
if r.deps.CoverStore != nil && originalCover != "" {
if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) {
if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil {
log.Warn("runner: catalogue refresh: cover download failed",
"slug", meta.Slug, "url", originalCover, "err", err)
// non-fatal
}
}
}
ok++
if ok%100 == 0 {
log.Info("runner: catalogue refresh progress",
"scraped", ok, "errors", errCount)
}
}
if err := <-errCh; err != nil {
log.Warn("runner: catalogue refresh: catalogue stream error", "err", err)
}
log.Info("runner: catalogue refresh finished",
"ok", ok, "skipped", skipped, "errors", errCount)
}
// downloadCover fetches the cover image from coverURL and stores it in MinIO
// under covers/{slug}.jpg. It retries up to 3 times with exponential backoff
// on transient errors (5xx, network failures).
func (r *Runner) downloadCover(ctx context.Context, slug, coverURL string) error {
const maxRetries = 3
delay := 2 * time.Second
var lastErr error
for attempt := 0; attempt < maxRetries; attempt++ {
if ctx.Err() != nil {
return ctx.Err()
}
if attempt > 0 {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(delay):
}
delay *= 2
}
data, err := fetchCoverBytes(ctx, coverURL)
if err != nil {
lastErr = err
continue
}
if err := r.deps.CoverStore.PutCover(ctx, slug, data, ""); err != nil {
return fmt.Errorf("put cover: %w", err)
}
return nil
}
return fmt.Errorf("download cover after %d retries: %w", maxRetries, lastErr)
}
// fetchCoverBytes performs a single HTTP GET for coverURL and returns the body.
func fetchCoverBytes(ctx context.Context, coverURL string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, coverURL, nil)
if err != nil {
return nil, fmt.Errorf("build request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
req.Header.Set("Referer", "https://novelfire.net/")
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("http get: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 500 {
_, _ = io.Copy(io.Discard, resp.Body)
return nil, fmt.Errorf("upstream %d for %s", resp.StatusCode, coverURL)
}
if resp.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, resp.Body)
return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, coverURL)
}
return io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5 MiB cap
}

View File

@@ -0,0 +1,21 @@
package runner
import (
"regexp"
"strings"
)
// stripMarkdown removes common markdown syntax from src, returning plain text
// suitable for TTS. Mirrors the helper in the scraper's server package.
func stripMarkdown(src string) string {
src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "")
src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "")
src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "")
src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "")
src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1")
src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "")
src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "")
src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "")
src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n")
return strings.TrimSpace(src)
}

View File

@@ -0,0 +1,92 @@
package runner
// metrics.go — lightweight HTTP metrics endpoint for the runner.
//
// GET /metrics returns a JSON document with live task counters and uptime.
// No external dependency (no Prometheus); plain net/http only.
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net"
"net/http"
"time"
)
// metricsServer serves GET /metrics for the runner process.
type metricsServer struct {
addr string
r *Runner
log *slog.Logger
}
func newMetricsServer(addr string, r *Runner, log *slog.Logger) *metricsServer {
return &metricsServer{addr: addr, r: r, log: log}
}
// ListenAndServe starts the HTTP server and blocks until ctx is cancelled or
// a fatal listen error occurs.
func (ms *metricsServer) ListenAndServe(ctx context.Context) error {
mux := http.NewServeMux()
mux.HandleFunc("GET /metrics", ms.handleMetrics)
mux.HandleFunc("GET /health", ms.handleHealth)
srv := &http.Server{
Addr: ms.addr,
Handler: mux,
ReadTimeout: 5 * time.Second,
WriteTimeout: 5 * time.Second,
BaseContext: func(_ net.Listener) context.Context { return ctx },
}
errCh := make(chan error, 1)
go func() {
ms.log.Info("runner: metrics server listening", "addr", ms.addr)
errCh <- srv.ListenAndServe()
}()
select {
case <-ctx.Done():
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = srv.Shutdown(shutCtx)
return nil
case err := <-errCh:
return fmt.Errorf("runner: metrics server: %w", err)
}
}
// handleMetrics handles GET /metrics.
// Response shape (JSON):
//
// {
// "tasks_running": N,
// "tasks_completed": N,
// "tasks_failed": N,
// "uptime_seconds": N
// }
func (ms *metricsServer) handleMetrics(w http.ResponseWriter, _ *http.Request) {
uptimeSec := int64(time.Since(ms.r.startedAt).Seconds())
metricsWriteJSON(w, 0, map[string]int64{
"tasks_running": ms.r.tasksRunning.Load(),
"tasks_completed": ms.r.tasksCompleted.Load(),
"tasks_failed": ms.r.tasksFailed.Load(),
"uptime_seconds": uptimeSec,
})
}
// handleHealth handles GET /health — simple liveness probe for the metrics server.
func (ms *metricsServer) handleHealth(w http.ResponseWriter, _ *http.Request) {
metricsWriteJSON(w, 0, map[string]string{"status": "ok"})
}
// metricsWriteJSON writes v as a JSON response with the given status code.
func metricsWriteJSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
if status != 0 {
w.WriteHeader(status)
}
_ = json.NewEncoder(w).Encode(v)
}

View File

@@ -0,0 +1,440 @@
// Package runner implements the worker loop that polls PocketBase for pending
// scrape and audio tasks, executes them, and reports results back.
//
// Design:
// - Run(ctx) loops on a ticker; each tick claims and dispatches pending tasks.
// - Scrape tasks are dispatched to the Orchestrator (one goroutine per task,
// up to MaxConcurrentScrape).
// - Audio tasks fetch chapter text, call Kokoro, upload to MinIO, and report
// the result back (up to MaxConcurrentAudio goroutines).
// - The runner is stateless between ticks; all state lives in PocketBase.
// - Atomic task counters are exposed via /metrics (see metrics.go).
// - Books are indexed in Meilisearch via an orchestrator.Config.PostMetadata
// hook injected at construction time.
package runner
import (
"context"
"fmt"
"log/slog"
"sync"
"sync/atomic"
"time"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/meili"
"github.com/libnovel/backend/internal/orchestrator"
"github.com/libnovel/backend/internal/scraper"
"github.com/libnovel/backend/internal/taskqueue"
)
// Config tunes the runner behaviour.
type Config struct {
// WorkerID uniquely identifies this runner instance in PocketBase records.
WorkerID string
// PollInterval is how often the runner checks for new tasks.
PollInterval time.Duration
// MaxConcurrentScrape limits simultaneous book-scrape goroutines.
MaxConcurrentScrape int
// MaxConcurrentAudio limits simultaneous audio-generation goroutines.
MaxConcurrentAudio int
// OrchestratorWorkers is the chapter-scraping parallelism inside each book run.
OrchestratorWorkers int
// HeartbeatInterval is how often active tasks PATCH their heartbeat_at
// timestamp to signal they are still alive. Defaults to 30s when 0.
HeartbeatInterval time.Duration
// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
// task is considered orphaned and reset to pending. Defaults to 2m when 0.
StaleTaskThreshold time.Duration
// BrowseRefreshInterval is how often the runner pre-fetches browse page
// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
BrowseRefreshInterval time.Duration
// CatalogueRefreshInterval is how often the runner walks the full catalogue,
// scrapes per-book metadata, downloads covers, and re-indexes everything in
// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
CatalogueRefreshInterval time.Duration
// MetricsAddr is the HTTP listen address for the /metrics endpoint.
// Defaults to ":9091". Set to "" to disable.
MetricsAddr string
}
// Dependencies are the external services the runner depends on.
type Dependencies struct {
// Consumer claims tasks from PocketBase.
Consumer taskqueue.Consumer
// BookWriter persists scraped data (used by orchestrator).
BookWriter bookstore.BookWriter
// BookReader reads chapter text for audio generation.
BookReader bookstore.BookReader
// AudioStore persists generated audio and checks key existence.
AudioStore bookstore.AudioStore
// BrowseStore stores browse page snapshots in MinIO.
BrowseStore bookstore.BrowseStore
// CoverStore stores book cover images in MinIO.
// If nil, cover downloads are skipped during catalogue refresh.
CoverStore bookstore.CoverStore
// SearchIndex indexes books in Meilisearch after scraping.
// If nil a no-op is used.
SearchIndex meili.Client
// Novel is the scraper implementation.
Novel scraper.NovelScraper
// Kokoro is the TTS client.
Kokoro kokoro.Client
// Log is the structured logger.
Log *slog.Logger
}
// Runner is the main worker process.
type Runner struct {
cfg Config
deps Dependencies
// Atomic task counters — read by /metrics without locking.
tasksRunning atomic.Int64
tasksCompleted atomic.Int64
tasksFailed atomic.Int64
startedAt time.Time
}
// New creates a Runner from cfg and deps.
func New(cfg Config, deps Dependencies) *Runner {
if cfg.PollInterval <= 0 {
cfg.PollInterval = 30 * time.Second
}
if cfg.MaxConcurrentScrape <= 0 {
cfg.MaxConcurrentScrape = 2
}
if cfg.MaxConcurrentAudio <= 0 {
cfg.MaxConcurrentAudio = 1
}
if cfg.WorkerID == "" {
cfg.WorkerID = "runner"
}
if cfg.HeartbeatInterval <= 0 {
cfg.HeartbeatInterval = 30 * time.Second
}
if cfg.StaleTaskThreshold <= 0 {
cfg.StaleTaskThreshold = 2 * time.Minute
}
if cfg.BrowseRefreshInterval <= 0 {
cfg.BrowseRefreshInterval = 6 * time.Hour
}
if cfg.CatalogueRefreshInterval <= 0 {
cfg.CatalogueRefreshInterval = 24 * time.Hour
}
if cfg.MetricsAddr == "" {
cfg.MetricsAddr = ":9091"
}
if deps.Log == nil {
deps.Log = slog.Default()
}
if deps.SearchIndex == nil {
deps.SearchIndex = meili.NoopClient{}
}
return &Runner{cfg: cfg, deps: deps, startedAt: time.Now()}
}
// Run starts the poll loop and the metrics HTTP server, blocking until ctx is
// cancelled.
func (r *Runner) Run(ctx context.Context) error {
r.deps.Log.Info("runner: starting",
"worker_id", r.cfg.WorkerID,
"poll_interval", r.cfg.PollInterval,
"max_scrape", r.cfg.MaxConcurrentScrape,
"max_audio", r.cfg.MaxConcurrentAudio,
"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
"catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval,
"metrics_addr", r.cfg.MetricsAddr,
)
// Start metrics HTTP server in background if configured.
if r.cfg.MetricsAddr != "" {
ms := newMetricsServer(r.cfg.MetricsAddr, r, r.deps.Log)
go func() {
if err := ms.ListenAndServe(ctx); err != nil {
r.deps.Log.Error("runner: metrics server error", "err", err)
}
}()
}
scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio)
var wg sync.WaitGroup
tick := time.NewTicker(r.cfg.PollInterval)
defer tick.Stop()
browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
defer browseTick.Stop()
catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval)
defer catalogueTick.Stop()
// Run one browse refresh immediately on startup.
go r.runBrowseRefresh(ctx)
// Run one catalogue refresh immediately on startup.
go r.runCatalogueRefresh(ctx)
// Run one poll immediately on startup, then on each tick.
for {
r.poll(ctx, scrapeSem, audioSem, &wg)
select {
case <-ctx.Done():
r.deps.Log.Info("runner: context cancelled, draining active tasks")
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case <-done:
r.deps.Log.Info("runner: all tasks drained, exiting")
case <-time.After(2 * time.Minute):
r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
}
return nil
case <-browseTick.C:
go r.runBrowseRefresh(ctx)
case <-catalogueTick.C:
go r.runCatalogueRefresh(ctx)
case <-tick.C:
}
}
}
// poll claims all available pending tasks and dispatches them to goroutines.
func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) {
// ── Reap orphaned tasks ───────────────────────────────────────────────
if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
r.deps.Log.Warn("runner: reap stale tasks failed", "err", err)
} else if n > 0 {
r.deps.Log.Info("runner: reaped stale tasks", "count", n)
}
// ── Scrape tasks ──────────────────────────────────────────────────────
for {
if ctx.Err() != nil {
return
}
task, ok, err := r.deps.Consumer.ClaimNextScrapeTask(ctx, r.cfg.WorkerID)
if err != nil {
r.deps.Log.Error("runner: ClaimNextScrapeTask failed", "err", err)
break
}
if !ok {
break
}
select {
case scrapeSem <- struct{}{}:
default:
r.deps.Log.Warn("runner: scrape semaphore full, will retry next tick",
"task_id", task.ID)
break
}
r.tasksRunning.Add(1)
wg.Add(1)
go func(t domain.ScrapeTask) {
defer wg.Done()
defer func() { <-scrapeSem }()
defer r.tasksRunning.Add(-1)
r.runScrapeTask(ctx, t)
}(task)
}
// ── Audio tasks ───────────────────────────────────────────────────────
for {
if ctx.Err() != nil {
return
}
task, ok, err := r.deps.Consumer.ClaimNextAudioTask(ctx, r.cfg.WorkerID)
if err != nil {
r.deps.Log.Error("runner: ClaimNextAudioTask failed", "err", err)
break
}
if !ok {
break
}
select {
case audioSem <- struct{}{}:
default:
r.deps.Log.Warn("runner: audio semaphore full, will retry next tick",
"task_id", task.ID)
break
}
r.tasksRunning.Add(1)
wg.Add(1)
go func(t domain.AudioTask) {
defer wg.Done()
defer func() { <-audioSem }()
defer r.tasksRunning.Add(-1)
r.runAudioTask(ctx, t)
}(task)
}
}
// newOrchestrator builds an orchestrator with the Meilisearch post-hook wired in.
func (r *Runner) newOrchestrator() *orchestrator.Orchestrator {
oCfg := orchestrator.Config{
Workers: r.cfg.OrchestratorWorkers,
PostMetadata: func(ctx context.Context, meta domain.BookMeta) {
if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
r.deps.Log.Warn("runner: meilisearch upsert failed",
"slug", meta.Slug, "err", err)
}
},
}
return orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log)
}
// runScrapeTask executes one scrape task end-to-end and reports the result.
func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
log := r.deps.Log.With("task_id", task.ID, "kind", task.Kind, "url", task.TargetURL)
log.Info("runner: scrape task starting")
hbCtx, hbCancel := context.WithCancel(ctx)
defer hbCancel()
go func() {
tick := time.NewTicker(r.cfg.HeartbeatInterval)
defer tick.Stop()
for {
select {
case <-hbCtx.Done():
return
case <-tick.C:
if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil {
log.Warn("runner: heartbeat failed", "err", err)
}
}
}
}()
o := r.newOrchestrator()
var result domain.ScrapeResult
switch task.Kind {
case "catalogue":
result = r.runCatalogueTask(ctx, task, o, log)
case "book", "book_range":
result = o.RunBook(ctx, task)
default:
result.ErrorMessage = fmt.Sprintf("unknown task kind: %q", task.Kind)
log.Warn("runner: unknown task kind")
}
if err := r.deps.Consumer.FinishScrapeTask(ctx, task.ID, result); err != nil {
log.Error("runner: FinishScrapeTask failed", "err", err)
}
if result.ErrorMessage != "" {
r.tasksFailed.Add(1)
} else {
r.tasksCompleted.Add(1)
}
log.Info("runner: scrape task finished",
"scraped", result.ChaptersScraped,
"skipped", result.ChaptersSkipped,
"errors", result.Errors,
)
}
// runCatalogueTask runs a full catalogue scrape.
func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o *orchestrator.Orchestrator, log *slog.Logger) domain.ScrapeResult {
entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
var result domain.ScrapeResult
for entry := range entries {
if ctx.Err() != nil {
break
}
bookTask := domain.ScrapeTask{
ID: task.ID,
Kind: "book",
TargetURL: entry.URL,
}
bookResult := o.RunBook(ctx, bookTask)
result.BooksFound += bookResult.BooksFound + 1
result.ChaptersScraped += bookResult.ChaptersScraped
result.ChaptersSkipped += bookResult.ChaptersSkipped
result.Errors += bookResult.Errors
}
if err := <-errCh; err != nil {
log.Warn("runner: catalogue scrape finished with error", "err", err)
result.Errors++
if result.ErrorMessage == "" {
result.ErrorMessage = err.Error()
}
}
return result
}
// runAudioTask executes one audio-generation task.
func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "voice", task.Voice)
log.Info("runner: audio task starting")
hbCtx, hbCancel := context.WithCancel(ctx)
defer hbCancel()
go func() {
tick := time.NewTicker(r.cfg.HeartbeatInterval)
defer tick.Stop()
for {
select {
case <-hbCtx.Done():
return
case <-tick.C:
if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil {
log.Warn("runner: heartbeat failed", "err", err)
}
}
}
}()
fail := func(msg string) {
log.Error("runner: audio task failed", "reason", msg)
r.tasksFailed.Add(1)
result := domain.AudioResult{ErrorMessage: msg}
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
log.Error("runner: FinishAudioTask failed", "err", err)
}
}
raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter)
if err != nil {
fail(fmt.Sprintf("read chapter: %v", err))
return
}
text := stripMarkdown(raw)
if text == "" {
fail("chapter text is empty after stripping markdown")
return
}
if r.deps.Kokoro == nil {
fail("kokoro client not configured")
return
}
audioData, err := r.deps.Kokoro.GenerateAudio(ctx, text, task.Voice)
if err != nil {
fail(fmt.Sprintf("kokoro generate: %v", err))
return
}
key := r.deps.AudioStore.AudioObjectKey(task.Slug, task.Chapter, task.Voice)
if err := r.deps.AudioStore.PutAudio(ctx, key, audioData); err != nil {
fail(fmt.Sprintf("put audio: %v", err))
return
}
r.tasksCompleted.Add(1)
result := domain.AudioResult{ObjectKey: key}
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
log.Error("runner: FinishAudioTask failed", "err", err)
}
log.Info("runner: audio task finished", "key", key)
}

View File

@@ -0,0 +1,365 @@
package runner_test
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/runner"
)
// ── Stub types ────────────────────────────────────────────────────────────────
// stubConsumer is a test double for taskqueue.Consumer.
type stubConsumer struct {
scrapeQueue []domain.ScrapeTask
audioQueue []domain.AudioTask
scrapeIdx int
audioIdx int
finished []string
failCalled []string
claimErr error
}
func (s *stubConsumer) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
if s.claimErr != nil {
return domain.ScrapeTask{}, false, s.claimErr
}
if s.scrapeIdx >= len(s.scrapeQueue) {
return domain.ScrapeTask{}, false, nil
}
t := s.scrapeQueue[s.scrapeIdx]
s.scrapeIdx++
return t, true, nil
}
func (s *stubConsumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
if s.claimErr != nil {
return domain.AudioTask{}, false, s.claimErr
}
if s.audioIdx >= len(s.audioQueue) {
return domain.AudioTask{}, false, nil
}
t := s.audioQueue[s.audioIdx]
s.audioIdx++
return t, true, nil
}
func (s *stubConsumer) FinishScrapeTask(_ context.Context, id string, _ domain.ScrapeResult) error {
s.finished = append(s.finished, id)
return nil
}
func (s *stubConsumer) FinishAudioTask(_ context.Context, id string, _ domain.AudioResult) error {
s.finished = append(s.finished, id)
return nil
}
func (s *stubConsumer) FailTask(_ context.Context, id, _ string) error {
s.failCalled = append(s.failCalled, id)
return nil
}
func (s *stubConsumer) HeartbeatTask(_ context.Context, _ string) error { return nil }
func (s *stubConsumer) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) {
return 0, nil
}
// stubBookWriter satisfies bookstore.BookWriter (no-op).
type stubBookWriter struct{}
func (s *stubBookWriter) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil }
func (s *stubBookWriter) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error {
return nil
}
func (s *stubBookWriter) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error {
return nil
}
func (s *stubBookWriter) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool {
return false
}
// stubBookReader satisfies bookstore.BookReader — returns a single chapter.
type stubBookReader struct {
text string
readErr error
}
func (s *stubBookReader) ReadChapter(_ context.Context, _ string, _ int) (string, error) {
return s.text, s.readErr
}
func (s *stubBookReader) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) {
return domain.BookMeta{}, false, nil
}
func (s *stubBookReader) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil }
func (s *stubBookReader) LocalSlugs(_ context.Context) (map[string]bool, error) { return nil, nil }
func (s *stubBookReader) MetadataMtime(_ context.Context, _ string) int64 { return 0 }
func (s *stubBookReader) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) {
return nil, nil
}
func (s *stubBookReader) CountChapters(_ context.Context, _ string) int { return 0 }
func (s *stubBookReader) ReindexChapters(_ context.Context, _ string) (int, error) {
return 0, nil
}
// stubAudioStore satisfies bookstore.AudioStore.
type stubAudioStore struct {
putCalled atomic.Int32
putErr error
}
func (s *stubAudioStore) AudioObjectKey(slug string, n int, voice string) string {
return slug + "/" + string(rune('0'+n)) + "/" + voice + ".mp3"
}
func (s *stubAudioStore) AudioExists(_ context.Context, _ string) bool { return false }
func (s *stubAudioStore) PutAudio(_ context.Context, _ string, _ []byte) error {
s.putCalled.Add(1)
return s.putErr
}
// stubNovelScraper satisfies scraper.NovelScraper minimally.
type stubNovelScraper struct {
entries []domain.CatalogueEntry
metaErr error
chapters []domain.ChapterRef
}
func (s *stubNovelScraper) ScrapeCatalogue(_ context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
ch := make(chan domain.CatalogueEntry, len(s.entries))
errCh := make(chan error, 1)
for _, e := range s.entries {
ch <- e
}
close(ch)
close(errCh)
return ch, errCh
}
func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) {
if s.metaErr != nil {
return domain.BookMeta{}, s.metaErr
}
return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil
}
func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
return s.chapters, nil
}
func (s *stubNovelScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
return domain.Chapter{Ref: ref, Text: "# Chapter\n\nSome text."}, nil
}
func (s *stubNovelScraper) ScrapeRanking(_ context.Context, _ int) (<-chan domain.BookMeta, <-chan error) {
ch := make(chan domain.BookMeta)
errCh := make(chan error, 1)
close(ch)
close(errCh)
return ch, errCh
}
func (s *stubNovelScraper) SourceName() string { return "stub" }
// stubKokoro satisfies kokoro.Client.
type stubKokoro struct {
data []byte
genErr error
called atomic.Int32
}
func (s *stubKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
s.called.Add(1)
return s.data, s.genErr
}
func (s *stubKokoro) ListVoices(_ context.Context) ([]string, error) {
return []string{"af_bella"}, nil
}
// ── stripMarkdown helper ──────────────────────────────────────────────────────
func TestStripMarkdownViaAudioTask(t *testing.T) {
// Verify markdown is stripped before sending to Kokoro.
// We inject chapter text with markdown; the kokoro stub verifies data flows.
consumer := &stubConsumer{
audioQueue: []domain.AudioTask{
{ID: "a1", Slug: "book", Chapter: 1, Voice: "af_bella", Status: domain.TaskStatusRunning},
},
}
bookReader := &stubBookReader{text: "## Chapter 1\n\nPlain **text** here."}
audioStore := &stubAudioStore{}
kokoroStub := &stubKokoro{data: []byte("mp3")}
cfg := runner.Config{
WorkerID: "test",
PollInterval: time.Hour, // long poll — we'll cancel manually
}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: bookReader,
AudioStore: audioStore,
Novel: &stubNovelScraper{},
Kokoro: kokoroStub,
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = r.Run(ctx)
if kokoroStub.called.Load() != 1 {
t.Errorf("expected Kokoro.GenerateAudio called once, got %d", kokoroStub.called.Load())
}
if audioStore.putCalled.Load() != 1 {
t.Errorf("expected PutAudio called once, got %d", audioStore.putCalled.Load())
}
}
func TestAudioTask_ReadChapterError(t *testing.T) {
consumer := &stubConsumer{
audioQueue: []domain.AudioTask{
{ID: "a2", Slug: "book", Chapter: 2, Voice: "af_bella", Status: domain.TaskStatusRunning},
},
}
bookReader := &stubBookReader{readErr: errors.New("chapter not found")}
audioStore := &stubAudioStore{}
kokoroStub := &stubKokoro{data: []byte("mp3")}
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: bookReader,
AudioStore: audioStore,
Novel: &stubNovelScraper{},
Kokoro: kokoroStub,
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = r.Run(ctx)
// Kokoro should not be called; FinishAudioTask should be called with error.
if kokoroStub.called.Load() != 0 {
t.Errorf("expected Kokoro not called, got %d", kokoroStub.called.Load())
}
if len(consumer.finished) != 1 {
t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished))
}
}
func TestAudioTask_KokoroError(t *testing.T) {
consumer := &stubConsumer{
audioQueue: []domain.AudioTask{
{ID: "a3", Slug: "book", Chapter: 3, Voice: "af_bella", Status: domain.TaskStatusRunning},
},
}
bookReader := &stubBookReader{text: "Chapter text."}
audioStore := &stubAudioStore{}
kokoroStub := &stubKokoro{genErr: errors.New("tts failed")}
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: bookReader,
AudioStore: audioStore,
Novel: &stubNovelScraper{},
Kokoro: kokoroStub,
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = r.Run(ctx)
if audioStore.putCalled.Load() != 0 {
t.Errorf("expected PutAudio not called, got %d", audioStore.putCalled.Load())
}
if len(consumer.finished) != 1 {
t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished))
}
}
func TestScrapeTask_BookKind(t *testing.T) {
consumer := &stubConsumer{
scrapeQueue: []domain.ScrapeTask{
{ID: "s1", Kind: "book", TargetURL: "https://example.com/book/test-book", Status: domain.TaskStatusRunning},
},
}
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: &stubBookReader{},
AudioStore: &stubAudioStore{},
Novel: &stubNovelScraper{},
Kokoro: &stubKokoro{},
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = r.Run(ctx)
if len(consumer.finished) != 1 || consumer.finished[0] != "s1" {
t.Errorf("expected task s1 finished, got %v", consumer.finished)
}
}
func TestScrapeTask_UnknownKind(t *testing.T) {
consumer := &stubConsumer{
scrapeQueue: []domain.ScrapeTask{
{ID: "s2", Kind: "unknown_kind", Status: domain.TaskStatusRunning},
},
}
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: &stubBookReader{},
AudioStore: &stubAudioStore{},
Novel: &stubNovelScraper{},
Kokoro: &stubKokoro{},
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
_ = r.Run(ctx)
// Unknown kind still finishes the task (with error message in result).
if len(consumer.finished) != 1 || consumer.finished[0] != "s2" {
t.Errorf("expected task s2 finished, got %v", consumer.finished)
}
}
func TestRun_CancelImmediately(t *testing.T) {
consumer := &stubConsumer{}
cfg := runner.Config{WorkerID: "test", PollInterval: 10 * time.Millisecond}
deps := runner.Dependencies{
Consumer: consumer,
BookWriter: &stubBookWriter{},
BookReader: &stubBookReader{},
AudioStore: &stubAudioStore{},
Novel: &stubNovelScraper{},
Kokoro: &stubKokoro{},
}
r := runner.New(cfg, deps)
ctx, cancel := context.WithCancel(context.Background())
cancel() // cancel before Run
err := r.Run(ctx)
if err != nil {
t.Errorf("expected nil on graceful shutdown, got %v", err)
}
}