feat(v3): add v3 stack — backend rewrite, renamed env vars, docs
- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
This commit is contained in:
176
v3/backend/internal/runner/browse_refresh.go
Normal file
176
v3/backend/internal/runner/browse_refresh.go
Normal file
@@ -0,0 +1,176 @@
|
||||
package runner
|
||||
|
||||
// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
|
||||
// browse page snapshots and stores them in MinIO.
|
||||
//
|
||||
// Design:
|
||||
// - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
|
||||
// - Fetches page 1 for each combination of the standard genre/sort/status
|
||||
// filter values and stores the parsed JSON blob in MinIO via BrowseStore.
|
||||
// - The backend's handleBrowse then serves from MinIO instead of calling
|
||||
// novelfire.net live, which avoids IP-based rate-limiting on the server.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
|
||||
type browseNovelListing struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Cover string `json:"cover"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// browseSnapshot is the JSON structure stored in MinIO.
|
||||
type browseSnapshot struct {
|
||||
Novels []browseNovelListing `json:"novels"`
|
||||
Page int `json:"page"`
|
||||
HasNext bool `json:"hasNext"`
|
||||
// CachedAt is the UTC time the snapshot was written (ISO 8601).
|
||||
CachedAt string `json:"cachedAt"`
|
||||
}
|
||||
|
||||
// browseCombos lists the filter combinations to pre-fetch.
|
||||
// Each entry is (genre, sort, status, novelType).
|
||||
var browseCombos = []struct{ genre, sort, status, novelType string }{
|
||||
{"all", "popular", "all", "all-novel"},
|
||||
{"all", "popular", "ongoing", "all-novel"},
|
||||
{"all", "popular", "completed", "all-novel"},
|
||||
{"all", "new", "all", "all-novel"},
|
||||
{"all", "new", "ongoing", "all-novel"},
|
||||
{"all", "new", "completed", "all-novel"},
|
||||
{"all", "top-rated", "all", "all-novel"},
|
||||
{"all", "top-rated", "ongoing", "all-novel"},
|
||||
{"all", "top-rated", "completed", "all-novel"},
|
||||
}
|
||||
|
||||
const novelFireBrowseBase = "https://novelfire.net"
|
||||
|
||||
// runBrowseRefresh fetches all browse combos from novelfire.net and stores
|
||||
// the results in MinIO. Errors per-combo are logged but do not abort the
|
||||
// whole refresh cycle.
|
||||
func (r *Runner) runBrowseRefresh(ctx context.Context) {
|
||||
if r.deps.BrowseStore == nil {
|
||||
r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
|
||||
return
|
||||
}
|
||||
|
||||
log := r.deps.Log.With("op", "browse_refresh")
|
||||
log.Info("runner: browse refresh starting", "combos", len(browseCombos))
|
||||
|
||||
ok, fail := 0, 0
|
||||
for _, c := range browseCombos {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
|
||||
if err != nil {
|
||||
log.Warn("runner: browse fetch failed",
|
||||
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
|
||||
fail++
|
||||
continue
|
||||
}
|
||||
|
||||
snap := browseSnapshot{
|
||||
Novels: novels,
|
||||
Page: 1,
|
||||
HasNext: hasNext,
|
||||
CachedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
data, _ := json.Marshal(snap)
|
||||
if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
|
||||
log.Warn("runner: browse put failed",
|
||||
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
|
||||
fail++
|
||||
continue
|
||||
}
|
||||
ok++
|
||||
}
|
||||
|
||||
log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
|
||||
}
|
||||
|
||||
// fetchBrowsePage calls novelfire.net and returns a list of novel listings
|
||||
// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
|
||||
func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
|
||||
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
|
||||
novelFireBrowseBase, genre, sort, status, novelType)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
|
||||
httpClient := &http.Client{Timeout: 45 * time.Second}
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
|
||||
}
|
||||
|
||||
return parseBrowseHTML(resp.Body)
|
||||
}
|
||||
|
||||
// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
|
||||
// in backend/handlers.go — kept separate to avoid coupling packages.
|
||||
func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
body := string(data)
|
||||
|
||||
hasNext := strings.Contains(body, `rel="next"`) ||
|
||||
strings.Contains(body, `aria-label="Next"`) ||
|
||||
strings.Contains(body, `class="next"`)
|
||||
|
||||
slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
|
||||
titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
|
||||
coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
|
||||
|
||||
slugMatches := slugRe.FindAllStringSubmatch(body, -1)
|
||||
titleMatches := titleRe.FindAllStringSubmatch(body, -1)
|
||||
coverMatches := coverRe.FindAllStringSubmatch(body, -1)
|
||||
|
||||
var novels []browseNovelListing
|
||||
seen := make(map[string]bool)
|
||||
for i, sm := range slugMatches {
|
||||
slug := sm[1]
|
||||
if seen[slug] {
|
||||
continue
|
||||
}
|
||||
seen[slug] = true
|
||||
|
||||
item := browseNovelListing{
|
||||
Slug: slug,
|
||||
URL: novelFireBrowseBase + "/book/" + slug,
|
||||
}
|
||||
if i < len(titleMatches) {
|
||||
item.Title = strings.TrimSpace(titleMatches[i][1])
|
||||
}
|
||||
if i < len(coverMatches) {
|
||||
item.Cover = coverMatches[i][1]
|
||||
}
|
||||
if item.Title != "" {
|
||||
novels = append(novels, item)
|
||||
}
|
||||
}
|
||||
|
||||
return novels, hasNext, nil
|
||||
}
|
||||
177
v3/backend/internal/runner/catalogue_refresh.go
Normal file
177
v3/backend/internal/runner/catalogue_refresh.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package runner
|
||||
|
||||
// catalogue_refresh.go — independent loop that walks the full novelfire.net
|
||||
// catalogue, scrapes per-book metadata, downloads cover images to MinIO, and
|
||||
// indexes every book in Meilisearch.
|
||||
//
|
||||
// Design:
|
||||
// - Runs on its own ticker (CatalogueRefreshInterval, default 24h) inside Run().
|
||||
// - Also fires once on startup.
|
||||
// - ScrapeCatalogue streams CatalogueEntry values over a channel — we iterate
|
||||
// and call ScrapeMetadata for each entry.
|
||||
// - Per-request random jitter (1–3s) prevents hammering novelfire.net.
|
||||
// - Cover images are fetched from the URL embedded in BookMeta.Cover and
|
||||
// stored in MinIO (browse bucket, key: covers/{slug}.jpg).
|
||||
// - WriteMetadata + UpsertBook are called for every successfully scraped book.
|
||||
// - Errors for individual books are logged and skipped; the loop continues.
|
||||
// - The cover URL stored in BookMeta.Cover is rewritten to the internal proxy
|
||||
// path (/api/cover/novelfire.net/{slug}) so the UI always fetches via the
|
||||
// backend, which will serve from MinIO.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// runCatalogueRefresh performs one full catalogue walk: scrapes metadata for
|
||||
// every book on novelfire.net, downloads covers to MinIO, and upserts to
|
||||
// Meilisearch. Errors for individual books are logged and skipped.
|
||||
func (r *Runner) runCatalogueRefresh(ctx context.Context) {
|
||||
if r.deps.Novel == nil {
|
||||
r.deps.Log.Warn("runner: catalogue refresh skipped — Novel scraper not configured")
|
||||
return
|
||||
}
|
||||
if r.deps.BookWriter == nil {
|
||||
r.deps.Log.Warn("runner: catalogue refresh skipped — BookWriter not configured")
|
||||
return
|
||||
}
|
||||
|
||||
log := r.deps.Log.With("op", "catalogue_refresh")
|
||||
log.Info("runner: catalogue refresh starting")
|
||||
|
||||
entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
|
||||
|
||||
ok, skipped, errCount := 0, 0, 0
|
||||
for entry := range entries {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
|
||||
// Random jitter between books to avoid rate-limiting.
|
||||
jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
break
|
||||
case <-time.After(jitter):
|
||||
}
|
||||
|
||||
meta, err := r.deps.Novel.ScrapeMetadata(ctx, entry.URL)
|
||||
if err != nil {
|
||||
log.Warn("runner: catalogue refresh: metadata scrape failed",
|
||||
"url", entry.URL, "err", err)
|
||||
errCount++
|
||||
continue
|
||||
}
|
||||
|
||||
// Rewrite cover URL to backend proxy path so UI never hits CDN directly.
|
||||
originalCover := meta.Cover
|
||||
meta.Cover = fmt.Sprintf("/api/cover/novelfire.net/%s", meta.Slug)
|
||||
|
||||
// Persist to PocketBase.
|
||||
if err := r.deps.BookWriter.WriteMetadata(ctx, meta); err != nil {
|
||||
log.Warn("runner: catalogue refresh: WriteMetadata failed",
|
||||
"slug", meta.Slug, "err", err)
|
||||
errCount++
|
||||
continue
|
||||
}
|
||||
|
||||
// Index in Meilisearch.
|
||||
if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
|
||||
log.Warn("runner: catalogue refresh: UpsertBook failed",
|
||||
"slug", meta.Slug, "err", err)
|
||||
// non-fatal — continue
|
||||
}
|
||||
|
||||
// Download and store cover image in MinIO if we have a cover URL
|
||||
// and a CoverStore is wired in.
|
||||
if r.deps.CoverStore != nil && originalCover != "" {
|
||||
if !r.deps.CoverStore.CoverExists(ctx, meta.Slug) {
|
||||
if err := r.downloadCover(ctx, meta.Slug, originalCover); err != nil {
|
||||
log.Warn("runner: catalogue refresh: cover download failed",
|
||||
"slug", meta.Slug, "url", originalCover, "err", err)
|
||||
// non-fatal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ok++
|
||||
if ok%100 == 0 {
|
||||
log.Info("runner: catalogue refresh progress",
|
||||
"scraped", ok, "errors", errCount)
|
||||
}
|
||||
}
|
||||
|
||||
if err := <-errCh; err != nil {
|
||||
log.Warn("runner: catalogue refresh: catalogue stream error", "err", err)
|
||||
}
|
||||
|
||||
log.Info("runner: catalogue refresh finished",
|
||||
"ok", ok, "skipped", skipped, "errors", errCount)
|
||||
}
|
||||
|
||||
// downloadCover fetches the cover image from coverURL and stores it in MinIO
|
||||
// under covers/{slug}.jpg. It retries up to 3 times with exponential backoff
|
||||
// on transient errors (5xx, network failures).
|
||||
func (r *Runner) downloadCover(ctx context.Context, slug, coverURL string) error {
|
||||
const maxRetries = 3
|
||||
delay := 2 * time.Second
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if attempt > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
delay *= 2
|
||||
}
|
||||
|
||||
data, err := fetchCoverBytes(ctx, coverURL)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
if err := r.deps.CoverStore.PutCover(ctx, slug, data, ""); err != nil {
|
||||
return fmt.Errorf("put cover: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("download cover after %d retries: %w", maxRetries, lastErr)
|
||||
}
|
||||
|
||||
// fetchCoverBytes performs a single HTTP GET for coverURL and returns the body.
|
||||
func fetchCoverBytes(ctx context.Context, coverURL string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, coverURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
|
||||
req.Header.Set("Referer", "https://novelfire.net/")
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http get: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 500 {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("upstream %d for %s", resp.StatusCode, coverURL)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, coverURL)
|
||||
}
|
||||
|
||||
return io.ReadAll(io.LimitReader(resp.Body, 5<<20)) // 5 MiB cap
|
||||
}
|
||||
21
v3/backend/internal/runner/helpers.go
Normal file
21
v3/backend/internal/runner/helpers.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// stripMarkdown removes common markdown syntax from src, returning plain text
|
||||
// suitable for TTS. Mirrors the helper in the scraper's server package.
|
||||
func stripMarkdown(src string) string {
|
||||
src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1")
|
||||
src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "")
|
||||
src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n")
|
||||
return strings.TrimSpace(src)
|
||||
}
|
||||
92
v3/backend/internal/runner/metrics.go
Normal file
92
v3/backend/internal/runner/metrics.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package runner
|
||||
|
||||
// metrics.go — lightweight HTTP metrics endpoint for the runner.
|
||||
//
|
||||
// GET /metrics returns a JSON document with live task counters and uptime.
|
||||
// No external dependency (no Prometheus); plain net/http only.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// metricsServer serves GET /metrics for the runner process.
|
||||
type metricsServer struct {
|
||||
addr string
|
||||
r *Runner
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
func newMetricsServer(addr string, r *Runner, log *slog.Logger) *metricsServer {
|
||||
return &metricsServer{addr: addr, r: r, log: log}
|
||||
}
|
||||
|
||||
// ListenAndServe starts the HTTP server and blocks until ctx is cancelled or
|
||||
// a fatal listen error occurs.
|
||||
func (ms *metricsServer) ListenAndServe(ctx context.Context) error {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /metrics", ms.handleMetrics)
|
||||
mux.HandleFunc("GET /health", ms.handleHealth)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: ms.addr,
|
||||
Handler: mux,
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 5 * time.Second,
|
||||
BaseContext: func(_ net.Listener) context.Context { return ctx },
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
ms.log.Info("runner: metrics server listening", "addr", ms.addr)
|
||||
errCh <- srv.ListenAndServe()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = srv.Shutdown(shutCtx)
|
||||
return nil
|
||||
case err := <-errCh:
|
||||
return fmt.Errorf("runner: metrics server: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// handleMetrics handles GET /metrics.
|
||||
// Response shape (JSON):
|
||||
//
|
||||
// {
|
||||
// "tasks_running": N,
|
||||
// "tasks_completed": N,
|
||||
// "tasks_failed": N,
|
||||
// "uptime_seconds": N
|
||||
// }
|
||||
func (ms *metricsServer) handleMetrics(w http.ResponseWriter, _ *http.Request) {
|
||||
uptimeSec := int64(time.Since(ms.r.startedAt).Seconds())
|
||||
metricsWriteJSON(w, 0, map[string]int64{
|
||||
"tasks_running": ms.r.tasksRunning.Load(),
|
||||
"tasks_completed": ms.r.tasksCompleted.Load(),
|
||||
"tasks_failed": ms.r.tasksFailed.Load(),
|
||||
"uptime_seconds": uptimeSec,
|
||||
})
|
||||
}
|
||||
|
||||
// handleHealth handles GET /health — simple liveness probe for the metrics server.
|
||||
func (ms *metricsServer) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
||||
metricsWriteJSON(w, 0, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// metricsWriteJSON writes v as a JSON response with the given status code.
|
||||
func metricsWriteJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if status != 0 {
|
||||
w.WriteHeader(status)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
440
v3/backend/internal/runner/runner.go
Normal file
440
v3/backend/internal/runner/runner.go
Normal file
@@ -0,0 +1,440 @@
|
||||
// Package runner implements the worker loop that polls PocketBase for pending
|
||||
// scrape and audio tasks, executes them, and reports results back.
|
||||
//
|
||||
// Design:
|
||||
// - Run(ctx) loops on a ticker; each tick claims and dispatches pending tasks.
|
||||
// - Scrape tasks are dispatched to the Orchestrator (one goroutine per task,
|
||||
// up to MaxConcurrentScrape).
|
||||
// - Audio tasks fetch chapter text, call Kokoro, upload to MinIO, and report
|
||||
// the result back (up to MaxConcurrentAudio goroutines).
|
||||
// - The runner is stateless between ticks; all state lives in PocketBase.
|
||||
// - Atomic task counters are exposed via /metrics (see metrics.go).
|
||||
// - Books are indexed in Meilisearch via an orchestrator.Config.PostMetadata
|
||||
// hook injected at construction time.
|
||||
package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/backend/internal/bookstore"
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
"github.com/libnovel/backend/internal/meili"
|
||||
"github.com/libnovel/backend/internal/orchestrator"
|
||||
"github.com/libnovel/backend/internal/scraper"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
)
|
||||
|
||||
// Config tunes the runner behaviour.
|
||||
type Config struct {
|
||||
// WorkerID uniquely identifies this runner instance in PocketBase records.
|
||||
WorkerID string
|
||||
// PollInterval is how often the runner checks for new tasks.
|
||||
PollInterval time.Duration
|
||||
// MaxConcurrentScrape limits simultaneous book-scrape goroutines.
|
||||
MaxConcurrentScrape int
|
||||
// MaxConcurrentAudio limits simultaneous audio-generation goroutines.
|
||||
MaxConcurrentAudio int
|
||||
// OrchestratorWorkers is the chapter-scraping parallelism inside each book run.
|
||||
OrchestratorWorkers int
|
||||
// HeartbeatInterval is how often active tasks PATCH their heartbeat_at
|
||||
// timestamp to signal they are still alive. Defaults to 30s when 0.
|
||||
HeartbeatInterval time.Duration
|
||||
// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
|
||||
// task is considered orphaned and reset to pending. Defaults to 2m when 0.
|
||||
StaleTaskThreshold time.Duration
|
||||
// BrowseRefreshInterval is how often the runner pre-fetches browse page
|
||||
// snapshots from novelfire.net and stores them in MinIO. Defaults to 6h.
|
||||
BrowseRefreshInterval time.Duration
|
||||
// CatalogueRefreshInterval is how often the runner walks the full catalogue,
|
||||
// scrapes per-book metadata, downloads covers, and re-indexes everything in
|
||||
// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
|
||||
CatalogueRefreshInterval time.Duration
|
||||
// MetricsAddr is the HTTP listen address for the /metrics endpoint.
|
||||
// Defaults to ":9091". Set to "" to disable.
|
||||
MetricsAddr string
|
||||
}
|
||||
|
||||
// Dependencies are the external services the runner depends on.
|
||||
type Dependencies struct {
|
||||
// Consumer claims tasks from PocketBase.
|
||||
Consumer taskqueue.Consumer
|
||||
// BookWriter persists scraped data (used by orchestrator).
|
||||
BookWriter bookstore.BookWriter
|
||||
// BookReader reads chapter text for audio generation.
|
||||
BookReader bookstore.BookReader
|
||||
// AudioStore persists generated audio and checks key existence.
|
||||
AudioStore bookstore.AudioStore
|
||||
// BrowseStore stores browse page snapshots in MinIO.
|
||||
BrowseStore bookstore.BrowseStore
|
||||
// CoverStore stores book cover images in MinIO.
|
||||
// If nil, cover downloads are skipped during catalogue refresh.
|
||||
CoverStore bookstore.CoverStore
|
||||
// SearchIndex indexes books in Meilisearch after scraping.
|
||||
// If nil a no-op is used.
|
||||
SearchIndex meili.Client
|
||||
// Novel is the scraper implementation.
|
||||
Novel scraper.NovelScraper
|
||||
// Kokoro is the TTS client.
|
||||
Kokoro kokoro.Client
|
||||
// Log is the structured logger.
|
||||
Log *slog.Logger
|
||||
}
|
||||
|
||||
// Runner is the main worker process.
|
||||
type Runner struct {
|
||||
cfg Config
|
||||
deps Dependencies
|
||||
|
||||
// Atomic task counters — read by /metrics without locking.
|
||||
tasksRunning atomic.Int64
|
||||
tasksCompleted atomic.Int64
|
||||
tasksFailed atomic.Int64
|
||||
|
||||
startedAt time.Time
|
||||
}
|
||||
|
||||
// New creates a Runner from cfg and deps.
|
||||
func New(cfg Config, deps Dependencies) *Runner {
|
||||
if cfg.PollInterval <= 0 {
|
||||
cfg.PollInterval = 30 * time.Second
|
||||
}
|
||||
if cfg.MaxConcurrentScrape <= 0 {
|
||||
cfg.MaxConcurrentScrape = 2
|
||||
}
|
||||
if cfg.MaxConcurrentAudio <= 0 {
|
||||
cfg.MaxConcurrentAudio = 1
|
||||
}
|
||||
if cfg.WorkerID == "" {
|
||||
cfg.WorkerID = "runner"
|
||||
}
|
||||
if cfg.HeartbeatInterval <= 0 {
|
||||
cfg.HeartbeatInterval = 30 * time.Second
|
||||
}
|
||||
if cfg.StaleTaskThreshold <= 0 {
|
||||
cfg.StaleTaskThreshold = 2 * time.Minute
|
||||
}
|
||||
if cfg.BrowseRefreshInterval <= 0 {
|
||||
cfg.BrowseRefreshInterval = 6 * time.Hour
|
||||
}
|
||||
if cfg.CatalogueRefreshInterval <= 0 {
|
||||
cfg.CatalogueRefreshInterval = 24 * time.Hour
|
||||
}
|
||||
if cfg.MetricsAddr == "" {
|
||||
cfg.MetricsAddr = ":9091"
|
||||
}
|
||||
if deps.Log == nil {
|
||||
deps.Log = slog.Default()
|
||||
}
|
||||
if deps.SearchIndex == nil {
|
||||
deps.SearchIndex = meili.NoopClient{}
|
||||
}
|
||||
return &Runner{cfg: cfg, deps: deps, startedAt: time.Now()}
|
||||
}
|
||||
|
||||
// Run starts the poll loop and the metrics HTTP server, blocking until ctx is
|
||||
// cancelled.
|
||||
func (r *Runner) Run(ctx context.Context) error {
|
||||
r.deps.Log.Info("runner: starting",
|
||||
"worker_id", r.cfg.WorkerID,
|
||||
"poll_interval", r.cfg.PollInterval,
|
||||
"max_scrape", r.cfg.MaxConcurrentScrape,
|
||||
"max_audio", r.cfg.MaxConcurrentAudio,
|
||||
"browse_refresh_interval", r.cfg.BrowseRefreshInterval,
|
||||
"catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval,
|
||||
"metrics_addr", r.cfg.MetricsAddr,
|
||||
)
|
||||
|
||||
// Start metrics HTTP server in background if configured.
|
||||
if r.cfg.MetricsAddr != "" {
|
||||
ms := newMetricsServer(r.cfg.MetricsAddr, r, r.deps.Log)
|
||||
go func() {
|
||||
if err := ms.ListenAndServe(ctx); err != nil {
|
||||
r.deps.Log.Error("runner: metrics server error", "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
|
||||
audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
tick := time.NewTicker(r.cfg.PollInterval)
|
||||
defer tick.Stop()
|
||||
|
||||
browseTick := time.NewTicker(r.cfg.BrowseRefreshInterval)
|
||||
defer browseTick.Stop()
|
||||
|
||||
catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval)
|
||||
defer catalogueTick.Stop()
|
||||
|
||||
// Run one browse refresh immediately on startup.
|
||||
go r.runBrowseRefresh(ctx)
|
||||
// Run one catalogue refresh immediately on startup.
|
||||
go r.runCatalogueRefresh(ctx)
|
||||
|
||||
// Run one poll immediately on startup, then on each tick.
|
||||
for {
|
||||
r.poll(ctx, scrapeSem, audioSem, &wg)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
r.deps.Log.Info("runner: context cancelled, draining active tasks")
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
r.deps.Log.Info("runner: all tasks drained, exiting")
|
||||
case <-time.After(2 * time.Minute):
|
||||
r.deps.Log.Warn("runner: drain timeout exceeded, forcing exit")
|
||||
}
|
||||
return nil
|
||||
case <-browseTick.C:
|
||||
go r.runBrowseRefresh(ctx)
|
||||
case <-catalogueTick.C:
|
||||
go r.runCatalogueRefresh(ctx)
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// poll claims all available pending tasks and dispatches them to goroutines.
|
||||
func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) {
|
||||
// ── Reap orphaned tasks ───────────────────────────────────────────────
|
||||
if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
|
||||
r.deps.Log.Warn("runner: reap stale tasks failed", "err", err)
|
||||
} else if n > 0 {
|
||||
r.deps.Log.Info("runner: reaped stale tasks", "count", n)
|
||||
}
|
||||
|
||||
// ── Scrape tasks ──────────────────────────────────────────────────────
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
task, ok, err := r.deps.Consumer.ClaimNextScrapeTask(ctx, r.cfg.WorkerID)
|
||||
if err != nil {
|
||||
r.deps.Log.Error("runner: ClaimNextScrapeTask failed", "err", err)
|
||||
break
|
||||
}
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case scrapeSem <- struct{}{}:
|
||||
default:
|
||||
r.deps.Log.Warn("runner: scrape semaphore full, will retry next tick",
|
||||
"task_id", task.ID)
|
||||
break
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
wg.Add(1)
|
||||
go func(t domain.ScrapeTask) {
|
||||
defer wg.Done()
|
||||
defer func() { <-scrapeSem }()
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runScrapeTask(ctx, t)
|
||||
}(task)
|
||||
}
|
||||
|
||||
// ── Audio tasks ───────────────────────────────────────────────────────
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
task, ok, err := r.deps.Consumer.ClaimNextAudioTask(ctx, r.cfg.WorkerID)
|
||||
if err != nil {
|
||||
r.deps.Log.Error("runner: ClaimNextAudioTask failed", "err", err)
|
||||
break
|
||||
}
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case audioSem <- struct{}{}:
|
||||
default:
|
||||
r.deps.Log.Warn("runner: audio semaphore full, will retry next tick",
|
||||
"task_id", task.ID)
|
||||
break
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
wg.Add(1)
|
||||
go func(t domain.AudioTask) {
|
||||
defer wg.Done()
|
||||
defer func() { <-audioSem }()
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runAudioTask(ctx, t)
|
||||
}(task)
|
||||
}
|
||||
}
|
||||
|
||||
// newOrchestrator builds an orchestrator with the Meilisearch post-hook wired in.
|
||||
func (r *Runner) newOrchestrator() *orchestrator.Orchestrator {
|
||||
oCfg := orchestrator.Config{
|
||||
Workers: r.cfg.OrchestratorWorkers,
|
||||
PostMetadata: func(ctx context.Context, meta domain.BookMeta) {
|
||||
if err := r.deps.SearchIndex.UpsertBook(ctx, meta); err != nil {
|
||||
r.deps.Log.Warn("runner: meilisearch upsert failed",
|
||||
"slug", meta.Slug, "err", err)
|
||||
}
|
||||
},
|
||||
}
|
||||
return orchestrator.New(oCfg, r.deps.Novel, r.deps.BookWriter, r.deps.Log)
|
||||
}
|
||||
|
||||
// runScrapeTask executes one scrape task end-to-end and reports the result.
|
||||
func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
|
||||
log := r.deps.Log.With("task_id", task.ID, "kind", task.Kind, "url", task.TargetURL)
|
||||
log.Info("runner: scrape task starting")
|
||||
|
||||
hbCtx, hbCancel := context.WithCancel(ctx)
|
||||
defer hbCancel()
|
||||
go func() {
|
||||
tick := time.NewTicker(r.cfg.HeartbeatInterval)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-hbCtx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil {
|
||||
log.Warn("runner: heartbeat failed", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
o := r.newOrchestrator()
|
||||
var result domain.ScrapeResult
|
||||
|
||||
switch task.Kind {
|
||||
case "catalogue":
|
||||
result = r.runCatalogueTask(ctx, task, o, log)
|
||||
case "book", "book_range":
|
||||
result = o.RunBook(ctx, task)
|
||||
default:
|
||||
result.ErrorMessage = fmt.Sprintf("unknown task kind: %q", task.Kind)
|
||||
log.Warn("runner: unknown task kind")
|
||||
}
|
||||
|
||||
if err := r.deps.Consumer.FinishScrapeTask(ctx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishScrapeTask failed", "err", err)
|
||||
}
|
||||
|
||||
if result.ErrorMessage != "" {
|
||||
r.tasksFailed.Add(1)
|
||||
} else {
|
||||
r.tasksCompleted.Add(1)
|
||||
}
|
||||
|
||||
log.Info("runner: scrape task finished",
|
||||
"scraped", result.ChaptersScraped,
|
||||
"skipped", result.ChaptersSkipped,
|
||||
"errors", result.Errors,
|
||||
)
|
||||
}
|
||||
|
||||
// runCatalogueTask runs a full catalogue scrape.
|
||||
func (r *Runner) runCatalogueTask(ctx context.Context, task domain.ScrapeTask, o *orchestrator.Orchestrator, log *slog.Logger) domain.ScrapeResult {
|
||||
entries, errCh := r.deps.Novel.ScrapeCatalogue(ctx)
|
||||
var result domain.ScrapeResult
|
||||
|
||||
for entry := range entries {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
bookTask := domain.ScrapeTask{
|
||||
ID: task.ID,
|
||||
Kind: "book",
|
||||
TargetURL: entry.URL,
|
||||
}
|
||||
bookResult := o.RunBook(ctx, bookTask)
|
||||
result.BooksFound += bookResult.BooksFound + 1
|
||||
result.ChaptersScraped += bookResult.ChaptersScraped
|
||||
result.ChaptersSkipped += bookResult.ChaptersSkipped
|
||||
result.Errors += bookResult.Errors
|
||||
}
|
||||
|
||||
if err := <-errCh; err != nil {
|
||||
log.Warn("runner: catalogue scrape finished with error", "err", err)
|
||||
result.Errors++
|
||||
if result.ErrorMessage == "" {
|
||||
result.ErrorMessage = err.Error()
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// runAudioTask executes one audio-generation task.
|
||||
func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
|
||||
log := r.deps.Log.With("task_id", task.ID, "slug", task.Slug, "chapter", task.Chapter, "voice", task.Voice)
|
||||
log.Info("runner: audio task starting")
|
||||
|
||||
hbCtx, hbCancel := context.WithCancel(ctx)
|
||||
defer hbCancel()
|
||||
go func() {
|
||||
tick := time.NewTicker(r.cfg.HeartbeatInterval)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-hbCtx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if err := r.deps.Consumer.HeartbeatTask(ctx, task.ID); err != nil {
|
||||
log.Warn("runner: heartbeat failed", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
fail := func(msg string) {
|
||||
log.Error("runner: audio task failed", "reason", msg)
|
||||
r.tasksFailed.Add(1)
|
||||
result := domain.AudioResult{ErrorMessage: msg}
|
||||
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishAudioTask failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter)
|
||||
if err != nil {
|
||||
fail(fmt.Sprintf("read chapter: %v", err))
|
||||
return
|
||||
}
|
||||
text := stripMarkdown(raw)
|
||||
if text == "" {
|
||||
fail("chapter text is empty after stripping markdown")
|
||||
return
|
||||
}
|
||||
|
||||
if r.deps.Kokoro == nil {
|
||||
fail("kokoro client not configured")
|
||||
return
|
||||
}
|
||||
audioData, err := r.deps.Kokoro.GenerateAudio(ctx, text, task.Voice)
|
||||
if err != nil {
|
||||
fail(fmt.Sprintf("kokoro generate: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
key := r.deps.AudioStore.AudioObjectKey(task.Slug, task.Chapter, task.Voice)
|
||||
if err := r.deps.AudioStore.PutAudio(ctx, key, audioData); err != nil {
|
||||
fail(fmt.Sprintf("put audio: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
r.tasksCompleted.Add(1)
|
||||
result := domain.AudioResult{ObjectKey: key}
|
||||
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
|
||||
log.Error("runner: FinishAudioTask failed", "err", err)
|
||||
}
|
||||
log.Info("runner: audio task finished", "key", key)
|
||||
}
|
||||
365
v3/backend/internal/runner/runner_test.go
Normal file
365
v3/backend/internal/runner/runner_test.go
Normal file
@@ -0,0 +1,365 @@
|
||||
package runner_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/runner"
|
||||
)
|
||||
|
||||
// ── Stub types ────────────────────────────────────────────────────────────────
|
||||
|
||||
// stubConsumer is a test double for taskqueue.Consumer.
|
||||
type stubConsumer struct {
|
||||
scrapeQueue []domain.ScrapeTask
|
||||
audioQueue []domain.AudioTask
|
||||
scrapeIdx int
|
||||
audioIdx int
|
||||
finished []string
|
||||
failCalled []string
|
||||
claimErr error
|
||||
}
|
||||
|
||||
func (s *stubConsumer) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
|
||||
if s.claimErr != nil {
|
||||
return domain.ScrapeTask{}, false, s.claimErr
|
||||
}
|
||||
if s.scrapeIdx >= len(s.scrapeQueue) {
|
||||
return domain.ScrapeTask{}, false, nil
|
||||
}
|
||||
t := s.scrapeQueue[s.scrapeIdx]
|
||||
s.scrapeIdx++
|
||||
return t, true, nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
|
||||
if s.claimErr != nil {
|
||||
return domain.AudioTask{}, false, s.claimErr
|
||||
}
|
||||
if s.audioIdx >= len(s.audioQueue) {
|
||||
return domain.AudioTask{}, false, nil
|
||||
}
|
||||
t := s.audioQueue[s.audioIdx]
|
||||
s.audioIdx++
|
||||
return t, true, nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) FinishScrapeTask(_ context.Context, id string, _ domain.ScrapeResult) error {
|
||||
s.finished = append(s.finished, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) FinishAudioTask(_ context.Context, id string, _ domain.AudioResult) error {
|
||||
s.finished = append(s.finished, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) FailTask(_ context.Context, id, _ string) error {
|
||||
s.failCalled = append(s.failCalled, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *stubConsumer) HeartbeatTask(_ context.Context, _ string) error { return nil }
|
||||
|
||||
func (s *stubConsumer) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// stubBookWriter satisfies bookstore.BookWriter (no-op).
|
||||
type stubBookWriter struct{}
|
||||
|
||||
func (s *stubBookWriter) WriteMetadata(_ context.Context, _ domain.BookMeta) error { return nil }
|
||||
func (s *stubBookWriter) WriteChapter(_ context.Context, _ string, _ domain.Chapter) error {
|
||||
return nil
|
||||
}
|
||||
func (s *stubBookWriter) WriteChapterRefs(_ context.Context, _ string, _ []domain.ChapterRef) error {
|
||||
return nil
|
||||
}
|
||||
func (s *stubBookWriter) ChapterExists(_ context.Context, _ string, _ domain.ChapterRef) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// stubBookReader satisfies bookstore.BookReader — returns a single chapter.
|
||||
type stubBookReader struct {
|
||||
text string
|
||||
readErr error
|
||||
}
|
||||
|
||||
func (s *stubBookReader) ReadChapter(_ context.Context, _ string, _ int) (string, error) {
|
||||
return s.text, s.readErr
|
||||
}
|
||||
func (s *stubBookReader) ReadMetadata(_ context.Context, _ string) (domain.BookMeta, bool, error) {
|
||||
return domain.BookMeta{}, false, nil
|
||||
}
|
||||
func (s *stubBookReader) ListBooks(_ context.Context) ([]domain.BookMeta, error) { return nil, nil }
|
||||
func (s *stubBookReader) LocalSlugs(_ context.Context) (map[string]bool, error) { return nil, nil }
|
||||
func (s *stubBookReader) MetadataMtime(_ context.Context, _ string) int64 { return 0 }
|
||||
func (s *stubBookReader) ListChapters(_ context.Context, _ string) ([]domain.ChapterInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *stubBookReader) CountChapters(_ context.Context, _ string) int { return 0 }
|
||||
func (s *stubBookReader) ReindexChapters(_ context.Context, _ string) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// stubAudioStore satisfies bookstore.AudioStore.
|
||||
type stubAudioStore struct {
|
||||
putCalled atomic.Int32
|
||||
putErr error
|
||||
}
|
||||
|
||||
func (s *stubAudioStore) AudioObjectKey(slug string, n int, voice string) string {
|
||||
return slug + "/" + string(rune('0'+n)) + "/" + voice + ".mp3"
|
||||
}
|
||||
func (s *stubAudioStore) AudioExists(_ context.Context, _ string) bool { return false }
|
||||
func (s *stubAudioStore) PutAudio(_ context.Context, _ string, _ []byte) error {
|
||||
s.putCalled.Add(1)
|
||||
return s.putErr
|
||||
}
|
||||
|
||||
// stubNovelScraper satisfies scraper.NovelScraper minimally.
|
||||
type stubNovelScraper struct {
|
||||
entries []domain.CatalogueEntry
|
||||
metaErr error
|
||||
chapters []domain.ChapterRef
|
||||
}
|
||||
|
||||
func (s *stubNovelScraper) ScrapeCatalogue(_ context.Context) (<-chan domain.CatalogueEntry, <-chan error) {
|
||||
ch := make(chan domain.CatalogueEntry, len(s.entries))
|
||||
errCh := make(chan error, 1)
|
||||
for _, e := range s.entries {
|
||||
ch <- e
|
||||
}
|
||||
close(ch)
|
||||
close(errCh)
|
||||
return ch, errCh
|
||||
}
|
||||
|
||||
func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMeta, error) {
|
||||
if s.metaErr != nil {
|
||||
return domain.BookMeta{}, s.metaErr
|
||||
}
|
||||
return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil
|
||||
}
|
||||
|
||||
func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
|
||||
return s.chapters, nil
|
||||
}
|
||||
|
||||
func (s *stubNovelScraper) ScrapeChapterText(_ context.Context, ref domain.ChapterRef) (domain.Chapter, error) {
|
||||
return domain.Chapter{Ref: ref, Text: "# Chapter\n\nSome text."}, nil
|
||||
}
|
||||
|
||||
func (s *stubNovelScraper) ScrapeRanking(_ context.Context, _ int) (<-chan domain.BookMeta, <-chan error) {
|
||||
ch := make(chan domain.BookMeta)
|
||||
errCh := make(chan error, 1)
|
||||
close(ch)
|
||||
close(errCh)
|
||||
return ch, errCh
|
||||
}
|
||||
|
||||
func (s *stubNovelScraper) SourceName() string { return "stub" }
|
||||
|
||||
// stubKokoro satisfies kokoro.Client.
|
||||
type stubKokoro struct {
|
||||
data []byte
|
||||
genErr error
|
||||
called atomic.Int32
|
||||
}
|
||||
|
||||
func (s *stubKokoro) GenerateAudio(_ context.Context, _, _ string) ([]byte, error) {
|
||||
s.called.Add(1)
|
||||
return s.data, s.genErr
|
||||
}
|
||||
|
||||
func (s *stubKokoro) ListVoices(_ context.Context) ([]string, error) {
|
||||
return []string{"af_bella"}, nil
|
||||
}
|
||||
|
||||
// ── stripMarkdown helper ──────────────────────────────────────────────────────
|
||||
|
||||
func TestStripMarkdownViaAudioTask(t *testing.T) {
|
||||
// Verify markdown is stripped before sending to Kokoro.
|
||||
// We inject chapter text with markdown; the kokoro stub verifies data flows.
|
||||
consumer := &stubConsumer{
|
||||
audioQueue: []domain.AudioTask{
|
||||
{ID: "a1", Slug: "book", Chapter: 1, Voice: "af_bella", Status: domain.TaskStatusRunning},
|
||||
},
|
||||
}
|
||||
bookReader := &stubBookReader{text: "## Chapter 1\n\nPlain **text** here."}
|
||||
audioStore := &stubAudioStore{}
|
||||
kokoroStub := &stubKokoro{data: []byte("mp3")}
|
||||
|
||||
cfg := runner.Config{
|
||||
WorkerID: "test",
|
||||
PollInterval: time.Hour, // long poll — we'll cancel manually
|
||||
}
|
||||
deps := runner.Dependencies{
|
||||
Consumer: consumer,
|
||||
BookWriter: &stubBookWriter{},
|
||||
BookReader: bookReader,
|
||||
AudioStore: audioStore,
|
||||
Novel: &stubNovelScraper{},
|
||||
Kokoro: kokoroStub,
|
||||
}
|
||||
|
||||
r := runner.New(cfg, deps)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
_ = r.Run(ctx)
|
||||
|
||||
if kokoroStub.called.Load() != 1 {
|
||||
t.Errorf("expected Kokoro.GenerateAudio called once, got %d", kokoroStub.called.Load())
|
||||
}
|
||||
if audioStore.putCalled.Load() != 1 {
|
||||
t.Errorf("expected PutAudio called once, got %d", audioStore.putCalled.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAudioTask_ReadChapterError(t *testing.T) {
|
||||
consumer := &stubConsumer{
|
||||
audioQueue: []domain.AudioTask{
|
||||
{ID: "a2", Slug: "book", Chapter: 2, Voice: "af_bella", Status: domain.TaskStatusRunning},
|
||||
},
|
||||
}
|
||||
bookReader := &stubBookReader{readErr: errors.New("chapter not found")}
|
||||
audioStore := &stubAudioStore{}
|
||||
kokoroStub := &stubKokoro{data: []byte("mp3")}
|
||||
|
||||
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
||||
deps := runner.Dependencies{
|
||||
Consumer: consumer,
|
||||
BookWriter: &stubBookWriter{},
|
||||
BookReader: bookReader,
|
||||
AudioStore: audioStore,
|
||||
Novel: &stubNovelScraper{},
|
||||
Kokoro: kokoroStub,
|
||||
}
|
||||
|
||||
r := runner.New(cfg, deps)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
_ = r.Run(ctx)
|
||||
|
||||
// Kokoro should not be called; FinishAudioTask should be called with error.
|
||||
if kokoroStub.called.Load() != 0 {
|
||||
t.Errorf("expected Kokoro not called, got %d", kokoroStub.called.Load())
|
||||
}
|
||||
if len(consumer.finished) != 1 {
|
||||
t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAudioTask_KokoroError(t *testing.T) {
|
||||
consumer := &stubConsumer{
|
||||
audioQueue: []domain.AudioTask{
|
||||
{ID: "a3", Slug: "book", Chapter: 3, Voice: "af_bella", Status: domain.TaskStatusRunning},
|
||||
},
|
||||
}
|
||||
bookReader := &stubBookReader{text: "Chapter text."}
|
||||
audioStore := &stubAudioStore{}
|
||||
kokoroStub := &stubKokoro{genErr: errors.New("tts failed")}
|
||||
|
||||
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
||||
deps := runner.Dependencies{
|
||||
Consumer: consumer,
|
||||
BookWriter: &stubBookWriter{},
|
||||
BookReader: bookReader,
|
||||
AudioStore: audioStore,
|
||||
Novel: &stubNovelScraper{},
|
||||
Kokoro: kokoroStub,
|
||||
}
|
||||
|
||||
r := runner.New(cfg, deps)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
_ = r.Run(ctx)
|
||||
|
||||
if audioStore.putCalled.Load() != 0 {
|
||||
t.Errorf("expected PutAudio not called, got %d", audioStore.putCalled.Load())
|
||||
}
|
||||
if len(consumer.finished) != 1 {
|
||||
t.Errorf("expected FinishAudioTask called once, got %d", len(consumer.finished))
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrapeTask_BookKind(t *testing.T) {
|
||||
consumer := &stubConsumer{
|
||||
scrapeQueue: []domain.ScrapeTask{
|
||||
{ID: "s1", Kind: "book", TargetURL: "https://example.com/book/test-book", Status: domain.TaskStatusRunning},
|
||||
},
|
||||
}
|
||||
|
||||
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
||||
deps := runner.Dependencies{
|
||||
Consumer: consumer,
|
||||
BookWriter: &stubBookWriter{},
|
||||
BookReader: &stubBookReader{},
|
||||
AudioStore: &stubAudioStore{},
|
||||
Novel: &stubNovelScraper{},
|
||||
Kokoro: &stubKokoro{},
|
||||
}
|
||||
|
||||
r := runner.New(cfg, deps)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
_ = r.Run(ctx)
|
||||
|
||||
if len(consumer.finished) != 1 || consumer.finished[0] != "s1" {
|
||||
t.Errorf("expected task s1 finished, got %v", consumer.finished)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrapeTask_UnknownKind(t *testing.T) {
|
||||
consumer := &stubConsumer{
|
||||
scrapeQueue: []domain.ScrapeTask{
|
||||
{ID: "s2", Kind: "unknown_kind", Status: domain.TaskStatusRunning},
|
||||
},
|
||||
}
|
||||
|
||||
cfg := runner.Config{WorkerID: "test", PollInterval: time.Hour}
|
||||
deps := runner.Dependencies{
|
||||
Consumer: consumer,
|
||||
BookWriter: &stubBookWriter{},
|
||||
BookReader: &stubBookReader{},
|
||||
AudioStore: &stubAudioStore{},
|
||||
Novel: &stubNovelScraper{},
|
||||
Kokoro: &stubKokoro{},
|
||||
}
|
||||
|
||||
r := runner.New(cfg, deps)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
_ = r.Run(ctx)
|
||||
|
||||
// Unknown kind still finishes the task (with error message in result).
|
||||
if len(consumer.finished) != 1 || consumer.finished[0] != "s2" {
|
||||
t.Errorf("expected task s2 finished, got %v", consumer.finished)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun_CancelImmediately(t *testing.T) {
|
||||
consumer := &stubConsumer{}
|
||||
cfg := runner.Config{WorkerID: "test", PollInterval: 10 * time.Millisecond}
|
||||
deps := runner.Dependencies{
|
||||
Consumer: consumer,
|
||||
BookWriter: &stubBookWriter{},
|
||||
BookReader: &stubBookReader{},
|
||||
AudioStore: &stubAudioStore{},
|
||||
Novel: &stubNovelScraper{},
|
||||
Kokoro: &stubKokoro{},
|
||||
}
|
||||
|
||||
r := runner.New(cfg, deps)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // cancel before Run
|
||||
|
||||
err := r.Run(ctx)
|
||||
if err != nil {
|
||||
t.Errorf("expected nil on graceful shutdown, got %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user