feat(queue): replace PocketBase polling with Asynq + Redis
Some checks failed
CI / Docker / caddy (pull_request) Failing after 50s
CI / Check ui (pull_request) Successful in 1m5s
Release / Check ui (push) Successful in 45s
Release / Test backend (push) Successful in 1m29s
CI / Docker / ui (pull_request) Successful in 1m28s
Release / Docker / backend (push) Successful in 3m14s
CI / Test backend (pull_request) Failing after 42s
CI / Docker / backend (pull_request) Has been skipped
CI / Docker / runner (pull_request) Has been skipped
Release / Docker / caddy (push) Successful in 6m48s
Release / Docker / ui (push) Successful in 2m8s
Release / Docker / runner (push) Successful in 2m51s
Release / Upload source maps (push) Failing after 53s
Release / Gitea Release (push) Has been skipped
Some checks failed
CI / Docker / caddy (pull_request) Failing after 50s
CI / Check ui (pull_request) Successful in 1m5s
Release / Check ui (push) Successful in 45s
Release / Test backend (push) Successful in 1m29s
CI / Docker / ui (pull_request) Successful in 1m28s
Release / Docker / backend (push) Successful in 3m14s
CI / Test backend (pull_request) Failing after 42s
CI / Docker / backend (pull_request) Has been skipped
CI / Docker / runner (pull_request) Has been skipped
Release / Docker / caddy (push) Successful in 6m48s
Release / Docker / ui (push) Successful in 2m8s
Release / Docker / runner (push) Successful in 2m51s
Release / Upload source maps (push) Failing after 53s
Release / Gitea Release (push) Has been skipped
Introduce a Redis-backed Asynq task queue so the runner consumes TTS
jobs pushed by the backend instead of polling PocketBase.
- backend/internal/asynqqueue: Producer and Consumer wrappers
- backend/internal/runner: AsynqRunner mux, per-instance Prometheus
registry (fixes duplicate-collector panic in tests), redisConnOpt
- backend/internal/config: REDIS_ADDR / REDIS_PASSWORD env vars
- backend/cmd/{backend,runner}/main.go: wire Redis when env set; fall
back to legacy poll mode when unset
- Caddyfile: caddy-l4 TCP proxy for redis.libnovel.cc:6380 → homelab
- caddy/Dockerfile: add --with github.com/mholt/caddy-l4
- docker-compose.yml: Caddy exposes 6380, backend/runner get Redis env
- homelab/runner/docker-compose.yml: Redis sidecar, runner depends_on
- homelab/otel/grafana: Grafana dashboards (backend, catalogue, runner)
and alerting rules / contact-points provisioning
This commit is contained in:
56
backend/internal/asynqqueue/consumer.go
Normal file
56
backend/internal/asynqqueue/consumer.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package asynqqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
)
|
||||
|
||||
// Consumer wraps the PocketBase-backed Consumer for result write-back only.
|
||||
//
|
||||
// When using Asynq, the runner no longer polls for work — Asynq delivers
|
||||
// tasks via the ServeMux handlers. The only Consumer operations the handlers
|
||||
// need are:
|
||||
// - FinishAudioTask / FinishScrapeTask — write result back to PocketBase
|
||||
// - FailTask — mark PocketBase record as failed
|
||||
//
|
||||
// ClaimNextAudioTask, ClaimNextScrapeTask, HeartbeatTask, and ReapStaleTasks
|
||||
// are all no-ops here because Asynq owns those responsibilities.
|
||||
type Consumer struct {
|
||||
pb taskqueue.Consumer // underlying PocketBase consumer (for write-back)
|
||||
}
|
||||
|
||||
// NewConsumer wraps an existing PocketBase Consumer.
|
||||
func NewConsumer(pb taskqueue.Consumer) *Consumer {
|
||||
return &Consumer{pb: pb}
|
||||
}
|
||||
|
||||
// ── Write-back (delegated to PocketBase) ──────────────────────────────────────
|
||||
|
||||
func (c *Consumer) FinishScrapeTask(ctx context.Context, id string, result domain.ScrapeResult) error {
|
||||
return c.pb.FinishScrapeTask(ctx, id, result)
|
||||
}
|
||||
|
||||
func (c *Consumer) FinishAudioTask(ctx context.Context, id string, result domain.AudioResult) error {
|
||||
return c.pb.FinishAudioTask(ctx, id, result)
|
||||
}
|
||||
|
||||
func (c *Consumer) FailTask(ctx context.Context, id, errMsg string) error {
|
||||
return c.pb.FailTask(ctx, id, errMsg)
|
||||
}
|
||||
|
||||
// ── No-ops (Asynq owns claiming / heartbeating / reaping) ───────────────────
|
||||
|
||||
func (c *Consumer) ClaimNextScrapeTask(_ context.Context, _ string) (domain.ScrapeTask, bool, error) {
|
||||
return domain.ScrapeTask{}, false, nil
|
||||
}
|
||||
|
||||
func (c *Consumer) ClaimNextAudioTask(_ context.Context, _ string) (domain.AudioTask, bool, error) {
|
||||
return domain.AudioTask{}, false, nil
|
||||
}
|
||||
|
||||
func (c *Consumer) HeartbeatTask(_ context.Context, _ string) error { return nil }
|
||||
|
||||
func (c *Consumer) ReapStaleTasks(_ context.Context, _ time.Duration) (int, error) { return 0, nil }
|
||||
90
backend/internal/asynqqueue/producer.go
Normal file
90
backend/internal/asynqqueue/producer.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package asynqqueue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
)
|
||||
|
||||
// Producer dual-writes every task: first to PocketBase (via pb, for audit /
|
||||
// UI status), then to Redis via Asynq so the runner picks it up immediately.
|
||||
type Producer struct {
|
||||
pb taskqueue.Producer // underlying PocketBase producer
|
||||
client *asynq.Client
|
||||
}
|
||||
|
||||
// NewProducer wraps an existing PocketBase Producer with Asynq dispatch.
|
||||
func NewProducer(pb taskqueue.Producer, redisOpt asynq.RedisConnOpt) *Producer {
|
||||
return &Producer{
|
||||
pb: pb,
|
||||
client: asynq.NewClient(redisOpt),
|
||||
}
|
||||
}
|
||||
|
||||
// Close shuts down the underlying Asynq client connection.
|
||||
func (p *Producer) Close() error {
|
||||
return p.client.Close()
|
||||
}
|
||||
|
||||
// CreateScrapeTask creates a PocketBase record then enqueues an Asynq job.
|
||||
func (p *Producer) CreateScrapeTask(ctx context.Context, kind, targetURL string, fromChapter, toChapter int) (string, error) {
|
||||
id, err := p.pb.CreateScrapeTask(ctx, kind, targetURL, fromChapter, toChapter)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
payload := ScrapePayload{
|
||||
PBTaskID: id,
|
||||
Kind: kind,
|
||||
TargetURL: targetURL,
|
||||
FromChapter: fromChapter,
|
||||
ToChapter: toChapter,
|
||||
}
|
||||
taskType := TypeScrapeBook
|
||||
if kind == "catalogue" {
|
||||
taskType = TypeScrapeCatalogue
|
||||
}
|
||||
if err := p.enqueue(ctx, taskType, payload); err != nil {
|
||||
// Non-fatal: PB record exists; runner will pick it up on next poll.
|
||||
return id, fmt.Errorf("asynq enqueue scrape (task still in PB): %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// CreateAudioTask creates a PocketBase record then enqueues an Asynq job.
|
||||
func (p *Producer) CreateAudioTask(ctx context.Context, slug string, chapter int, voice string) (string, error) {
|
||||
id, err := p.pb.CreateAudioTask(ctx, slug, chapter, voice)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
payload := AudioPayload{
|
||||
PBTaskID: id,
|
||||
Slug: slug,
|
||||
Chapter: chapter,
|
||||
Voice: voice,
|
||||
}
|
||||
if err := p.enqueue(ctx, TypeAudioGenerate, payload); err != nil {
|
||||
return id, fmt.Errorf("asynq enqueue audio (task still in PB): %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// CancelTask delegates to PocketBase; Asynq jobs may already be running and
|
||||
// cannot be reliably cancelled, so we only update the audit record.
|
||||
func (p *Producer) CancelTask(ctx context.Context, id string) error {
|
||||
return p.pb.CancelTask(ctx, id)
|
||||
}
|
||||
|
||||
// enqueue serialises payload and dispatches it to Asynq.
|
||||
func (p *Producer) enqueue(_ context.Context, taskType string, payload any) error {
|
||||
b, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal payload: %w", err)
|
||||
}
|
||||
_, err = p.client.Enqueue(asynq.NewTask(taskType, b))
|
||||
return err
|
||||
}
|
||||
46
backend/internal/asynqqueue/tasks.go
Normal file
46
backend/internal/asynqqueue/tasks.go
Normal file
@@ -0,0 +1,46 @@
|
||||
// Package asynqqueue provides Asynq-backed implementations of the
|
||||
// taskqueue.Producer and taskqueue.Consumer interfaces.
|
||||
//
|
||||
// Architecture:
|
||||
// - Producer: dual-writes — creates a PocketBase record for audit/UI, then
|
||||
// enqueues an Asynq job so the runner picks it up immediately (sub-ms).
|
||||
// - Consumer: thin wrapper used only for result write-back (FinishAudioTask,
|
||||
// FinishScrapeTask, FailTask). ClaimNext*/Heartbeat/Reap are no-ops because
|
||||
// Asynq owns those responsibilities.
|
||||
// - Handlers: asynq.HandlerFunc wrappers that decode job payloads and invoke
|
||||
// the existing runner logic (runScrapeTask / runAudioTask).
|
||||
//
|
||||
// Fallback: when REDIS_ADDR is empty the caller should use the plain
|
||||
// storage.Store (PocketBase-polling) implementation unchanged.
|
||||
package asynqqueue
|
||||
|
||||
// Queue names — keep all jobs on the default queue for now.
|
||||
// Add separate queues (e.g. "audio", "scrape") later if you need priority.
|
||||
const QueueDefault = "default"
|
||||
|
||||
// Task type constants used for Asynq routing.
|
||||
const (
|
||||
TypeAudioGenerate = "audio:generate"
|
||||
TypeScrapeBook = "scrape:book"
|
||||
TypeScrapeCatalogue = "scrape:catalogue"
|
||||
)
|
||||
|
||||
// AudioPayload is the Asynq job payload for audio generation tasks.
|
||||
type AudioPayload struct {
|
||||
// PBTaskID is the PocketBase record ID created before enqueueing.
|
||||
// The handler uses it to write results back via Consumer.FinishAudioTask.
|
||||
PBTaskID string `json:"pb_task_id"`
|
||||
Slug string `json:"slug"`
|
||||
Chapter int `json:"chapter"`
|
||||
Voice string `json:"voice"`
|
||||
}
|
||||
|
||||
// ScrapePayload is the Asynq job payload for scrape tasks.
|
||||
type ScrapePayload struct {
|
||||
// PBTaskID is the PocketBase record ID created before enqueueing.
|
||||
PBTaskID string `json:"pb_task_id"`
|
||||
Kind string `json:"kind"` // "catalogue", "book", or "book_range"
|
||||
TargetURL string `json:"target_url"` // empty for catalogue tasks
|
||||
FromChapter int `json:"from_chapter"` // 0 unless Kind=="book_range"
|
||||
ToChapter int `json:"to_chapter"` // 0 unless Kind=="book_range"
|
||||
}
|
||||
@@ -86,6 +86,19 @@ type Valkey struct {
|
||||
Addr string
|
||||
}
|
||||
|
||||
// Redis holds connection settings for the Asynq task queue Redis instance.
|
||||
// This is separate from Valkey (presign cache) — it may point to the same
|
||||
// Redis or a dedicated one. An empty Addr falls back to PocketBase polling.
|
||||
type Redis struct {
|
||||
// Addr is the host:port (or rediss://... URL) of the Redis instance.
|
||||
// Use rediss:// scheme for TLS (e.g. rediss://:password@redis.libnovel.cc:6380).
|
||||
// An empty string disables Asynq and falls back to PocketBase polling.
|
||||
Addr string
|
||||
// Password is the Redis AUTH password.
|
||||
// Not needed when Addr is a full rediss:// URL that includes the password.
|
||||
Password string
|
||||
}
|
||||
|
||||
// Runner holds settings specific to the runner/worker binary.
|
||||
type Runner struct {
|
||||
// PollInterval is how often the runner checks PocketBase for pending tasks.
|
||||
@@ -125,6 +138,7 @@ type Config struct {
|
||||
Runner Runner
|
||||
Meilisearch Meilisearch
|
||||
Valkey Valkey
|
||||
Redis Redis
|
||||
// LogLevel is one of "debug", "info", "warn", "error".
|
||||
LogLevel string
|
||||
}
|
||||
@@ -192,6 +206,11 @@ func Load() Config {
|
||||
Valkey: Valkey{
|
||||
Addr: envOr("VALKEY_ADDR", ""),
|
||||
},
|
||||
|
||||
Redis: Redis{
|
||||
Addr: envOr("REDIS_ADDR", ""),
|
||||
Password: envOr("REDIS_PASSWORD", ""),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
149
backend/internal/runner/asynq_runner.go
Normal file
149
backend/internal/runner/asynq_runner.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package runner
|
||||
|
||||
// asynq_runner.go — Asynq-based task dispatch for the runner.
|
||||
//
|
||||
// When cfg.RedisAddr is set, Run() calls runAsynq() instead of runPoll().
|
||||
// The Asynq server replaces the polling loop: it listens on Redis for tasks
|
||||
// enqueued by the backend Producer and delivers them immediately.
|
||||
//
|
||||
// Handlers in this file decode Asynq job payloads and call the existing
|
||||
// runScrapeTask / runAudioTask methods, keeping all execution logic in one place.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
asynqmetrics "github.com/hibiken/asynq/x/metrics"
|
||||
"github.com/libnovel/backend/internal/asynqqueue"
|
||||
"github.com/libnovel/backend/internal/domain"
|
||||
)
|
||||
|
||||
// runAsynq starts an Asynq server that replaces the PocketBase poll loop.
|
||||
// It also starts the periodic catalogue refresh ticker.
|
||||
// Blocks until ctx is cancelled.
|
||||
func (r *Runner) runAsynq(ctx context.Context) error {
|
||||
redisOpt, err := r.redisConnOpt()
|
||||
if err != nil {
|
||||
return fmt.Errorf("runner: parse redis addr: %w", err)
|
||||
}
|
||||
|
||||
srv := asynq.NewServer(redisOpt, asynq.Config{
|
||||
// Allocate concurrency slots for each task type.
|
||||
// Total concurrency = scrape + audio slots.
|
||||
Concurrency: r.cfg.MaxConcurrentScrape + r.cfg.MaxConcurrentAudio,
|
||||
Queues: map[string]int{
|
||||
asynqqueue.QueueDefault: 1,
|
||||
},
|
||||
// Let Asynq handle retries with exponential back-off.
|
||||
RetryDelayFunc: asynq.DefaultRetryDelayFunc,
|
||||
// Log errors from handlers via the existing structured logger.
|
||||
ErrorHandler: asynq.ErrorHandlerFunc(func(_ context.Context, task *asynq.Task, err error) {
|
||||
r.deps.Log.Error("runner: asynq task failed",
|
||||
"type", task.Type(),
|
||||
"err", err,
|
||||
)
|
||||
}),
|
||||
})
|
||||
|
||||
mux := asynq.NewServeMux()
|
||||
mux.HandleFunc(asynqqueue.TypeAudioGenerate, r.handleAudioTask)
|
||||
mux.HandleFunc(asynqqueue.TypeScrapeBook, r.handleScrapeTask)
|
||||
mux.HandleFunc(asynqqueue.TypeScrapeCatalogue, r.handleScrapeTask)
|
||||
|
||||
// Register Asynq queue metrics with the default Prometheus registry so
|
||||
// the /metrics endpoint (metrics.go) can expose them.
|
||||
inspector := asynq.NewInspector(redisOpt)
|
||||
collector := asynqmetrics.NewQueueMetricsCollector(inspector)
|
||||
if err := r.metricsRegistry.Register(collector); err != nil {
|
||||
r.deps.Log.Warn("runner: could not register asynq prometheus collector", "err", err)
|
||||
}
|
||||
|
||||
// Start the periodic catalogue refresh.
|
||||
catalogueTick := time.NewTicker(r.cfg.CatalogueRefreshInterval)
|
||||
defer catalogueTick.Stop()
|
||||
if !r.cfg.SkipInitialCatalogueRefresh {
|
||||
go r.runCatalogueRefresh(ctx)
|
||||
} else {
|
||||
r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)")
|
||||
}
|
||||
|
||||
r.deps.Log.Info("runner: asynq mode active", "redis_addr", r.cfg.RedisAddr)
|
||||
|
||||
// Run catalogue refresh ticker in the background.
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-catalogueTick.C:
|
||||
go r.runCatalogueRefresh(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Start Asynq server (non-blocking).
|
||||
if err := srv.Start(mux); err != nil {
|
||||
return fmt.Errorf("runner: asynq server start: %w", err)
|
||||
}
|
||||
|
||||
// Block until context is cancelled, then gracefully stop.
|
||||
<-ctx.Done()
|
||||
r.deps.Log.Info("runner: context cancelled, shutting down asynq server")
|
||||
srv.Shutdown()
|
||||
return nil
|
||||
}
|
||||
|
||||
// redisConnOpt parses cfg.RedisAddr into an asynq.RedisConnOpt.
|
||||
// Supports full "redis://" / "rediss://" URLs and plain "host:port".
|
||||
func (r *Runner) redisConnOpt() (asynq.RedisConnOpt, error) {
|
||||
addr := r.cfg.RedisAddr
|
||||
// ParseRedisURI handles redis:// and rediss:// schemes.
|
||||
if len(addr) > 7 && (addr[:8] == "redis://" || addr[:9] == "rediss://") {
|
||||
return asynq.ParseRedisURI(addr)
|
||||
}
|
||||
// Plain "host:port" — use RedisClientOpt directly.
|
||||
return asynq.RedisClientOpt{
|
||||
Addr: addr,
|
||||
Password: r.cfg.RedisPassword,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// handleScrapeTask is the Asynq handler for TypeScrapeBook and TypeScrapeCatalogue.
|
||||
func (r *Runner) handleScrapeTask(ctx context.Context, t *asynq.Task) error {
|
||||
var p asynqqueue.ScrapePayload
|
||||
if err := json.Unmarshal(t.Payload(), &p); err != nil {
|
||||
return fmt.Errorf("unmarshal scrape payload: %w", err)
|
||||
}
|
||||
task := domain.ScrapeTask{
|
||||
ID: p.PBTaskID,
|
||||
Kind: p.Kind,
|
||||
TargetURL: p.TargetURL,
|
||||
FromChapter: p.FromChapter,
|
||||
ToChapter: p.ToChapter,
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runScrapeTask(ctx, task)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleAudioTask is the Asynq handler for TypeAudioGenerate.
|
||||
func (r *Runner) handleAudioTask(ctx context.Context, t *asynq.Task) error {
|
||||
var p asynqqueue.AudioPayload
|
||||
if err := json.Unmarshal(t.Payload(), &p); err != nil {
|
||||
return fmt.Errorf("unmarshal audio payload: %w", err)
|
||||
}
|
||||
task := domain.AudioTask{
|
||||
ID: p.PBTaskID,
|
||||
Slug: p.Slug,
|
||||
Chapter: p.Chapter,
|
||||
Voice: p.Voice,
|
||||
}
|
||||
r.tasksRunning.Add(1)
|
||||
defer r.tasksRunning.Add(-1)
|
||||
r.runAudioTask(ctx, task)
|
||||
return nil
|
||||
}
|
||||
@@ -1,21 +1,28 @@
|
||||
package runner
|
||||
|
||||
// metrics.go — lightweight HTTP metrics endpoint for the runner.
|
||||
// metrics.go — Prometheus metrics HTTP endpoint for the runner.
|
||||
//
|
||||
// GET /metrics returns a JSON document with live task counters and uptime.
|
||||
// No external dependency (no Prometheus); plain net/http only.
|
||||
// GET /metrics returns a Prometheus text/plain scrape response.
|
||||
// Exposes:
|
||||
// - Standard Go runtime metrics (via promhttp)
|
||||
// - Runner task counters (tasks_running, tasks_completed, tasks_failed)
|
||||
// - Asynq queue metrics (registered in asynq_runner.go when Redis is enabled)
|
||||
//
|
||||
// GET /health — simple liveness probe.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
// metricsServer serves GET /metrics for the runner process.
|
||||
// metricsServer serves GET /metrics and GET /health for the runner process.
|
||||
type metricsServer struct {
|
||||
addr string
|
||||
r *Runner
|
||||
@@ -23,21 +30,62 @@ type metricsServer struct {
|
||||
}
|
||||
|
||||
func newMetricsServer(addr string, r *Runner, log *slog.Logger) *metricsServer {
|
||||
return &metricsServer{addr: addr, r: r, log: log}
|
||||
ms := &metricsServer{addr: addr, r: r, log: log}
|
||||
ms.registerCollectors()
|
||||
return ms
|
||||
}
|
||||
|
||||
// registerCollectors registers runner-specific Prometheus collectors.
|
||||
// Called once at construction; Asynq queue collector is registered separately
|
||||
// in asynq_runner.go after the Redis connection is established.
|
||||
func (ms *metricsServer) registerCollectors() {
|
||||
// Runner task gauges / counters backed by the atomic fields on Runner.
|
||||
ms.r.metricsRegistry.MustRegister(prometheus.NewGaugeFunc(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "runner",
|
||||
Name: "tasks_running",
|
||||
Help: "Number of tasks currently being processed.",
|
||||
},
|
||||
func() float64 { return float64(ms.r.tasksRunning.Load()) },
|
||||
))
|
||||
ms.r.metricsRegistry.MustRegister(prometheus.NewCounterFunc(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "runner",
|
||||
Name: "tasks_completed_total",
|
||||
Help: "Total number of tasks completed successfully since startup.",
|
||||
},
|
||||
func() float64 { return float64(ms.r.tasksCompleted.Load()) },
|
||||
))
|
||||
ms.r.metricsRegistry.MustRegister(prometheus.NewCounterFunc(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: "runner",
|
||||
Name: "tasks_failed_total",
|
||||
Help: "Total number of tasks that ended in failure since startup.",
|
||||
},
|
||||
func() float64 { return float64(ms.r.tasksFailed.Load()) },
|
||||
))
|
||||
ms.r.metricsRegistry.MustRegister(prometheus.NewGaugeFunc(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: "runner",
|
||||
Name: "uptime_seconds",
|
||||
Help: "Seconds since the runner process started.",
|
||||
},
|
||||
func() float64 { return time.Since(ms.r.startedAt).Seconds() },
|
||||
))
|
||||
}
|
||||
|
||||
// ListenAndServe starts the HTTP server and blocks until ctx is cancelled or
|
||||
// a fatal listen error occurs.
|
||||
func (ms *metricsServer) ListenAndServe(ctx context.Context) error {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /metrics", ms.handleMetrics)
|
||||
mux.Handle("GET /metrics", promhttp.HandlerFor(ms.r.metricsRegistry, promhttp.HandlerOpts{}))
|
||||
mux.HandleFunc("GET /health", ms.handleHealth)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: ms.addr,
|
||||
Handler: mux,
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
BaseContext: func(_ net.Listener) context.Context { return ctx },
|
||||
}
|
||||
|
||||
@@ -58,35 +106,8 @@ func (ms *metricsServer) ListenAndServe(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// handleMetrics handles GET /metrics.
|
||||
// Response shape (JSON):
|
||||
//
|
||||
// {
|
||||
// "tasks_running": N,
|
||||
// "tasks_completed": N,
|
||||
// "tasks_failed": N,
|
||||
// "uptime_seconds": N
|
||||
// }
|
||||
func (ms *metricsServer) handleMetrics(w http.ResponseWriter, _ *http.Request) {
|
||||
uptimeSec := int64(time.Since(ms.r.startedAt).Seconds())
|
||||
metricsWriteJSON(w, 0, map[string]int64{
|
||||
"tasks_running": ms.r.tasksRunning.Load(),
|
||||
"tasks_completed": ms.r.tasksCompleted.Load(),
|
||||
"tasks_failed": ms.r.tasksFailed.Load(),
|
||||
"uptime_seconds": uptimeSec,
|
||||
})
|
||||
}
|
||||
|
||||
// handleHealth handles GET /health — simple liveness probe for the metrics server.
|
||||
// handleHealth handles GET /health — simple liveness probe.
|
||||
func (ms *metricsServer) handleHealth(w http.ResponseWriter, _ *http.Request) {
|
||||
metricsWriteJSON(w, 0, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// metricsWriteJSON writes v as a JSON response with the given status code.
|
||||
func metricsWriteJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if status != 0 {
|
||||
w.WriteHeader(status)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
_, _ = w.Write([]byte(`{"status":"ok"}`))
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ import (
|
||||
"github.com/libnovel/backend/internal/pockettts"
|
||||
"github.com/libnovel/backend/internal/scraper"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// Config tunes the runner behaviour.
|
||||
@@ -41,6 +42,7 @@ type Config struct {
|
||||
// WorkerID uniquely identifies this runner instance in PocketBase records.
|
||||
WorkerID string
|
||||
// PollInterval is how often the runner checks for new tasks.
|
||||
// Only used in PocketBase-polling mode (RedisAddr == "").
|
||||
PollInterval time.Duration
|
||||
// MaxConcurrentScrape limits simultaneous book-scrape goroutines.
|
||||
MaxConcurrentScrape int
|
||||
@@ -50,9 +52,11 @@ type Config struct {
|
||||
OrchestratorWorkers int
|
||||
// HeartbeatInterval is how often active tasks PATCH their heartbeat_at
|
||||
// timestamp to signal they are still alive. Defaults to 30s when 0.
|
||||
// Only used in PocketBase-polling mode.
|
||||
HeartbeatInterval time.Duration
|
||||
// StaleTaskThreshold is how old a heartbeat must be (or absent) before the
|
||||
// task is considered orphaned and reset to pending. Defaults to 2m when 0.
|
||||
// Only used in PocketBase-polling mode.
|
||||
StaleTaskThreshold time.Duration
|
||||
// CatalogueRefreshInterval is how often the runner walks the full catalogue,
|
||||
// scrapes per-book metadata, downloads covers, and re-indexes everything in
|
||||
@@ -66,6 +70,15 @@ type Config struct {
|
||||
// MetricsAddr is the HTTP listen address for the /metrics endpoint.
|
||||
// Defaults to ":9091". Set to "" to disable.
|
||||
MetricsAddr string
|
||||
// RedisAddr is the address of the Redis instance used for Asynq task
|
||||
// dispatch. When set the runner switches from PocketBase-polling mode to
|
||||
// Asynq ServeMux mode (immediate task delivery, no polling).
|
||||
// Supports plain "host:port" or a full "rediss://..." URL.
|
||||
// When empty the runner falls back to PocketBase polling.
|
||||
RedisAddr string
|
||||
// RedisPassword is the Redis AUTH password.
|
||||
// Not required when RedisAddr is a full URL that includes credentials.
|
||||
RedisPassword string
|
||||
}
|
||||
|
||||
// Dependencies are the external services the runner depends on.
|
||||
@@ -99,6 +112,8 @@ type Runner struct {
|
||||
cfg Config
|
||||
deps Dependencies
|
||||
|
||||
metricsRegistry *prometheus.Registry
|
||||
|
||||
// Atomic task counters — read by /metrics without locking.
|
||||
tasksRunning atomic.Int64
|
||||
tasksCompleted atomic.Int64
|
||||
@@ -139,15 +154,18 @@ func New(cfg Config, deps Dependencies) *Runner {
|
||||
if deps.SearchIndex == nil {
|
||||
deps.SearchIndex = meili.NoopClient{}
|
||||
}
|
||||
return &Runner{cfg: cfg, deps: deps, startedAt: time.Now()}
|
||||
return &Runner{cfg: cfg, deps: deps, startedAt: time.Now(), metricsRegistry: prometheus.NewRegistry()}
|
||||
}
|
||||
|
||||
// Run starts the poll loop and the metrics HTTP server, blocking until ctx is
|
||||
// cancelled.
|
||||
// Run starts the worker loop and the metrics HTTP server, blocking until ctx
|
||||
// is cancelled.
|
||||
//
|
||||
// When cfg.RedisAddr is set the runner uses Asynq (immediate task delivery).
|
||||
// Otherwise it falls back to PocketBase polling (legacy mode).
|
||||
func (r *Runner) Run(ctx context.Context) error {
|
||||
r.deps.Log.Info("runner: starting",
|
||||
"worker_id", r.cfg.WorkerID,
|
||||
"poll_interval", r.cfg.PollInterval,
|
||||
"mode", r.mode(),
|
||||
"max_scrape", r.cfg.MaxConcurrentScrape,
|
||||
"max_audio", r.cfg.MaxConcurrentAudio,
|
||||
"catalogue_refresh_interval", r.cfg.CatalogueRefreshInterval,
|
||||
@@ -164,6 +182,23 @@ func (r *Runner) Run(ctx context.Context) error {
|
||||
}()
|
||||
}
|
||||
|
||||
if r.cfg.RedisAddr != "" {
|
||||
return r.runAsynq(ctx)
|
||||
}
|
||||
return r.runPoll(ctx)
|
||||
}
|
||||
|
||||
// mode returns a short string describing the active dispatch mode.
|
||||
func (r *Runner) mode() string {
|
||||
if r.cfg.RedisAddr != "" {
|
||||
return "asynq"
|
||||
}
|
||||
return "poll"
|
||||
}
|
||||
|
||||
// runPoll is the legacy PocketBase-polling dispatch loop.
|
||||
// Used when cfg.RedisAddr is empty.
|
||||
func (r *Runner) runPoll(ctx context.Context) error {
|
||||
scrapeSem := make(chan struct{}, r.cfg.MaxConcurrentScrape)
|
||||
audioSem := make(chan struct{}, r.cfg.MaxConcurrentAudio)
|
||||
var wg sync.WaitGroup
|
||||
@@ -181,6 +216,8 @@ func (r *Runner) Run(ctx context.Context) error {
|
||||
r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)")
|
||||
}
|
||||
|
||||
r.deps.Log.Info("runner: poll mode active", "poll_interval", r.cfg.PollInterval)
|
||||
|
||||
// Run one poll immediately on startup, then on each tick.
|
||||
for {
|
||||
r.poll(ctx, scrapeSem, audioSem, &wg)
|
||||
|
||||
Reference in New Issue
Block a user