feat(queue): replace PocketBase polling with Asynq + Redis

Introduce a Redis-backed Asynq task queue so the runner consumes TTS
jobs pushed by the backend instead of polling PocketBase.

- backend/internal/asynqqueue: Producer and Consumer wrappers
- backend/internal/runner: AsynqRunner mux, per-instance Prometheus
  registry (fixes duplicate-collector panic in tests), redisConnOpt
- backend/internal/config: REDIS_ADDR / REDIS_PASSWORD env vars
- backend/cmd/{backend,runner}/main.go: wire Redis when env set; fall
  back to legacy poll mode when unset
- Caddyfile: caddy-l4 TCP proxy for redis.libnovel.cc:6380 → homelab
- caddy/Dockerfile: add --with github.com/mholt/caddy-l4
- docker-compose.yml: Caddy exposes 6380, backend/runner get Redis env
- homelab/runner/docker-compose.yml: Redis sidecar, runner depends_on
- homelab/otel/grafana: Grafana dashboards (backend, catalogue, runner)
  and alerting rules / contact-points provisioning

backend/internal/runner/metrics.go

@@ -1,21 +1,28 @@
 package runner
 
-// metrics.go — lightweight HTTP metrics endpoint for the runner.
+// metrics.go — Prometheus metrics HTTP endpoint for the runner.
 //
-// GET /metrics returns a JSON document with live task counters and uptime.
-// No external dependency (no Prometheus); plain net/http only.
+// GET /metrics returns a Prometheus text/plain scrape response.
+// Exposes:
+//   - Standard Go runtime metrics (via promhttp)
+//   - Runner task counters (tasks_running, tasks_completed, tasks_failed)
+//   - Asynq queue metrics (registered in asynq_runner.go when Redis is enabled)
+//
+// GET /health — simple liveness probe.
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"log/slog"
 	"net"
 	"net/http"
 	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
-// metricsServer serves GET /metrics for the runner process.
+// metricsServer serves GET /metrics and GET /health for the runner process.
 type metricsServer struct {
 	addr string
 	r    *Runner
@@ -23,21 +30,62 @@ type metricsServer struct {
 }
 
 func newMetricsServer(addr string, r *Runner, log *slog.Logger) *metricsServer {
-	return &metricsServer{addr: addr, r: r, log: log}
+	ms := &metricsServer{addr: addr, r: r, log: log}
+	ms.registerCollectors()
+	return ms
+}
+
+// registerCollectors registers runner-specific Prometheus collectors.
+// Called once at construction; Asynq queue collector is registered separately
+// in asynq_runner.go after the Redis connection is established.
+func (ms *metricsServer) registerCollectors() {
+	// Runner task gauges / counters backed by the atomic fields on Runner.
+	ms.r.metricsRegistry.MustRegister(prometheus.NewGaugeFunc(
+		prometheus.GaugeOpts{
+			Namespace: "runner",
+			Name:      "tasks_running",
+			Help:      "Number of tasks currently being processed.",
+		},
+		func() float64 { return float64(ms.r.tasksRunning.Load()) },
+	))
+	ms.r.metricsRegistry.MustRegister(prometheus.NewCounterFunc(
+		prometheus.CounterOpts{
+			Namespace: "runner",
+			Name:      "tasks_completed_total",
+			Help:      "Total number of tasks completed successfully since startup.",
+		},
+		func() float64 { return float64(ms.r.tasksCompleted.Load()) },
+	))
+	ms.r.metricsRegistry.MustRegister(prometheus.NewCounterFunc(
+		prometheus.CounterOpts{
+			Namespace: "runner",
+			Name:      "tasks_failed_total",
+			Help:      "Total number of tasks that ended in failure since startup.",
+		},
+		func() float64 { return float64(ms.r.tasksFailed.Load()) },
+	))
+	ms.r.metricsRegistry.MustRegister(prometheus.NewGaugeFunc(
+		prometheus.GaugeOpts{
+			Namespace: "runner",
+			Name:      "uptime_seconds",
+			Help:      "Seconds since the runner process started.",
+		},
+		func() float64 { return time.Since(ms.r.startedAt).Seconds() },
+	))
 }
 
 // ListenAndServe starts the HTTP server and blocks until ctx is cancelled or
 // a fatal listen error occurs.
 func (ms *metricsServer) ListenAndServe(ctx context.Context) error {
 	mux := http.NewServeMux()
-	mux.HandleFunc("GET /metrics", ms.handleMetrics)
+	mux.Handle("GET /metrics", promhttp.HandlerFor(ms.r.metricsRegistry, promhttp.HandlerOpts{}))
 	mux.HandleFunc("GET /health", ms.handleHealth)
 
 	srv := &http.Server{
 		Addr:         ms.addr,
 		Handler:      mux,
 		ReadTimeout:  5 * time.Second,
-		WriteTimeout: 5 * time.Second,
+		WriteTimeout: 10 * time.Second,
 		BaseContext:  func(_ net.Listener) context.Context { return ctx },
 	}
 
@@ -58,35 +106,8 @@ func (ms *metricsServer) ListenAndServe(ctx context.Context) error {
 	}
 }
 
-// handleMetrics handles GET /metrics.
-// Response shape (JSON):
-//
-//	{
-//	  "tasks_running":   N,
-//	  "tasks_completed": N,
-//	  "tasks_failed":    N,
-//	  "uptime_seconds":  N
-//	}
-func (ms *metricsServer) handleMetrics(w http.ResponseWriter, _ *http.Request) {
-	uptimeSec := int64(time.Since(ms.r.startedAt).Seconds())
-	metricsWriteJSON(w, 0, map[string]int64{
-		"tasks_running":   ms.r.tasksRunning.Load(),
-		"tasks_completed": ms.r.tasksCompleted.Load(),
-		"tasks_failed":    ms.r.tasksFailed.Load(),
-		"uptime_seconds":  uptimeSec,
-	})
-}
-
-// handleHealth handles GET /health — simple liveness probe for the metrics server.
+// handleHealth handles GET /health — simple liveness probe.
 func (ms *metricsServer) handleHealth(w http.ResponseWriter, _ *http.Request) {
-	metricsWriteJSON(w, 0, map[string]string{"status": "ok"})
-}
-
-// metricsWriteJSON writes v as a JSON response with the given status code.
-func metricsWriteJSON(w http.ResponseWriter, status int, v any) {
 	w.Header().Set("Content-Type", "application/json")
-	if status != 0 {
-		w.WriteHeader(status)
-	}
-	_ = json.NewEncoder(w).Encode(v)
+	_, _ = w.Write([]byte(`{"status":"ok"}`))
 }