fix(pipeline): add redis+libretranslate to homelab, make Asynq enqueue errors non-fatal
Some checks failed
CI / Backend (pull_request) Successful in 35s
CI / UI (push) Failing after 37s
CI / Backend (push) Successful in 53s
CI / UI (pull_request) Failing after 27s

- homelab/docker-compose.yml: add redis:7-alpine service (port 6379 bound to host
  so Caddy TLS proxy on prod can reach it), add libretranslate service, add
  redis_data and libretranslate_data volumes
- asynqqueue/producer.go: Asynq enqueue failures are now logged as warnings instead
  of returned as errors — PB record already exists so runner picks it up via poll
- backend/main.go: pass logger to NewProducer

Root cause: Redis was not reachable at 192.168.0.109:6379 because the redis
container had no host port binding. Caddy TLS proxy terminates TLS but could
not TCP-connect to the backend Redis.
This commit is contained in:
Admin
2026-03-31 10:18:13 +05:00
parent 004cb95e56
commit 4e70a2981d
3 changed files with 56 additions and 4 deletions

View File

@@ -134,7 +134,7 @@ func run() error {
if parseErr != nil { if parseErr != nil {
return fmt.Errorf("parse REDIS_ADDR: %w", parseErr) return fmt.Errorf("parse REDIS_ADDR: %w", parseErr)
} }
asynqProducer := asynqqueue.NewProducer(store, redisOpt) asynqProducer := asynqqueue.NewProducer(store, redisOpt, log)
defer asynqProducer.Close() //nolint:errcheck defer asynqProducer.Close() //nolint:errcheck
producer = asynqProducer producer = asynqProducer
log.Info("backend: asynq task dispatch enabled", "addr", cfg.Redis.Addr) log.Info("backend: asynq task dispatch enabled", "addr", cfg.Redis.Addr)

View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"log/slog"
"github.com/hibiken/asynq" "github.com/hibiken/asynq"
"github.com/libnovel/backend/internal/taskqueue" "github.com/libnovel/backend/internal/taskqueue"
@@ -14,13 +15,15 @@ import (
type Producer struct { type Producer struct {
pb taskqueue.Producer // underlying PocketBase producer pb taskqueue.Producer // underlying PocketBase producer
client *asynq.Client client *asynq.Client
log *slog.Logger
} }
// NewProducer wraps an existing PocketBase Producer with Asynq dispatch. // NewProducer wraps an existing PocketBase Producer with Asynq dispatch.
func NewProducer(pb taskqueue.Producer, redisOpt asynq.RedisConnOpt) *Producer { func NewProducer(pb taskqueue.Producer, redisOpt asynq.RedisConnOpt, log *slog.Logger) *Producer {
return &Producer{ return &Producer{
pb: pb, pb: pb,
client: asynq.NewClient(redisOpt), client: asynq.NewClient(redisOpt),
log: log,
} }
} }
@@ -49,7 +52,9 @@ func (p *Producer) CreateScrapeTask(ctx context.Context, kind, targetURL string,
} }
if err := p.enqueue(ctx, taskType, payload); err != nil { if err := p.enqueue(ctx, taskType, payload); err != nil {
// Non-fatal: PB record exists; runner will pick it up on next poll. // Non-fatal: PB record exists; runner will pick it up on next poll.
return id, fmt.Errorf("asynq enqueue scrape (task still in PB): %w", err) p.log.Warn("asynq enqueue scrape failed (task still in PB, runner will poll)",
"task_id", id, "err", err)
return id, nil
} }
return id, nil return id, nil
} }
@@ -68,7 +73,10 @@ func (p *Producer) CreateAudioTask(ctx context.Context, slug string, chapter int
Voice: voice, Voice: voice,
} }
if err := p.enqueue(ctx, TypeAudioGenerate, payload); err != nil { if err := p.enqueue(ctx, TypeAudioGenerate, payload); err != nil {
return id, fmt.Errorf("asynq enqueue audio (task still in PB): %w", err) // Non-fatal: PB record exists; runner will pick it up on next poll.
p.log.Warn("asynq enqueue audio failed (task still in PB, runner will poll)",
"task_id", id, "err", err)
return id, nil
} }
return id, nil return id, nil
} }

View File

@@ -289,6 +289,48 @@ services:
timeout: 5s timeout: 5s
retries: 5 retries: 5
# ── Redis (Asynq task queue) ────────────────────────────────────────────────
# Dedicated Redis instance for Asynq job dispatch.
# The prod backend enqueues jobs via redis.libnovel.cc:6380 (Caddy TLS proxy →
# host:6379). The runner reads from this instance directly on the Docker network.
# Port is bound to 0.0.0.0:6379 so the Caddy layer4 proxy on prod can reach it.
redis:
image: redis:7-alpine
restart: unless-stopped
command: ["redis-server", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD}"]
ports:
- "6379:6379"
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
interval: 10s
timeout: 5s
retries: 5
# ── LibreTranslate ──────────────────────────────────────────────────────────
# Self-hosted machine translation. Runner connects via http://libretranslate:5000.
# Only English → configured target languages are loaded to save RAM.
libretranslate:
image: libretranslate/libretranslate:latest
restart: unless-stopped
environment:
LT_API_KEYS: "true"
LT_API_KEYS_DB_PATH: "/app/db/api_keys.db"
LT_LOAD_ONLY: "en,ru,id,pt,fr"
LT_DISABLE_WEB_UI: "true"
LT_UPDATE_MODELS: "false"
expose:
- "5000"
volumes:
- libretranslate_data:/app/db
healthcheck:
test: ["CMD", "curl", "-sf", "http://localhost:5000/languages"]
interval: 30s
timeout: 10s
retries: 5
start_period: 60s
# ── Valkey ────────────────────────────────────────────────────────────────── # ── Valkey ──────────────────────────────────────────────────────────────────
# Used by GlitchTip for task queuing. # Used by GlitchTip for task queuing.
valkey: valkey:
@@ -469,6 +511,8 @@ services:
volumes: volumes:
postgres_data: postgres_data:
redis_data:
libretranslate_data:
valkey_data: valkey_data:
uptime_kuma_data: uptime_kuma_data:
gotify_data: gotify_data: