From 4e70a2981dfb5bc30b380485c26a76d05e2f02f7 Mon Sep 17 00:00:00 2001
From: Admin <admin@kalekber.cc>
Date: Tue, 31 Mar 2026 10:18:13 +0500
Subject: [PATCH] fix(pipeline): add redis+libretranslate to homelab, make
 Asynq enqueue errors non-fatal
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- homelab/docker-compose.yml: add redis:7-alpine service (port 6379 bound to host
  so Caddy TLS proxy on prod can reach it), add libretranslate service, add
  redis_data and libretranslate_data volumes
- asynqqueue/producer.go: Asynq enqueue failures are now logged as warnings instead
  of returned as errors — PB record already exists so runner picks it up via poll
- backend/main.go: pass logger to NewProducer

Root cause: Redis was not reachable at 192.168.0.109:6379 because the redis
container had no host port binding. Caddy TLS proxy terminates TLS but could
not TCP-connect to the backend Redis.
---
 backend/cmd/backend/main.go             |  2 +-
 backend/internal/asynqqueue/producer.go | 14 ++++++--
 homelab/docker-compose.yml              | 44 +++++++++++++++++++++++++
 3 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/backend/cmd/backend/main.go b/backend/cmd/backend/main.go
index de09c45..9b67acf 100644
--- a/backend/cmd/backend/main.go
+++ b/backend/cmd/backend/main.go
@@ -134,7 +134,7 @@ func run() error {
 		if parseErr != nil {
 			return fmt.Errorf("parse REDIS_ADDR: %w", parseErr)
 		}
-		asynqProducer := asynqqueue.NewProducer(store, redisOpt)
+		asynqProducer := asynqqueue.NewProducer(store, redisOpt, log)
 		defer asynqProducer.Close() //nolint:errcheck
 		producer = asynqProducer
 		log.Info("backend: asynq task dispatch enabled", "addr", cfg.Redis.Addr)
diff --git a/backend/internal/asynqqueue/producer.go b/backend/internal/asynqqueue/producer.go
index b2c5568..8fe8688 100644
--- a/backend/internal/asynqqueue/producer.go
+++ b/backend/internal/asynqqueue/producer.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log/slog"
 
 	"github.com/hibiken/asynq"
 	"github.com/libnovel/backend/internal/taskqueue"
@@ -14,13 +15,15 @@ import (
 type Producer struct {
 	pb     taskqueue.Producer // underlying PocketBase producer
 	client *asynq.Client
+	log    *slog.Logger
 }
 
 // NewProducer wraps an existing PocketBase Producer with Asynq dispatch.
-func NewProducer(pb taskqueue.Producer, redisOpt asynq.RedisConnOpt) *Producer {
+func NewProducer(pb taskqueue.Producer, redisOpt asynq.RedisConnOpt, log *slog.Logger) *Producer {
 	return &Producer{
 		pb:     pb,
 		client: asynq.NewClient(redisOpt),
+		log:    log,
 	}
 }
 
@@ -49,7 +52,9 @@ func (p *Producer) CreateScrapeTask(ctx context.Context, kind, targetURL string,
 	}
 	if err := p.enqueue(ctx, taskType, payload); err != nil {
 		// Non-fatal: PB record exists; runner will pick it up on next poll.
-		return id, fmt.Errorf("asynq enqueue scrape (task still in PB): %w", err)
+		p.log.Warn("asynq enqueue scrape failed (task still in PB, runner will poll)",
+			"task_id", id, "err", err)
+		return id, nil
 	}
 	return id, nil
 }
@@ -68,7 +73,10 @@ func (p *Producer) CreateAudioTask(ctx context.Context, slug string, chapter int
 		Voice:    voice,
 	}
 	if err := p.enqueue(ctx, TypeAudioGenerate, payload); err != nil {
-		return id, fmt.Errorf("asynq enqueue audio (task still in PB): %w", err)
+		// Non-fatal: PB record exists; runner will pick it up on next poll.
+		p.log.Warn("asynq enqueue audio failed (task still in PB, runner will poll)",
+			"task_id", id, "err", err)
+		return id, nil
 	}
 	return id, nil
 }
diff --git a/homelab/docker-compose.yml b/homelab/docker-compose.yml
index a45f407..f8347dc 100644
--- a/homelab/docker-compose.yml
+++ b/homelab/docker-compose.yml
@@ -289,6 +289,48 @@ services:
       timeout: 5s
       retries: 5
 
+  # ── Redis (Asynq task queue) ────────────────────────────────────────────────
+  # Dedicated Redis instance for Asynq job dispatch.
+  # The prod backend enqueues jobs via redis.libnovel.cc:6380 (Caddy TLS proxy →
+  # host:6379). The runner reads from this instance directly on the Docker network.
+  # Port is bound to 0.0.0.0:6379 so the Caddy layer4 proxy on prod can reach it.
+  redis:
+    image: redis:7-alpine
+    restart: unless-stopped
+    command: ["redis-server", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD}"]
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # ── LibreTranslate ──────────────────────────────────────────────────────────
+  # Self-hosted machine translation. Runner connects via http://libretranslate:5000.
+  # Only English → configured target languages are loaded to save RAM.
+  libretranslate:
+    image: libretranslate/libretranslate:latest
+    restart: unless-stopped
+    environment:
+      LT_API_KEYS: "true"
+      LT_API_KEYS_DB_PATH: "/app/db/api_keys.db"
+      LT_LOAD_ONLY: "en,ru,id,pt,fr"
+      LT_DISABLE_WEB_UI: "true"
+      LT_UPDATE_MODELS: "false"
+    expose:
+      - "5000"
+    volumes:
+      - libretranslate_data:/app/db
+    healthcheck:
+      test: ["CMD", "curl", "-sf", "http://localhost:5000/languages"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+
   # ── Valkey ──────────────────────────────────────────────────────────────────
   # Used by GlitchTip for task queuing.
   valkey:
@@ -469,6 +511,8 @@ services:
 
 volumes:
   postgres_data:
+  redis_data:
+  libretranslate_data:
   valkey_data:
   uptime_kuma_data:
   gotify_data: