Migrate tooling to homelab; add OTel observability stack

- Remove GlitchTip, Umami, Fider, Gotify, Uptime Kuma, Dozzle, Watchtower
  from prod docker-compose (now run on homelab)
- Add dozzle-agent on prod (127.0.0.1:7007) for homelab Dozzle to connect to
- Remove corresponding subdomain blocks from Caddyfile (now routed via
  Cloudflare Tunnel from homelab)
- Add homelab/docker-compose.yml: unified homelab stack with all migrated
  tooling services plus full OTel stack (Tempo 2.6.1, Loki, Prometheus,
  OTel Collector, Grafana)
- Add homelab/otel/: Tempo, Loki, Prometheus, OTel Collector configs +
  Grafana provisioning (datasources + dashboards)
- Add homelab/dozzle/users.yml for Dozzle auth

Caddyfile

@@ -204,41 +204,11 @@
 }
 
 # ── Tooling subdomains ────────────────────────────────────────────────────────
-feedback.libnovel.cc {
+# feedback.libnovel.cc, errors.libnovel.cc, analytics.libnovel.cc,
-	import security_headers
+# logs.libnovel.cc, uptime.libnovel.cc, push.libnovel.cc, grafana.libnovel.cc
-	reverse_proxy fider:3000
+# are now routed via Cloudflare Tunnel directly to the homelab (192.168.0.109).
-}
+# No Caddy rules needed here — Cloudflare handles TLS termination and routing.
-
+
-# ── GlitchTip: error tracking ─────────────────────────────────────────────────
-errors.libnovel.cc {
-	import security_headers
-	reverse_proxy glitchtip-web:8000
-}
-
-# ── Umami: page analytics ─────────────────────────────────────────────────────
-analytics.libnovel.cc {
-	import security_headers
-	reverse_proxy umami:3000
-}
-
-# ── Dozzle: Docker log viewer ─────────────────────────────────────────────────
-logs.libnovel.cc {
-	import security_headers
-	reverse_proxy dozzle:8080
-}
-
-# ── Uptime Kuma: uptime monitoring ────────────────────────────────────────────
-uptime.libnovel.cc {
-	import security_headers
-	reverse_proxy uptime-kuma:3001
-}
-
-# ── Gotify: push notifications ────────────────────────────────────────────────
-push.libnovel.cc {
-	import security_headers
-	reverse_proxy gotify:80
-}
-
 # ── PocketBase: exposed for homelab runner task polling ───────────────────────
 # Allows the homelab runner to claim tasks and write results via the PB API.
 # Admin UI is also accessible here for convenience.

docker-compose.yml

@@ -299,6 +299,19 @@ services:
       timeout: 10s
       retries: 5
 
+  # ─── Dozzle agent ────────────────────────────────────────────────────────────
+  # Exposes prod container logs to the Dozzle instance on the homelab.
+  # The homelab Dozzle connects here via DOZZLE_REMOTE_AGENT.
+  # Port 7007 is bound to localhost only — not reachable from the internet.
+  dozzle-agent:
+    image: amir20/dozzle:latest
+    restart: unless-stopped
+    command: agent
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    ports:
+      - "127.0.0.1:7007:7007"
+
   # ─── CrowdSec bouncer registration ───────────────────────────────────────────
   # One-shot: registers the Caddy bouncer with the CrowdSec LAPI and writes the
   # generated API key to crowdsec/.crowdsec.env, which Caddy reads via env_file.
@@ -380,203 +393,6 @@ services:
       WATCHTOWER_NOTIFICATION_URL: "${WATCHTOWER_NOTIFICATION_URL}"
       DOCKER_API_VERSION: "1.44"
 
-  # ─── Shared PostgreSQL (Fider + GlitchTip + Umami) ───────────────────────────
-  # A single Postgres instance hosting three separate databases.
-  # PocketBase uses its own embedded SQLite; this postgres is only for the
-  # three new services below.
-  postgres:
-    image: postgres:16-alpine
-    restart: unless-stopped
-    environment:
-      POSTGRES_USER: "${POSTGRES_USER}"
-      POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
-      POSTGRES_DB: postgres
-    expose:
-      - "5432"
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    healthcheck:
-      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
-  # ─── Postgres database initialisation ────────────────────────────────────────
-  # One-shot: creates the fider, glitchtip, and umami databases if missing.
-  postgres-init:
-    image: postgres:16-alpine
-    depends_on:
-      postgres:
-        condition: service_healthy
-    environment:
-      PGPASSWORD: "${POSTGRES_PASSWORD}"
-    entrypoint: >
-      /bin/sh -c "
-        psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='fider'\" | grep -q 1 ||
-          psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE fider\";
-        psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='glitchtip'\" | grep -q 1 ||
-          psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE glitchtip\";
-        psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='umami'\" | grep -q 1 ||
-          psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE umami\";
-        echo 'postgres-init: databases ready';
-      "
-    restart: "no"
-
-  # ─── Fider (user feedback & feature requests) ─────────────────────────────────
-  fider:
-    image: getfider/fider:stable
-    restart: unless-stopped
-    depends_on:
-      postgres-init:
-        condition: service_completed_successfully
-      postgres:
-        condition: service_healthy
-    expose:
-      - "3000"
-    environment:
-      BASE_URL: "${FIDER_BASE_URL}"
-      DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/fider?sslmode=disable"
-      JWT_SECRET: "${FIDER_JWT_SECRET}"
-      # Email: Resend SMTP
-      EMAIL_NOREPLY: "noreply@libnovel.cc"
-      EMAIL_SMTP_HOST: "${FIDER_SMTP_HOST}"
-      EMAIL_SMTP_PORT: "${FIDER_SMTP_PORT}"
-      EMAIL_SMTP_USERNAME: "${FIDER_SMTP_USER}"
-      EMAIL_SMTP_PASSWORD: "${FIDER_SMTP_PASSWORD}"
-      EMAIL_SMTP_ENABLE_STARTTLS: "false"
-
-  # ─── GlitchTip DB migration (one-shot) ───────────────────────────────────────
-  glitchtip-migrate:
-    image: glitchtip/glitchtip:latest
-    depends_on:
-      postgres-init:
-        condition: service_completed_successfully
-      postgres:
-        condition: service_healthy
-    environment:
-      DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip"
-      SECRET_KEY: "${GLITCHTIP_SECRET_KEY}"
-      GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}"
-      EMAIL_URL: "${GLITCHTIP_EMAIL_URL}"
-      DEFAULT_FROM_EMAIL: "noreply@libnovel.cc"
-      VALKEY_URL: "redis://valkey:6379/1"
-    command: "./manage.py migrate"
-    restart: "no"
-
-  # ─── GlitchTip web (error tracking UI + API) ─────────────────────────────────
-  glitchtip-web:
-    image: glitchtip/glitchtip:latest
-    restart: unless-stopped
-    depends_on:
-      glitchtip-migrate:
-        condition: service_completed_successfully
-      valkey:
-        condition: service_healthy
-    expose:
-      - "8000"
-    environment:
-      DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip"
-      SECRET_KEY: "${GLITCHTIP_SECRET_KEY}"
-      GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}"
-      EMAIL_URL: "${GLITCHTIP_EMAIL_URL}"
-      DEFAULT_FROM_EMAIL: "noreply@libnovel.cc"
-      VALKEY_URL: "redis://valkey:6379/1"
-      PORT: "8000"
-      ENABLE_USER_REGISTRATION: "false"
-    healthcheck:
-      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/0/')"]
-      interval: 15s
-      timeout: 5s
-      retries: 5
-
-  # ─── GlitchTip worker (background task processor) ─────────────────────────────
-  glitchtip-worker:
-    image: glitchtip/glitchtip:latest
-    restart: unless-stopped
-    depends_on:
-      glitchtip-migrate:
-        condition: service_completed_successfully
-      valkey:
-        condition: service_healthy
-    environment:
-      DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip"
-      SECRET_KEY: "${GLITCHTIP_SECRET_KEY}"
-      GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}"
-      EMAIL_URL: "${GLITCHTIP_EMAIL_URL}"
-      DEFAULT_FROM_EMAIL: "noreply@libnovel.cc"
-      VALKEY_URL: "redis://valkey:6379/1"
-      SERVER_ROLE: "worker"
-
-  # ─── Umami (page analytics) ───────────────────────────────────────────────────
-  umami:
-    image: ghcr.io/umami-software/umami:postgresql-latest
-    restart: unless-stopped
-    depends_on:
-      postgres-init:
-        condition: service_completed_successfully
-      postgres:
-        condition: service_healthy
-    expose:
-      - "3000"
-    environment:
-      DATABASE_URL: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/umami"
-      APP_SECRET: "${UMAMI_APP_SECRET}"
-    healthcheck:
-      test: ["CMD", "curl", "-sf", "http://localhost:3000/api/heartbeat"]
-      interval: 15s
-      timeout: 5s
-      retries: 5
-
-  # ─── Dozzle (Docker log viewer) ───────────────────────────────────────────────
-  dozzle:
-    image: amir20/dozzle:latest
-    restart: unless-stopped
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - ./dozzle/users.yml:/data/users.yml:ro
-    expose:
-      - "8080"
-    environment:
-      DOZZLE_AUTH_PROVIDER: simple
-      DOZZLE_HOSTNAME: "logs.libnovel.cc"
-    healthcheck:
-      test: ["CMD", "/dozzle", "healthcheck"]
-      interval: 15s
-      timeout: 5s
-      retries: 5
-
-  # ─── Uptime Kuma (uptime monitoring) ──────────────────────────────────────────
-  uptime-kuma:
-    image: louislam/uptime-kuma:1
-    restart: unless-stopped
-    volumes:
-      - uptime_kuma_data:/app/data
-    expose:
-      - "3001"
-    healthcheck:
-      test: ["CMD", "extra/healthcheck"]
-      interval: 15s
-      timeout: 5s
-      retries: 5
-
-  # ─── Gotify (push notifications) ──────────────────────────────────────────────
-  gotify:
-    image: gotify/server:latest
-    restart: unless-stopped
-    volumes:
-      - gotify_data:/app/data
-    expose:
-      - "80"
-    environment:
-      GOTIFY_DEFAULTUSER_NAME: "${GOTIFY_ADMIN_USER}"
-      GOTIFY_DEFAULTUSER_PASS: "${GOTIFY_ADMIN_PASS}"
-      GOTIFY_SERVER_PORT: "80"
-    healthcheck:
-      test: ["CMD", "curl", "-sf", "http://localhost:80/health"]
-      interval: 15s
-      timeout: 5s
-      retries: 5
-
 volumes:
   minio_data:
   pb_data:
@@ -586,6 +402,3 @@ volumes:
   caddy_config:
   caddy_logs:
   crowdsec_data:
-  postgres_data:
-  uptime_kuma_data:
-  gotify_data:

homelab/docker-compose.yml

@@ -0,0 +1,411 @@
+# LibNovel homelab
+#
+# Runs on 192.168.0.109. Hosts:
+#   - libnovel runner (background task worker)
+#   - tooling: GlitchTip, Umami, Fider, Dozzle, Uptime Kuma, Gotify
+#   - observability: OTel Collector, Tempo, Loki, Prometheus, Grafana
+#   - cloudflared tunnel (public subdomains via Cloudflare Zero Trust)
+#   - shared Postgres for tooling DBs
+#
+# All secrets come from Doppler (project=libnovel, config=prd_homelab).
+# Run with: doppler run -- docker compose up -d
+#
+# Public subdomains (via Cloudflare Tunnel — no ports exposed to internet):
+#   errors.libnovel.cc    → glitchtip-web:8000
+#   analytics.libnovel.cc → umami:3000
+#   feedback.libnovel.cc  → fider:3000
+#   logs.libnovel.cc      → dozzle:8080
+#   uptime.libnovel.cc    → uptime-kuma:3001
+#   push.libnovel.cc      → gotify:80
+#   grafana.libnovel.cc   → grafana:3000
+
+services:
+
+  # ── Cloudflare Tunnel ───────────────────────────────────────────────────────
+  # Outbound-only encrypted tunnel to Cloudflare.
+  # Routes all public subdomains to their respective containers on this network.
+  # No inbound ports needed — cloudflared initiates all connections outward.
+  cloudflared:
+    image: cloudflare/cloudflared:latest
+    restart: unless-stopped
+    command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN}
+    environment:
+      CLOUDFLARE_TUNNEL_TOKEN: "${CLOUDFLARE_TUNNEL_TOKEN}"
+
+  # ── LibNovel Runner ─────────────────────────────────────────────────────────
+  # Background task worker. Connects to prod PocketBase, MinIO, Meilisearch
+  # via their public subdomains (pb.libnovel.cc, storage.libnovel.cc, etc.)
+  runner:
+    image: kalekber/libnovel-runner:latest
+    restart: unless-stopped
+    stop_grace_period: 135s
+    labels:
+      com.centurylinklabs.watchtower.enable: "true"
+    environment:
+      POCKETBASE_URL: "https://pb.libnovel.cc"
+      POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}"
+      POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}"
+
+      MINIO_ENDPOINT: "storage.libnovel.cc"
+      MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}"
+      MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD}"
+      MINIO_USE_SSL: "true"
+      MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT}"
+      MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL}"
+
+      MEILI_URL: "${MEILI_URL}"
+      MEILI_API_KEY: "${MEILI_API_KEY}"
+      VALKEY_ADDR: ""
+      GODEBUG: "preferIPv4=1"
+
+      KOKORO_URL: "${KOKORO_URL}"
+      KOKORO_VOICE: "${KOKORO_VOICE}"
+
+      RUNNER_WORKER_ID: "${RUNNER_WORKER_ID}"
+      RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL}"
+      RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE}"
+      RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO}"
+      RUNNER_TIMEOUT: "${RUNNER_TIMEOUT}"
+      RUNNER_METRICS_ADDR: "${RUNNER_METRICS_ADDR}"
+      RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH: "true"
+
+      LOG_LEVEL: "${LOG_LEVEL}"
+      GLITCHTIP_DSN: "${GLITCHTIP_DSN}"
+
+      # OTel — send runner traces/metrics to the local collector
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"
+      OTEL_SERVICE_NAME: "libnovel-runner"
+
+    healthcheck:
+      test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"]
+      interval: 60s
+      timeout: 5s
+      retries: 3
+
+  # ── Shared Postgres ─────────────────────────────────────────────────────────
+  # Hosts glitchtip, umami, and fider databases.
+  postgres:
+    image: postgres:16-alpine
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: "${POSTGRES_USER}"
+      POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
+      POSTGRES_DB: postgres
+    expose:
+      - "5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # ── Postgres database initialisation ────────────────────────────────────────
+  postgres-init:
+    image: postgres:16-alpine
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PGPASSWORD: "${POSTGRES_PASSWORD}"
+    entrypoint: >
+      /bin/sh -c "
+        psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='fider'\" | grep -q 1 ||
+          psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE fider\";
+        psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='glitchtip'\" | grep -q 1 ||
+          psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE glitchtip\";
+        psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='umami'\" | grep -q 1 ||
+          psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE umami\";
+        echo 'postgres-init: databases ready';
+      "
+    restart: "no"
+
+  # ── GlitchTip DB migration ──────────────────────────────────────────────────
+  glitchtip-migrate:
+    image: glitchtip/glitchtip:latest
+    depends_on:
+      postgres-init:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+    environment:
+      DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip"
+      SECRET_KEY: "${GLITCHTIP_SECRET_KEY}"
+      GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}"
+      EMAIL_URL: "${GLITCHTIP_EMAIL_URL}"
+      DEFAULT_FROM_EMAIL: "noreply@libnovel.cc"
+      VALKEY_URL: "redis://valkey:6379/1"
+    command: "./manage.py migrate"
+    restart: "no"
+
+  # ── GlitchTip web ───────────────────────────────────────────────────────────
+  glitchtip-web:
+    image: glitchtip/glitchtip:latest
+    restart: unless-stopped
+    depends_on:
+      glitchtip-migrate:
+        condition: service_completed_successfully
+    expose:
+      - "8000"
+    environment:
+      DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip"
+      SECRET_KEY: "${GLITCHTIP_SECRET_KEY}"
+      GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}"
+      EMAIL_URL: "${GLITCHTIP_EMAIL_URL}"
+      DEFAULT_FROM_EMAIL: "noreply@libnovel.cc"
+      VALKEY_URL: "redis://valkey:6379/1"
+      PORT: "8000"
+      ENABLE_USER_REGISTRATION: "false"
+    healthcheck:
+      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/0/')"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+
+  # ── GlitchTip worker ────────────────────────────────────────────────────────
+  glitchtip-worker:
+    image: glitchtip/glitchtip:latest
+    restart: unless-stopped
+    depends_on:
+      glitchtip-migrate:
+        condition: service_completed_successfully
+    environment:
+      DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip"
+      SECRET_KEY: "${GLITCHTIP_SECRET_KEY}"
+      GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}"
+      EMAIL_URL: "${GLITCHTIP_EMAIL_URL}"
+      DEFAULT_FROM_EMAIL: "noreply@libnovel.cc"
+      VALKEY_URL: "redis://valkey:6379/1"
+      SERVER_ROLE: "worker"
+
+  # ── Umami ───────────────────────────────────────────────────────────────────
+  umami:
+    image: ghcr.io/umami-software/umami:postgresql-latest
+    restart: unless-stopped
+    depends_on:
+      postgres-init:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+    expose:
+      - "3000"
+    environment:
+      DATABASE_URL: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/umami"
+      APP_SECRET: "${UMAMI_APP_SECRET}"
+    healthcheck:
+      test: ["CMD", "curl", "-sf", "http://localhost:3000/api/heartbeat"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+
+  # ── Fider ───────────────────────────────────────────────────────────────────
+  fider:
+    image: getfider/fider:stable
+    restart: unless-stopped
+    depends_on:
+      postgres-init:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+    expose:
+      - "3000"
+    environment:
+      BASE_URL: "${FIDER_BASE_URL}"
+      DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/fider?sslmode=disable"
+      JWT_SECRET: "${FIDER_JWT_SECRET}"
+      EMAIL_NOREPLY: "noreply@libnovel.cc"
+      EMAIL_SMTP_HOST: "${FIDER_SMTP_HOST}"
+      EMAIL_SMTP_PORT: "${FIDER_SMTP_PORT}"
+      EMAIL_SMTP_USERNAME: "${FIDER_SMTP_USER}"
+      EMAIL_SMTP_PASSWORD: "${FIDER_SMTP_PASSWORD}"
+      EMAIL_SMTP_ENABLE_STARTTLS: "false"
+
+  # ── Dozzle ──────────────────────────────────────────────────────────────────
+  # Watches both homelab and prod containers.
+  # Prod agent runs on 165.22.70.138:7007 (added separately to prod compose).
+  dozzle:
+    image: amir20/dozzle:latest
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./dozzle/users.yml:/data/users.yml:ro
+    expose:
+      - "8080"
+    environment:
+      DOZZLE_AUTH_PROVIDER: simple
+      DOZZLE_HOSTNAME: "logs.libnovel.cc"
+      DOZZLE_REMOTE_AGENT: "prod@165.22.70.138:7007"
+    healthcheck:
+      test: ["CMD", "/dozzle", "healthcheck"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+
+  # ── Uptime Kuma ─────────────────────────────────────────────────────────────
+  uptime-kuma:
+    image: louislam/uptime-kuma:1
+    restart: unless-stopped
+    volumes:
+      - uptime_kuma_data:/app/data
+    expose:
+      - "3001"
+    healthcheck:
+      test: ["CMD", "extra/healthcheck"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+
+  # ── Gotify ──────────────────────────────────────────────────────────────────
+  gotify:
+    image: gotify/server:latest
+    restart: unless-stopped
+    volumes:
+      - gotify_data:/app/data
+    expose:
+      - "80"
+    environment:
+      GOTIFY_DEFAULTUSER_NAME: "${GOTIFY_ADMIN_USER}"
+      GOTIFY_DEFAULTUSER_PASS: "${GOTIFY_ADMIN_PASS}"
+      GOTIFY_SERVER_PORT: "80"
+    healthcheck:
+      test: ["CMD", "curl", "-sf", "http://localhost:80/health"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+
+  # ── Valkey ──────────────────────────────────────────────────────────────────
+  # Used by GlitchTip for task queuing.
+  valkey:
+    image: valkey/valkey:7-alpine
+    restart: unless-stopped
+    expose:
+      - "6379"
+    volumes:
+      - valkey_data:/data
+    healthcheck:
+      test: ["CMD", "valkey-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # ── OTel Collector ──────────────────────────────────────────────────────────
+  # Receives OTLP from backend/ui/runner, fans out to Tempo + Prometheus + Loki.
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    restart: unless-stopped
+    volumes:
+      - ./otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
+    expose:
+      - "4317"  # OTLP gRPC
+      - "4318"  # OTLP HTTP
+      - "8888"  # Collector self-metrics (scraped by Prometheus)
+    depends_on:
+      - tempo
+      - prometheus
+      - loki
+    # No healthcheck — distroless image has no shell or curl
+
+  # ── Tempo ───────────────────────────────────────────────────────────────────
+  # Distributed trace storage. Receives OTLP from the collector.
+  tempo:
+    image: grafana/tempo:2.6.1
+    restart: unless-stopped
+    command: ["-config.file=/etc/tempo.yaml"]
+    volumes:
+      - ./otel/tempo.yaml:/etc/tempo.yaml:ro
+      - tempo_data:/var/tempo
+    expose:
+      - "3200"  # Tempo query API (queried by Grafana)
+      - "4317"  # OTLP gRPC ingest (collector → tempo)
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:3200/ready"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+
+  # ── Prometheus ──────────────────────────────────────────────────────────────
+  # Scrapes metrics from backend (via prod), runner, and otel-collector.
+  prometheus:
+    image: prom/prometheus:latest
+    restart: unless-stopped
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yaml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention.time=30d"
+      - "--web.enable-remote-write-receiver"
+    volumes:
+      - ./otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
+      - prometheus_data:/prometheus
+    expose:
+      - "9090"
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:9090/-/healthy"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+
+  # ── Loki ────────────────────────────────────────────────────────────────────
+  # Log aggregation. Receives logs from OTel collector. Replaces manual Dozzle
+  # tailing for structured log search.
+  loki:
+    image: grafana/loki:latest
+    restart: unless-stopped
+    command: ["-config.file=/etc/loki/loki.yaml"]
+    volumes:
+      - ./otel/loki.yaml:/etc/loki/loki.yaml:ro
+      - loki_data:/loki
+    expose:
+      - "3100"
+    # No healthcheck — distroless image has no shell or curl
+
+  # ── Grafana ─────────────────────────────────────────────────────────────────
+  # Single UI for traces (Tempo), metrics (Prometheus), and logs (Loki).
+  # Accessible at grafana.libnovel.cc via Cloudflare Tunnel.
+  grafana:
+    image: grafana/grafana:latest
+    restart: unless-stopped
+    depends_on:
+      - tempo
+      - prometheus
+      - loki
+    expose:
+      - "3000"
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./otel/grafana/provisioning:/etc/grafana/provisioning:ro
+    environment:
+      GF_SERVER_ROOT_URL: "https://grafana.libnovel.cc"
+      GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER}"
+      GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD}"
+      GF_AUTH_ANONYMOUS_ENABLED: "false"
+      GF_FEATURE_TOGGLES_ENABLE: "traceqlEditor"
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+
+  # ── Watchtower ──────────────────────────────────────────────────────────────
+  # Auto-updates runner image when CI pushes a new tag.
+  # Only watches services with the watchtower label.
+  watchtower:
+    image: containrrr/watchtower:latest
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: --label-enable --interval 300 --cleanup
+    environment:
+      WATCHTOWER_NOTIFICATIONS: "${WATCHTOWER_NOTIFICATIONS}"
+      WATCHTOWER_NOTIFICATION_URL: "${WATCHTOWER_NOTIFICATION_URL}"
+      DOCKER_API_VERSION: "1.44"
+
+volumes:
+  postgres_data:
+  valkey_data:
+  uptime_kuma_data:
+  gotify_data:
+  tempo_data:
+  prometheus_data:
+  loki_data:
+  grafana_data:

homelab/dozzle/users.yml

@@ -0,0 +1,5 @@
+users:
+  admin:
+    name: admin
+    email: admin@libnovel.cc
+    password: "$2y$10$4jqLza2grpxnQn0EGux2C.UmlSxRmOvH/J1ySzOBxMZgW6cA2TnmK"

homelab/otel/collector.yaml

@@ -0,0 +1,68 @@
+# OTel Collector config
+#
+# Receivers:  OTLP (gRPC + HTTP) from backend, ui, runner
+# Processors: batch for efficiency, resource detection for host metadata
+# Exporters:  Tempo (traces), Prometheus (metrics), Loki (logs)
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+    timeout: 5s
+    send_batch_size: 512
+
+  # Attach host metadata to all telemetry
+  resourcedetection:
+    detectors: [env, system]
+    timeout: 5s
+
+exporters:
+  # Traces → Tempo
+  otlp/tempo:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+
+  # Metrics → Prometheus (remote write)
+  prometheusremotewrite:
+    endpoint: "http://prometheus:9090/api/v1/write"
+    tls:
+      insecure_skip_verify: true
+
+  # Logs → Loki (via OTLP HTTP endpoint)
+  otlphttp/loki:
+    endpoint: "http://loki:3100/otlp"
+    tls:
+      insecure: true
+
+  # Collector self-observability (optional debug)
+  debug:
+    verbosity: basic
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+  pprof:
+    endpoint: 0.0.0.0:1777
+
+service:
+  extensions: [health_check, pprof]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [resourcedetection, batch]
+      exporters: [otlp/tempo]
+    metrics:
+      receivers: [otlp]
+      processors: [resourcedetection, batch]
+      exporters: [prometheusremotewrite]
+    logs:
+      receivers: [otlp]
+      processors: [resourcedetection, batch]
+      exporters: [otlphttp/loki]

homelab/otel/grafana/provisioning/dashboards/dashboards.yaml

@@ -0,0 +1,13 @@
+# Grafana dashboard provisioning
+# Points Grafana at the local dashboards directory.
+# Drop any .json dashboard file into homelab/otel/grafana/provisioning/dashboards/
+# and it will appear in Grafana automatically on restart.
+
+apiVersion: 1
+
+providers:
+  - name: libnovel
+    folder: LibNovel
+    type: file
+    options:
+      path: /etc/grafana/provisioning/dashboards

homelab/otel/grafana/provisioning/datasources/datasources.yaml

@@ -0,0 +1,53 @@
+# Grafana datasource provisioning
+# Auto-configures Tempo, Prometheus, and Loki on first start.
+# No manual setup needed in the UI.
+
+apiVersion: 1
+
+datasources:
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    url: http://tempo:3200
+    access: proxy
+    isDefault: false
+    jsonData:
+      httpMethod: GET
+      serviceMap:
+        datasourceUid: prometheus
+      nodeGraph:
+        enabled: true
+      traceQuery:
+        timeShiftEnabled: true
+        spanStartTimeShift: "1h"
+        spanEndTimeShift: "-1h"
+      spanBar:
+        type: "Tag"
+        tag: "http.url"
+      lokiSearch:
+        datasourceUid: loki
+
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    url: http://prometheus:9090
+    access: proxy
+    isDefault: true
+    jsonData:
+      httpMethod: POST
+      exemplarTraceIdDestinations:
+        - name: traceID
+          datasourceUid: tempo
+
+  - name: Loki
+    type: loki
+    uid: loki
+    url: http://loki:3100
+    access: proxy
+    isDefault: false
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: '"traceID":"(\w+)"'
+          name: TraceID
+          url: "$${__value.raw}"

homelab/otel/loki.yaml

@@ -0,0 +1,38 @@
+# Loki config — minimal single-node setup
+# Receives logs from OTel Collector. 30-day retention.
+
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: 720h  # 30 days
+
+compactor:
+  working_directory: /loki/compactor
+  delete_request_store: filesystem
+  retention_enabled: true

homelab/otel/prometheus.yaml

@@ -0,0 +1,22 @@
+# Prometheus config
+# Scrapes OTel collector self-metrics and runner metrics endpoint.
+# Backend metrics come in via OTel remote-write — no direct scrape needed.
+
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    environment: production
+
+scrape_configs:
+  # OTel Collector self-metrics
+  - job_name: otel-collector
+    static_configs:
+      - targets: ["otel-collector:8888"]
+
+  # Runner JSON metrics endpoint (native format, no Prometheus client yet)
+  # Will be replaced by OTLP once runner is instrumented with OTel SDK.
+  - job_name: libnovel-runner
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["runner:9091"]

homelab/otel/tempo.yaml

@@ -0,0 +1,45 @@
+# Tempo config — minimal single-node setup
+# Stores traces locally. Grafana queries via the HTTP API on port 3200.
+
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+
+ingester:
+  trace_idle_period: 10s
+  max_block_bytes: 104857600  # 100MB
+  max_block_duration: 30m
+
+compactor:
+  compaction:
+    block_retention: 720h  # 30 days
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /var/tempo/blocks
+    wal:
+      path: /var/tempo/wal
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics]
+      generate_native_histograms: both