From a39f660a3723caf9ef06d0b642afc1cacf60fdd7 Mon Sep 17 00:00:00 2001 From: Admin Date: Thu, 26 Mar 2026 21:22:43 +0500 Subject: [PATCH] Migrate tooling to homelab; add OTel observability stack - Remove GlitchTip, Umami, Fider, Gotify, Uptime Kuma, Dozzle, Watchtower from prod docker-compose (now run on homelab) - Add dozzle-agent on prod (127.0.0.1:7007) for homelab Dozzle to connect to - Remove corresponding subdomain blocks from Caddyfile (now routed via Cloudflare Tunnel from homelab) - Add homelab/docker-compose.yml: unified homelab stack with all migrated tooling services plus full OTel stack (Tempo 2.6.1, Loki, Prometheus, OTel Collector, Grafana) - Add homelab/otel/: Tempo, Loki, Prometheus, OTel Collector configs + Grafana provisioning (datasources + dashboards) - Add homelab/dozzle/users.yml for Dozzle auth --- Caddyfile | 40 +- docker-compose.yml | 213 +-------- homelab/docker-compose.yml | 411 ++++++++++++++++++ homelab/dozzle/users.yml | 5 + homelab/otel/collector.yaml | 68 +++ .../provisioning/dashboards/dashboards.yaml | 13 + .../provisioning/datasources/datasources.yaml | 53 +++ homelab/otel/loki.yaml | 38 ++ homelab/otel/prometheus.yaml | 22 + homelab/otel/tempo.yaml | 45 ++ 10 files changed, 673 insertions(+), 235 deletions(-) create mode 100644 homelab/docker-compose.yml create mode 100644 homelab/dozzle/users.yml create mode 100644 homelab/otel/collector.yaml create mode 100644 homelab/otel/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 homelab/otel/grafana/provisioning/datasources/datasources.yaml create mode 100644 homelab/otel/loki.yaml create mode 100644 homelab/otel/prometheus.yaml create mode 100644 homelab/otel/tempo.yaml diff --git a/Caddyfile b/Caddyfile index 9df85fb..6379318 100644 --- a/Caddyfile +++ b/Caddyfile @@ -204,41 +204,11 @@ } } -# ── Fider: user feedback & feature requests ─────────────────────────────────── -feedback.libnovel.cc { - import security_headers - reverse_proxy fider:3000 -} - -# ── GlitchTip: error tracking ───────────────────────────────────────────────── -errors.libnovel.cc { - import security_headers - reverse_proxy glitchtip-web:8000 -} - -# ── Umami: page analytics ───────────────────────────────────────────────────── -analytics.libnovel.cc { - import security_headers - reverse_proxy umami:3000 -} - -# ── Dozzle: Docker log viewer ───────────────────────────────────────────────── -logs.libnovel.cc { - import security_headers - reverse_proxy dozzle:8080 -} - -# ── Uptime Kuma: uptime monitoring ──────────────────────────────────────────── -uptime.libnovel.cc { - import security_headers - reverse_proxy uptime-kuma:3001 -} - -# ── Gotify: push notifications ──────────────────────────────────────────────── -push.libnovel.cc { - import security_headers - reverse_proxy gotify:80 -} +# ── Tooling subdomains ──────────────────────────────────────────────────────── +# feedback.libnovel.cc, errors.libnovel.cc, analytics.libnovel.cc, +# logs.libnovel.cc, uptime.libnovel.cc, push.libnovel.cc, grafana.libnovel.cc +# are now routed via Cloudflare Tunnel directly to the homelab (192.168.0.109). +# No Caddy rules needed here — Cloudflare handles TLS termination and routing. # ── PocketBase: exposed for homelab runner task polling ─────────────────────── # Allows the homelab runner to claim tasks and write results via the PB API. diff --git a/docker-compose.yml b/docker-compose.yml index 36c8b8c..4a93a2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -299,6 +299,19 @@ services: timeout: 10s retries: 5 + # ─── Dozzle agent ──────────────────────────────────────────────────────────── + # Exposes prod container logs to the Dozzle instance on the homelab. + # The homelab Dozzle connects here via DOZZLE_REMOTE_AGENT. + # Port 7007 is bound to localhost only — not reachable from the internet. + dozzle-agent: + image: amir20/dozzle:latest + restart: unless-stopped + command: agent + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "127.0.0.1:7007:7007" + # ─── CrowdSec bouncer registration ─────────────────────────────────────────── # One-shot: registers the Caddy bouncer with the CrowdSec LAPI and writes the # generated API key to crowdsec/.crowdsec.env, which Caddy reads via env_file. @@ -380,203 +393,6 @@ services: WATCHTOWER_NOTIFICATION_URL: "${WATCHTOWER_NOTIFICATION_URL}" DOCKER_API_VERSION: "1.44" - # ─── Shared PostgreSQL (Fider + GlitchTip + Umami) ─────────────────────────── - # A single Postgres instance hosting three separate databases. - # PocketBase uses its own embedded SQLite; this postgres is only for the - # three new services below. - postgres: - image: postgres:16-alpine - restart: unless-stopped - environment: - POSTGRES_USER: "${POSTGRES_USER}" - POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" - POSTGRES_DB: postgres - expose: - - "5432" - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"] - interval: 10s - timeout: 5s - retries: 5 - - # ─── Postgres database initialisation ──────────────────────────────────────── - # One-shot: creates the fider, glitchtip, and umami databases if missing. - postgres-init: - image: postgres:16-alpine - depends_on: - postgres: - condition: service_healthy - environment: - PGPASSWORD: "${POSTGRES_PASSWORD}" - entrypoint: > - /bin/sh -c " - psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='fider'\" | grep -q 1 || - psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE fider\"; - psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='glitchtip'\" | grep -q 1 || - psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE glitchtip\"; - psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='umami'\" | grep -q 1 || - psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE umami\"; - echo 'postgres-init: databases ready'; - " - restart: "no" - - # ─── Fider (user feedback & feature requests) ───────────────────────────────── - fider: - image: getfider/fider:stable - restart: unless-stopped - depends_on: - postgres-init: - condition: service_completed_successfully - postgres: - condition: service_healthy - expose: - - "3000" - environment: - BASE_URL: "${FIDER_BASE_URL}" - DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/fider?sslmode=disable" - JWT_SECRET: "${FIDER_JWT_SECRET}" - # Email: Resend SMTP - EMAIL_NOREPLY: "noreply@libnovel.cc" - EMAIL_SMTP_HOST: "${FIDER_SMTP_HOST}" - EMAIL_SMTP_PORT: "${FIDER_SMTP_PORT}" - EMAIL_SMTP_USERNAME: "${FIDER_SMTP_USER}" - EMAIL_SMTP_PASSWORD: "${FIDER_SMTP_PASSWORD}" - EMAIL_SMTP_ENABLE_STARTTLS: "false" - - # ─── GlitchTip DB migration (one-shot) ─────────────────────────────────────── - glitchtip-migrate: - image: glitchtip/glitchtip:latest - depends_on: - postgres-init: - condition: service_completed_successfully - postgres: - condition: service_healthy - environment: - DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip" - SECRET_KEY: "${GLITCHTIP_SECRET_KEY}" - GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}" - EMAIL_URL: "${GLITCHTIP_EMAIL_URL}" - DEFAULT_FROM_EMAIL: "noreply@libnovel.cc" - VALKEY_URL: "redis://valkey:6379/1" - command: "./manage.py migrate" - restart: "no" - - # ─── GlitchTip web (error tracking UI + API) ───────────────────────────────── - glitchtip-web: - image: glitchtip/glitchtip:latest - restart: unless-stopped - depends_on: - glitchtip-migrate: - condition: service_completed_successfully - valkey: - condition: service_healthy - expose: - - "8000" - environment: - DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip" - SECRET_KEY: "${GLITCHTIP_SECRET_KEY}" - GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}" - EMAIL_URL: "${GLITCHTIP_EMAIL_URL}" - DEFAULT_FROM_EMAIL: "noreply@libnovel.cc" - VALKEY_URL: "redis://valkey:6379/1" - PORT: "8000" - ENABLE_USER_REGISTRATION: "false" - healthcheck: - test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/0/')"] - interval: 15s - timeout: 5s - retries: 5 - - # ─── GlitchTip worker (background task processor) ───────────────────────────── - glitchtip-worker: - image: glitchtip/glitchtip:latest - restart: unless-stopped - depends_on: - glitchtip-migrate: - condition: service_completed_successfully - valkey: - condition: service_healthy - environment: - DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip" - SECRET_KEY: "${GLITCHTIP_SECRET_KEY}" - GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}" - EMAIL_URL: "${GLITCHTIP_EMAIL_URL}" - DEFAULT_FROM_EMAIL: "noreply@libnovel.cc" - VALKEY_URL: "redis://valkey:6379/1" - SERVER_ROLE: "worker" - - # ─── Umami (page analytics) ─────────────────────────────────────────────────── - umami: - image: ghcr.io/umami-software/umami:postgresql-latest - restart: unless-stopped - depends_on: - postgres-init: - condition: service_completed_successfully - postgres: - condition: service_healthy - expose: - - "3000" - environment: - DATABASE_URL: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/umami" - APP_SECRET: "${UMAMI_APP_SECRET}" - healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:3000/api/heartbeat"] - interval: 15s - timeout: 5s - retries: 5 - - # ─── Dozzle (Docker log viewer) ─────────────────────────────────────────────── - dozzle: - image: amir20/dozzle:latest - restart: unless-stopped - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./dozzle/users.yml:/data/users.yml:ro - expose: - - "8080" - environment: - DOZZLE_AUTH_PROVIDER: simple - DOZZLE_HOSTNAME: "logs.libnovel.cc" - healthcheck: - test: ["CMD", "/dozzle", "healthcheck"] - interval: 15s - timeout: 5s - retries: 5 - - # ─── Uptime Kuma (uptime monitoring) ────────────────────────────────────────── - uptime-kuma: - image: louislam/uptime-kuma:1 - restart: unless-stopped - volumes: - - uptime_kuma_data:/app/data - expose: - - "3001" - healthcheck: - test: ["CMD", "extra/healthcheck"] - interval: 15s - timeout: 5s - retries: 5 - - # ─── Gotify (push notifications) ────────────────────────────────────────────── - gotify: - image: gotify/server:latest - restart: unless-stopped - volumes: - - gotify_data:/app/data - expose: - - "80" - environment: - GOTIFY_DEFAULTUSER_NAME: "${GOTIFY_ADMIN_USER}" - GOTIFY_DEFAULTUSER_PASS: "${GOTIFY_ADMIN_PASS}" - GOTIFY_SERVER_PORT: "80" - healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:80/health"] - interval: 15s - timeout: 5s - retries: 5 - volumes: minio_data: pb_data: @@ -586,6 +402,3 @@ volumes: caddy_config: caddy_logs: crowdsec_data: - postgres_data: - uptime_kuma_data: - gotify_data: diff --git a/homelab/docker-compose.yml b/homelab/docker-compose.yml new file mode 100644 index 0000000..42aca57 --- /dev/null +++ b/homelab/docker-compose.yml @@ -0,0 +1,411 @@ +# LibNovel homelab +# +# Runs on 192.168.0.109. Hosts: +# - libnovel runner (background task worker) +# - tooling: GlitchTip, Umami, Fider, Dozzle, Uptime Kuma, Gotify +# - observability: OTel Collector, Tempo, Loki, Prometheus, Grafana +# - cloudflared tunnel (public subdomains via Cloudflare Zero Trust) +# - shared Postgres for tooling DBs +# +# All secrets come from Doppler (project=libnovel, config=prd_homelab). +# Run with: doppler run -- docker compose up -d +# +# Public subdomains (via Cloudflare Tunnel — no ports exposed to internet): +# errors.libnovel.cc → glitchtip-web:8000 +# analytics.libnovel.cc → umami:3000 +# feedback.libnovel.cc → fider:3000 +# logs.libnovel.cc → dozzle:8080 +# uptime.libnovel.cc → uptime-kuma:3001 +# push.libnovel.cc → gotify:80 +# grafana.libnovel.cc → grafana:3000 + +services: + + # ── Cloudflare Tunnel ─────────────────────────────────────────────────────── + # Outbound-only encrypted tunnel to Cloudflare. + # Routes all public subdomains to their respective containers on this network. + # No inbound ports needed — cloudflared initiates all connections outward. + cloudflared: + image: cloudflare/cloudflared:latest + restart: unless-stopped + command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN} + environment: + CLOUDFLARE_TUNNEL_TOKEN: "${CLOUDFLARE_TUNNEL_TOKEN}" + + # ── LibNovel Runner ───────────────────────────────────────────────────────── + # Background task worker. Connects to prod PocketBase, MinIO, Meilisearch + # via their public subdomains (pb.libnovel.cc, storage.libnovel.cc, etc.) + runner: + image: kalekber/libnovel-runner:latest + restart: unless-stopped + stop_grace_period: 135s + labels: + com.centurylinklabs.watchtower.enable: "true" + environment: + POCKETBASE_URL: "https://pb.libnovel.cc" + POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL}" + POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD}" + + MINIO_ENDPOINT: "storage.libnovel.cc" + MINIO_ACCESS_KEY: "${MINIO_ROOT_USER}" + MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD}" + MINIO_USE_SSL: "true" + MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT}" + MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL}" + + MEILI_URL: "${MEILI_URL}" + MEILI_API_KEY: "${MEILI_API_KEY}" + VALKEY_ADDR: "" + GODEBUG: "preferIPv4=1" + + KOKORO_URL: "${KOKORO_URL}" + KOKORO_VOICE: "${KOKORO_VOICE}" + + RUNNER_WORKER_ID: "${RUNNER_WORKER_ID}" + RUNNER_POLL_INTERVAL: "${RUNNER_POLL_INTERVAL}" + RUNNER_MAX_CONCURRENT_SCRAPE: "${RUNNER_MAX_CONCURRENT_SCRAPE}" + RUNNER_MAX_CONCURRENT_AUDIO: "${RUNNER_MAX_CONCURRENT_AUDIO}" + RUNNER_TIMEOUT: "${RUNNER_TIMEOUT}" + RUNNER_METRICS_ADDR: "${RUNNER_METRICS_ADDR}" + RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH: "true" + + LOG_LEVEL: "${LOG_LEVEL}" + GLITCHTIP_DSN: "${GLITCHTIP_DSN}" + + # OTel — send runner traces/metrics to the local collector + OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317" + OTEL_SERVICE_NAME: "libnovel-runner" + + healthcheck: + test: ["CMD", "/healthcheck", "file", "/tmp/runner.alive", "120"] + interval: 60s + timeout: 5s + retries: 3 + + # ── Shared Postgres ───────────────────────────────────────────────────────── + # Hosts glitchtip, umami, and fider databases. + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: "${POSTGRES_USER}" + POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" + POSTGRES_DB: postgres + expose: + - "5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"] + interval: 10s + timeout: 5s + retries: 5 + + # ── Postgres database initialisation ──────────────────────────────────────── + postgres-init: + image: postgres:16-alpine + depends_on: + postgres: + condition: service_healthy + environment: + PGPASSWORD: "${POSTGRES_PASSWORD}" + entrypoint: > + /bin/sh -c " + psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='fider'\" | grep -q 1 || + psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE fider\"; + psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='glitchtip'\" | grep -q 1 || + psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE glitchtip\"; + psql -h postgres -U ${POSTGRES_USER} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='umami'\" | grep -q 1 || + psql -h postgres -U ${POSTGRES_USER} -d postgres -c \"CREATE DATABASE umami\"; + echo 'postgres-init: databases ready'; + " + restart: "no" + + # ── GlitchTip DB migration ────────────────────────────────────────────────── + glitchtip-migrate: + image: glitchtip/glitchtip:latest + depends_on: + postgres-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip" + SECRET_KEY: "${GLITCHTIP_SECRET_KEY}" + GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}" + EMAIL_URL: "${GLITCHTIP_EMAIL_URL}" + DEFAULT_FROM_EMAIL: "noreply@libnovel.cc" + VALKEY_URL: "redis://valkey:6379/1" + command: "./manage.py migrate" + restart: "no" + + # ── GlitchTip web ─────────────────────────────────────────────────────────── + glitchtip-web: + image: glitchtip/glitchtip:latest + restart: unless-stopped + depends_on: + glitchtip-migrate: + condition: service_completed_successfully + expose: + - "8000" + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip" + SECRET_KEY: "${GLITCHTIP_SECRET_KEY}" + GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}" + EMAIL_URL: "${GLITCHTIP_EMAIL_URL}" + DEFAULT_FROM_EMAIL: "noreply@libnovel.cc" + VALKEY_URL: "redis://valkey:6379/1" + PORT: "8000" + ENABLE_USER_REGISTRATION: "false" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/0/')"] + interval: 15s + timeout: 5s + retries: 5 + + # ── GlitchTip worker ──────────────────────────────────────────────────────── + glitchtip-worker: + image: glitchtip/glitchtip:latest + restart: unless-stopped + depends_on: + glitchtip-migrate: + condition: service_completed_successfully + environment: + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/glitchtip" + SECRET_KEY: "${GLITCHTIP_SECRET_KEY}" + GLITCHTIP_DOMAIN: "${GLITCHTIP_DOMAIN}" + EMAIL_URL: "${GLITCHTIP_EMAIL_URL}" + DEFAULT_FROM_EMAIL: "noreply@libnovel.cc" + VALKEY_URL: "redis://valkey:6379/1" + SERVER_ROLE: "worker" + + # ── Umami ─────────────────────────────────────────────────────────────────── + umami: + image: ghcr.io/umami-software/umami:postgresql-latest + restart: unless-stopped + depends_on: + postgres-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + expose: + - "3000" + environment: + DATABASE_URL: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/umami" + APP_SECRET: "${UMAMI_APP_SECRET}" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:3000/api/heartbeat"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Fider ─────────────────────────────────────────────────────────────────── + fider: + image: getfider/fider:stable + restart: unless-stopped + depends_on: + postgres-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + expose: + - "3000" + environment: + BASE_URL: "${FIDER_BASE_URL}" + DATABASE_URL: "postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/fider?sslmode=disable" + JWT_SECRET: "${FIDER_JWT_SECRET}" + EMAIL_NOREPLY: "noreply@libnovel.cc" + EMAIL_SMTP_HOST: "${FIDER_SMTP_HOST}" + EMAIL_SMTP_PORT: "${FIDER_SMTP_PORT}" + EMAIL_SMTP_USERNAME: "${FIDER_SMTP_USER}" + EMAIL_SMTP_PASSWORD: "${FIDER_SMTP_PASSWORD}" + EMAIL_SMTP_ENABLE_STARTTLS: "false" + + # ── Dozzle ────────────────────────────────────────────────────────────────── + # Watches both homelab and prod containers. + # Prod agent runs on 165.22.70.138:7007 (added separately to prod compose). + dozzle: + image: amir20/dozzle:latest + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./dozzle/users.yml:/data/users.yml:ro + expose: + - "8080" + environment: + DOZZLE_AUTH_PROVIDER: simple + DOZZLE_HOSTNAME: "logs.libnovel.cc" + DOZZLE_REMOTE_AGENT: "prod@165.22.70.138:7007" + healthcheck: + test: ["CMD", "/dozzle", "healthcheck"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Uptime Kuma ───────────────────────────────────────────────────────────── + uptime-kuma: + image: louislam/uptime-kuma:1 + restart: unless-stopped + volumes: + - uptime_kuma_data:/app/data + expose: + - "3001" + healthcheck: + test: ["CMD", "extra/healthcheck"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Gotify ────────────────────────────────────────────────────────────────── + gotify: + image: gotify/server:latest + restart: unless-stopped + volumes: + - gotify_data:/app/data + expose: + - "80" + environment: + GOTIFY_DEFAULTUSER_NAME: "${GOTIFY_ADMIN_USER}" + GOTIFY_DEFAULTUSER_PASS: "${GOTIFY_ADMIN_PASS}" + GOTIFY_SERVER_PORT: "80" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:80/health"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Valkey ────────────────────────────────────────────────────────────────── + # Used by GlitchTip for task queuing. + valkey: + image: valkey/valkey:7-alpine + restart: unless-stopped + expose: + - "6379" + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # ── OTel Collector ────────────────────────────────────────────────────────── + # Receives OTLP from backend/ui/runner, fans out to Tempo + Prometheus + Loki. + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + restart: unless-stopped + volumes: + - ./otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro + expose: + - "4317" # OTLP gRPC + - "4318" # OTLP HTTP + - "8888" # Collector self-metrics (scraped by Prometheus) + depends_on: + - tempo + - prometheus + - loki + # No healthcheck — distroless image has no shell or curl + + # ── Tempo ─────────────────────────────────────────────────────────────────── + # Distributed trace storage. Receives OTLP from the collector. + tempo: + image: grafana/tempo:2.6.1 + restart: unless-stopped + command: ["-config.file=/etc/tempo.yaml"] + volumes: + - ./otel/tempo.yaml:/etc/tempo.yaml:ro + - tempo_data:/var/tempo + expose: + - "3200" # Tempo query API (queried by Grafana) + - "4317" # OTLP gRPC ingest (collector → tempo) + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3200/ready"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Prometheus ────────────────────────────────────────────────────────────── + # Scrapes metrics from backend (via prod), runner, and otel-collector. + prometheus: + image: prom/prometheus:latest + restart: unless-stopped + command: + - "--config.file=/etc/prometheus/prometheus.yaml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=30d" + - "--web.enable-remote-write-receiver" + volumes: + - ./otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - prometheus_data:/prometheus + expose: + - "9090" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:9090/-/healthy"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Loki ──────────────────────────────────────────────────────────────────── + # Log aggregation. Receives logs from OTel collector. Replaces manual Dozzle + # tailing for structured log search. + loki: + image: grafana/loki:latest + restart: unless-stopped + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./otel/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + expose: + - "3100" + # No healthcheck — distroless image has no shell or curl + + # ── Grafana ───────────────────────────────────────────────────────────────── + # Single UI for traces (Tempo), metrics (Prometheus), and logs (Loki). + # Accessible at grafana.libnovel.cc via Cloudflare Tunnel. + grafana: + image: grafana/grafana:latest + restart: unless-stopped + depends_on: + - tempo + - prometheus + - loki + expose: + - "3000" + volumes: + - grafana_data:/var/lib/grafana + - ./otel/grafana/provisioning:/etc/grafana/provisioning:ro + environment: + GF_SERVER_ROOT_URL: "https://grafana.libnovel.cc" + GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER}" + GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD}" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_FEATURE_TOGGLES_ENABLE: "traceqlEditor" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"] + interval: 15s + timeout: 5s + retries: 5 + + # ── Watchtower ────────────────────────────────────────────────────────────── + # Auto-updates runner image when CI pushes a new tag. + # Only watches services with the watchtower label. + watchtower: + image: containrrr/watchtower:latest + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --label-enable --interval 300 --cleanup + environment: + WATCHTOWER_NOTIFICATIONS: "${WATCHTOWER_NOTIFICATIONS}" + WATCHTOWER_NOTIFICATION_URL: "${WATCHTOWER_NOTIFICATION_URL}" + DOCKER_API_VERSION: "1.44" + +volumes: + postgres_data: + valkey_data: + uptime_kuma_data: + gotify_data: + tempo_data: + prometheus_data: + loki_data: + grafana_data: diff --git a/homelab/dozzle/users.yml b/homelab/dozzle/users.yml new file mode 100644 index 0000000..c53b951 --- /dev/null +++ b/homelab/dozzle/users.yml @@ -0,0 +1,5 @@ +users: + admin: + name: admin + email: admin@libnovel.cc + password: "$2y$10$4jqLza2grpxnQn0EGux2C.UmlSxRmOvH/J1ySzOBxMZgW6cA2TnmK" diff --git a/homelab/otel/collector.yaml b/homelab/otel/collector.yaml new file mode 100644 index 0000000..521c71c --- /dev/null +++ b/homelab/otel/collector.yaml @@ -0,0 +1,68 @@ +# OTel Collector config +# +# Receivers: OTLP (gRPC + HTTP) from backend, ui, runner +# Processors: batch for efficiency, resource detection for host metadata +# Exporters: Tempo (traces), Prometheus (metrics), Loki (logs) + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + timeout: 5s + send_batch_size: 512 + + # Attach host metadata to all telemetry + resourcedetection: + detectors: [env, system] + timeout: 5s + +exporters: + # Traces → Tempo + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + + # Metrics → Prometheus (remote write) + prometheusremotewrite: + endpoint: "http://prometheus:9090/api/v1/write" + tls: + insecure_skip_verify: true + + # Logs → Loki (via OTLP HTTP endpoint) + otlphttp/loki: + endpoint: "http://loki:3100/otlp" + tls: + insecure: true + + # Collector self-observability (optional debug) + debug: + verbosity: basic + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + pprof: + endpoint: 0.0.0.0:1777 + +service: + extensions: [health_check, pprof] + pipelines: + traces: + receivers: [otlp] + processors: [resourcedetection, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [resourcedetection, batch] + exporters: [prometheusremotewrite] + logs: + receivers: [otlp] + processors: [resourcedetection, batch] + exporters: [otlphttp/loki] diff --git a/homelab/otel/grafana/provisioning/dashboards/dashboards.yaml b/homelab/otel/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..b03f6e3 --- /dev/null +++ b/homelab/otel/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +# Grafana dashboard provisioning +# Points Grafana at the local dashboards directory. +# Drop any .json dashboard file into homelab/otel/grafana/provisioning/dashboards/ +# and it will appear in Grafana automatically on restart. + +apiVersion: 1 + +providers: + - name: libnovel + folder: LibNovel + type: file + options: + path: /etc/grafana/provisioning/dashboards diff --git a/homelab/otel/grafana/provisioning/datasources/datasources.yaml b/homelab/otel/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..3179610 --- /dev/null +++ b/homelab/otel/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,53 @@ +# Grafana datasource provisioning +# Auto-configures Tempo, Prometheus, and Loki on first start. +# No manual setup needed in the UI. + +apiVersion: 1 + +datasources: + - name: Tempo + type: tempo + uid: tempo + url: http://tempo:3200 + access: proxy + isDefault: false + jsonData: + httpMethod: GET + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + traceQuery: + timeShiftEnabled: true + spanStartTimeShift: "1h" + spanEndTimeShift: "-1h" + spanBar: + type: "Tag" + tag: "http.url" + lokiSearch: + datasourceUid: loki + + - name: Prometheus + type: prometheus + uid: prometheus + url: http://prometheus:9090 + access: proxy + isDefault: true + jsonData: + httpMethod: POST + exemplarTraceIdDestinations: + - name: traceID + datasourceUid: tempo + + - name: Loki + type: loki + uid: loki + url: http://loki:3100 + access: proxy + isDefault: false + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: '"traceID":"(\w+)"' + name: TraceID + url: "$${__value.raw}" diff --git a/homelab/otel/loki.yaml b/homelab/otel/loki.yaml new file mode 100644 index 0000000..e27d820 --- /dev/null +++ b/homelab/otel/loki.yaml @@ -0,0 +1,38 @@ +# Loki config — minimal single-node setup +# Receives logs from OTel Collector. 30-day retention. + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: 720h # 30 days + +compactor: + working_directory: /loki/compactor + delete_request_store: filesystem + retention_enabled: true diff --git a/homelab/otel/prometheus.yaml b/homelab/otel/prometheus.yaml new file mode 100644 index 0000000..4fff3fd --- /dev/null +++ b/homelab/otel/prometheus.yaml @@ -0,0 +1,22 @@ +# Prometheus config +# Scrapes OTel collector self-metrics and runner metrics endpoint. +# Backend metrics come in via OTel remote-write — no direct scrape needed. + +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + environment: production + +scrape_configs: + # OTel Collector self-metrics + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + + # Runner JSON metrics endpoint (native format, no Prometheus client yet) + # Will be replaced by OTLP once runner is instrumented with OTel SDK. + - job_name: libnovel-runner + metrics_path: /metrics + static_configs: + - targets: ["runner:9091"] diff --git a/homelab/otel/tempo.yaml b/homelab/otel/tempo.yaml new file mode 100644 index 0000000..a70b725 --- /dev/null +++ b/homelab/otel/tempo.yaml @@ -0,0 +1,45 @@ +# Tempo config — minimal single-node setup +# Stores traces locally. Grafana queries via the HTTP API on port 3200. + +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + trace_idle_period: 10s + max_block_bytes: 104857600 # 100MB + max_block_duration: 30m + +compactor: + compaction: + block_retention: 720h # 30 days + +storage: + trace: + backend: local + local: + path: /var/tempo/blocks + wal: + path: /var/tempo/wal + +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] + generate_native_histograms: both