feat(observability+tts): OTel logs for backend/runner/ui; add kokoro-fastapi (GPU) and pocket-tts (CPU) to homelab
Some checks failed
Release / Check ui (push) Successful in 47s
Release / Docker / caddy (push) Successful in 48s
Release / Test backend (push) Successful in 3m24s
CI / Check ui (pull_request) Successful in 52s
CI / Test backend (pull_request) Successful in 3m6s
CI / Docker / caddy (pull_request) Failing after 38s
Release / Upload source maps (push) Failing after 33s
Release / Docker / ui (push) Failing after 2m41s
Release / Docker / backend (push) Successful in 2m48s
Release / Docker / runner (push) Failing after 1m1s
Release / Gitea Release (push) Has been skipped
CI / Docker / ui (pull_request) Successful in 1m48s
CI / Docker / backend (pull_request) Successful in 1m56s
CI / Docker / runner (pull_request) Successful in 1m45s

- otelsetup.Init now returns a *slog.Logger wired to the OTLP log exporter
  so all slog output is shipped to Loki with embedded trace IDs
- backend and runner both adopt the new OTel-bridged logger
- runner.runScrapeTask and runAudioTask now emit structured OTel spans
- ui/hooks.server.ts adds BatchLogRecordProcessor alongside existing trace exporter
- homelab: add kokoro-fastapi GPU service (ghcr.io/remsky/kokoro-fastapi-gpu)
  using deploy.resources.reservations for NVIDIA GPU, exposed internally on :8880
- homelab: add pocket-tts CPU service (ghcr.io/kyutai-labs/pocket-tts) on :8000
- runner KOKORO_URL hardcoded to http://kokoro-fastapi:8880 (fixes DNS failure
  for the stale kokoro.kalekber.cc hostname)
This commit is contained in:
Admin
2026-03-27 16:05:22 +05:00
parent 7e5e0495cf
commit 4831c74acc
10 changed files with 164 additions and 25 deletions

View File

@@ -5,12 +5,13 @@
// OTEL_EXPORTER_OTLP_ENDPOINT — OTLP/HTTP endpoint, e.g. http://otel-collector:4318
// OTEL_SERVICE_NAME — service name reported in traces (default: "backend")
//
// When OTEL_EXPORTER_OTLP_ENDPOINT is empty the function is a no-op and
// returns a nil shutdown func, so the caller never needs to branch on it.
// When OTEL_EXPORTER_OTLP_ENDPOINT is empty the function is a no-op: it
// returns a nil shutdown func and the default slog.Logger, so callers never
// need to branch on it.
//
// Usage in main.go:
//
// shutdown, err := otelsetup.Init(ctx, version)
// shutdown, log, err := otelsetup.Init(ctx, version)
// if err != nil { return err }
// if shutdown != nil { defer shutdown() }
package otelsetup
@@ -18,22 +19,31 @@ package otelsetup
import (
"context"
"fmt"
"log/slog"
"os"
"time"
"go.opentelemetry.io/contrib/bridges/otelslog"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
otellog "go.opentelemetry.io/otel/log/global"
"go.opentelemetry.io/otel/sdk/log"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
)
// Init sets up a TracerProvider that exports spans via OTLP/HTTP.
// Returns a shutdown function (or nil if OTel is disabled) and any error.
func Init(ctx context.Context, version string) (shutdown func(), err error) {
// Init sets up TracerProvider and LoggerProvider that export via OTLP/HTTP.
//
// Returns:
// - shutdown: flushes and stops both providers (nil when OTel is disabled).
// - logger: an slog.Logger bridged to OTel logs (falls back to default when disabled).
// - err: non-nil only on SDK initialisation failure.
func Init(ctx context.Context, version string) (shutdown func(), logger *slog.Logger, err error) {
endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
if endpoint == "" {
return nil, nil // OTel disabled — not an error
return nil, slog.Default(), nil // OTel disabled — not an error
}
serviceName := os.Getenv("OTEL_SERVICE_NAME")
@@ -41,14 +51,7 @@ func Init(ctx context.Context, version string) (shutdown func(), err error) {
serviceName = "backend"
}
exp, err := otlptracehttp.New(ctx,
otlptracehttp.WithEndpoint(endpoint),
otlptracehttp.WithInsecure(), // collector is on the internal Docker network
)
if err != nil {
return nil, fmt.Errorf("otelsetup: create OTLP exporter: %w", err)
}
// ── Shared resource ───────────────────────────────────────────────────────
res, err := resource.New(ctx,
resource.WithAttributes(
semconv.ServiceName(serviceName),
@@ -56,19 +59,50 @@ func Init(ctx context.Context, version string) (shutdown func(), err error) {
),
)
if err != nil {
return nil, fmt.Errorf("otelsetup: create resource: %w", err)
return nil, slog.Default(), fmt.Errorf("otelsetup: create resource: %w", err)
}
// ── Trace provider ────────────────────────────────────────────────────────
traceExp, err := otlptracehttp.New(ctx,
otlptracehttp.WithEndpoint(endpoint),
otlptracehttp.WithInsecure(), // collector is on the internal Docker network
)
if err != nil {
return nil, slog.Default(), fmt.Errorf("otelsetup: create OTLP trace exporter: %w", err)
}
tp := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exp),
sdktrace.WithBatcher(traceExp),
sdktrace.WithResource(res),
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.2))),
)
otel.SetTracerProvider(tp)
return func() {
// ── Log provider ──────────────────────────────────────────────────────────
logExp, err := otlploghttp.New(ctx,
otlploghttp.WithEndpoint(endpoint),
otlploghttp.WithInsecure(),
)
if err != nil {
return nil, slog.Default(), fmt.Errorf("otelsetup: create OTLP log exporter: %w", err)
}
lp := log.NewLoggerProvider(
log.WithProcessor(log.NewBatchProcessor(logExp)),
log.WithResource(res),
)
otellog.SetLoggerProvider(lp)
// Bridge slog → OTel logs. Structured fields and trace IDs are forwarded
// automatically; Grafana can correlate log lines with Tempo traces.
otelLogger := otelslog.NewLogger(serviceName)
shutdown = func() {
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = tp.Shutdown(shutCtx)
}, nil
_ = lp.Shutdown(shutCtx)
}
return shutdown, otelLogger, nil
}