Add OTel distributed tracing to backend, ui, and runner
Some checks failed
CI / Check ui (pull_request) Failing after 44s
CI / Docker / ui (pull_request) Has been skipped
CI / Test backend (pull_request) Successful in 3m30s
CI / Docker / backend (pull_request) Successful in 2m28s
CI / Docker / caddy (pull_request) Successful in 5m22s
CI / Docker / runner (pull_request) Successful in 1m52s
Some checks failed
CI / Check ui (pull_request) Failing after 44s
CI / Docker / ui (pull_request) Has been skipped
CI / Test backend (pull_request) Successful in 3m30s
CI / Docker / backend (pull_request) Successful in 2m28s
CI / Docker / caddy (pull_request) Successful in 5m22s
CI / Docker / runner (pull_request) Successful in 1m52s
Go backend: - Add OTel SDK + otelhttp middleware deps (go.mod) - New internal/otelsetup package: init OTLP/HTTP TracerProvider from env vars - cmd/backend/main.go: call otelsetup.Init() after logger + ctx setup - internal/backend/server.go: wrap mux with otelhttp.NewHandler() before sentryhttp, so all HTTP spans are recorded SvelteKit UI: - Add @opentelemetry/sdk-node, exporter-trace-otlp-http, resources, semantic-conventions - hooks.server.ts: init NodeSDK when OTEL_EXPORTER_OTLP_ENDPOINT is set; graceful shutdown on SIGTERM/SIGINT Config: - docker-compose.yml: pass OTEL_EXPORTER_OTLP_ENDPOINT + OTEL_SERVICE_NAME to backend, runner, and ui services - homelab/docker-compose.yml: fix runner OTel endpoint to HTTP port 4318 - Doppler prd: OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.libnovel.cc - Doppler prd_homelab: OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 All services no-op gracefully when the env var is unset (local dev).
This commit is contained in:
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/libnovel/backend/internal/kokoro"
|
||||
"github.com/libnovel/backend/internal/meili"
|
||||
"github.com/libnovel/backend/internal/taskqueue"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
)
|
||||
|
||||
// Dependencies holds all external services the backend server depends on.
|
||||
@@ -170,9 +171,17 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
||||
mux.HandleFunc("DELETE /api/progress/{slug}", s.handleDeleteProgress)
|
||||
|
||||
// Wrap mux with OTel tracing (no-op when no TracerProvider is set),
|
||||
// then with Sentry for panic recovery and error reporting.
|
||||
var handler http.Handler = mux
|
||||
handler = otelhttp.NewHandler(handler, "libnovel.backend",
|
||||
otelhttp.WithMessageEvents(otelhttp.ReadEvents, otelhttp.WriteEvents),
|
||||
)
|
||||
handler = sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle(handler)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: s.cfg.Addr,
|
||||
Handler: sentryhttp.New(sentryhttp.Options{Repanic: true}).Handle(mux),
|
||||
Handler: handler,
|
||||
ReadTimeout: 15 * time.Second,
|
||||
WriteTimeout: 60 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
|
||||
74
backend/internal/otelsetup/otelsetup.go
Normal file
74
backend/internal/otelsetup/otelsetup.go
Normal file
@@ -0,0 +1,74 @@
|
||||
// Package otelsetup initialises the OpenTelemetry SDK for the LibNovel backend.
|
||||
//
|
||||
// It reads two environment variables:
|
||||
//
|
||||
// OTEL_EXPORTER_OTLP_ENDPOINT — OTLP/HTTP endpoint, e.g. http://otel-collector:4318
|
||||
// OTEL_SERVICE_NAME — service name reported in traces (default: "backend")
|
||||
//
|
||||
// When OTEL_EXPORTER_OTLP_ENDPOINT is empty the function is a no-op and
|
||||
// returns a nil shutdown func, so the caller never needs to branch on it.
|
||||
//
|
||||
// Usage in main.go:
|
||||
//
|
||||
// shutdown, err := otelsetup.Init(ctx, version)
|
||||
// if err != nil { return err }
|
||||
// if shutdown != nil { defer shutdown() }
|
||||
package otelsetup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
|
||||
)
|
||||
|
||||
// Init sets up a TracerProvider that exports spans via OTLP/HTTP.
|
||||
// Returns a shutdown function (or nil if OTel is disabled) and any error.
|
||||
func Init(ctx context.Context, version string) (shutdown func(), err error) {
|
||||
endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
|
||||
if endpoint == "" {
|
||||
return nil, nil // OTel disabled — not an error
|
||||
}
|
||||
|
||||
serviceName := os.Getenv("OTEL_SERVICE_NAME")
|
||||
if serviceName == "" {
|
||||
serviceName = "backend"
|
||||
}
|
||||
|
||||
exp, err := otlptracehttp.New(ctx,
|
||||
otlptracehttp.WithEndpoint(endpoint),
|
||||
otlptracehttp.WithInsecure(), // collector is on the internal Docker network
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("otelsetup: create OTLP exporter: %w", err)
|
||||
}
|
||||
|
||||
res, err := resource.New(ctx,
|
||||
resource.WithAttributes(
|
||||
semconv.ServiceName(serviceName),
|
||||
semconv.ServiceVersion(version),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("otelsetup: create resource: %w", err)
|
||||
}
|
||||
|
||||
tp := sdktrace.NewTracerProvider(
|
||||
sdktrace.WithBatcher(exp),
|
||||
sdktrace.WithResource(res),
|
||||
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.2))),
|
||||
)
|
||||
otel.SetTracerProvider(tp)
|
||||
|
||||
return func() {
|
||||
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = tp.Shutdown(shutCtx)
|
||||
}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user