feat: add Grafana Faro RUM, fix dashboards, add Grafana to admin nav
All checks were successful
Release / Test backend (push) Successful in 40s
Release / Check ui (push) Successful in 1m5s
Release / Docker / caddy (push) Successful in 1m9s
Release / Docker / backend (push) Successful in 3m14s
Release / Docker / runner (push) Successful in 4m7s
Release / Upload source maps (push) Successful in 2m11s
Release / Docker / ui (push) Successful in 2m23s
Release / Gitea Release (push) Successful in 40s

- Add @grafana/faro-web-sdk to UI; wire initializeFaro in hooks.client.ts
  gated on PUBLIC_FARO_COLLECTOR_URL (no-op in dev)
- Add Grafana Alloy service (faro.receiver) to homelab compose;
  Faro endpoint → alloy:12347 (faro.libnovel.cc via cloudflared)
- Add PUBLIC_FARO_COLLECTOR_URL env var to docker-compose.yml UI service
- Add Web Vitals dashboard (web-vitals.json): LCP/INP/CLS/TTFB/FCP p75
  stats + LCP/TTFB time-series + Faro exception logs from Loki
- Fix runner.json: strip libnovel_ prefix from all metric names
- Fix backend.json: replace 5 dead http_client_* panels with
  spanmetrics-based equivalents (Request Rate by Span Name + Latency
  by Span Name p95)
- Fix OTel collector: add service.telemetry.metrics.address: 0.0.0.0:8888
  so Prometheus can scrape collector self-metrics
- Add Grafana link to admin nav external tools; add admin_nav_grafana
  message key to all 5 locale files; recompile paraglide
This commit is contained in:
Admin
2026-04-05 12:58:16 +05:00
parent eb8a92f0c1
commit 4d3b91af30
18 changed files with 593 additions and 62 deletions

View File

@@ -18,6 +18,7 @@
# uptime.libnovel.cc → uptime-kuma:3001
# push.libnovel.cc → gotify:80
# grafana.libnovel.cc → grafana:3000
# faro.libnovel.cc → alloy:12347
services:
@@ -346,6 +347,23 @@ services:
timeout: 5s
retries: 5
# ── Grafana Alloy (Faro RUM receiver) ───────────────────────────────────────
# Receives browser telemetry from @grafana/faro-web-sdk (Web Vitals, traces,
# errors). Exposes POST /collect at faro.libnovel.cc via cloudflared.
# Forwards traces to otel-collector (→ Tempo) and logs to Loki directly.
alloy:
image: grafana/alloy:latest
restart: unless-stopped
command: ["run", "--server.http.listen-addr=0.0.0.0:12348", "/etc/alloy/alloy.river"]
volumes:
- ./otel/alloy.river:/etc/alloy/alloy.river:ro
expose:
- "12347" # Faro HTTP receiver (POST /collect)
- "12348" # Alloy UI / health endpoint
depends_on:
- otel-collector
- loki
# ── OTel Collector ──────────────────────────────────────────────────────────
# Receives OTLP from backend/ui/runner, fans out to Tempo + Prometheus + Loki.
otel-collector:

43
homelab/otel/alloy.river Normal file
View File

@@ -0,0 +1,43 @@
// Grafana Alloy — Faro RUM receiver
//
// Receives browser telemetry (Web Vitals, traces, logs, exceptions) from the
// LibNovel SvelteKit frontend via the @grafana/faro-web-sdk.
//
// Pipeline:
// faro.receiver → receives HTTP POST /collect from browsers
// otelcol.exporter.otlphttp → forwards traces to OTel Collector → Tempo
// loki.write → forwards logs/exceptions to Loki
//
// The Faro endpoint is exposed publicly at faro.libnovel.cc via cloudflared.
// CORS is configured to allow requests from libnovel.cc.
faro.receiver "faro" {
server {
listen_address = "0.0.0.0"
listen_port = 12347
cors_allowed_origins = ["https://libnovel.cc", "https://www.libnovel.cc"]
}
output {
logs = [loki.write.faro.receiver]
traces = [otelcol.exporter.otlphttp.faro.input]
}
}
// Forward Faro traces to the OTel Collector (which routes to Tempo)
otelcol.exporter.otlphttp "faro" {
client {
endpoint = "http://otel-collector:4318"
tls {
insecure = true
}
}
}
// Forward Faro logs/exceptions directly to Loki
loki.write "faro" {
endpoint {
url = "http://loki:3100/loki/api/v1/push"
}
}

View File

@@ -53,6 +53,9 @@ extensions:
service:
extensions: [health_check, pprof]
telemetry:
metrics:
address: 0.0.0.0:8888
pipelines:
traces:
receivers: [otlp]

View File

@@ -1,7 +1,7 @@
{
"uid": "libnovel-backend",
"title": "Backend API",
"description": "Request rate, error rate, and latency for the LibNovel backend. Powered by Tempo span metrics and UI OTel instrumentation.",
"description": "Request rate, error rate, and latency for the LibNovel backend. Powered by Tempo span metrics.",
"tags": ["libnovel", "backend", "api"],
"timezone": "browser",
"refresh": "30s",
@@ -173,7 +173,7 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\", http_response_status_code=~\"5..\"}[5m])) * 60",
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"backend\", status_code=\"STATUS_CODE_ERROR\"}[5m])) * 60",
"legendFormat": "5xx/min",
"instant": true
}
@@ -182,7 +182,7 @@
{
"id": 10,
"type": "timeseries",
"title": "Request Rate by Status",
"title": "Request Rate (total vs errors)",
"gridPos": { "x": 0, "y": 4, "w": 12, "h": 8 },
"options": {
"tooltip": { "mode": "multi" },
@@ -191,27 +191,21 @@
"fieldConfig": {
"defaults": { "unit": "reqps", "custom": { "lineWidth": 2, "fillOpacity": 10 } },
"overrides": [
{ "matcher": { "id": "byFrameRefID", "options": "errors" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }
{ "matcher": { "id": "byName", "options": "errors" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }
]
},
"targets": [
{
"refId": "success",
"refId": "total",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\", http_response_status_code=~\"2..\"}[5m]))",
"legendFormat": "2xx"
},
{
"refId": "notfound",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\", http_response_status_code=~\"4..\"}[5m]))",
"legendFormat": "4xx"
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"backend\"}[5m]))",
"legendFormat": "total"
},
{
"refId": "errors",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\", http_response_status_code=~\"5..\"}[5m]))",
"legendFormat": "5xx"
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"backend\", status_code=\"STATUS_CODE_ERROR\"}[5m]))",
"legendFormat": "errors"
}
]
},
@@ -248,50 +242,30 @@
{
"id": 12,
"type": "timeseries",
"title": "Requests / min by HTTP method (UI → Backend)",
"title": "Request Rate by Span Name (top operations)",
"gridPos": { "x": 0, "y": 12, "w": 12, "h": 8 },
"description": "Throughput broken down by HTTP route / span name from Tempo span metrics.",
"options": {
"tooltip": { "mode": "multi" },
"legend": { "displayMode": "list", "placement": "bottom" }
},
"fieldConfig": {
"defaults": { "unit": "short", "custom": { "lineWidth": 2, "fillOpacity": 5 } }
"defaults": { "unit": "reqps", "custom": { "lineWidth": 2, "fillOpacity": 5 } }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"backend\"}[5m])) by (http_request_method) * 60",
"legendFormat": "{{http_request_method}}"
"expr": "topk(10, sum(rate(traces_spanmetrics_calls_total{service=\"backend\"}[5m])) by (span_name))",
"legendFormat": "{{span_name}}"
}
]
},
{
"id": 13,
"type": "timeseries",
"title": "Requests / min — UI → PocketBase",
"title": "Latency by Span Name (p95)",
"gridPos": { "x": 12, "y": 12, "w": 12, "h": 8 },
"description": "Traffic from SvelteKit server to PocketBase (auth, collections, etc.).",
"options": {
"tooltip": { "mode": "multi" },
"legend": { "displayMode": "list", "placement": "bottom" }
},
"fieldConfig": {
"defaults": { "unit": "short", "custom": { "lineWidth": 2, "fillOpacity": 5 } }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(rate(http_client_request_duration_seconds_count{job=\"ui\", server_address=\"pocketbase\"}[5m])) by (http_request_method, http_response_status_code) * 60",
"legendFormat": "{{http_request_method}} {{http_response_status_code}}"
}
]
},
{
"id": 14,
"type": "timeseries",
"title": "UI → Backend Latency (p50 / p95)",
"gridPos": { "x": 0, "y": 20, "w": 12, "h": 8 },
"description": "HTTP client latency as seen from the SvelteKit SSR layer calling backend.",
"description": "p95 latency per operation — helps identify slow endpoints.",
"options": {
"tooltip": { "mode": "multi" },
"legend": { "displayMode": "list", "placement": "bottom" }
@@ -302,13 +276,8 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.50, sum(rate(http_client_request_duration_seconds_bucket{job=\"ui\", server_address=\"backend\"}[5m])) by (le))",
"legendFormat": "p50"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum(rate(http_client_request_duration_seconds_bucket{job=\"ui\", server_address=\"backend\"}[5m])) by (le))",
"legendFormat": "p95"
"expr": "topk(10, histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"backend\"}[5m])) by (le, span_name)))",
"legendFormat": "{{span_name}}"
}
]
},
@@ -316,7 +285,7 @@
"id": 20,
"type": "logs",
"title": "Backend Errors",
"gridPos": { "x": 0, "y": 28, "w": 24, "h": 10 },
"gridPos": { "x": 0, "y": 20, "w": 24, "h": 10 },
"options": {
"showTime": true,
"showLabels": false,

View File

@@ -35,7 +35,7 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "libnovel_runner_tasks_running",
"expr": "runner_tasks_running",
"legendFormat": "running",
"instant": true
}
@@ -61,7 +61,7 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "libnovel_runner_tasks_completed_total",
"expr": "runner_tasks_completed_total",
"legendFormat": "completed",
"instant": true
}
@@ -93,7 +93,7 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "libnovel_runner_tasks_failed_total",
"expr": "runner_tasks_failed_total",
"legendFormat": "failed",
"instant": true
}
@@ -126,7 +126,7 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "libnovel_runner_uptime_seconds",
"expr": "runner_uptime_seconds",
"legendFormat": "uptime",
"instant": true
}
@@ -159,7 +159,7 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "libnovel_runner_tasks_failed_total / clamp_min(libnovel_runner_tasks_completed_total + libnovel_runner_tasks_failed_total, 1)",
"expr": "runner_tasks_failed_total / clamp_min(runner_tasks_completed_total + runner_tasks_failed_total, 1)",
"legendFormat": "failure rate",
"instant": true
}
@@ -215,17 +215,17 @@
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(libnovel_runner_tasks_completed_total[5m]) * 60",
"expr": "rate(runner_tasks_completed_total[5m]) * 60",
"legendFormat": "completed"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(libnovel_runner_tasks_failed_total[5m]) * 60",
"expr": "rate(runner_tasks_failed_total[5m]) * 60",
"legendFormat": "failed"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "libnovel_runner_tasks_running",
"expr": "runner_tasks_running",
"legendFormat": "running"
}
]

View File

@@ -0,0 +1,284 @@
{
"uid": "libnovel-web-vitals",
"title": "Web Vitals (RUM)",
"description": "Real User Monitoring — Core Web Vitals (LCP, CLS, INP, TTFB, FCP) from @grafana/faro-web-sdk. Data flows: browser → Alloy faro.receiver → Tempo (traces) + Loki (logs).",
"tags": ["libnovel", "frontend", "rum", "web-vitals"],
"timezone": "browser",
"refresh": "1m",
"time": { "from": "now-24h", "to": "now" },
"schemaVersion": 39,
"panels": [
{
"id": 1,
"type": "stat",
"title": "LCP — p75 (Largest Contentful Paint)",
"description": "Good < 2.5 s, needs improvement < 4 s, poor ≥ 4 s.",
"gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 },
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
"fieldConfig": {
"defaults": {
"unit": "ms",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 2500 },
{ "color": "red", "value": 4000 }
]
}
}
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.75, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*lcp|LCP\"}[1h])) by (le)) * 1000",
"legendFormat": "LCP p75",
"instant": true
}
]
},
{
"id": 2,
"type": "stat",
"title": "INP — p75 (Interaction to Next Paint)",
"description": "Good < 200 ms, needs improvement < 500 ms, poor ≥ 500 ms.",
"gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 },
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
"fieldConfig": {
"defaults": {
"unit": "ms",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 200 },
{ "color": "red", "value": 500 }
]
}
}
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.75, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*inp|INP\"}[1h])) by (le)) * 1000",
"legendFormat": "INP p75",
"instant": true
}
]
},
{
"id": 3,
"type": "stat",
"title": "CLS — p75 (Cumulative Layout Shift)",
"description": "Good < 0.1, needs improvement < 0.25, poor ≥ 0.25.",
"gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 },
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 3,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 0.1 },
{ "color": "red", "value": 0.25 }
]
}
}
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.75, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*cls|CLS\"}[1h])) by (le))",
"legendFormat": "CLS p75",
"instant": true
}
]
},
{
"id": 4,
"type": "stat",
"title": "TTFB — p75 (Time to First Byte)",
"description": "Good < 800 ms, needs improvement < 1800 ms, poor ≥ 1800 ms.",
"gridPos": { "x": 12, "y": 0, "w": 4, "h": 4 },
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
"fieldConfig": {
"defaults": {
"unit": "ms",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 800 },
{ "color": "red", "value": 1800 }
]
}
}
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.75, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*ttfb|TTFB\"}[1h])) by (le)) * 1000",
"legendFormat": "TTFB p75",
"instant": true
}
]
},
{
"id": 5,
"type": "stat",
"title": "FCP — p75 (First Contentful Paint)",
"description": "Good < 1.8 s, needs improvement < 3 s, poor ≥ 3 s.",
"gridPos": { "x": 16, "y": 0, "w": 4, "h": 4 },
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
"fieldConfig": {
"defaults": {
"unit": "ms",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1800 },
{ "color": "red", "value": 3000 }
]
}
}
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.75, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*fcp|FCP\"}[1h])) by (le)) * 1000",
"legendFormat": "FCP p75",
"instant": true
}
]
},
{
"id": 6,
"type": "stat",
"title": "Active Sessions (30 min)",
"gridPos": { "x": 20, "y": 0, "w": 4, "h": 4 },
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area" },
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }
}
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(rate(traces_spanmetrics_calls_total{service=\"libnovel-ui\"}[30m]))",
"legendFormat": "sessions",
"instant": true
}
]
},
{
"id": 10,
"type": "timeseries",
"title": "LCP over time (p50 / p75 / p95)",
"gridPos": { "x": 0, "y": 4, "w": 12, "h": 8 },
"options": { "tooltip": { "mode": "multi" }, "legend": { "displayMode": "list", "placement": "bottom" } },
"fieldConfig": {
"defaults": { "unit": "ms", "custom": { "lineWidth": 2, "fillOpacity": 10 } },
"overrides": [
{ "matcher": { "id": "byName", "options": "Good (2.5s)" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }, { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [4, 4] } }] },
{ "matcher": { "id": "byName", "options": "Poor (4s)" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }, { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [4, 4] } }] }
]
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*lcp|LCP\"}[5m])) by (le)) * 1000",
"legendFormat": "p50"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.75, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*lcp|LCP\"}[5m])) by (le)) * 1000",
"legendFormat": "p75"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*lcp|LCP\"}[5m])) by (le)) * 1000",
"legendFormat": "p95"
},
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "2500", "legendFormat": "Good (2.5s)" },
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "4000", "legendFormat": "Poor (4s)" }
]
},
{
"id": 11,
"type": "timeseries",
"title": "TTFB over time (p50 / p75 / p95)",
"gridPos": { "x": 12, "y": 4, "w": 12, "h": 8 },
"options": { "tooltip": { "mode": "multi" }, "legend": { "displayMode": "list", "placement": "bottom" } },
"fieldConfig": {
"defaults": { "unit": "ms", "custom": { "lineWidth": 2, "fillOpacity": 10 } }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*ttfb|TTFB\"}[5m])) by (le)) * 1000",
"legendFormat": "p50"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.75, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*ttfb|TTFB\"}[5m])) by (le)) * 1000",
"legendFormat": "p75"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"libnovel-ui\", span_name=~\"faro.*ttfb|TTFB\"}[5m])) by (le)) * 1000",
"legendFormat": "p95"
}
]
},
{
"id": 20,
"type": "logs",
"title": "Frontend Errors & Exceptions",
"description": "JS exceptions and console errors captured by Faro and shipped to Loki.",
"gridPos": { "x": 0, "y": 12, "w": 24, "h": 10 },
"options": {
"showTime": true,
"showLabels": true,
"wrapLogMessage": true,
"prettifyLogMessage": true,
"enableLogDetails": true,
"sortOrder": "Descending",
"dedupStrategy": "none"
},
"targets": [
{
"datasource": { "type": "loki", "uid": "loki" },
"expr": "{service_name=\"libnovel-ui\"} | json | kind =~ `(exception|error)`",
"legendFormat": ""
}
]
},
{
"id": 21,
"type": "logs",
"title": "Frontend Logs (all Faro events)",
"gridPos": { "x": 0, "y": 22, "w": 24, "h": 10 },
"options": {
"showTime": true,
"showLabels": false,
"wrapLogMessage": true,
"prettifyLogMessage": true,
"enableLogDetails": true,
"sortOrder": "Descending",
"dedupStrategy": "none"
},
"targets": [
{
"datasource": { "type": "loki", "uid": "loki" },
"expr": "{service_name=\"libnovel-ui\"}",
"legendFormat": ""
}
]
}
]
}