/** * @file metrics.js * @description Prometheus / OpenMetrics text-exposition endpoint (GET /api/metrics) * so this monitoring dashboard can itself be scraped into Prometheus / Grafana. * * The dashboard already tracks everything an operator wants on a wall board — * live sessions, agent states, event throughput, token burn, connected realtime * clients — but only over its own websocket + REST surface. This route re-exposes * those same counters in the standard Prometheus text format (v0.0.4) so they can * flow into an existing observability stack alongside the rest of a team's infra. * * All values are read straight from the same prepared statements the REST API * uses (`server/db.js`), so the numbers line up exactly with the UI. The endpoint * is READ-ONLY, allocates nothing persistent, and — being mounted under `/api` — * sits behind the same guards as every other route: the DNS-rebinding Host-header * guard and the optional `DASHBOARD_TOKEN` guard. A scraper that reaches the * server as anything other than loopback (e.g. Prometheus in Docker via * `host.docker.internal`) must therefore be allowlisted with * `DASHBOARD_ALLOWED_HOSTS` (and send the token when one is set), so an instance * never leaks operational data to an unexpected origin. * * @author Nguyễn Ngọc Trí Vĩ */ const { Router } = require("express"); const { stmts, db } = require("../db"); const { getConnectionCount } = require("../websocket"); const { calculateCost, dailyTokenRows } = require("./pricing"); const router = Router(); // How long the event loop was held up — what an operator sees as a laggy // terminal: every keystroke's echo waits for it. Sampled every 20 ms and reset // each minute, so the gauges describe the last minute or so. const LOOP_WINDOW_MS = 60_000; const loopDelay = require("perf_hooks").monitorEventLoopDelay({ resolution: 20 }); loopDelay.enable(); let loopWindow = null; // the last full minute, reported until the next one ends setInterval(() => { loopWindow = { p50: loopDelay.percentile(50), p99: loopDelay.percentile(99), max: loopDelay.max, }; loopDelay.reset(); }, LOOP_WINDOW_MS).unref(); // Resolved once at load — the dashboard version, surfaced as a build-info label // (the canonical Prometheus idiom for exposing a version string as a metric). const APP_VERSION = (() => { try { return require("../../package.json").version || "0.0.0"; } catch { return "0.0.0"; } })(); // Statuses are enumerated (not just whatever the DB currently holds) so a metric // series never silently disappears when its count hits zero — a gauge that drops // out of the exposition breaks rate()/alerting downstream. const SESSION_STATUSES = ["active", "completed", "error", "abandoned"]; const AGENT_STATUSES = ["working", "waiting", "completed", "error"]; /** Escape a Prometheus label value (backslash, double-quote, newline). */ function escapeLabelValue(value) { return String(value).replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/"/g, '\\"'); } /** * Render one metric family (HELP + TYPE header, then one line per sample) into * the `out` line array. Each sample is `{ value, labels? }`. Non-finite values * are coerced to 0 so a bad read can never emit an unparseable exposition line. */ function appendMetric(out, name, help, type, samples) { out.push(`# HELP ${name} ${help}`); out.push(`# TYPE ${name} ${type}`); for (const sample of samples) { const labels = sample.labels ? "{" + Object.entries(sample.labels) .map(([k, v]) => `${k}="${escapeLabelValue(v)}"`) .join(",") + "}" : ""; const value = Number.isFinite(sample.value) ? sample.value : 0; out.push(`${name}${labels} ${value}`); } } // GET /api/metrics — Prometheus exposition of the dashboard's live counters. router.get("/", (_req, res) => { const out = []; appendMetric(out, "agenthub_up", "1 when the dashboard API is serving this scrape.", "gauge", [ { value: 1 }, ]); appendMetric( out, "agenthub_build_info", "Dashboard build info; the value is always 1, the version rides on the label.", "gauge", [{ labels: { version: APP_VERSION }, value: 1 }] ); appendMetric( out, "agenthub_process_uptime_seconds", "Uptime of the dashboard server process in seconds.", "gauge", [{ value: Math.round(process.uptime()) }] ); appendMetric( out, "agenthub_process_resident_memory_bytes", "Resident set size (RSS) of the dashboard server process in bytes.", "gauge", [{ value: process.memoryUsage().rss }] ); const loop = loopWindow || { p50: loopDelay.percentile(50), p99: loopDelay.percentile(99), max: loopDelay.max, }; appendMetric( out, "agenthub_event_loop_delay_seconds", "Event-loop delay over the last minute (p50, p99, max): how long requests, hooks and terminal echo waited.", "gauge", [ { labels: { quantile: "0.5" }, value: loop.p50 / 1e9 }, { labels: { quantile: "0.99" }, value: loop.p99 / 1e9 }, { labels: { quantile: "1" }, value: loop.max / 1e9 }, ] ); // Sessions by status. const sessionCounts = new Map(stmts.sessionStatusCounts.all().map((r) => [r.status, r.count])); appendMetric( out, "agenthub_sessions", "Number of sessions by lifecycle status.", "gauge", SESSION_STATUSES.map((status) => ({ labels: { status }, value: sessionCounts.get(status) || 0, })) ); // Agents by status. const agentCounts = new Map(stmts.agentStatusCounts.all().map((r) => [r.status, r.count])); appendMetric( out, "agenthub_agents", "Number of agents (main + subagents) by status.", "gauge", AGENT_STATUSES.map((status) => ({ labels: { status }, value: agentCounts.get(status) || 0 })) ); // Event throughput (monotonic — a counter). appendMetric( out, "agenthub_events_total", "Total hook and synthetic events recorded since the database was created.", "counter", [{ value: stmts.countEvents.get().count }] ); // Connected realtime (WebSocket) clients. let clients = 0; try { clients = getConnectionCount(); } catch { /* websocket not up (e.g. under test without a server) — report 0 */ } appendMetric( out, "agenthub_websocket_clients", "Currently connected realtime (WebSocket) dashboard clients.", "gauge", [{ value: clients }] ); // Remote Data Sources, split by whether background auto-sync is enabled. let enabledSources = 0; let totalSources = 0; try { const rows = stmts.listRemoteSources.all(); totalSources = rows.length; enabledSources = rows.filter((r) => r.enabled).length; } catch { /* remote_sources table absent on a very old DB — report 0 */ } appendMetric( out, "agenthub_remote_sources", "Configured Remote Data Sources, split by auto-sync enabled state.", "gauge", [ { labels: { enabled: "true" }, value: enabledSources }, { labels: { enabled: "false" }, value: totalSources - enabledSources }, ] ); // Cumulative token usage by kind (baseline_* preserves pre-compaction totals, // matching how the pricing endpoints total usage). const tokens = db .prepare( `SELECT COALESCE(SUM(input_tokens + baseline_input), 0) AS input, COALESCE(SUM(output_tokens + baseline_output), 0) AS output, COALESCE(SUM(cache_read_tokens + baseline_cache_read), 0) AS cache_read, COALESCE(SUM(cache_write_tokens + baseline_cache_write), 0) AS cache_write FROM token_usage` ) .get(); appendMetric( out, "agenthub_tokens_total", "Cumulative token usage across all sessions, by kind.", "counter", [ { labels: { kind: "input" }, value: tokens.input }, { labels: { kind: "output" }, value: tokens.output }, { labels: { kind: "cache_read" }, value: tokens.cache_read }, { labels: { kind: "cache_write" }, value: tokens.cache_write }, ] ); // Cumulative cost in USD by AI tool and model, priced exactly as // GET /api/pricing/cost prices it (the per-day rate). The code-execution // container time, billed once per org, has no model and is not included. const rules = stmts.listPricing.all(); const rowsByTool = new Map(); for (const r of dailyTokenRows("+0 minutes", "", [])) { const tool = r.tool || "claude"; if (!rowsByTool.has(tool)) rowsByTool.set(tool, []); rowsByTool.get(tool).push(r); } const costSamples = []; for (const [tool, rows] of rowsByTool) { const byModel = new Map(); for (const b of calculateCost(rows, rules).breakdown) { byModel.set(b.model, (byModel.get(b.model) || 0) + b.cost); } for (const [model, value] of byModel) { costSamples.push({ labels: { tool, model }, value: Math.round(value * 10000) / 10000 }); } } appendMetric( out, "agenthub_cost_usd_total", "Cumulative token cost in USD across all sessions, by AI tool and model.", "counter", costSamples ); res.set("Content-Type", "text/plain; version=0.0.4; charset=utf-8"); res.send(out.join("\n") + "\n"); }); module.exports = router;