AgentHub/server/__tests__/cost-by-tool.test.js

/**
 * @file Tests for the AI-tool data scope (`?tool=` on cost / analytics / stats /
 * workflows), the per-tool cost split `by_tool`, the Prometheus cost metric, the
 * flex tier and OpenAI priority rates synced into the fast columns.
 * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
 */
"use strict";

const { describe, it, before, after } = require("node:test");
const assert = require("node:assert/strict");
const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");
const http = require("node:http");

const ROOT = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "cost-by-tool-")));
process.env.DASHBOARD_DB_PATH = path.join(ROOT, "t.db");
process.env.CLAUDE_HOME = path.join(ROOT, "claude");
process.env.DASHBOARD_DATA_DIR = path.join(ROOT, "data");
process.env.DASHBOARD_REMOTE_SYNC_MS = "0";
process.env.DASHBOARD_LIVENESS_PROBE = "0";

const { createApp } = require("../index");
const { db, stmts } = require("../db");
const { calculateCost } = require("../routes/pricing");
const { toRow } = require("../lib/pricing-sync");

const M = 1_000_000;
let server;
let base;

function call(url) {
  return new Promise((resolve, reject) => {
    http
      .get(`${base}${url}`, (res) => {
        let text = "";
        res.on("data", (c) => (text += c));
        res.on("end", () =>
          resolve({
            status: res.statusCode,
            text,
            body: res.headers["content-type"]?.includes("json") ? JSON.parse(text) : null,
          })
        );
      })
      .on("error", reject);
  });
}

function session(id, tool) {
  db.prepare(
    "INSERT INTO sessions (id, name, status, started_at, updated_at, tool) VALUES (?, ?, 'completed', '2026-10-01T00:00:00Z', '2026-10-01T00:00:00Z', ?)"
  ).run(id, id, tool);
}
const usage = (id, model, input) =>
  stmts.setTokenUsage.run(id, model, "standard", "global", "standard", input, 0, 0, 0, 0, 0, 0, 0);

before(async () => {
  server = http.createServer(createApp());
  await new Promise((r) => server.listen(0, "127.0.0.1", r));
  base = `http://127.0.0.1:${server.address().port}`;
  // $1 / $2 per million input tokens.
  for (const [pattern, rate] of [
    ["tooltest-claude%", 1],
    ["tooltest-gpt%", 2],
  ]) {
    db.prepare(
      "INSERT OR REPLACE INTO model_pricing (model_pattern, display_name, input_per_mtok, output_per_mtok, cache_read_per_mtok, cache_write_per_mtok) VALUES (?, ?, ?, 0, 0, 0)"
    ).run(pattern, pattern, rate);
  }
  session("c1", "claude");
  usage("c1", "tooltest-claude-1", 3 * M);
  // Pre-compaction usage lives in the baseline columns; every cost path counts it.
  db.prepare("UPDATE token_usage SET baseline_input = ? WHERE session_id = 'c1'").run(1 * M);
  session("x1", "codex");
  usage("x1", "tooltest-gpt-1", 1 * M);
});
after(() => {
  server.close();
  fs.rmSync(ROOT, { recursive: true, force: true });
});

describe("?tool= and by_tool", () => {
  it("splits the total per tool, most expensive first", async () => {
    const r = await call("/api/pricing/cost");
    assert.equal(r.body.total_cost, 6);
    assert.deepEqual(r.body.by_tool, [
      { tool: "claude", cost: 4 },
      { tool: "codex", cost: 2 },
    ]);
  });
  it("narrows cost, analytics, stats and workflows to one tool", async () => {
    const cost = await call("/api/pricing/cost?tool=codex");
    assert.equal(cost.body.total_cost, 2);
    const an = await call("/api/analytics?tool=codex");
    assert.equal(an.body.total_cost, 2);
    assert.equal(an.body.overview.total_sessions, 1);
    const st = await call("/api/stats?tool=codex");
    assert.equal(st.body.total_sessions, 1);
    const wf = await call("/api/workflows?tool=claude");
    assert.equal(wf.status, 200);
  });
  it("analytics total_cost equals /api/pricing/cost, baselines included", async () => {
    const an = await call("/api/analytics");
    const cost = await call("/api/pricing/cost");
    assert.equal(an.body.total_cost, cost.body.total_cost);
  });
  it("answers 400 for a malformed tool", async () => {
    for (const url of [
      "/api/pricing/cost?tool=a%20b",
      "/api/analytics?tool=%3Bdrop",
      "/api/stats?tool=a%2Fb",
      "/api/workflows?tool=a%27",
    ]) {
      assert.equal((await call(url)).status, 400, url);
    }
  });
});

describe("agenthub_cost_usd_total", () => {
  it("exposes cost per tool and model", async () => {
    const r = await call("/api/metrics");
    assert.match(r.text, /agenthub_cost_usd_total\{tool="claude",model="tooltest-claude-1"\} 4\n/);
    assert.match(r.text, /agenthub_cost_usd_total\{tool="codex",model="tooltest-gpt-1"\} 2\n/);
  });
});

describe("tiers", () => {
  it("prices the flex tier at half, like batch", () => {
    const rules = [{ model_pattern: "m%", input_per_mtok: 2 }];
    const row = {
      model: "m1",
      input_tokens: M,
      output_tokens: 0,
      cache_read_tokens: 0,
      cache_write_tokens: 0,
    };
    assert.equal(calculateCost([row], rules).total_cost, 2);
    assert.equal(calculateCost([{ ...row, service_tier: "flex" }], rules).total_cost, 1);
  });
  it("syncs an OpenAI model's priority rates into the fast columns", () => {
    const row = toRow("gpt-5.6", {
      litellm_provider: "openai",
      input_cost_per_token: 4e-6,
      output_cost_per_token: 2e-5,
      input_cost_per_token_priority: 8e-6,
      output_cost_per_token_priority: 4e-5,
    });
    assert.equal(row.fastIn, 8);
    assert.equal(row.fastOut, 40);
    const plain = toRow("gpt-5.6", {
      litellm_provider: "openai",
      input_cost_per_token: 4e-6,
      output_cost_per_token: 2e-5,
    });
    assert.equal(plain.fastIn, 0);
  });
});