AgentHub/server/__tests__/pricing-sync.test.js

/**
 * @file Tests for the automatic pricing sync: a model seen in token_usage with
 * no row of its own gets one from the price list, an edited row is never
 * overwritten, a bad list or a failed fetch changes nothing, and an OpenAI
 * model (a Codex session's) is priced from its `openai` entry too.
 * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
 */

const { describe, it, beforeEach } = require("node:test");
const assert = require("node:assert/strict");
const path = require("path");
const os = require("os");

process.env.DASHBOARD_DB_PATH = path.join(
  os.tmpdir(),
  `pricing-sync-${Date.now()}-${process.pid}.db`
);
const { db } = require("../db");
const {
  syncPricing,
  familyOf,
  displayName,
  toRow,
  _resetCache,
  _setCache,
} = require("../lib/pricing-sync");

// The real LiteLLM entry for Opus 5.5, trimmed.
const LIST = {
  "claude-opus-9-9": {
    litellm_provider: "anthropic",
    input_cost_per_token: 0.000004,
    output_cost_per_token: 0.00002,
    cache_read_input_token_cost: 2e-7,
    cache_creation_input_token_cost: 0.000005,
    cache_creation_input_token_cost_above_1hr: 0.000008,
    supports_fast_mode: true,
    provider_specific_entry: { fast: 2 },
  },
};

function useModel(model) {
  db.prepare("INSERT OR IGNORE INTO sessions (id, status, started_at) VALUES (?, 'active', ?)").run(
    `s-${model}`,
    new Date().toISOString()
  );
  db.prepare("INSERT INTO token_usage (session_id, model) VALUES (?, ?)").run(`s-${model}`, model);
}
const row = (p) => db.prepare("SELECT * FROM model_pricing WHERE model_pattern = ?").get(p);

beforeEach(() => {
  _resetCache();
  db.prepare("DELETE FROM token_usage").run();
  db.prepare(
    "DELETE FROM model_pricing WHERE model_pattern LIKE 'claude-opus-9%' OR model_pattern LIKE 'gpt-9%'"
  ).run();
});

describe("pricing sync", () => {
  it("reduces an id to its family and names it", () => {
    assert.equal(familyOf("claude-opus-5-5-20260101"), "claude-opus-5-5");
    assert.equal(familyOf("gpt-5.6-sol"), "gpt-5.6-sol");
    assert.equal(familyOf("gpt-5.4-2026-03-05"), "gpt-5.4");
    assert.equal(familyOf("o3-mini"), "o3-mini");
    assert.equal(familyOf("clinepass/cline-pass/deepseek-v4-pro"), null);
    assert.equal(familyOf("Highest"), null);
    assert.equal(familyOf("claude-x%"), null);
    assert.equal(displayName("claude-opus-5-5"), "Claude Opus 5.5");
    assert.equal(displayName("gpt-5.6-sol"), "GPT-5.6 Sol");
    assert.equal(displayName("o3-mini"), "o3 Mini");
  });

  it("prices a Gemini model from Vertex AI's entry, never another vendor's", async () => {
    assert.equal(familyOf("gemini-3.8-flash"), "gemini-3.8-flash");
    assert.equal(displayName("gemini-3.8-flash"), "Gemini 3.8 Flash");
    useModel("gemini-3.8-flash");
    const list = {
      "gemini-3.8-flash": {
        litellm_provider: "vertex_ai-language-models",
        input_cost_per_token: 7.5e-7,
        output_cost_per_token: 3.75e-6,
        cache_read_input_token_cost: 7.5e-8,
      },
    };
    assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), ["gemini-3.8-flash%"]);
    const r = row("gemini-3.8-flash%");
    assert.equal(r.display_name, "Gemini 3.8 Flash");
    assert.deepEqual(
      [r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok],
      [0.75, 3.75, 0.075]
    );
    assert.equal(r.cache_write_per_mtok, 0);
    const other = {
      "gemini-3.8-flash": { ...list["gemini-3.8-flash"], litellm_provider: "openrouter" },
    };
    assert.equal(toRow("gemini-3.8-flash", other["gemini-3.8-flash"]), null);
  });

  it("adds a row for a used model that only a broader pattern matched", async () => {
    db.prepare(
      "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok) VALUES ('claude-opus-9%', 'Broad', 5)"
    ).run();
    useModel("claude-opus-9-9-20260101");
    const added = await syncPricing(db, { fetchList: async () => LIST });
    assert.deepEqual(added, ["claude-opus-9-9%"]);
    const r = row("claude-opus-9-9%");
    assert.equal(r.display_name, "Claude Opus 9.9");
    assert.deepEqual([r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok], [4, 20, 0.2]);
    assert.deepEqual([r.cache_write_per_mtok, r.cache_write_1h_per_mtok], [5, 8]);
    assert.deepEqual([r.fast_input_per_mtok, r.fast_output_per_mtok], [8, 40]);
  });

  it("never overwrites a row that exists, and fetches nothing when none is missing", async () => {
    db.prepare(
      "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok) VALUES ('claude-opus-9-9%', 'Mine', 1)"
    ).run();
    useModel("claude-opus-9-9");
    let fetched = false;
    await syncPricing(db, {
      fetchList: async () => {
        fetched = true;
        return LIST;
      },
    });
    assert.equal(fetched, false);
    assert.equal(row("claude-opus-9-9%").input_per_mtok, 1);
  });

  it("skips absurd rates and survives a failed fetch", async () => {
    useModel("claude-opus-9-9");
    const bad = { "claude-opus-9-9": { ...LIST["claude-opus-9-9"], input_cost_per_token: 5 } };
    assert.deepEqual(await syncPricing(db, { fetchList: async () => bad }), []);
    _resetCache();
    const failed = await syncPricing(db, {
      fetchList: async () => {
        throw new Error("offline");
      },
    });
    assert.deepEqual(failed, []);
    assert.equal(row("claude-opus-9-9%"), undefined);
    // A failure is not retried by the next caller, only after the check interval.
    let calls = 0;
    const counting = async () => {
      calls += 1;
      return LIST;
    };
    await syncPricing(db, { fetchList: counting });
    assert.equal(calls, 0);
    await syncPricing(db, { fetchList: counting, now: Date.now() + 16 * 60_000 });
    assert.equal(calls, 1);
  });

  it("prices an OpenAI model from its openai entry, with no cache-write charge", async () => {
    useModel("gpt-9.1-sol");
    const list = {
      "gpt-9.1-sol": {
        litellm_provider: "openai",
        input_cost_per_token: 0.000004,
        output_cost_per_token: 0.00002,
        cache_read_input_token_cost: 4e-7,
      },
    };
    assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), ["gpt-9.1-sol%"]);
    const r = row("gpt-9.1-sol%");
    assert.equal(r.display_name, "GPT-9.1 Sol");
    assert.deepEqual([r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok], [4, 20, 0.4]);
    assert.deepEqual([r.cache_write_per_mtok, r.cache_write_1h_per_mtok], [0, 0]);
    assert.deepEqual([r.fast_input_per_mtok, r.fast_output_per_mtok], [0, 0]);
  });

  it("refuses an entry whose provider does not match the model's vendor", async () => {
    useModel("gpt-9.2");
    const list = {
      "gpt-9.2": {
        litellm_provider: "azure",
        input_cost_per_token: 1e-6,
        output_cost_per_token: 1e-6,
      },
    };
    assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), []);
  });

  it("POST /api/settings/reset-pricing brings a synced model's row straight back", async () => {
    const express = require("express");
    const http = require("node:http");
    const app = express();
    app.use(express.json());
    app.use("/api/settings", require("../routes/settings"));
    const server = http.createServer(app);
    await new Promise((r) => server.listen(0, "127.0.0.1", r));
    const saved = process.env.AGENTHUB_PRICING_SYNC;
    process.env.AGENTHUB_PRICING_SYNC = "1";
    try {
      useModel("gpt-9.3");
      _setCache({
        "gpt-9.3": {
          litellm_provider: "openai",
          input_cost_per_token: 1e-6,
          output_cost_per_token: 8e-6,
        },
      });
      const res = await fetch(
        `http://127.0.0.1:${server.address().port}/api/settings/reset-pricing`,
        {
          method: "POST",
        }
      );
      const body = await res.json();
      assert.ok(body.pricing.some((p) => p.model_pattern === "gpt-9.3%"));
      assert.ok(body.pricing.some((p) => p.model_pattern.startsWith("claude-")));
    } finally {
      if (saved === undefined) delete process.env.AGENTHUB_PRICING_SYNC;
      else process.env.AGENTHUB_PRICING_SYNC = saved;
      server.close();
    }
  });

  it("POST /api/settings/reset-pricing with scope claude keeps every other row", async () => {
    const express = require("express");
    const http = require("node:http");
    const app = express();
    app.use(express.json());
    app.use("/api/settings", require("../routes/settings"));
    const server = http.createServer(app);
    await new Promise((r) => server.listen(0, "127.0.0.1", r));
    try {
      const add = db.prepare(
        "INSERT OR REPLACE INTO model_pricing (model_pattern, display_name, input_per_mtok, output_per_mtok) VALUES (?, ?, ?, ?)"
      );
      add.run("glm-4%", "GLM 4 (router)", 1, 2);
      add.run("claude-opus-5-5%", "edited", 999, 999);
      const res = await fetch(
        `http://127.0.0.1:${server.address().port}/api/settings/reset-pricing`,
        {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify({ scope: "claude" }),
        }
      );
      const { pricing } = await res.json();
      assert.equal(pricing.find((p) => p.model_pattern === "glm-4%")?.input_per_mtok, 1);
      assert.notEqual(
        pricing.find((p) => p.model_pattern === "claude-opus-5-5%")?.input_per_mtok,
        999
      );
    } finally {
      server.close();
    }
  });
});