/** * @file Tests for the automatic pricing sync: a model seen in token_usage with * no row of its own gets one from the price list, an edited row is never * overwritten, a bad list or a failed fetch changes nothing, and an OpenAI * model (a Codex session's) is priced from its `openai` entry too. * @author Nguyễn Ngọc Trí Vĩ */ const { describe, it, beforeEach } = require("node:test"); const assert = require("node:assert/strict"); const path = require("path"); const os = require("os"); process.env.DASHBOARD_DB_PATH = path.join( os.tmpdir(), `pricing-sync-${Date.now()}-${process.pid}.db` ); const { db } = require("../db"); const { syncPricing, familyOf, displayName, toRow, _resetCache, _setCache, } = require("../lib/pricing-sync"); // The real LiteLLM entry for Opus 5.5, trimmed. const LIST = { "claude-opus-9-9": { litellm_provider: "anthropic", input_cost_per_token: 0.000004, output_cost_per_token: 0.00002, cache_read_input_token_cost: 2e-7, cache_creation_input_token_cost: 0.000005, cache_creation_input_token_cost_above_1hr: 0.000008, supports_fast_mode: true, provider_specific_entry: { fast: 2 }, }, }; function useModel(model) { db.prepare("INSERT OR IGNORE INTO sessions (id, status, started_at) VALUES (?, 'active', ?)").run( `s-${model}`, new Date().toISOString() ); db.prepare("INSERT INTO token_usage (session_id, model) VALUES (?, ?)").run(`s-${model}`, model); } const row = (p) => db.prepare("SELECT * FROM model_pricing WHERE model_pattern = ?").get(p); beforeEach(() => { _resetCache(); db.prepare("DELETE FROM token_usage").run(); db.prepare( "DELETE FROM model_pricing WHERE model_pattern LIKE 'claude-opus-9%' OR model_pattern LIKE 'gpt-9%'" ).run(); }); describe("pricing sync", () => { it("reduces an id to its family and names it", () => { assert.equal(familyOf("claude-opus-5-5-20260101"), "claude-opus-5-5"); assert.equal(familyOf("gpt-5.6-sol"), "gpt-5.6-sol"); assert.equal(familyOf("gpt-5.4-2026-03-05"), "gpt-5.4"); assert.equal(familyOf("o3-mini"), "o3-mini"); assert.equal(familyOf("clinepass/cline-pass/deepseek-v4-pro"), null); assert.equal(familyOf("Highest"), null); assert.equal(familyOf("claude-x%"), null); assert.equal(displayName("claude-opus-5-5"), "Claude Opus 5.5"); assert.equal(displayName("gpt-5.6-sol"), "GPT-5.6 Sol"); assert.equal(displayName("o3-mini"), "o3 Mini"); }); it("prices a Gemini model from Vertex AI's entry, never another vendor's", async () => { assert.equal(familyOf("gemini-3.8-flash"), "gemini-3.8-flash"); assert.equal(displayName("gemini-3.8-flash"), "Gemini 3.8 Flash"); useModel("gemini-3.8-flash"); const list = { "gemini-3.8-flash": { litellm_provider: "vertex_ai-language-models", input_cost_per_token: 7.5e-7, output_cost_per_token: 3.75e-6, cache_read_input_token_cost: 7.5e-8, }, }; assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), ["gemini-3.8-flash%"]); const r = row("gemini-3.8-flash%"); assert.equal(r.display_name, "Gemini 3.8 Flash"); assert.deepEqual( [r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok], [0.75, 3.75, 0.075] ); assert.equal(r.cache_write_per_mtok, 0); const other = { "gemini-3.8-flash": { ...list["gemini-3.8-flash"], litellm_provider: "openrouter" }, }; assert.equal(toRow("gemini-3.8-flash", other["gemini-3.8-flash"]), null); }); it("adds a row for a used model that only a broader pattern matched", async () => { db.prepare( "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok) VALUES ('claude-opus-9%', 'Broad', 5)" ).run(); useModel("claude-opus-9-9-20260101"); const added = await syncPricing(db, { fetchList: async () => LIST }); assert.deepEqual(added, ["claude-opus-9-9%"]); const r = row("claude-opus-9-9%"); assert.equal(r.display_name, "Claude Opus 9.9"); assert.deepEqual([r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok], [4, 20, 0.2]); assert.deepEqual([r.cache_write_per_mtok, r.cache_write_1h_per_mtok], [5, 8]); assert.deepEqual([r.fast_input_per_mtok, r.fast_output_per_mtok], [8, 40]); }); it("never overwrites a row that exists, and fetches nothing when none is missing", async () => { db.prepare( "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok) VALUES ('claude-opus-9-9%', 'Mine', 1)" ).run(); useModel("claude-opus-9-9"); let fetched = false; await syncPricing(db, { fetchList: async () => { fetched = true; return LIST; }, }); assert.equal(fetched, false); assert.equal(row("claude-opus-9-9%").input_per_mtok, 1); }); it("skips absurd rates and survives a failed fetch", async () => { useModel("claude-opus-9-9"); const bad = { "claude-opus-9-9": { ...LIST["claude-opus-9-9"], input_cost_per_token: 5 } }; assert.deepEqual(await syncPricing(db, { fetchList: async () => bad }), []); _resetCache(); const failed = await syncPricing(db, { fetchList: async () => { throw new Error("offline"); }, }); assert.deepEqual(failed, []); assert.equal(row("claude-opus-9-9%"), undefined); // A failure is not retried by the next caller, only after the check interval. let calls = 0; const counting = async () => { calls += 1; return LIST; }; await syncPricing(db, { fetchList: counting }); assert.equal(calls, 0); await syncPricing(db, { fetchList: counting, now: Date.now() + 16 * 60_000 }); assert.equal(calls, 1); }); it("prices an OpenAI model from its openai entry, with no cache-write charge", async () => { useModel("gpt-9.1-sol"); const list = { "gpt-9.1-sol": { litellm_provider: "openai", input_cost_per_token: 0.000004, output_cost_per_token: 0.00002, cache_read_input_token_cost: 4e-7, }, }; assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), ["gpt-9.1-sol%"]); const r = row("gpt-9.1-sol%"); assert.equal(r.display_name, "GPT-9.1 Sol"); assert.deepEqual([r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok], [4, 20, 0.4]); assert.deepEqual([r.cache_write_per_mtok, r.cache_write_1h_per_mtok], [0, 0]); assert.deepEqual([r.fast_input_per_mtok, r.fast_output_per_mtok], [0, 0]); }); it("refuses an entry whose provider does not match the model's vendor", async () => { useModel("gpt-9.2"); const list = { "gpt-9.2": { litellm_provider: "azure", input_cost_per_token: 1e-6, output_cost_per_token: 1e-6, }, }; assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), []); }); it("POST /api/settings/reset-pricing brings a synced model's row straight back", async () => { const express = require("express"); const http = require("node:http"); const app = express(); app.use(express.json()); app.use("/api/settings", require("../routes/settings")); const server = http.createServer(app); await new Promise((r) => server.listen(0, "127.0.0.1", r)); const saved = process.env.AGENTHUB_PRICING_SYNC; process.env.AGENTHUB_PRICING_SYNC = "1"; try { useModel("gpt-9.3"); _setCache({ "gpt-9.3": { litellm_provider: "openai", input_cost_per_token: 1e-6, output_cost_per_token: 8e-6, }, }); const res = await fetch( `http://127.0.0.1:${server.address().port}/api/settings/reset-pricing`, { method: "POST", } ); const body = await res.json(); assert.ok(body.pricing.some((p) => p.model_pattern === "gpt-9.3%")); assert.ok(body.pricing.some((p) => p.model_pattern.startsWith("claude-"))); } finally { if (saved === undefined) delete process.env.AGENTHUB_PRICING_SYNC; else process.env.AGENTHUB_PRICING_SYNC = saved; server.close(); } }); it("POST /api/settings/reset-pricing with scope claude keeps every other row", async () => { const express = require("express"); const http = require("node:http"); const app = express(); app.use(express.json()); app.use("/api/settings", require("../routes/settings")); const server = http.createServer(app); await new Promise((r) => server.listen(0, "127.0.0.1", r)); try { const add = db.prepare( "INSERT OR REPLACE INTO model_pricing (model_pattern, display_name, input_per_mtok, output_per_mtok) VALUES (?, ?, ?, ?)" ); add.run("glm-4%", "GLM 4 (router)", 1, 2); add.run("claude-opus-5-5%", "edited", 999, 999); const res = await fetch( `http://127.0.0.1:${server.address().port}/api/settings/reset-pricing`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ scope: "claude" }), } ); const { pricing } = await res.json(); assert.equal(pricing.find((p) => p.model_pattern === "glm-4%")?.input_per_mtok, 1); assert.notEqual( pricing.find((p) => p.model_pattern === "claude-opus-5-5%")?.input_per_mtok, 999 ); } finally { server.close(); } }); });