/** * @file Tests for the automatic pricing sync: a model seen in token_usage with * no row of its own gets one from the price list, a current model gets one * before it is used, the sync's own rows follow a rate change while a user's * row and a deleted pattern are never touched, a bad list or a failed fetch * changes nothing, and an OpenAI model (a Codex session's) is priced from its * `openai` entry too, and a Claude entry's above-100k rates fill the long_* * columns. * @author Nguyễn Ngọc Trí Vĩ */ const { describe, it, beforeEach } = require("node:test"); const assert = require("node:assert/strict"); const path = require("path"); const os = require("os"); process.env.DASHBOARD_DB_PATH = path.join( os.tmpdir(), `pricing-sync-${Date.now()}-${process.pid}.db` ); const { db } = require("../db"); const { syncPricing, familyOf, displayName, toRow, _resetCache, _setCache, isCurrent, } = require("../lib/pricing-sync"); // The real LiteLLM entry for Opus 5.5, trimmed. const LIST = { "claude-opus-9-9": { litellm_provider: "anthropic", input_cost_per_token: 0.000004, output_cost_per_token: 0.00002, cache_read_input_token_cost: 2e-7, cache_creation_input_token_cost: 0.000005, cache_creation_input_token_cost_above_1hr: 0.000008, supports_fast_mode: true, provider_specific_entry: { fast: 2 }, }, }; function useModel(model) { db.prepare("INSERT OR IGNORE INTO sessions (id, status, started_at) VALUES (?, 'active', ?)").run( `s-${model}`, new Date().toISOString() ); db.prepare("INSERT INTO token_usage (session_id, model) VALUES (?, ?)").run(`s-${model}`, model); } const row = (p) => db.prepare("SELECT * FROM model_pricing WHERE model_pattern = ?").get(p); beforeEach(() => { _resetCache(); db.prepare("DELETE FROM token_usage").run(); db.prepare( "DELETE FROM model_pricing WHERE model_pattern LIKE 'claude-opus-9%' OR model_pattern LIKE 'gpt-9%' OR model_pattern LIKE 'gemini-9%' OR model_pattern LIKE 'gpt-4o%'" ).run(); db.prepare("DELETE FROM model_pricing_dismissed").run(); }); describe("pricing sync", () => { it("never adds a retired Claude model in advance, only once it is used", () => { const today = "2026-10-01"; for (const k of ["claude-opus-4-1", "claude-opus-4", "claude-sonnet-4"]) { assert.equal(isCurrent(k, { mode: "chat" }, today), false, k); } assert.equal(isCurrent("claude-opus-4-5", { mode: "chat" }, today), true); }); it("reduces an id to its family and names it", () => { assert.equal(familyOf("claude-opus-5-5-20260101"), "claude-opus-5-5"); assert.equal(familyOf("gpt-5.6-sol"), "gpt-5.6-sol"); assert.equal(familyOf("gpt-5.4-2026-03-05"), "gpt-5.4"); assert.equal(familyOf("o3-mini"), "o3-mini"); assert.equal(familyOf("clinepass/cline-pass/deepseek-v4-pro"), null); assert.equal(familyOf("Highest"), null); assert.equal(familyOf("claude-x%"), null); assert.equal(displayName("claude-opus-5-5"), "Claude Opus 5.5"); assert.equal(displayName("gpt-5.6-sol"), "GPT-5.6 Sol"); assert.equal(displayName("o3-mini"), "o3 Mini"); }); it("prices a Gemini model from Vertex AI's entry, never another vendor's", async () => { assert.equal(familyOf("gemini-3.8-flash"), "gemini-3.8-flash"); assert.equal(displayName("gemini-3.8-flash"), "Gemini 3.8 Flash"); useModel("gemini-3.8-flash"); const list = { "gemini-3.8-flash": { litellm_provider: "vertex_ai-language-models", input_cost_per_token: 7.5e-7, output_cost_per_token: 3.75e-6, cache_read_input_token_cost: 7.5e-8, }, }; assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), ["gemini-3.8-flash%"]); const r = row("gemini-3.8-flash%"); assert.equal(r.display_name, "Gemini 3.8 Flash"); assert.deepEqual( [r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok], [0.75, 3.75, 0.075] ); assert.equal(r.cache_write_per_mtok, 0); const other = { "gemini-3.8-flash": { ...list["gemini-3.8-flash"], litellm_provider: "openrouter" }, }; assert.equal(toRow("gemini-3.8-flash", other["gemini-3.8-flash"]), null); }); it("adds a row for a used model that only a broader pattern matched", async () => { db.prepare( "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok) VALUES ('claude-opus-9%', 'Broad', 5)" ).run(); useModel("claude-opus-9-9-20260101"); const added = await syncPricing(db, { fetchList: async () => LIST }); assert.deepEqual(added, ["claude-opus-9-9%"]); const r = row("claude-opus-9-9%"); assert.equal(r.display_name, "Claude Opus 9.9"); assert.deepEqual([r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok], [4, 20, 0.2]); assert.deepEqual([r.cache_write_per_mtok, r.cache_write_1h_per_mtok], [5, 8]); assert.deepEqual([r.fast_input_per_mtok, r.fast_output_per_mtok], [8, 40]); }); it("never overwrites a row the user priced, and fetches once a day", async () => { db.prepare( "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok, source) VALUES ('claude-opus-9-9%', 'Mine', 1, 'user')" ).run(); useModel("claude-opus-9-9"); let calls = 0; const counting = async () => { calls += 1; return LIST; }; await syncPricing(db, { fetchList: counting }); await syncPricing(db, { fetchList: counting, now: Date.now() + 60 * 60_000 }); assert.equal(calls, 1); await syncPricing(db, { fetchList: counting, now: Date.now() + 25 * 60 * 60_000 }); assert.equal(calls, 2); assert.equal(row("claude-opus-9-9%").input_per_mtok, 1); }); it("adds a current model before it is ever used, and skips old or non-chat ones", async () => { const chat = (extra = {}) => ({ litellm_provider: "openai", mode: "chat", input_cost_per_token: 1e-6, output_cost_per_token: 8e-6, ...extra, }); const list = { "claude-opus-9-9": { ...LIST["claude-opus-9-9"], mode: "chat" }, "gpt-9.1": chat(), "gpt-9.1-audio": chat(), "gpt-9.1-2026-01-01": chat(), "gpt-9.2": chat({ mode: "embedding" }), "gpt-9.3": chat({ deprecation_date: "2020-01-01" }), "gpt-4o": chat(), "gemini-9.1-pro": chat({ litellm_provider: "gemini" }), }; const added = await syncPricing(db, { fetchList: async () => list }); assert.deepEqual(added.sort(), ["claude-opus-9-9%", "gemini-9.1-pro%", "gpt-9.1%"]); assert.equal(row("gpt-9.1%").source, "sync"); assert.equal(row("gpt-4o%"), undefined); }); it("follows a price change on its own rows, never on the user's", async () => { const entry = (input) => ({ litellm_provider: "openai", mode: "chat", input_cost_per_token: input, output_cost_per_token: 8e-6, }); await syncPricing(db, { fetchList: async () => ({ "gpt-9.1": entry(1e-6) }) }); assert.equal(row("gpt-9.1%").input_per_mtok, 1); db.prepare( "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok, output_per_mtok, source) VALUES ('gpt-9.2%', 'Mine', 7, 8, 'user')" ).run(); _resetCache(); await syncPricing(db, { fetchList: async () => ({ "gpt-9.1": entry(2e-6), "gpt-9.2": entry(2e-6) }), }); assert.equal(row("gpt-9.1%").input_per_mtok, 2); assert.equal(row("gpt-9.2%").input_per_mtok, 7); }); it("adopts a row from before the marker only while it still equals the list", async () => { const entry = { litellm_provider: "openai", mode: "chat", input_cost_per_token: 1e-6, output_cost_per_token: 8e-6, }; const legacy = db.prepare( "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok, output_per_mtok, cache_read_per_mtok) VALUES (?, ?, ?, ?, ?)" ); legacy.run("gpt-9.1%", "GPT-9.1", 1, 8, 0.1); // what the old sync inserted legacy.run("gpt-9.2%", "Edited", 5, 8, 0.1); // edited since await syncPricing(db, { fetchList: async () => ({ "gpt-9.1": entry, "gpt-9.2": entry }) }); assert.equal(row("gpt-9.1%").source, "sync"); assert.equal(row("gpt-9.2%").source, null); assert.equal(row("gpt-9.2%").input_per_mtok, 5); }); it("keeps a deleted row deleted", async () => { db.prepare("INSERT INTO model_pricing_dismissed (model_pattern) VALUES ('gpt-9.1%')").run(); useModel("gpt-9.1"); const list = { "gpt-9.1": { litellm_provider: "openai", mode: "chat", input_cost_per_token: 1e-6, output_cost_per_token: 8e-6, }, }; assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), []); assert.equal(row("gpt-9.1%"), undefined); }); it("PUT marks a row the user's, DELETE dismisses it, reset forgets the dismissal", async () => { const express = require("express"); const http = require("node:http"); const app = express(); app.use(express.json()); app.use("/api/pricing", require("../routes/pricing")); app.use("/api/settings", require("../routes/settings")); const server = http.createServer(app); await new Promise((r) => server.listen(0, "127.0.0.1", r)); const base = `http://127.0.0.1:${server.address().port}`; try { db.prepare( "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok, source) VALUES ('gpt-9.1%', 'GPT-9.1', 1, 'sync')" ).run(); await fetch(`${base}/api/pricing`, { method: "PUT", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model_pattern: "gpt-9.1%", display_name: "GPT-9.1", input_per_mtok: 3, output_per_mtok: 9, }), }); assert.equal(row("gpt-9.1%").source, "user"); await fetch(`${base}/api/pricing/${encodeURIComponent("gpt-9.1%")}`, { method: "DELETE" }); assert.ok( db.prepare("SELECT 1 FROM model_pricing_dismissed WHERE model_pattern = 'gpt-9.1%'").get() ); await fetch(`${base}/api/settings/reset-pricing`, { method: "POST" }); assert.equal(db.prepare("SELECT COUNT(*) AS n FROM model_pricing_dismissed").get().n, 0); } finally { server.close(); } }); it("skips absurd rates and survives a failed fetch", async () => { useModel("claude-opus-9-9"); const bad = { "claude-opus-9-9": { ...LIST["claude-opus-9-9"], input_cost_per_token: 5 } }; assert.deepEqual(await syncPricing(db, { fetchList: async () => bad }), []); _resetCache(); const failed = await syncPricing(db, { fetchList: async () => { throw new Error("offline"); }, }); assert.deepEqual(failed, []); assert.equal(row("claude-opus-9-9%"), undefined); // A failure is not retried by the next caller, only after the check interval. let calls = 0; const counting = async () => { calls += 1; return LIST; }; await syncPricing(db, { fetchList: counting }); assert.equal(calls, 0); await syncPricing(db, { fetchList: counting, now: Date.now() + 16 * 60_000 }); assert.equal(calls, 1); }); it("prices an OpenAI model from its openai entry, with no cache-write charge", async () => { useModel("gpt-9.1-sol"); const list = { "gpt-9.1-sol": { litellm_provider: "openai", input_cost_per_token: 0.000004, output_cost_per_token: 0.00002, cache_read_input_token_cost: 4e-7, }, }; assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), ["gpt-9.1-sol%"]); const r = row("gpt-9.1-sol%"); assert.equal(r.display_name, "GPT-9.1 Sol"); assert.deepEqual([r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok], [4, 20, 0.4]); assert.deepEqual([r.cache_write_per_mtok, r.cache_write_1h_per_mtok], [0, 0]); assert.deepEqual([r.fast_input_per_mtok, r.fast_output_per_mtok], [0, 0]); }); it("refuses an entry whose provider does not match the model's vendor", async () => { useModel("gpt-9.2"); const list = { "gpt-9.2": { litellm_provider: "azure", input_cost_per_token: 1e-6, output_cost_per_token: 1e-6, }, }; assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), []); }); it("POST /api/settings/reset-pricing brings a synced model's row straight back", async () => { const express = require("express"); const http = require("node:http"); const app = express(); app.use(express.json()); app.use("/api/settings", require("../routes/settings")); const server = http.createServer(app); await new Promise((r) => server.listen(0, "127.0.0.1", r)); const saved = process.env.AGENTHUB_PRICING_SYNC; process.env.AGENTHUB_PRICING_SYNC = "1"; try { useModel("gpt-9.3"); _setCache({ "gpt-9.3": { litellm_provider: "openai", input_cost_per_token: 1e-6, output_cost_per_token: 8e-6, }, }); const res = await fetch( `http://127.0.0.1:${server.address().port}/api/settings/reset-pricing`, { method: "POST", } ); const body = await res.json(); assert.ok(body.pricing.some((p) => p.model_pattern === "gpt-9.3%")); assert.ok(body.pricing.some((p) => p.model_pattern.startsWith("claude-"))); } finally { if (saved === undefined) delete process.env.AGENTHUB_PRICING_SYNC; else process.env.AGENTHUB_PRICING_SYNC = saved; server.close(); } }); it("POST /api/settings/reset-pricing re-adds a current model nobody has used", async () => { const express = require("express"); const http = require("node:http"); const app = express(); app.use(express.json()); app.use("/api/settings", require("../routes/settings")); const server = http.createServer(app); await new Promise((r) => server.listen(0, "127.0.0.1", r)); const saved = process.env.AGENTHUB_PRICING_SYNC; process.env.AGENTHUB_PRICING_SYNC = "1"; try { _setCache({ "gpt-9.7": { litellm_provider: "openai", mode: "chat", input_cost_per_token: 1e-6, output_cost_per_token: 8e-6, }, }); // Applied once: the list is cached and current, nothing is missing. await syncPricing(db); assert.ok(db.prepare("SELECT 1 FROM model_pricing WHERE model_pattern = 'gpt-9.7%'").get()); const res = await fetch( `http://127.0.0.1:${server.address().port}/api/settings/reset-pricing`, { method: "POST" } ); const body = await res.json(); assert.ok(body.pricing.some((p) => p.model_pattern === "gpt-9.7%")); } finally { if (saved === undefined) delete process.env.AGENTHUB_PRICING_SYNC; else process.env.AGENTHUB_PRICING_SYNC = saved; server.close(); } }); it("does not bring a deleted built-in row back at the next start", () => { const pattern = db .prepare( "SELECT model_pattern FROM model_pricing WHERE model_pattern LIKE 'claude-%' LIMIT 1" ) .get().model_pattern; db.prepare("DELETE FROM model_pricing WHERE model_pattern = ?").run(pattern); db.prepare("INSERT OR IGNORE INTO model_pricing_dismissed (model_pattern) VALUES (?)").run( pattern ); require("../db").topUpDefaultPricing(); assert.equal( db.prepare("SELECT 1 FROM model_pricing WHERE model_pattern = ?").get(pattern), undefined ); db.prepare("DELETE FROM model_pricing_dismissed WHERE model_pattern = ?").run(pattern); }); it("POST /api/settings/reset-pricing with scope claude keeps every other row", async () => { const express = require("express"); const http = require("node:http"); const app = express(); app.use(express.json()); app.use("/api/settings", require("../routes/settings")); const server = http.createServer(app); await new Promise((r) => server.listen(0, "127.0.0.1", r)); try { const add = db.prepare( "INSERT OR REPLACE INTO model_pricing (model_pattern, display_name, input_per_mtok, output_per_mtok) VALUES (?, ?, ?, ?)" ); add.run("glm-4%", "GLM 4 (router)", 1, 2); add.run("claude-opus-5-5%", "edited", 999, 999); const res = await fetch( `http://127.0.0.1:${server.address().port}/api/settings/reset-pricing`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ scope: "claude" }), } ); const { pricing } = await res.json(); assert.equal(pricing.find((p) => p.model_pattern === "glm-4%")?.input_per_mtok, 1); assert.notEqual( pricing.find((p) => p.model_pattern === "claude-opus-5-5%")?.input_per_mtok, 999 ); } finally { server.close(); } }); }); describe("long-context rates", () => { it("takes Anthropic's above-100k input/output rates into the long columns", () => { const e = { ...LIST["claude-opus-9-9"], input_cost_per_token_above_100k_tokens: 0.00002, output_cost_per_token_above_100k_tokens: 0.0001, }; const r = toRow("claude-opus-9-9", e); assert.equal(r.longIn, 20); assert.equal(r.longOut, 100); assert.equal(toRow("claude-opus-9-9", LIST["claude-opus-9-9"]).longIn, 0); }); it("writes them, and fills them into a sync row that predates them", async () => { const e = { ...LIST["claude-opus-9-9"], input_cost_per_token_above_100k_tokens: 0.00002, output_cost_per_token_above_100k_tokens: 0.0001, }; useModel("claude-opus-9-9"); await syncPricing(db, { fetchList: async () => ({ "claude-opus-9-9": LIST["claude-opus-9-9"] }), }); assert.equal(row("claude-opus-9-9%").long_input_per_mtok, 0); _resetCache(); await syncPricing(db, { fetchList: async () => ({ "claude-opus-9-9": e }) }); assert.equal(row("claude-opus-9-9%").long_input_per_mtok, 20); assert.equal(row("claude-opus-9-9%").long_output_per_mtok, 100); }); });