AgentHub/server/__tests__/pricing-sync.test.js

/**
 * @file Tests for the automatic pricing sync: a model seen in token_usage with
 * no row of its own gets one from the price list, a current model gets one
 * before it is used, the sync's own rows follow a rate change while a user's
 * row and a deleted pattern are never touched, a bad list or a failed fetch
 * changes nothing, and an OpenAI model (a Codex session's) is priced from its
 * `openai` entry too, and a Claude entry's above-100k rates fill the long_*
 * columns.
 * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
 */

const { describe, it, beforeEach } = require("node:test");
const assert = require("node:assert/strict");
const path = require("path");
const os = require("os");

process.env.DASHBOARD_DB_PATH = path.join(
  os.tmpdir(),
  `pricing-sync-${Date.now()}-${process.pid}.db`
);
const { db } = require("../db");
const {
  syncPricing,
  familyOf,
  displayName,
  toRow,
  _resetCache,
  _setCache,
  isCurrent,
} = require("../lib/pricing-sync");

// The real LiteLLM entry for Opus 5.5, trimmed.
const LIST = {
  "claude-opus-9-9": {
    litellm_provider: "anthropic",
    input_cost_per_token: 0.000004,
    output_cost_per_token: 0.00002,
    cache_read_input_token_cost: 2e-7,
    cache_creation_input_token_cost: 0.000005,
    cache_creation_input_token_cost_above_1hr: 0.000008,
    supports_fast_mode: true,
    provider_specific_entry: { fast: 2 },
  },
};

function useModel(model) {
  db.prepare("INSERT OR IGNORE INTO sessions (id, status, started_at) VALUES (?, 'active', ?)").run(
    `s-${model}`,
    new Date().toISOString()
  );
  db.prepare("INSERT INTO token_usage (session_id, model) VALUES (?, ?)").run(`s-${model}`, model);
}
const row = (p) => db.prepare("SELECT * FROM model_pricing WHERE model_pattern = ?").get(p);

beforeEach(() => {
  _resetCache();
  db.prepare("DELETE FROM token_usage").run();
  db.prepare(
    "DELETE FROM model_pricing WHERE model_pattern LIKE 'claude-opus-9%' OR model_pattern LIKE 'gpt-9%' OR model_pattern LIKE 'gemini-9%' OR model_pattern LIKE 'gpt-4o%'"
  ).run();
  db.prepare("DELETE FROM model_pricing_dismissed").run();
});

describe("pricing sync", () => {
  it("never adds a retired Claude model in advance, only once it is used", () => {
    const today = "2026-10-01";
    for (const k of ["claude-opus-4-1", "claude-opus-4", "claude-sonnet-4"]) {
      assert.equal(isCurrent(k, { mode: "chat" }, today), false, k);
    }
    assert.equal(isCurrent("claude-opus-4-5", { mode: "chat" }, today), true);
  });

  it("reduces an id to its family and names it", () => {
    assert.equal(familyOf("claude-opus-5-5-20260101"), "claude-opus-5-5");
    assert.equal(familyOf("gpt-5.6-sol"), "gpt-5.6-sol");
    assert.equal(familyOf("gpt-5.4-2026-03-05"), "gpt-5.4");
    assert.equal(familyOf("o3-mini"), "o3-mini");
    assert.equal(familyOf("clinepass/cline-pass/deepseek-v4-pro"), null);
    assert.equal(familyOf("Highest"), null);
    assert.equal(familyOf("claude-x%"), null);
    assert.equal(displayName("claude-opus-5-5"), "Claude Opus 5.5");
    assert.equal(displayName("gpt-5.6-sol"), "GPT-5.6 Sol");
    assert.equal(displayName("o3-mini"), "o3 Mini");
  });

  it("prices a Gemini model from Vertex AI's entry, never another vendor's", async () => {
    assert.equal(familyOf("gemini-3.8-flash"), "gemini-3.8-flash");
    assert.equal(displayName("gemini-3.8-flash"), "Gemini 3.8 Flash");
    useModel("gemini-3.8-flash");
    const list = {
      "gemini-3.8-flash": {
        litellm_provider: "vertex_ai-language-models",
        input_cost_per_token: 7.5e-7,
        output_cost_per_token: 3.75e-6,
        cache_read_input_token_cost: 7.5e-8,
      },
    };
    assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), ["gemini-3.8-flash%"]);
    const r = row("gemini-3.8-flash%");
    assert.equal(r.display_name, "Gemini 3.8 Flash");
    assert.deepEqual(
      [r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok],
      [0.75, 3.75, 0.075]
    );
    assert.equal(r.cache_write_per_mtok, 0);
    const other = {
      "gemini-3.8-flash": { ...list["gemini-3.8-flash"], litellm_provider: "openrouter" },
    };
    assert.equal(toRow("gemini-3.8-flash", other["gemini-3.8-flash"]), null);
  });

  it("adds a row for a used model that only a broader pattern matched", async () => {
    db.prepare(
      "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok) VALUES ('claude-opus-9%', 'Broad', 5)"
    ).run();
    useModel("claude-opus-9-9-20260101");
    const added = await syncPricing(db, { fetchList: async () => LIST });
    assert.deepEqual(added, ["claude-opus-9-9%"]);
    const r = row("claude-opus-9-9%");
    assert.equal(r.display_name, "Claude Opus 9.9");
    assert.deepEqual([r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok], [4, 20, 0.2]);
    assert.deepEqual([r.cache_write_per_mtok, r.cache_write_1h_per_mtok], [5, 8]);
    assert.deepEqual([r.fast_input_per_mtok, r.fast_output_per_mtok], [8, 40]);
  });

  it("never overwrites a row the user priced, and fetches once a day", async () => {
    db.prepare(
      "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok, source) VALUES ('claude-opus-9-9%', 'Mine', 1, 'user')"
    ).run();
    useModel("claude-opus-9-9");
    let calls = 0;
    const counting = async () => {
      calls += 1;
      return LIST;
    };
    await syncPricing(db, { fetchList: counting });
    await syncPricing(db, { fetchList: counting, now: Date.now() + 60 * 60_000 });
    assert.equal(calls, 1);
    await syncPricing(db, { fetchList: counting, now: Date.now() + 25 * 60 * 60_000 });
    assert.equal(calls, 2);
    assert.equal(row("claude-opus-9-9%").input_per_mtok, 1);
  });

  it("adds a current model before it is ever used, and skips old or non-chat ones", async () => {
    const chat = (extra = {}) => ({
      litellm_provider: "openai",
      mode: "chat",
      input_cost_per_token: 1e-6,
      output_cost_per_token: 8e-6,
      ...extra,
    });
    const list = {
      "claude-opus-9-9": { ...LIST["claude-opus-9-9"], mode: "chat" },
      "gpt-9.1": chat(),
      "gpt-9.1-audio": chat(),
      "gpt-9.1-2026-01-01": chat(),
      "gpt-9.2": chat({ mode: "embedding" }),
      "gpt-9.3": chat({ deprecation_date: "2020-01-01" }),
      "gpt-4o": chat(),
      "gemini-9.1-pro": chat({ litellm_provider: "gemini" }),
    };
    const added = await syncPricing(db, { fetchList: async () => list });
    assert.deepEqual(added.sort(), ["claude-opus-9-9%", "gemini-9.1-pro%", "gpt-9.1%"]);
    assert.equal(row("gpt-9.1%").source, "sync");
    assert.equal(row("gpt-4o%"), undefined);
  });

  it("follows a price change on its own rows, never on the user's", async () => {
    const entry = (input) => ({
      litellm_provider: "openai",
      mode: "chat",
      input_cost_per_token: input,
      output_cost_per_token: 8e-6,
    });
    await syncPricing(db, { fetchList: async () => ({ "gpt-9.1": entry(1e-6) }) });
    assert.equal(row("gpt-9.1%").input_per_mtok, 1);
    db.prepare(
      "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok, output_per_mtok, source) VALUES ('gpt-9.2%', 'Mine', 7, 8, 'user')"
    ).run();
    _resetCache();
    await syncPricing(db, {
      fetchList: async () => ({ "gpt-9.1": entry(2e-6), "gpt-9.2": entry(2e-6) }),
    });
    assert.equal(row("gpt-9.1%").input_per_mtok, 2);
    assert.equal(row("gpt-9.2%").input_per_mtok, 7);
  });

  it("adopts a row from before the marker only while it still equals the list", async () => {
    const entry = {
      litellm_provider: "openai",
      mode: "chat",
      input_cost_per_token: 1e-6,
      output_cost_per_token: 8e-6,
    };
    const legacy = db.prepare(
      "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok, output_per_mtok, cache_read_per_mtok) VALUES (?, ?, ?, ?, ?)"
    );
    legacy.run("gpt-9.1%", "GPT-9.1", 1, 8, 0.1); // what the old sync inserted
    legacy.run("gpt-9.2%", "Edited", 5, 8, 0.1); // edited since
    await syncPricing(db, { fetchList: async () => ({ "gpt-9.1": entry, "gpt-9.2": entry }) });
    assert.equal(row("gpt-9.1%").source, "sync");
    assert.equal(row("gpt-9.2%").source, null);
    assert.equal(row("gpt-9.2%").input_per_mtok, 5);
  });

  it("keeps a deleted row deleted", async () => {
    db.prepare("INSERT INTO model_pricing_dismissed (model_pattern) VALUES ('gpt-9.1%')").run();
    useModel("gpt-9.1");
    const list = {
      "gpt-9.1": {
        litellm_provider: "openai",
        mode: "chat",
        input_cost_per_token: 1e-6,
        output_cost_per_token: 8e-6,
      },
    };
    assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), []);
    assert.equal(row("gpt-9.1%"), undefined);
  });

  it("PUT marks a row the user's, DELETE dismisses it, reset forgets the dismissal", async () => {
    const express = require("express");
    const http = require("node:http");
    const app = express();
    app.use(express.json());
    app.use("/api/pricing", require("../routes/pricing"));
    app.use("/api/settings", require("../routes/settings"));
    const server = http.createServer(app);
    await new Promise((r) => server.listen(0, "127.0.0.1", r));
    const base = `http://127.0.0.1:${server.address().port}`;
    try {
      db.prepare(
        "INSERT INTO model_pricing (model_pattern, display_name, input_per_mtok, source) VALUES ('gpt-9.1%', 'GPT-9.1', 1, 'sync')"
      ).run();
      await fetch(`${base}/api/pricing`, {
        method: "PUT",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({
          model_pattern: "gpt-9.1%",
          display_name: "GPT-9.1",
          input_per_mtok: 3,
          output_per_mtok: 9,
        }),
      });
      assert.equal(row("gpt-9.1%").source, "user");
      await fetch(`${base}/api/pricing/${encodeURIComponent("gpt-9.1%")}`, { method: "DELETE" });
      assert.ok(
        db.prepare("SELECT 1 FROM model_pricing_dismissed WHERE model_pattern = 'gpt-9.1%'").get()
      );
      await fetch(`${base}/api/settings/reset-pricing`, { method: "POST" });
      assert.equal(db.prepare("SELECT COUNT(*) AS n FROM model_pricing_dismissed").get().n, 0);
    } finally {
      server.close();
    }
  });

  it("skips absurd rates and survives a failed fetch", async () => {
    useModel("claude-opus-9-9");
    const bad = { "claude-opus-9-9": { ...LIST["claude-opus-9-9"], input_cost_per_token: 5 } };
    assert.deepEqual(await syncPricing(db, { fetchList: async () => bad }), []);
    _resetCache();
    const failed = await syncPricing(db, {
      fetchList: async () => {
        throw new Error("offline");
      },
    });
    assert.deepEqual(failed, []);
    assert.equal(row("claude-opus-9-9%"), undefined);
    // A failure is not retried by the next caller, only after the check interval.
    let calls = 0;
    const counting = async () => {
      calls += 1;
      return LIST;
    };
    await syncPricing(db, { fetchList: counting });
    assert.equal(calls, 0);
    await syncPricing(db, { fetchList: counting, now: Date.now() + 16 * 60_000 });
    assert.equal(calls, 1);
  });

  it("prices an OpenAI model from its openai entry, with no cache-write charge", async () => {
    useModel("gpt-9.1-sol");
    const list = {
      "gpt-9.1-sol": {
        litellm_provider: "openai",
        input_cost_per_token: 0.000004,
        output_cost_per_token: 0.00002,
        cache_read_input_token_cost: 4e-7,
      },
    };
    assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), ["gpt-9.1-sol%"]);
    const r = row("gpt-9.1-sol%");
    assert.equal(r.display_name, "GPT-9.1 Sol");
    assert.deepEqual([r.input_per_mtok, r.output_per_mtok, r.cache_read_per_mtok], [4, 20, 0.4]);
    assert.deepEqual([r.cache_write_per_mtok, r.cache_write_1h_per_mtok], [0, 0]);
    assert.deepEqual([r.fast_input_per_mtok, r.fast_output_per_mtok], [0, 0]);
  });

  it("refuses an entry whose provider does not match the model's vendor", async () => {
    useModel("gpt-9.2");
    const list = {
      "gpt-9.2": {
        litellm_provider: "azure",
        input_cost_per_token: 1e-6,
        output_cost_per_token: 1e-6,
      },
    };
    assert.deepEqual(await syncPricing(db, { fetchList: async () => list }), []);
  });

  it("POST /api/settings/reset-pricing brings a synced model's row straight back", async () => {
    const express = require("express");
    const http = require("node:http");
    const app = express();
    app.use(express.json());
    app.use("/api/settings", require("../routes/settings"));
    const server = http.createServer(app);
    await new Promise((r) => server.listen(0, "127.0.0.1", r));
    const saved = process.env.AGENTHUB_PRICING_SYNC;
    process.env.AGENTHUB_PRICING_SYNC = "1";
    try {
      useModel("gpt-9.3");
      _setCache({
        "gpt-9.3": {
          litellm_provider: "openai",
          input_cost_per_token: 1e-6,
          output_cost_per_token: 8e-6,
        },
      });
      const res = await fetch(
        `http://127.0.0.1:${server.address().port}/api/settings/reset-pricing`,
        {
          method: "POST",
        }
      );
      const body = await res.json();
      assert.ok(body.pricing.some((p) => p.model_pattern === "gpt-9.3%"));
      assert.ok(body.pricing.some((p) => p.model_pattern.startsWith("claude-")));
    } finally {
      if (saved === undefined) delete process.env.AGENTHUB_PRICING_SYNC;
      else process.env.AGENTHUB_PRICING_SYNC = saved;
      server.close();
    }
  });

  it("POST /api/settings/reset-pricing re-adds a current model nobody has used", async () => {
    const express = require("express");
    const http = require("node:http");
    const app = express();
    app.use(express.json());
    app.use("/api/settings", require("../routes/settings"));
    const server = http.createServer(app);
    await new Promise((r) => server.listen(0, "127.0.0.1", r));
    const saved = process.env.AGENTHUB_PRICING_SYNC;
    process.env.AGENTHUB_PRICING_SYNC = "1";
    try {
      _setCache({
        "gpt-9.7": {
          litellm_provider: "openai",
          mode: "chat",
          input_cost_per_token: 1e-6,
          output_cost_per_token: 8e-6,
        },
      });
      // Applied once: the list is cached and current, nothing is missing.
      await syncPricing(db);
      assert.ok(db.prepare("SELECT 1 FROM model_pricing WHERE model_pattern = 'gpt-9.7%'").get());
      const res = await fetch(
        `http://127.0.0.1:${server.address().port}/api/settings/reset-pricing`,
        { method: "POST" }
      );
      const body = await res.json();
      assert.ok(body.pricing.some((p) => p.model_pattern === "gpt-9.7%"));
    } finally {
      if (saved === undefined) delete process.env.AGENTHUB_PRICING_SYNC;
      else process.env.AGENTHUB_PRICING_SYNC = saved;
      server.close();
    }
  });

  it("does not bring a deleted built-in row back at the next start", () => {
    const pattern = db
      .prepare(
        "SELECT model_pattern FROM model_pricing WHERE model_pattern LIKE 'claude-%' LIMIT 1"
      )
      .get().model_pattern;
    db.prepare("DELETE FROM model_pricing WHERE model_pattern = ?").run(pattern);
    db.prepare("INSERT OR IGNORE INTO model_pricing_dismissed (model_pattern) VALUES (?)").run(
      pattern
    );
    require("../db").topUpDefaultPricing();
    assert.equal(
      db.prepare("SELECT 1 FROM model_pricing WHERE model_pattern = ?").get(pattern),
      undefined
    );
    db.prepare("DELETE FROM model_pricing_dismissed WHERE model_pattern = ?").run(pattern);
  });

  it("POST /api/settings/reset-pricing with scope claude keeps every other row", async () => {
    const express = require("express");
    const http = require("node:http");
    const app = express();
    app.use(express.json());
    app.use("/api/settings", require("../routes/settings"));
    const server = http.createServer(app);
    await new Promise((r) => server.listen(0, "127.0.0.1", r));
    try {
      const add = db.prepare(
        "INSERT OR REPLACE INTO model_pricing (model_pattern, display_name, input_per_mtok, output_per_mtok) VALUES (?, ?, ?, ?)"
      );
      add.run("glm-4%", "GLM 4 (router)", 1, 2);
      add.run("claude-opus-5-5%", "edited", 999, 999);
      const res = await fetch(
        `http://127.0.0.1:${server.address().port}/api/settings/reset-pricing`,
        {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify({ scope: "claude" }),
        }
      );
      const { pricing } = await res.json();
      assert.equal(pricing.find((p) => p.model_pattern === "glm-4%")?.input_per_mtok, 1);
      assert.notEqual(
        pricing.find((p) => p.model_pattern === "claude-opus-5-5%")?.input_per_mtok,
        999
      );
    } finally {
      server.close();
    }
  });
});

describe("long-context rates", () => {
  it("takes Anthropic's above-100k input/output rates into the long columns", () => {
    const e = {
      ...LIST["claude-opus-9-9"],
      input_cost_per_token_above_100k_tokens: 0.00002,
      output_cost_per_token_above_100k_tokens: 0.0001,
    };
    const r = toRow("claude-opus-9-9", e);
    assert.equal(r.longIn, 20);
    assert.equal(r.longOut, 100);
    assert.equal(toRow("claude-opus-9-9", LIST["claude-opus-9-9"]).longIn, 0);
  });

  it("writes them, and fills them into a sync row that predates them", async () => {
    const e = {
      ...LIST["claude-opus-9-9"],
      input_cost_per_token_above_100k_tokens: 0.00002,
      output_cost_per_token_above_100k_tokens: 0.0001,
    };
    useModel("claude-opus-9-9");
    await syncPricing(db, {
      fetchList: async () => ({ "claude-opus-9-9": LIST["claude-opus-9-9"] }),
    });
    assert.equal(row("claude-opus-9-9%").long_input_per_mtok, 0);
    _resetCache();
    await syncPricing(db, { fetchList: async () => ({ "claude-opus-9-9": e }) });
    assert.equal(row("claude-opus-9-9%").long_input_per_mtok, 20);
    assert.equal(row("claude-opus-9-9%").long_output_per_mtok, 100);
  });
});