perf: hourly update check only with a browser open; opt-in retention, incremental auto-vacuum and Compact
- update-scheduler: default interval 1 h (DASHBOARD_UPDATE_CHECK_INTERVAL_MS and its 60 s floor kept); a scheduled tick with no /ws client is skipped, and the next browser to connect gets one catch-up check when the last one is stale (websocket.js onConnection). Manual check unchanged. - db-retention.js: shared session purge (used by POST /api/settings/cleanup too), opt-in AGENTHUB_RETENTION_DAYS applied by the hourly maintenance in 25-session transactions with yields (max 5000/run), bounded incremental_vacuum when the freelist is large, and a manual Compact (VACUUM in a worker, converting to auto_vacuum=INCREMENTAL). - db.js: new databases are INCREMENTAL; an existing one up to 32 MB is converted at startup. - GET/PUT /api/settings/retention, POST /api/settings/compact (confirm: true); Settings → Data → Session cleanup gets the Automatic cleanup control and Compact now (en + vi).
This commit is contained in:
@@ -0,0 +1,175 @@
|
||||
/**
|
||||
* @file Tests server/lib/db-retention.js and its Settings routes — a new
|
||||
* database is INCREMENTAL auto_vacuum, retention is off by default and purges
|
||||
* nothing then, when on it removes only finished sessions older than the
|
||||
* cutoff (with their events, agents and token rows) in bounded batches, the
|
||||
* free-page reclaim shrinks the freelist, GET/PUT /api/settings/retention
|
||||
* validate and persist the setting, and POST /api/settings/compact needs
|
||||
* confirm and runs a VACUUM in a worker. The database and settings file are
|
||||
* temp paths.
|
||||
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
*/
|
||||
|
||||
const os = require("node:os");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const { describe, it, before, after } = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
|
||||
const ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "agenthub-db-retention-"));
|
||||
process.env.DASHBOARD_DB_PATH = path.join(ROOT, "dashboard.db");
|
||||
process.env.AGENTHUB_SETTINGS_FILE = path.join(ROOT, "settings.env");
|
||||
process.env.DASHBOARD_LIVENESS_PROBE = "0";
|
||||
delete process.env.AGENTHUB_RETENTION_DAYS;
|
||||
|
||||
const { createApp, startServer } = require("../index");
|
||||
const { db } = require("../db");
|
||||
const retention = require("../lib/db-retention");
|
||||
|
||||
let server;
|
||||
let BASE;
|
||||
before(async () => {
|
||||
server = await startServer(createApp(), 0);
|
||||
BASE = `http://127.0.0.1:${server.address().port}`;
|
||||
});
|
||||
after(() => {
|
||||
server?.close();
|
||||
delete process.env.AGENTHUB_RETENTION_DAYS;
|
||||
fs.rmSync(ROOT, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function call(method, urlPath, body) {
|
||||
const res = await fetch(BASE + urlPath, {
|
||||
method,
|
||||
headers: body ? { "content-type": "application/json" } : {},
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
return { status: res.status, body: await res.json() };
|
||||
}
|
||||
|
||||
const daysAgo = (n) => new Date(Date.now() - n * 86400 * 1000).toISOString();
|
||||
|
||||
function seed(id, status, startedAt, events = 3) {
|
||||
db.prepare(
|
||||
"INSERT INTO sessions (id, name, status, cwd, started_at, updated_at) VALUES (?, ?, ?, '/tmp', ?, ?)"
|
||||
).run(id, id, status, startedAt, startedAt);
|
||||
const ins = db.prepare(
|
||||
"INSERT INTO events (session_id, event_type, summary, data, created_at) VALUES (?, 'PostToolUse', 's', ?, ?)"
|
||||
);
|
||||
for (let i = 0; i < events; i++)
|
||||
ins.run(id, JSON.stringify({ pad: "x".repeat(2000) }), startedAt);
|
||||
}
|
||||
const exists = (id) => !!db.prepare("SELECT 1 FROM sessions WHERE id = ?").get(id);
|
||||
const eventCount = (id) =>
|
||||
db.prepare("SELECT COUNT(*) AS n FROM events WHERE session_id = ?").get(id).n;
|
||||
|
||||
describe("auto_vacuum", () => {
|
||||
it("a new database is INCREMENTAL", () => {
|
||||
assert.equal(db.pragma("auto_vacuum", { simple: true }), 2);
|
||||
assert.equal(retention.spaceInfo(db).auto_vacuum, "incremental");
|
||||
});
|
||||
});
|
||||
|
||||
describe("retentionConfig", () => {
|
||||
it("is off by default and for anything out of range", () => {
|
||||
assert.equal(retention.retentionConfig({}).days, 0);
|
||||
for (const v of ["", "0", "-3", "1.5", "abc", "99999"]) {
|
||||
assert.equal(retention.retentionConfig({ AGENTHUB_RETENTION_DAYS: v }).days, 0, v);
|
||||
}
|
||||
assert.equal(retention.retentionConfig({ AGENTHUB_RETENTION_DAYS: "30" }).days, 30);
|
||||
});
|
||||
});
|
||||
|
||||
describe("scheduled maintenance", () => {
|
||||
it("purges nothing while retention is off", async () => {
|
||||
seed("ret-off-old", "completed", daysAgo(400));
|
||||
const out = await retention.runScheduledMaintenance({ db });
|
||||
assert.equal(out.retention, undefined);
|
||||
assert.ok(exists("ret-off-old"));
|
||||
db.prepare("DELETE FROM sessions WHERE id = 'ret-off-old'").run();
|
||||
});
|
||||
|
||||
it("purges only finished sessions older than the cutoff, with their rows", async () => {
|
||||
seed("ret-old-done", "completed", daysAgo(100));
|
||||
seed("ret-old-err", "error", daysAgo(100));
|
||||
seed("ret-old-active", "active", daysAgo(100));
|
||||
seed("ret-new-done", "completed", daysAgo(5));
|
||||
db.prepare(
|
||||
"INSERT INTO token_usage (session_id, model, input_tokens) VALUES ('ret-old-done', 'm', 1)"
|
||||
).run();
|
||||
process.env.AGENTHUB_RETENTION_DAYS = "30";
|
||||
try {
|
||||
const out = await retention.runScheduledMaintenance({ db });
|
||||
assert.equal(out.retention.purged_sessions, 2);
|
||||
assert.equal(out.retention.purged_events, 6);
|
||||
} finally {
|
||||
delete process.env.AGENTHUB_RETENTION_DAYS;
|
||||
}
|
||||
assert.ok(!exists("ret-old-done"));
|
||||
assert.ok(!exists("ret-old-err"));
|
||||
assert.equal(eventCount("ret-old-done"), 0);
|
||||
assert.equal(
|
||||
db.prepare("SELECT COUNT(*) AS n FROM token_usage WHERE session_id = 'ret-old-done'").get().n,
|
||||
0
|
||||
);
|
||||
assert.ok(exists("ret-old-active"), "an active session is never purged");
|
||||
assert.ok(exists("ret-new-done"), "a recent session is kept");
|
||||
assert.equal(eventCount("ret-new-done"), 3);
|
||||
});
|
||||
|
||||
it("works in bounded batches and stops at the per-run cap", async () => {
|
||||
for (let i = 0; i < 7; i++) seed(`ret-batch-${i}`, "abandoned", daysAgo(50), 1);
|
||||
const first = await retention.applyRetention({ db, days: 10, batch: 2, maxSessions: 5 });
|
||||
assert.equal(first.purged_sessions, 5);
|
||||
const second = await retention.applyRetention({ db, days: 10, batch: 2, maxSessions: 5 });
|
||||
assert.equal(second.purged_sessions, 2);
|
||||
const third = await retention.applyRetention({ db, days: 10 });
|
||||
assert.equal(third.purged_sessions, 0);
|
||||
});
|
||||
|
||||
it("returns free pages to the OS", async () => {
|
||||
for (let i = 0; i < 40; i++) seed(`ret-big-${i}`, "completed", daysAgo(60), 20);
|
||||
await retention.applyRetention({ db, days: 30 });
|
||||
const before = db.pragma("freelist_count", { simple: true });
|
||||
assert.ok(before > 50, `expected free pages, got ${before}`);
|
||||
const out = await retention.reclaimFreePages({ db, minPages: 10, slice: 16 });
|
||||
assert.ok(out.reclaimed_pages > 0);
|
||||
assert.ok(db.pragma("freelist_count", { simple: true }) < before);
|
||||
});
|
||||
});
|
||||
|
||||
describe("routes", () => {
|
||||
it("GET reports retention off by default", async () => {
|
||||
const r = await call("GET", "/api/settings/retention");
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal(r.body.days, 0);
|
||||
assert.equal(r.body.compacting, false);
|
||||
assert.equal(r.body.space.auto_vacuum, "incremental");
|
||||
});
|
||||
|
||||
it("PUT validates and persists, 0 turns it off", async () => {
|
||||
for (const bad of [-1, 1.5, "7", 99999, null]) {
|
||||
const r = await call("PUT", "/api/settings/retention", { days: bad });
|
||||
assert.equal(r.status, 400, String(bad));
|
||||
}
|
||||
let r = await call("PUT", "/api/settings/retention", { days: 90 });
|
||||
assert.equal(r.status, 200);
|
||||
assert.equal(r.body.days, 90);
|
||||
assert.match(fs.readFileSync(process.env.AGENTHUB_SETTINGS_FILE, "utf8"), /RETENTION_DAYS=90/);
|
||||
r = await call("PUT", "/api/settings/retention", { days: 0 });
|
||||
assert.equal(r.body.days, 0);
|
||||
});
|
||||
|
||||
it("POST /compact needs confirm and then vacuums", async () => {
|
||||
let r = await call("POST", "/api/settings/compact", {});
|
||||
assert.equal(r.status, 400);
|
||||
r = await call("POST", "/api/settings/compact", { confirm: true });
|
||||
assert.equal(r.status, 202);
|
||||
for (let i = 0; i < 200 && retention.isCompacting(); i++) {
|
||||
await new Promise((res) => setTimeout(res, 50));
|
||||
}
|
||||
const last = retention.lastCompactResult();
|
||||
assert.equal(last?.ok, true, last?.error);
|
||||
assert.equal(db.pragma("auto_vacuum", { simple: true }), 2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* @file Tests server/update-scheduler.js — the default interval is an hour
|
||||
* (the env override and its 60 s floor still apply), a scheduled tick with no
|
||||
* browser connected runs no check, and a browser connecting after that gets
|
||||
* one catch-up check (only when the last check is stale). The check, the
|
||||
* connection count and the connection event are injected; nothing touches git
|
||||
* or the network.
|
||||
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
*/
|
||||
|
||||
const { describe, it, afterEach } = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
|
||||
delete process.env.DASHBOARD_UPDATE_CHECK;
|
||||
const { startUpdateScheduler, intervalMs } = require("../update-scheduler");
|
||||
|
||||
const wait = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
let handle = null;
|
||||
afterEach(() => {
|
||||
handle?.stop();
|
||||
handle = null;
|
||||
delete process.env.DASHBOARD_UPDATE_CHECK_INTERVAL_MS;
|
||||
});
|
||||
|
||||
function harness(clients) {
|
||||
const h = { checks: 0, clients, listeners: [] };
|
||||
handle = startUpdateScheduler({
|
||||
broadcast: () => {},
|
||||
getConnectionCount: () => h.clients,
|
||||
onConnection: (fn) => {
|
||||
h.listeners.push(fn);
|
||||
return () => {};
|
||||
},
|
||||
getStatus: async () => {
|
||||
h.checks++;
|
||||
return { update_available: false };
|
||||
},
|
||||
initialDelayMs: 10,
|
||||
});
|
||||
h.connect = () => h.listeners.forEach((fn) => fn());
|
||||
return h;
|
||||
}
|
||||
|
||||
describe("intervalMs", () => {
|
||||
it("defaults to one hour and keeps the override with its floor", () => {
|
||||
delete process.env.DASHBOARD_UPDATE_CHECK_INTERVAL_MS;
|
||||
assert.equal(intervalMs(), 60 * 60 * 1000);
|
||||
process.env.DASHBOARD_UPDATE_CHECK_INTERVAL_MS = "120000";
|
||||
assert.equal(intervalMs(), 120000);
|
||||
process.env.DASHBOARD_UPDATE_CHECK_INTERVAL_MS = "1000";
|
||||
assert.equal(intervalMs(), 60 * 60 * 1000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("startUpdateScheduler", () => {
|
||||
it("checks on the first tick when a browser is connected", async () => {
|
||||
const h = harness(1);
|
||||
await wait(40);
|
||||
assert.equal(h.checks, 1);
|
||||
});
|
||||
|
||||
it("skips the tick with no browser, then checks once when one connects", async () => {
|
||||
const h = harness(0);
|
||||
await wait(40);
|
||||
assert.equal(h.checks, 0);
|
||||
h.clients = 1;
|
||||
h.connect();
|
||||
h.connect(); // a second tab in the same window: still one check
|
||||
await wait(3_300);
|
||||
assert.equal(h.checks, 1);
|
||||
h.connect(); // fresh check: no catch-up
|
||||
await wait(3_300);
|
||||
assert.equal(h.checks, 1);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user