perf: hourly update check only with a browser open; opt-in retention, incremental auto-vacuum and Compact
- update-scheduler: default interval 1 h (DASHBOARD_UPDATE_CHECK_INTERVAL_MS and its 60 s floor kept); a scheduled tick with no /ws client is skipped, and the next browser to connect gets one catch-up check when the last one is stale (websocket.js onConnection). Manual check unchanged. - db-retention.js: shared session purge (used by POST /api/settings/cleanup too), opt-in AGENTHUB_RETENTION_DAYS applied by the hourly maintenance in 25-session transactions with yields (max 5000/run), bounded incremental_vacuum when the freelist is large, and a manual Compact (VACUUM in a worker, converting to auto_vacuum=INCREMENTAL). - db.js: new databases are INCREMENTAL; an existing one up to 32 MB is converted at startup. - GET/PUT /api/settings/retention, POST /api/settings/compact (confirm: true); Settings → Data → Session cleanup gets the Automatic cleanup control and Compact now (en + vi).
This commit is contained in:
+66
-41
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* @file Express router for settings-related endpoints, providing system info, database statistics, hook status, and operations to clear data, re-import sessions, reinstall hooks, reset pricing, export data, and perform cleanup of stale sessions, plus the privacy controls (secret redaction, ignored folders), the push-ingest token and the daily database backup (server/lib/db-backup.js). This allows the frontend to manage and maintain the agent monitoring system effectively.
|
||||
* @file Express router for settings-related endpoints, providing system info, database statistics, hook status, and operations to clear data, re-import sessions, reinstall hooks, reset pricing, export data, and perform cleanup of stale sessions, plus the privacy controls (secret redaction, ignored folders), the push-ingest token, the daily database backup (server/lib/db-backup.js) and the opt-in retention + Compact (server/lib/db-retention.js). This allows the frontend to manage and maintain the agent monitoring system effectively.
|
||||
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
*/
|
||||
|
||||
@@ -28,6 +28,7 @@ const { getCodexHome, setCodexHome } = require("../lib/codex-home");
|
||||
const { envOwned, applySetting } = require("../lib/settings-env");
|
||||
const privacy = require("../lib/privacy");
|
||||
const dbBackup = require("../lib/db-backup");
|
||||
const retention = require("../lib/db-retention");
|
||||
const pushToken = require("../lib/push-token");
|
||||
const pricingSync = require("../lib/pricing-sync");
|
||||
const {
|
||||
@@ -678,10 +679,66 @@ function pushTokenWrite(fn) {
|
||||
router.post("/push-token", sameOriginGuard, pushTokenWrite(pushToken.rotatePushToken));
|
||||
router.delete("/push-token", sameOriginGuard, pushTokenWrite(pushToken.revokePushToken));
|
||||
|
||||
// GET/PUT /api/settings/retention, POST /api/settings/compact — database size
|
||||
// upkeep (server/lib/db-retention.js). Retention is OFF by default: `days` 0
|
||||
// turns it off, 1–3650 makes the hourly maintenance purge finished sessions
|
||||
// older than that, the manual purge's rule. It persists to the settings file;
|
||||
// one the environment set is `env_owned` and refused. Compact (a full VACUUM
|
||||
// in a worker) answers 202 and the page polls GET while `compacting`.
|
||||
function retentionState() {
|
||||
let space = null;
|
||||
try {
|
||||
space = retention.spaceInfo(db);
|
||||
} catch {
|
||||
/* pragma unavailable: the card shows no numbers */
|
||||
}
|
||||
return {
|
||||
...retention.retentionConfig(),
|
||||
max_days: retention.MAX_DAYS,
|
||||
env_owned: { days: envOwned.has(retention.KEYS.days) },
|
||||
last_run: retention.lastMaintenance(),
|
||||
space,
|
||||
db_size: getDbSize(),
|
||||
compacting: retention.isCompacting(),
|
||||
last_compact: retention.lastCompactResult(),
|
||||
};
|
||||
}
|
||||
router.get("/retention", (_req, res) => res.json(retentionState()));
|
||||
router.put("/retention", sameOriginGuard, (req, res) => {
|
||||
const days = req.body?.days;
|
||||
if (!Number.isInteger(days) || days < 0 || days > retention.MAX_DAYS) {
|
||||
return res.status(400).json({
|
||||
error: {
|
||||
code: "INVALID_INPUT",
|
||||
message: `days must be a whole number from 0 (off) to ${retention.MAX_DAYS}`,
|
||||
},
|
||||
});
|
||||
}
|
||||
if (envOwned.has(retention.KEYS.days)) {
|
||||
return res.status(409).json({
|
||||
error: { code: "EENVOWNED", message: `${retention.KEYS.days} is set by the environment` },
|
||||
});
|
||||
}
|
||||
try {
|
||||
applySetting(retention.KEYS.days, days === 0 ? "" : String(days));
|
||||
} catch (err) {
|
||||
return res.status(500).json({ error: { code: "ESAVE", message: err.message } });
|
||||
}
|
||||
res.json({ ok: true, ...retentionState() });
|
||||
});
|
||||
router.post("/compact", sameOriginGuard, (req, res) => {
|
||||
if (req.body?.confirm !== true) {
|
||||
return res.status(400).json({
|
||||
error: { code: "ECONFIRM", message: "compacting needs confirm: true" },
|
||||
});
|
||||
}
|
||||
retention.runCompact().catch(() => {}); // the outcome is `last_compact` in GET
|
||||
res.status(202).json({ ok: true, ...retentionState() });
|
||||
});
|
||||
|
||||
// POST /api/settings/cleanup — abandon stale sessions, purge old data.
|
||||
// `purge_days` is a whole number ≥ 1 (0.1 days would purge almost everything);
|
||||
// `dry_run: true` answers the same counts and changes nothing.
|
||||
const PURGE_CHUNK = 500;
|
||||
router.post("/cleanup", (req, res) => {
|
||||
const { abandon_hours, purge_days } = req.body;
|
||||
const dryRun = req.body.dry_run === true;
|
||||
@@ -739,11 +796,7 @@ router.post("/cleanup", (req, res) => {
|
||||
if (purge_days && typeof purge_days === "number" && purge_days > 0) {
|
||||
const cutoff = new Date(Date.now() - purge_days * 86400 * 1000).toISOString();
|
||||
// Only purge completed/error/abandoned sessions, never active
|
||||
const toDelete = db
|
||||
.prepare(
|
||||
"SELECT id FROM sessions WHERE status IN ('completed','error','abandoned') AND started_at < ?"
|
||||
)
|
||||
.all(cutoff);
|
||||
const toDelete = retention.purgeableBefore(db, cutoff);
|
||||
|
||||
if (dryRun) {
|
||||
// Exactly the ids the real run deletes: those already purgeable, plus
|
||||
@@ -752,43 +805,15 @@ router.post("/cleanup", (req, res) => {
|
||||
...toDelete.map((r) => r.id),
|
||||
...wouldAbandon.filter((r) => r.started_at < cutoff).map((r) => r.id),
|
||||
];
|
||||
for (let i = 0; i < ids.length; i += PURGE_CHUNK) {
|
||||
const chunk = ids.slice(i, i + PURGE_CHUNK);
|
||||
const placeholders = chunk.map(() => "?").join(",");
|
||||
result.purged_events += db
|
||||
.prepare(`SELECT COUNT(*) AS n FROM events WHERE session_id IN (${placeholders})`)
|
||||
.get(...chunk).n;
|
||||
result.purged_agents += db
|
||||
.prepare(`SELECT COUNT(*) AS n FROM agents WHERE session_id IN (${placeholders})`)
|
||||
.get(...chunk).n;
|
||||
}
|
||||
result.purged_sessions = ids.length;
|
||||
Object.assign(result, retention.countSessionIds(db, ids));
|
||||
return res.json({ ok: true, ...result });
|
||||
}
|
||||
|
||||
// One transaction, in chunks: a single IN (...) list of every id would
|
||||
// pass SQLite's bound-variable limit on a large history.
|
||||
db.transaction(() => {
|
||||
const ids = toDelete.map((r) => r.id);
|
||||
for (let i = 0; i < ids.length; i += PURGE_CHUNK) {
|
||||
const chunk = ids.slice(i, i + PURGE_CHUNK);
|
||||
const placeholders = chunk.map(() => "?").join(",");
|
||||
// Cascading deletes handle agents/events, but token_usage FK might not cascade on all setups
|
||||
result.purged_events += db
|
||||
.prepare(`DELETE FROM events WHERE session_id IN (${placeholders})`)
|
||||
.run(...chunk).changes;
|
||||
result.purged_agents += db
|
||||
.prepare(`DELETE FROM agents WHERE session_id IN (${placeholders})`)
|
||||
.run(...chunk).changes;
|
||||
db.prepare(`DELETE FROM token_usage WHERE session_id IN (${placeholders})`).run(...chunk);
|
||||
db.prepare(`DELETE FROM token_usage_hourly WHERE session_id IN (${placeholders})`).run(
|
||||
...chunk
|
||||
);
|
||||
forgetHourly(chunk);
|
||||
db.prepare(`DELETE FROM sessions WHERE id IN (${placeholders})`).run(...chunk);
|
||||
}
|
||||
result.purged_sessions = ids.length;
|
||||
})();
|
||||
// One transaction; the deletion itself is shared with the automatic
|
||||
// retention (server/lib/db-retention.js).
|
||||
const ids = toDelete.map((r) => r.id);
|
||||
const purged = db.transaction(() => retention.purgeSessionIds(db, ids))();
|
||||
Object.assign(result, purged, { purged_sessions: ids.length });
|
||||
}
|
||||
|
||||
res.json({ ok: true, ...result });
|
||||
|
||||
Reference in New Issue
Block a user