From e07993c536d2f5a758c074a4eec0ca0fc1723530 Mon Sep 17 00:00:00 2001 From: nntrivi2001 Date: Fri, 16 Oct 2026 10:20:38 +0700 Subject: [PATCH] perf: hourly update check only with a browser open; opt-in retention, incremental auto-vacuum and Compact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - update-scheduler: default interval 1 h (DASHBOARD_UPDATE_CHECK_INTERVAL_MS and its 60 s floor kept); a scheduled tick with no /ws client is skipped, and the next browser to connect gets one catch-up check when the last one is stale (websocket.js onConnection). Manual check unchanged. - db-retention.js: shared session purge (used by POST /api/settings/cleanup too), opt-in AGENTHUB_RETENTION_DAYS applied by the hourly maintenance in 25-session transactions with yields (max 5000/run), bounded incremental_vacuum when the freelist is large, and a manual Compact (VACUUM in a worker, converting to auto_vacuum=INCREMENTAL). - db.js: new databases are INCREMENTAL; an existing one up to 32 MB is converted at startup. - GET/PUT /api/settings/retention, POST /api/settings/compact (confirm: true); Settings → Data → Session cleanup gets the Automatic cleanup control and Compact now (en + vi). --- .env.example | 14 +- ARCHITECTURE.md | 2 +- .../components/settings/RetentionSettings.tsx | 180 ++++++++++ client/src/i18n/locales/en/settings.json | 17 + client/src/i18n/locales/vi/settings.json | 17 + client/src/lib/api.ts | 47 +++ client/src/pages/Settings.tsx | 5 +- client/src/pages/__tests__/Settings.test.tsx | 12 + .../screens.snapshot.test.tsx.snap | 102 ++++++ .../pages/__tests__/screens.snapshot.test.tsx | 12 + docs/API.md | 10 + docs/DATABASE.md | 2 +- server/__tests__/db-retention.test.js | 175 +++++++++ server/__tests__/update-scheduler.test.js | 76 ++++ server/db.js | 36 +- server/index.js | 10 +- server/lib/db-retention.js | 337 ++++++++++++++++++ server/openapi-extra/misc.js | 114 ++++++ server/routes/settings.js | 107 +++--- server/update-scheduler.js | 88 ++++- server/websocket.js | 18 + 21 files changed, 1323 insertions(+), 58 deletions(-) create mode 100644 client/src/components/settings/RetentionSettings.tsx create mode 100644 server/__tests__/db-retention.test.js create mode 100644 server/__tests__/update-scheduler.test.js create mode 100644 server/lib/db-retention.js diff --git a/.env.example b/.env.example index f9264920..fc93639a 100644 --- a/.env.example +++ b/.env.example @@ -72,9 +72,21 @@ # DASHBOARD_SESSION_SYNC_MS=30000 # Hourly database upkeep: a non-blocking WAL checkpoint (the WAL then shrinks -# to 64 MB) and PRAGMA optimize. 0 turns it off. Never VACUUMs. +# to 64 MB), PRAGMA optimize, the opt-in retention below and a bounded +# incremental_vacuum. 0 turns it off. Never a full VACUUM (that is Settings → +# Data → Compact, on a click). # DASHBOARD_DB_MAINTAIN_MS=3600000 +# Automatic retention (OFF unless set): finished sessions that started more +# than N days ago (1–3650) are purged by the hourly upkeep. Usually set from +# Settings → Data → Session cleanup, which writes it to the settings file. +# AGENTHUB_RETENTION_DAYS=90 + +# Update check interval (ms, default 3600000 = 1 h, minimum 60000). A tick +# with no browser connected is skipped; the next browser to connect gets one. +# DASHBOARD_UPDATE_CHECK=0 turns the scheduled check off. +# DASHBOARD_UPDATE_CHECK_INTERVAL_MS=3600000 + # Poll interval (ms) for Antigravity CLI sessions and tokens, read from agy's # own ~/.gemini/antigravity-cli/conversations/*.db (0 disables; default: 15000). # DASHBOARD_ANTIGRAVITY_POLL_MS=15000 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0917ae55..a5e064b1 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2225,7 +2225,7 @@ sequenceDiagram | Component | Responsibility | | --- | --- | | **`server/lib/update-check.js`** | `getUpdatesStatus(root?, { skipFetch?, latestRelease? })` + `applyUpdate(root?)`. Runs every git call via `execFile` (no shell, 10s–120s timeouts) and reads the newest release over `fetch` with an `AbortSignal.timeout` — `fetchLatestRelease` NEVER throws, resolving to `{ok:false, reason}` (`no_remote`, `remote_not_http`, `no_releases`, `unreachable`, `bad_payload`, `http_`) so a status poll degrades instead of erroring. `releaseApiUrl` derives the endpoint from a repository URL (https, scp-style and `ssh://` forms) and returns null for anything not owner/repo shaped, which is what keeps local-path remotes off the network entirely. `fetchLatestRelease` finds that URL via the `origin` remote; `fetchReleaseFromUrl` takes one directly, which is how the **git-free plugin path** works: `pluginMarketplaceUrl(mkt, plugin)` reads `known_marketplaces.json` → the marketplace's `.claude-plugin/marketplace.json` → the entry's `source.url`, returning null (never a guessed host) for a `github` `owner/repo` shorthand and never throwing. **Order matters:** `detectPluginInstall` runs FIRST in `getUpdatesStatus`, ahead of the `.git` probe — a plugin cache unpacked from an archive has no `.git`, and probing first is the 1.5.1 bug that made `update_available` permanently false. `compareVersions` orders numeric triples and strips a leading `v`, since this repo carries both `v1.5.0` and `1.4.6`. `isShallowClone` gates every commit-distance calculation; `detectPluginInstall` reads the marketplace and plugin name out of the cache path. **Branch- and fork-aware** as before: resolves `/master`/`/main`/`/HEAD`, reads the current branch and its tracked upstream, and shapes `manual_command` per situation — the `claude plugin ...` commands for `plugin_managed`; `git fetch --tags && git merge --ff-only ` for `release_available`; `git pull --ff-only` when the branch tracks the canonical ref; `git fetch && git merge --ff-only ` for forks; `git fetch ` only on a feature branch or detached HEAD. Non-git installs, missing remotes, fetch failures, and unresolvable upstream refs are returned as soft payloads — never throws. | -| **`server/update-scheduler.js`** | Ticks the lib every `DASHBOARD_UPDATE_CHECK_INTERVAL_MS` (default 300 000, floor 60 000). First tick is scheduled 8s after server start with `.unref()` so it doesn't block shutdown. Broadcasts only when the fingerprint `{update_available, remote_sha, commits_behind, fetch_error, manual_command}` changes — `manual_command` is included so situation transitions (e.g. user switches branches, or adds an `upstream` remote) trigger a re-broadcast even when the SHA and commit count are unchanged. Emits a framed message to stdout on "up-to-date → behind" transitions, and only suggests "restart the dashboard" when the printed command actually rewrites the working tree. `DASHBOARD_UPDATE_CHECK=0\|false\|off` disables the scheduler entirely. | +| **`server/update-scheduler.js`** | Ticks the lib every `DASHBOARD_UPDATE_CHECK_INTERVAL_MS` (default 3 600 000 = 1 h, floor 60 000). A tick with no `/ws` browser connected (`getConnectionCount`) is skipped, and the first browser to connect afterwards (`onConnection` in `server/websocket.js`) gets one check 3 s later if the last one is older than the interval; the manual `POST /api/updates/check` is unaffected. First tick is scheduled 8s after server start with `.unref()` so it doesn't block shutdown. Broadcasts only when the fingerprint `{update_available, remote_sha, commits_behind, fetch_error, manual_command}` changes — `manual_command` is included so situation transitions (e.g. user switches branches, or adds an `upstream` remote) trigger a re-broadcast even when the SHA and commit count are unchanged. Emits a framed message to stdout on "up-to-date → behind" transitions, and only suggests "restart the dashboard" when the printed command actually rewrites the working tree. `DASHBOARD_UPDATE_CHECK=0\|false\|off` disables the scheduler entirely. | | **`server/routes/updates.js`** | Three endpoints: `GET /status` (read-only check), `POST /check` (check + broadcast), `POST /apply`. `/apply` has **two modes**, chosen by how this copy was installed, never by the request body: **checkout** — fast-forward onto the release tag, `npm run setup`, `npm run build` in production (both through the bootstrap's `npmCommand` with `NODE_ENV` dropped — `npmInvocation` in `server/lib/update-check.js` — since the server's `NODE_ENV=production` makes npm omit the client's devDependency build toolchain, and a bare `npm.cmd` is EINVAL under `execFile` on Windows), then `scheduleRestart()`; **plugin** — `applyUpdate`'s `plugin_managed` refusal is caught and `schedulePluginUpdate()` runs instead, answering `200 {applied:true, mode:"plugin"}`. The plugin branch fires only when there is genuinely something to pull (`update_available` plus a marketplace and plugin name parsed from the cache path) — a `plugin_managed` refusal with nothing newer published stays a 409 rather than spawning a chain that would stop the server for nothing. Otherwise 409 `UPDATE_NOT_APPLICABLE` with `reason` `"up_to_date"`, `"not_fast_forwardable"`, or `"plugin_managed"`. Both modes write the response BEFORE anything stops the process. Mounted under `/api/*`, so all three ride the default loopback bind as their trust boundary and are gated by `DASHBOARD_TOKEN` when one is set. | | **`server/lib/plugin-update.js`** + **`scripts/plugin-update-helper.js`** | The plugin-cache update chain. The lib only spawns the helper detached (cwd = the runtime dir, **not** the plugin root, which is about to be superseded) and clears any previous verdict first so a stale success can't be read back as this run's result. All work is in the helper because its last step SIGTERMs the server that asked for the update — the same split as `self-restart.js`, one step further. The helper re-validates the marketplace and plugin names against `/^[A-Za-z0-9][\w.-]*$/` (they reach an argv across a process boundary), resolves `claude` on PATH or at `~/.local/bin/claude` (a hook-spawned server does not inherit the login shell's PATH), picks the newest cache directory that isn't the old root and does contain `scripts/plugin-bootstrap.js`, and records **every** outcome — including an unexpected throw — to `plugin-update.json` under `~/.claude/agenthub/runtime/`, outside the cache so it survives the swap. Steps: `validate`, `resolve_claude`, `marketplace_update`, `plugin_update`, `already_current` (`ok:true`; nothing was pulled and the old server was left running), `bootstrap`, `done`. `readLastPluginUpdate()` surfaces it as `last_plugin_update` on the status payload — without it a failed chain is indistinguishable from a slow one. | | **`UpdateNotifier.tsx`** | Modal. Hydrates from `api.updates.status()` on mount and mirrors the payload back into the local `eventBus` so the Sidebar can listen without a second git fetch. Subscribes to `update_status` WS frames for ongoing sync. Keeps the dismissal key in `localStorage` (`agent-monitor-update-dismissed-sha`) and in React state — the **release tag** when one was read, else the upstream SHA, so a newer release re-opens the prompt while a dismissed one stays dismissed; a window event `dashboard:reset-update-dismissal` from the Sidebar clears both. ESC / backdrop click dismisses. | diff --git a/client/src/components/settings/RetentionSettings.tsx b/client/src/components/settings/RetentionSettings.tsx new file mode 100644 index 00000000..5af6a006 --- /dev/null +++ b/client/src/components/settings/RetentionSettings.tsx @@ -0,0 +1,180 @@ +/** + * @file RetentionSettings.tsx + * @description Settings → Data → Session cleanup, under the manual purge: the + * OPT-IN automatic cleanup (`AGENTHUB_RETENTION_DAYS`, 0 = off, the default) + * that the hourly maintenance applies, the database's size and free space, and + * **Compact now** — a full VACUUM behind a confirm, polled through + * `GET /api/settings/retention` while it runs, then toasted. Backed by + * `server/lib/db-retention.js`; a value the environment set is read-only. + * @author Nguyễn Ngọc Trí Vĩ + */ + +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { AlertTriangle, Minimize2, RefreshCw } from "lucide-react"; +import { api } from "../../lib/api"; +import type { RetentionState } from "../../lib/api"; +import { confirmDialog } from "../../lib/confirm"; +import { formatBytes, formatDateTime } from "../../lib/format"; +import { toast } from "../../lib/toast"; + +export function RetentionSettings() { + const { t } = useTranslation("settings"); + const [state, setState] = useState(null); + const [days, setDays] = useState("0"); + const [saving, setSaving] = useState(false); + + useEffect(() => { + api.settings.retention + .get() + .then((s) => { + setState(s); + setDays(String(s.days)); + }) + .catch(() => undefined); + }, []); + + // A Compact takes a while on a large database: poll until it ends. + const compacting = !!state?.compacting; + useEffect(() => { + if (!compacting) return; + const id = window.setInterval(() => { + api.settings.retention + .get() + .then((s) => { + setState(s); // the draft keeps its value + if (s.compacting || !s.last_compact) return; + if (s.last_compact.ok) { + toast.success( + t("retention.compactDone", { + before: formatBytes(s.last_compact.size_before ?? 0), + after: formatBytes(s.last_compact.size_after ?? 0), + }) + ); + } else { + toast.error(t("retention.compactFailed", { error: s.last_compact.error })); + } + }) + .catch(() => undefined); + }, 2000); + return () => window.clearInterval(id); + }, [compacting, t]); + + const max = state?.max_days ?? 3650; + const n = Number(days); + const valid = days.trim() !== "" && Number.isInteger(n) && n >= 0 && n <= max; + const changed = state !== null && valid && n !== state.days; + const owned = !!state?.env_owned.days; + + async function save() { + setSaving(true); + try { + const s = await api.settings.retention.save(n); + setState(s); + setDays(String(s.days)); + toast.success(t("retention.saved")); + } catch (err) { + toast.error(err); + } finally { + setSaving(false); + } + } + + async function compact() { + const ok = await confirmDialog({ + title: t("retention.compactConfirmTitle"), + message: t("retention.compactConfirmBody"), + confirmLabel: t("retention.compact"), + }); + if (!ok) return; + try { + setState(await api.settings.retention.compact()); + } catch (err) { + toast.error(err); + } + } + + const purged = state?.last_run?.retention?.purged_sessions; + + return ( +
+
+

{t("retention.title")}

+

{t("retention.description")}

+
+
+
+ +
+ setDays(e.target.value)} + aria-invalid={!valid || undefined} + aria-describedby={!valid ? "retention-days-error" : undefined} + className="input w-20 text-sm text-right font-mono" + /> + {t("common:days")} +
+ {!valid && ( +

+ {t("retention.invalid", { max })} +

+ )} +
+ +
+

+ {state && state.days > 0 ? t("retention.on", { days: state.days }) : t("retention.off")} + {state?.last_run && purged !== undefined && ( + <> {t("retention.lastRun", { time: formatDateTime(state.last_run.at), n: purged })} + )} +

+ {owned && ( +

+ + {t("privacy.envOwned")} +

+ )} +
+ + {state?.space && ( + + {t("retention.space", { + size: formatBytes(state.db_size), + free: formatBytes(state.space.free_bytes), + mode: state.space.auto_vacuum, + })} + + )} +
+
+ ); +} diff --git a/client/src/i18n/locales/en/settings.json b/client/src/i18n/locales/en/settings.json index 476fa4fe..9e60ed4b 100644 --- a/client/src/i18n/locales/en/settings.json +++ b/client/src/i18n/locales/en/settings.json @@ -1217,6 +1217,23 @@ "ignoreHint": "One absolute path per line. Hooks from these folders, and anything under them, are dropped whole; history import and the Codex reader skip their sessions. AGENTHUB_IGNORE_CWD sets the same list.", "envOwned": "A value here is set by the environment the dashboard started in, and cannot be changed from this page." }, + "retention": { + "title": "Automatic cleanup", + "description": "Off by default. When on, the hourly maintenance deletes finished sessions (completed, error, abandoned) that started more than this many days ago — with their events, agents and token usage — a little at a time.", + "days": "Delete sessions older than", + "off": "Automatic cleanup is off.", + "on": "Sessions older than {{days}} days are deleted every hour.", + "invalid": "A whole number from 0 (off) to {{max}}", + "saved": "Automatic cleanup saved", + "lastRun": "Last run {{time}}: {{n}} session(s) deleted", + "space": "Database {{size}} · {{free}} free inside it · auto-vacuum: {{mode}}", + "compact": "Compact now", + "compacting": "Compacting…", + "compactConfirmTitle": "Compact the database?", + "compactConfirmBody": "This rewrites the whole database file to give free space back to the disk and turns on incremental auto-vacuum. On a large database it takes a while, and new hook events wait until it finishes.", + "compactDone": "Database compacted: {{before}} → {{after}}", + "compactFailed": "Compact failed: {{error}}" + }, "backup": { "title": "Daily backup", "description": "A full copy of the database once a day, into a folder you choose — best on another disk, so a failed drive does not take the backups with it.", diff --git a/client/src/i18n/locales/vi/settings.json b/client/src/i18n/locales/vi/settings.json index fe387658..b8ae6732 100644 --- a/client/src/i18n/locales/vi/settings.json +++ b/client/src/i18n/locales/vi/settings.json @@ -1217,6 +1217,23 @@ "ignoreHint": "Mỗi dòng một đường dẫn tuyệt đối. Hook từ các thư mục này, và mọi thứ bên trong, bị bỏ hoàn toàn; việc nhập lịch sử và bộ đọc Codex cũng bỏ qua phiên của chúng. AGENTHUB_IGNORE_CWD đặt cùng danh sách này.", "envOwned": "Một giá trị ở đây do môi trường khởi động dashboard đặt, và không thể đổi từ trang này." }, + "retention": { + "title": "Tự động dọn dẹp", + "description": "Mặc định tắt. Khi bật, bảo trì hằng giờ xóa các phiên đã kết thúc (hoàn tất, lỗi, bỏ dở) bắt đầu cách đây hơn số ngày này — cùng sự kiện, agent và token của chúng — từng ít một.", + "days": "Xóa phiên cũ hơn", + "off": "Tự động dọn dẹp đang tắt.", + "on": "Phiên cũ hơn {{days}} ngày được xóa mỗi giờ.", + "invalid": "Số nguyên từ 0 (tắt) đến {{max}}", + "saved": "Đã lưu tự động dọn dẹp", + "lastRun": "Lần chạy gần nhất {{time}}: đã xóa {{n}} phiên", + "space": "Cơ sở dữ liệu {{size}} · {{free}} trống bên trong · auto-vacuum: {{mode}}", + "compact": "Nén ngay", + "compacting": "Đang nén…", + "compactConfirmTitle": "Nén cơ sở dữ liệu?", + "compactConfirmBody": "Thao tác này ghi lại toàn bộ tệp cơ sở dữ liệu để trả dung lượng trống cho ổ đĩa và bật auto-vacuum tăng dần. Với cơ sở dữ liệu lớn sẽ mất một lúc, và sự kiện hook mới phải chờ đến khi xong.", + "compactDone": "Đã nén cơ sở dữ liệu: {{before}} → {{after}}", + "compactFailed": "Nén thất bại: {{error}}" + }, "backup": { "title": "Backup hằng ngày", "description": "Mỗi ngày một bản sao đầy đủ của cơ sở dữ liệu vào thư mục bạn chọn — tốt nhất là trên một ổ đĩa khác, để hỏng một ổ không mất luôn bản backup.", diff --git a/client/src/lib/api.ts b/client/src/lib/api.ts index 3fb0ae6e..f1355930 100644 --- a/client/src/lib/api.ts +++ b/client/src/lib/api.ts @@ -1649,6 +1649,23 @@ export const api = { /** POST /api/settings/backup/run — starts one; poll `get` while `running` */ run: () => request("/settings/backup/run", { method: "POST" }), }, + /** Opt-in automatic retention and Compact (`server/lib/db-retention.js`). */ + retention: { + /** GET /api/settings/retention */ + get: () => request("/settings/retention"), + /** PUT /api/settings/retention — `days` 0 turns it off */ + save: (days: number) => + request("/settings/retention", { + method: "PUT", + body: JSON.stringify({ days }), + }), + /** POST /api/settings/compact — a full VACUUM; poll `get` while `compacting` */ + compact: () => + request("/settings/compact", { + method: "POST", + body: JSON.stringify({ confirm: true }), + }), + }, /** The bearer token for `POST /api/hooks/ingest-batch`. Only its hash is * stored: `rotate` returns the new token this once, `get` never does. */ pushToken: { @@ -5461,6 +5478,36 @@ export interface BackupState { env_owned: { dir: boolean; hour: boolean; keep: boolean }; } +/** GET /api/settings/retention. `days` 0 = automatic retention off; + * `last_run`/`last_compact` are this server process's own. */ +export interface RetentionState { + days: number; + max_days: number; + env_owned: { days: boolean }; + last_run: { + at: string; + retention?: { purged_sessions: number; purged_events: number; error?: string }; + reclaim?: { reclaimed_pages: number; error?: string }; + } | null; + space: { + auto_vacuum: "none" | "full" | "incremental"; + page_size: number; + page_count: number; + freelist_pages: number; + free_bytes: number; + } | null; + db_size: number; + compacting: boolean; + last_compact: { + at: string; + ok: boolean; + size_before?: number; + size_after?: number; + ms?: number; + error?: string; + } | null; +} + /** GET /api/settings/push-token — whether push ingest is on; never the token. */ export interface PushTokenStatus { enabled: boolean; diff --git a/client/src/pages/Settings.tsx b/client/src/pages/Settings.tsx index 86f1b02c..1a6cf034 100644 --- a/client/src/pages/Settings.tsx +++ b/client/src/pages/Settings.tsx @@ -1,6 +1,6 @@ /** * @file Settings.tsx - * @description The install-wide settings, as three pages of the Settings nav (`SettingsHub.tsx`): `section="data"` — total cost, database overview, workspace housekeeping (dry run first, confirm before a real run), import, remote sources, push ingest (`components/settings/PushIngestSettings.tsx`), privacy (`components/settings/PrivacySettings.tsx`), the daily database backup (`components/settings/BackupSettings.tsx`), export, this browser's preferences backup, session cleanup (whole days ≥ 1, a dry-run count in its confirm) and the danger zone (typed confirmation) —, `section="about"` (the System page) — the start-at-login switch and Quit (`AutostartSettings`), then the server process (release, uptime, platform, CPU, memory, live connections), how AgentHub is installed (the update check's cached status) and the database file — and `section="notifications"` — browser notifications (per-browser, with a reset) and alerts (server-wide). Server info (`/api/settings/info`) is read only by Data and About, refreshed on activity at most every 10 s. Language, theme and the pet live on the Appearance page (`components/settings/AppearanceSettings.tsx`); per-tool settings live in that tool's page under `/settings/agents/`. + * @description The install-wide settings, as three pages of the Settings nav (`SettingsHub.tsx`): `section="data"` — total cost, database overview, workspace housekeeping (dry run first, confirm before a real run), import, remote sources, push ingest (`components/settings/PushIngestSettings.tsx`), privacy (`components/settings/PrivacySettings.tsx`), the daily database backup (`components/settings/BackupSettings.tsx`), export, this browser's preferences backup, session cleanup (whole days ≥ 1, a dry-run count in its confirm; under it the opt-in automatic cleanup and Compact, `components/settings/RetentionSettings.tsx`) and the danger zone (typed confirmation) —, `section="about"` (the System page) — the start-at-login switch and Quit (`AutostartSettings`), then the server process (release, uptime, platform, CPU, memory, live connections), how AgentHub is installed (the update check's cached status) and the database file — and `section="notifications"` — browser notifications (per-browser, with a reset) and alerts (server-wide). Server info (`/api/settings/info`) is read only by Data and About, refreshed on activity at most every 10 s. Language, theme and the pet live on the Appearance page (`components/settings/AppearanceSettings.tsx`); per-tool settings live in that tool's page under `/settings/agents/`. * @author Nguyễn Ngọc Trí Vĩ */ /* ============================================================================= @@ -103,6 +103,7 @@ import { RemoteSources } from "../components/RemoteSources"; import { PushIngestSettings } from "../components/settings/PushIngestSettings"; import { PrivacySettings } from "../components/settings/PrivacySettings"; import { BackupSettings } from "../components/settings/BackupSettings"; +import { RetentionSettings } from "../components/settings/RetentionSettings"; import { AutostartSettings } from "../components/settings/AutostartSettings"; import { usePermissions } from "../components/AuthGate"; import { Skeleton } from "../components/Skeleton"; @@ -1054,6 +1055,8 @@ export function Settings({ {actionBanner(["cleanup"])} + + {/* ─── THIS BROWSER'S PREFERENCES ─── */} diff --git a/client/src/pages/__tests__/Settings.test.tsx b/client/src/pages/__tests__/Settings.test.tsx index 2da66d7e..7cfa1db7 100644 --- a/client/src/pages/__tests__/Settings.test.tsx +++ b/client/src/pages/__tests__/Settings.test.tsx @@ -77,6 +77,18 @@ vi.mock("../../lib/api", async (importOriginal) => { env_owned: { dir: false, hour: false, keep: false }, })), }, + retention: { + get: vi.fn(async () => ({ + days: 0, + max_days: 3650, + env_owned: { days: false }, + last_run: null, + space: null, + db_size: 0, + compacting: false, + last_compact: null, + })), + }, }, }, }; diff --git a/client/src/pages/__tests__/__snapshots__/screens.snapshot.test.tsx.snap b/client/src/pages/__tests__/__snapshots__/screens.snapshot.test.tsx.snap index 80adb12a..710285d1 100644 --- a/client/src/pages/__tests__/__snapshots__/screens.snapshot.test.tsx.snap +++ b/client/src/pages/__tests__/__snapshots__/screens.snapshot.test.tsx.snap @@ -9357,6 +9357,108 @@ exit 0 Run Cleanup +
+
+

+ Automatic cleanup +

+

+ Off by default. When on, the hourly maintenance deletes finished sessions (completed, error, abandoned) that started more than this many days ago — with their events, agents and token usage — a little at a time. +

+
+
+
+ +
+ + + days + +
+
+ +
+

+ Automatic cleanup is off. +

+
+ +
+
{ env_owned: { dir: false, hour: false, keep: false }, }), }, + retention: { + get: r({ + days: 0, + max_days: 3650, + env_owned: { days: false }, + last_run: null, + space: null, + db_size: 0, + compacting: false, + last_compact: null, + }), + }, exportData: () => "/api/settings/export", cleanup: r({ ok: true, diff --git a/docs/API.md b/docs/API.md index a3219ed4..7b4b6297 100644 --- a/docs/API.md +++ b/docs/API.md @@ -3057,6 +3057,16 @@ POST /api/settings/backup/run 202 — started (or already running); poll GE Settings → Data → **Daily backup** (`server/lib/db-backup.js`). Once a day, at `hour` (0–23, the server's local time, default 2), the database is copied to `dir` as `agenthub-YYYYMMDD-HHMMSS.db`: `VACUUM INTO` on a read-only connection in a worker thread, so the copy is a consistent snapshot (writes from another dashboard on the same database never restart it), compacted, and the event loop keeps serving while it runs (≈30 s for a 1.25 GB database). It is written as `.partial` and renamed when complete. Beyond `keep` (1–365, default 7) the oldest are deleted — only regular files matching that exact name; nothing else in the folder, and no symlink, is ever touched. A day missed while the server was down runs at the next start; a failed run is retried the next day or by **Back up now**, never every minute. The three values persist to the settings file as `AGENTHUB_BACKUP_DIR` / `AGENTHUB_BACKUP_HOUR` / `AGENTHUB_BACKUP_KEEP`; one the environment set is `env_owned` (`409 EENVOWNED`). Writes are same-origin, and admin-only in server mode. `400 INVALID_PATH` for a relative, missing, non-directory or unwritable folder, `400 INVALID_INPUT` for an hour or count out of range, `400 ENOBACKUPDIR` for a run with no folder. The scheduler beat is `DASHBOARD_BACKUP_TICK_MS` (default 60000, `0` = off). Restoring is manual, on purpose: `agenthub stop`, delete `dashboard.db-wal` and `dashboard.db-shm`, copy a backup over `dashboard.db` (its path is on Settings → System), `agenthub start`. + +### Retention and Compact + +``` +GET /api/settings/retention { days, max_days, env_owned: { days }, last_run, space: { auto_vacuum, page_size, page_count, freelist_pages, free_bytes }, db_size, compacting, last_compact } +PUT /api/settings/retention { days } → the same state; 0 turns it off (the default) +POST /api/settings/compact { confirm: true } → 202 — started (or already running); poll GET while `compacting` +``` + +Settings → Data → **Session cleanup** (`server/lib/db-retention.js`). Automatic retention is OFF unless `days` (1–3650, `AGENTHUB_RETENTION_DAYS` in the settings file) is set; nothing is deleted by default. When set, the hourly maintenance (`DASHBOARD_DB_MAINTAIN_MS`) purges finished (completed/error/abandoned) sessions that started more than `days` ago — the same rule and the same deletion (`purgeSessionIds`: events, agents, token usage, the session) as `POST /api/settings/cleanup` with `purge_days` — 25 sessions per transaction with a yield between, at most 5000 per run. A new database is created with `auto_vacuum = INCREMENTAL`, an existing one up to 32 MB is converted by a VACUUM at startup; the same hourly run then returns free pages to the OS (`incremental_vacuum`, in 1024-page slices, at most 65536 pages per run) once more than 1024 are free. A larger, older database is converted only by **Compact**: a full VACUUM in a worker thread followed by a TRUNCATE checkpoint — it holds the write lock while it runs (tens of seconds on a GB-size file), so hooks wait meanwhile; it runs only on a click. In server mode both writes are an admin's (`/settings` is in `ADMIN_WRITE_PREFIXES`). --- ## WebSocket API diff --git a/docs/DATABASE.md b/docs/DATABASE.md index d094779b..685a8066 100644 --- a/docs/DATABASE.md +++ b/docs/DATABASE.md @@ -915,7 +915,7 @@ worker's own writable connection. SQLite's automatic checkpoint runs inside whic per commit under load. A failed checkpoint switches the automatic one back on. A second process on the same file (another dashboard, the CLI) keeps its own setting. -**Upkeep.** `maintainDatabase()` (exported by `server/db.js`) runs hourly from `server/index.js` (`DASHBOARD_DB_MAINTAIN_MS`, `0` = off): a `wal_checkpoint(PASSIVE)` — it never waits on a reader, such as the other dashboard — after which the next write shrinks the WAL to `journal_size_limit`, and `PRAGMA optimize`. Nothing runs `VACUUM`: it rewrites the whole file under an exclusive lock, so space freed by a cleanup is reused for new rows instead of returned to the disk. +**Upkeep.** `maintainDatabase()` (exported by `server/db.js`) runs hourly from `server/index.js` (`DASHBOARD_DB_MAINTAIN_MS`, `0` = off): a `wal_checkpoint(PASSIVE)` — it never waits on a reader, such as the other dashboard — after which the next write shrinks the WAL to `journal_size_limit`, and `PRAGMA optimize`. Then `runScheduledMaintenance` (`server/lib/db-retention.js`) applies the OPT-IN retention (`AGENTHUB_RETENTION_DAYS`, off by default: finished sessions older than N days, purged 25 per transaction like the manual cleanup) and, when the database is in `auto_vacuum = INCREMENTAL` and more than 1024 pages are free, a bounded `incremental_vacuum` (1024-page slices, ≤ 65536 pages per run) that returns them to the disk. A new database is created INCREMENTAL; an existing one up to 32 MB is converted by one `VACUUM` at startup (`convertAutoVacuum`), a larger one only by Settings → Data → **Compact** (`POST /api/settings/compact`: a full `VACUUM` in a worker thread, write lock held while it runs). No automatic path runs a full `VACUUM` on a large file. See docs/API.md § Retention and Compact. **Growth.** `events.data` is the hook payload TRIMMED by `trim_event_data(?)` (`server/lib/event-payload.js`, a SQL function registered on the connection in `server/db.js`, used by every event `INSERT` and by the duplicate guard): Edit/MultiEdit/NotebookEdit/Write drop `tool_response.originalFile`/`oldString`/`newString` (Write also `content`), Read drops `tool_response.file.content`/`base64`, and any string inside `tool_input`/`tool_response` over 8 KB keeps its first 6 KB and last 2 KB; `structuredPatch` and every field outside those two are kept, and `_trimmed` lists what was dropped. The full text stays in the session transcript and AgentHub's snapshot of it under `/transcripts/`, which the Conversation tab reads. Rows stored before this are trimmed once in the background (`sweepEventPayloads`, 200 rows per transaction, `user_version` 4 after marker 3); measured on a 1.2 GB install it took `events.data` from 913 MB to ~400 MB. Freed pages are reused, not returned to the disk (no `VACUUM`). No ROW is pruned automatically. The only deletions are the user's: Settings → clear data / cleanup, a workspace's purge, `scripts/clear-data.js`. Clear data runs in ONE transaction (foreign keys off around it, so workflows are deleted explicitly), cleanup deletes in 500-id chunks inside one transaction. `GET /api/settings/export` streams `events` in keyset pages of 500 instead of loading the table. diff --git a/server/__tests__/db-retention.test.js b/server/__tests__/db-retention.test.js new file mode 100644 index 00000000..3f487cef --- /dev/null +++ b/server/__tests__/db-retention.test.js @@ -0,0 +1,175 @@ +/** + * @file Tests server/lib/db-retention.js and its Settings routes — a new + * database is INCREMENTAL auto_vacuum, retention is off by default and purges + * nothing then, when on it removes only finished sessions older than the + * cutoff (with their events, agents and token rows) in bounded batches, the + * free-page reclaim shrinks the freelist, GET/PUT /api/settings/retention + * validate and persist the setting, and POST /api/settings/compact needs + * confirm and runs a VACUUM in a worker. The database and settings file are + * temp paths. + * @author Nguyễn Ngọc Trí Vĩ + */ + +const os = require("node:os"); +const fs = require("node:fs"); +const path = require("node:path"); +const { describe, it, before, after } = require("node:test"); +const assert = require("node:assert/strict"); + +const ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "agenthub-db-retention-")); +process.env.DASHBOARD_DB_PATH = path.join(ROOT, "dashboard.db"); +process.env.AGENTHUB_SETTINGS_FILE = path.join(ROOT, "settings.env"); +process.env.DASHBOARD_LIVENESS_PROBE = "0"; +delete process.env.AGENTHUB_RETENTION_DAYS; + +const { createApp, startServer } = require("../index"); +const { db } = require("../db"); +const retention = require("../lib/db-retention"); + +let server; +let BASE; +before(async () => { + server = await startServer(createApp(), 0); + BASE = `http://127.0.0.1:${server.address().port}`; +}); +after(() => { + server?.close(); + delete process.env.AGENTHUB_RETENTION_DAYS; + fs.rmSync(ROOT, { recursive: true, force: true }); +}); + +async function call(method, urlPath, body) { + const res = await fetch(BASE + urlPath, { + method, + headers: body ? { "content-type": "application/json" } : {}, + body: body ? JSON.stringify(body) : undefined, + }); + return { status: res.status, body: await res.json() }; +} + +const daysAgo = (n) => new Date(Date.now() - n * 86400 * 1000).toISOString(); + +function seed(id, status, startedAt, events = 3) { + db.prepare( + "INSERT INTO sessions (id, name, status, cwd, started_at, updated_at) VALUES (?, ?, ?, '/tmp', ?, ?)" + ).run(id, id, status, startedAt, startedAt); + const ins = db.prepare( + "INSERT INTO events (session_id, event_type, summary, data, created_at) VALUES (?, 'PostToolUse', 's', ?, ?)" + ); + for (let i = 0; i < events; i++) + ins.run(id, JSON.stringify({ pad: "x".repeat(2000) }), startedAt); +} +const exists = (id) => !!db.prepare("SELECT 1 FROM sessions WHERE id = ?").get(id); +const eventCount = (id) => + db.prepare("SELECT COUNT(*) AS n FROM events WHERE session_id = ?").get(id).n; + +describe("auto_vacuum", () => { + it("a new database is INCREMENTAL", () => { + assert.equal(db.pragma("auto_vacuum", { simple: true }), 2); + assert.equal(retention.spaceInfo(db).auto_vacuum, "incremental"); + }); +}); + +describe("retentionConfig", () => { + it("is off by default and for anything out of range", () => { + assert.equal(retention.retentionConfig({}).days, 0); + for (const v of ["", "0", "-3", "1.5", "abc", "99999"]) { + assert.equal(retention.retentionConfig({ AGENTHUB_RETENTION_DAYS: v }).days, 0, v); + } + assert.equal(retention.retentionConfig({ AGENTHUB_RETENTION_DAYS: "30" }).days, 30); + }); +}); + +describe("scheduled maintenance", () => { + it("purges nothing while retention is off", async () => { + seed("ret-off-old", "completed", daysAgo(400)); + const out = await retention.runScheduledMaintenance({ db }); + assert.equal(out.retention, undefined); + assert.ok(exists("ret-off-old")); + db.prepare("DELETE FROM sessions WHERE id = 'ret-off-old'").run(); + }); + + it("purges only finished sessions older than the cutoff, with their rows", async () => { + seed("ret-old-done", "completed", daysAgo(100)); + seed("ret-old-err", "error", daysAgo(100)); + seed("ret-old-active", "active", daysAgo(100)); + seed("ret-new-done", "completed", daysAgo(5)); + db.prepare( + "INSERT INTO token_usage (session_id, model, input_tokens) VALUES ('ret-old-done', 'm', 1)" + ).run(); + process.env.AGENTHUB_RETENTION_DAYS = "30"; + try { + const out = await retention.runScheduledMaintenance({ db }); + assert.equal(out.retention.purged_sessions, 2); + assert.equal(out.retention.purged_events, 6); + } finally { + delete process.env.AGENTHUB_RETENTION_DAYS; + } + assert.ok(!exists("ret-old-done")); + assert.ok(!exists("ret-old-err")); + assert.equal(eventCount("ret-old-done"), 0); + assert.equal( + db.prepare("SELECT COUNT(*) AS n FROM token_usage WHERE session_id = 'ret-old-done'").get().n, + 0 + ); + assert.ok(exists("ret-old-active"), "an active session is never purged"); + assert.ok(exists("ret-new-done"), "a recent session is kept"); + assert.equal(eventCount("ret-new-done"), 3); + }); + + it("works in bounded batches and stops at the per-run cap", async () => { + for (let i = 0; i < 7; i++) seed(`ret-batch-${i}`, "abandoned", daysAgo(50), 1); + const first = await retention.applyRetention({ db, days: 10, batch: 2, maxSessions: 5 }); + assert.equal(first.purged_sessions, 5); + const second = await retention.applyRetention({ db, days: 10, batch: 2, maxSessions: 5 }); + assert.equal(second.purged_sessions, 2); + const third = await retention.applyRetention({ db, days: 10 }); + assert.equal(third.purged_sessions, 0); + }); + + it("returns free pages to the OS", async () => { + for (let i = 0; i < 40; i++) seed(`ret-big-${i}`, "completed", daysAgo(60), 20); + await retention.applyRetention({ db, days: 30 }); + const before = db.pragma("freelist_count", { simple: true }); + assert.ok(before > 50, `expected free pages, got ${before}`); + const out = await retention.reclaimFreePages({ db, minPages: 10, slice: 16 }); + assert.ok(out.reclaimed_pages > 0); + assert.ok(db.pragma("freelist_count", { simple: true }) < before); + }); +}); + +describe("routes", () => { + it("GET reports retention off by default", async () => { + const r = await call("GET", "/api/settings/retention"); + assert.equal(r.status, 200); + assert.equal(r.body.days, 0); + assert.equal(r.body.compacting, false); + assert.equal(r.body.space.auto_vacuum, "incremental"); + }); + + it("PUT validates and persists, 0 turns it off", async () => { + for (const bad of [-1, 1.5, "7", 99999, null]) { + const r = await call("PUT", "/api/settings/retention", { days: bad }); + assert.equal(r.status, 400, String(bad)); + } + let r = await call("PUT", "/api/settings/retention", { days: 90 }); + assert.equal(r.status, 200); + assert.equal(r.body.days, 90); + assert.match(fs.readFileSync(process.env.AGENTHUB_SETTINGS_FILE, "utf8"), /RETENTION_DAYS=90/); + r = await call("PUT", "/api/settings/retention", { days: 0 }); + assert.equal(r.body.days, 0); + }); + + it("POST /compact needs confirm and then vacuums", async () => { + let r = await call("POST", "/api/settings/compact", {}); + assert.equal(r.status, 400); + r = await call("POST", "/api/settings/compact", { confirm: true }); + assert.equal(r.status, 202); + for (let i = 0; i < 200 && retention.isCompacting(); i++) { + await new Promise((res) => setTimeout(res, 50)); + } + const last = retention.lastCompactResult(); + assert.equal(last?.ok, true, last?.error); + assert.equal(db.pragma("auto_vacuum", { simple: true }), 2); + }); +}); diff --git a/server/__tests__/update-scheduler.test.js b/server/__tests__/update-scheduler.test.js new file mode 100644 index 00000000..ed073949 --- /dev/null +++ b/server/__tests__/update-scheduler.test.js @@ -0,0 +1,76 @@ +/** + * @file Tests server/update-scheduler.js — the default interval is an hour + * (the env override and its 60 s floor still apply), a scheduled tick with no + * browser connected runs no check, and a browser connecting after that gets + * one catch-up check (only when the last check is stale). The check, the + * connection count and the connection event are injected; nothing touches git + * or the network. + * @author Nguyễn Ngọc Trí Vĩ + */ + +const { describe, it, afterEach } = require("node:test"); +const assert = require("node:assert/strict"); + +delete process.env.DASHBOARD_UPDATE_CHECK; +const { startUpdateScheduler, intervalMs } = require("../update-scheduler"); + +const wait = (ms) => new Promise((r) => setTimeout(r, ms)); + +let handle = null; +afterEach(() => { + handle?.stop(); + handle = null; + delete process.env.DASHBOARD_UPDATE_CHECK_INTERVAL_MS; +}); + +function harness(clients) { + const h = { checks: 0, clients, listeners: [] }; + handle = startUpdateScheduler({ + broadcast: () => {}, + getConnectionCount: () => h.clients, + onConnection: (fn) => { + h.listeners.push(fn); + return () => {}; + }, + getStatus: async () => { + h.checks++; + return { update_available: false }; + }, + initialDelayMs: 10, + }); + h.connect = () => h.listeners.forEach((fn) => fn()); + return h; +} + +describe("intervalMs", () => { + it("defaults to one hour and keeps the override with its floor", () => { + delete process.env.DASHBOARD_UPDATE_CHECK_INTERVAL_MS; + assert.equal(intervalMs(), 60 * 60 * 1000); + process.env.DASHBOARD_UPDATE_CHECK_INTERVAL_MS = "120000"; + assert.equal(intervalMs(), 120000); + process.env.DASHBOARD_UPDATE_CHECK_INTERVAL_MS = "1000"; + assert.equal(intervalMs(), 60 * 60 * 1000); + }); +}); + +describe("startUpdateScheduler", () => { + it("checks on the first tick when a browser is connected", async () => { + const h = harness(1); + await wait(40); + assert.equal(h.checks, 1); + }); + + it("skips the tick with no browser, then checks once when one connects", async () => { + const h = harness(0); + await wait(40); + assert.equal(h.checks, 0); + h.clients = 1; + h.connect(); + h.connect(); // a second tab in the same window: still one check + await wait(3_300); + assert.equal(h.checks, 1); + h.connect(); // fresh check: no catch-up + await wait(3_300); + assert.equal(h.checks, 1); + }); +}); diff --git a/server/db.js b/server/db.js index 627a60b5..fb9a1d61 100644 --- a/server/db.js +++ b/server/db.js @@ -141,6 +141,14 @@ migrateLegacyDatabase(DB_PATH); const db = new Database(DB_PATH); +// Freed pages go back to the OS only with auto_vacuum. On a NEW database this +// takes effect at once — it must come before `journal_mode = WAL`, which +// writes the header; an existing one is converted further down, by a VACUUM, +// only while it is small (`convertAutoVacuum`), else by Settings → Data → +// Compact. `maintainDatabase` then runs a bounded `incremental_vacuum`. +if (db.pragma("auto_vacuum", { simple: true }) === 0) { + db.pragma("auto_vacuum = INCREMENTAL"); +} db.pragma("journal_mode = WAL"); db.pragma("foreign_keys = ON"); db.pragma("busy_timeout = 5000"); @@ -1866,6 +1874,28 @@ for (const col of ["display_name", "email", "avatar_at"]) { } } +// An existing database made before auto_vacuum: the setting is stored only by +// a VACUUM, which rewrites the whole file under an exclusive lock. Done here, +// at startup and under the migration lock, only for a file up to +// AUTO_VACUUM_CONVERT_MAX_BYTES (well under a second); a bigger one is +// converted by the manual Compact (server/lib/db-retention.js). Best effort: a +// second process holding a read open makes it fail, and the next start retries. +const AUTO_VACUUM_CONVERT_MAX_BYTES = 32 * 1024 * 1024; +function convertAutoVacuum() { + try { + if (db.pragma("auto_vacuum", { simple: true }) !== 0) return false; + const bytes = + db.pragma("page_count", { simple: true }) * db.pragma("page_size", { simple: true }); + if (bytes > AUTO_VACUUM_CONVERT_MAX_BYTES) return false; + db.pragma("auto_vacuum = INCREMENTAL"); + db.exec("VACUUM"); + return db.pragma("auto_vacuum", { simple: true }) === 2; + } catch { + return false; + } +} +convertAutoVacuum(); + migrationLock.exec("COMMIT"); migrationLock.close(); @@ -2535,7 +2565,9 @@ const stmts = { * on a reader — the other dashboard's), after which the next write truncates * the WAL to `journal_size_limit`; and `PRAGMA optimize`, which refreshes the * planner's statistics only where they went stale. Never VACUUM: it rewrites - * the whole file under an exclusive lock. + * the whole file under an exclusive lock. The opt-in retention and the + * bounded `incremental_vacuum` run after this, from + * `server/lib/db-retention.js`'s `runScheduledMaintenance`. */ function maintainDatabase() { const out = {}; @@ -2561,4 +2593,6 @@ module.exports = { applyIntroPricing, topUpDefaultPricing, maintainDatabase, + convertAutoVacuum, + AUTO_VACUUM_CONVERT_MAX_BYTES, }; diff --git a/server/index.js b/server/index.js index 7e503bbf..7cfddc44 100644 --- a/server/index.js +++ b/server/index.js @@ -500,11 +500,17 @@ function startBackgroundServices() { .catch(() => {}); // Hourly DB upkeep: a non-blocking checkpoint (the WAL then shrinks to - // journal_size_limit) and PRAGMA optimize. Off with DASHBOARD_DB_MAINTAIN_MS=0. + // journal_size_limit), PRAGMA optimize, the opt-in retention + // (AGENTHUB_RETENTION_DAYS, off by default) and a bounded incremental_vacuum + // (server/lib/db-retention.js). Off with DASHBOARD_DB_MAINTAIN_MS=0. { const every = Number(process.env.DASHBOARD_DB_MAINTAIN_MS ?? 60 * 60 * 1000); if (every > 0) { - const t = setInterval(() => require("./db").maintainDatabase(), every); + const t = setInterval(() => { + require("./lib/db-retention") + .runScheduledMaintenance() + .catch(() => {}); + }, every); if (t.unref) t.unref(); } } diff --git a/server/lib/db-retention.js b/server/lib/db-retention.js new file mode 100644 index 00000000..3865c01c --- /dev/null +++ b/server/lib/db-retention.js @@ -0,0 +1,337 @@ +/** + * @file Database size upkeep (Settings → Data → Session cleanup): the shared + * session purge, the OPT-IN automatic retention, the bounded reclaim of free + * pages, and the manual Compact. + * + * - `purgeSessionIds` is THE deletion of whole sessions — events, agents, + * token usage (daily and hourly) and the session row — used by both the + * manual `POST /api/settings/cleanup` and the automatic retention, so the two + * can never leave different leftovers. + * - Retention is OFF unless `AGENTHUB_RETENTION_DAYS` (settings file, 1–3650) + * is set; it never deletes data by default. When set, the hourly + * `runScheduledMaintenance` purges finished (completed/error/abandoned) + * sessions that STARTED more than that many days ago — the manual purge's + * rule — in transactions of `RETENTION_BATCH` sessions, yielding to the event + * loop between them, and at most `RETENTION_MAX_PER_RUN` per run, so a first + * run on a long history never blocks hooks for long. Each batch deletes by + * `idx_events_session`, never a scan of the events table. + * - With `auto_vacuum = INCREMENTAL` (server/db.js sets it on a new database + * and converts a small existing one at startup), the same run returns free + * pages to the OS once the freelist passes `RECLAIM_MIN_PAGES`, in slices of + * `RECLAIM_SLICE_PAGES`, at most `RECLAIM_MAX_PAGES` per run. + * - `runCompact` (`POST /api/settings/compact`) is the one full VACUUM: in a + * worker thread on its own connection, setting `auto_vacuum = INCREMENTAL` + * first so a large existing database is converted, then a TRUNCATE + * checkpoint so the file actually shrinks. It holds the write lock for its + * whole run (tens of seconds on a GB-size file), so hooks wait or are retried + * meanwhile — it only ever runs on a click. This file is also its worker's + * entry point. + * @author Nguyễn Ngọc Trí Vĩ + */ + +"use strict"; + +const { Worker, isMainThread, workerData, parentPort } = require("node:worker_threads"); + +const KEYS = { days: "AGENTHUB_RETENTION_DAYS" }; +const MAX_DAYS = 3650; +const RETENTION_BATCH = 25; +const RETENTION_MAX_PER_RUN = 5000; +const RECLAIM_MIN_PAGES = 1024; +const RECLAIM_SLICE_PAGES = 1024; +const RECLAIM_MAX_PAGES = 65536; +const PURGE_CHUNK = 500; +const PURGEABLE = "status IN ('completed','error','abandoned')"; + +const coded = (message, code) => Object.assign(new Error(message), { code }); +const yieldLoop = () => new Promise((r) => setImmediate(r)); + +/** `{ days }` — 0 means off. Anything outside 1–MAX_DAYS reads as off. */ +function retentionConfig(env = process.env) { + const raw = env[KEYS.days]; + const n = Number(raw); + const days = raw !== undefined && raw !== "" && Number.isInteger(n) && n >= 1 && n <= MAX_DAYS; + return { days: days ? n : 0 }; +} + +/** + * Delete these sessions and everything hung off them, in chunks (a single + * IN (...) of every id would pass SQLite's bound-variable limit). Runs in the + * CALLER's transaction when there is one. Returns the rows removed. + */ +function purgeSessionIds(db, ids) { + const { forgetHourly } = require("./hourly-usage"); + const out = { purged_sessions: 0, purged_events: 0, purged_agents: 0 }; + for (let i = 0; i < ids.length; i += PURGE_CHUNK) { + const chunk = ids.slice(i, i + PURGE_CHUNK); + const placeholders = chunk.map(() => "?").join(","); + // Cascading deletes handle agents/events, but token_usage FK might not cascade on all setups + out.purged_events += db + .prepare(`DELETE FROM events WHERE session_id IN (${placeholders})`) + .run(...chunk).changes; + out.purged_agents += db + .prepare(`DELETE FROM agents WHERE session_id IN (${placeholders})`) + .run(...chunk).changes; + db.prepare(`DELETE FROM token_usage WHERE session_id IN (${placeholders})`).run(...chunk); + db.prepare(`DELETE FROM token_usage_hourly WHERE session_id IN (${placeholders})`).run( + ...chunk + ); + forgetHourly(chunk); + out.purged_sessions += db + .prepare(`DELETE FROM sessions WHERE id IN (${placeholders})`) + .run(...chunk).changes; + } + return out; +} + +/** What `purgeSessionIds` would delete, without deleting it (the dry run). */ +function countSessionIds(db, ids) { + const out = { purged_sessions: ids.length, purged_events: 0, purged_agents: 0 }; + for (let i = 0; i < ids.length; i += PURGE_CHUNK) { + const chunk = ids.slice(i, i + PURGE_CHUNK); + const placeholders = chunk.map(() => "?").join(","); + out.purged_events += db + .prepare(`SELECT COUNT(*) AS n FROM events WHERE session_id IN (${placeholders})`) + .get(...chunk).n; + out.purged_agents += db + .prepare(`SELECT COUNT(*) AS n FROM agents WHERE session_id IN (${placeholders})`) + .get(...chunk).n; + } + return out; +} + +/** Finished sessions that started before `cutoff` (ISO), at most `limit`. */ +function purgeableBefore(db, cutoff, limit = -1) { + return db + .prepare(`SELECT id, started_at FROM sessions WHERE ${PURGEABLE} AND started_at < ? LIMIT ?`) + .all(cutoff, limit); +} + +/** + * Purge finished sessions older than `days`, a batch per transaction with a + * yield between batches. Resolves to the totals; never throws. + */ +async function applyRetention({ + db = require("../db").db, + days, + now = Date.now(), + batch = RETENTION_BATCH, + maxSessions = RETENTION_MAX_PER_RUN, +} = {}) { + const out = { purged_sessions: 0, purged_events: 0, purged_agents: 0 }; + if (!Number.isInteger(days) || days < 1) return out; + const cutoff = new Date(now - days * 86400 * 1000).toISOString(); + try { + while (out.purged_sessions < maxSessions) { + const ids = purgeableBefore( + db, + cutoff, + Math.min(batch, maxSessions - out.purged_sessions) + ).map((r) => r.id); + if (ids.length === 0) break; + const r = db.transaction(() => purgeSessionIds(db, ids))(); + out.purged_sessions += r.purged_sessions; + out.purged_events += r.purged_events; + out.purged_agents += r.purged_agents; + if (r.purged_sessions === 0) break; // nothing removable: never spin + await yieldLoop(); + } + } catch (err) { + out.error = err.message; + } + return out; +} + +/** One `incremental_vacuum(n)`, stepped to the end on either driver. */ +function incrementalVacuum(db, pages) { + const sql = `PRAGMA incremental_vacuum(${Math.max(1, Math.floor(pages))})`; + try { + db.prepare(sql).all(); // node:sqlite: one row per freed page + } catch (err) { + if (!/does not return data/i.test(err.message)) throw err; + db.pragma(`incremental_vacuum(${Math.max(1, Math.floor(pages))})`); // better-sqlite3 + } +} + +/** + * Hand free pages back to the OS, a slice at a time. A no-op unless the + * database is in INCREMENTAL auto_vacuum and its freelist is over `minPages`. + */ +async function reclaimFreePages({ + db = require("../db").db, + minPages = RECLAIM_MIN_PAGES, + slice = RECLAIM_SLICE_PAGES, + maxPages = RECLAIM_MAX_PAGES, +} = {}) { + const out = { reclaimed_pages: 0 }; + try { + if (db.pragma("auto_vacuum", { simple: true }) !== 2) return out; + let free = db.pragma("freelist_count", { simple: true }); + if (free < minPages) return out; + while (free > 0 && out.reclaimed_pages < maxPages) { + incrementalVacuum(db, Math.min(slice, free, maxPages - out.reclaimed_pages)); + const after = db.pragma("freelist_count", { simple: true }); + if (after >= free) break; + out.reclaimed_pages += free - after; + free = after; + await yieldLoop(); + } + } catch (err) { + out.error = err.message; + } + return out; +} + +let lastRun = null; +let maintaining = null; + +/** + * The hourly upkeep: `maintainDatabase` (checkpoint + optimize), then the + * opt-in retention, then the bounded free-page reclaim. A call while one runs + * joins it. + */ +function runScheduledMaintenance(opts = {}) { + if (maintaining) return maintaining; + maintaining = maintain(opts).finally(() => { + maintaining = null; + }); + return maintaining; +} + +async function maintain({ db } = {}) { + const dbm = require("../db"); + const conn = db || dbm.db; + const started = new Date(); + const out = { at: started.toISOString() }; + try { + if (!db) dbm.maintainDatabase(); + } catch { + /* maintainDatabase reports its own errors in its result */ + } + const { days } = retentionConfig(); + if (days) { + out.retention = await applyRetention({ db: conn, days }); + if (out.retention.purged_sessions > 0) { + console.log( + `[retention] purged ${out.retention.purged_sessions} session(s) older than ${days} days` + ); + } + } + out.reclaim = await reclaimFreePages({ db: conn }); + out.ms = Date.now() - started; + lastRun = out; + return out; +} + +/** Page counts for the Settings card. */ +function spaceInfo(db = require("../db").db) { + const pageSize = db.pragma("page_size", { simple: true }); + const pageCount = db.pragma("page_count", { simple: true }); + const free = db.pragma("freelist_count", { simple: true }); + const mode = db.pragma("auto_vacuum", { simple: true }); + return { + auto_vacuum: mode === 2 ? "incremental" : mode === 1 ? "full" : "none", + page_size: pageSize, + page_count: pageCount, + freelist_pages: free, + free_bytes: free * pageSize, + }; +} + +let compacting = null; +let lastCompact = null; + +function vacuumInWorker(dbPath) { + return new Promise((resolve, reject) => { + const w = new Worker(__filename, { workerData: { agenthubCompactWorker: true, dbPath } }); + w.once("message", (m) => (m.error ? reject(coded(m.error.message, m.error.code)) : resolve(m))); + w.once("error", reject); + w.once( + "exit", + (code) => code !== 0 && reject(coded(`compact worker exited (${code})`, "ECOMPACT")) + ); + }); +} + +/** Start (or join) the manual Compact. The outcome is `lastCompactResult()`. */ +function runCompact() { + if (compacting) return compacting; + const fs = require("node:fs"); + const { DB_PATH } = require("../db"); + const size = () => { + try { + return fs.statSync(DB_PATH).size; + } catch { + return 0; + } + }; + compacting = (async () => { + const started = Date.now(); + const before = size(); + try { + await vacuumInWorker(DB_PATH); + lastCompact = { + at: new Date(started).toISOString(), + ok: true, + size_before: before, + size_after: size(), + ms: Date.now() - started, + }; + return lastCompact; + } catch (err) { + lastCompact = { at: new Date(started).toISOString(), ok: false, error: err.message }; + throw err; + } + })().finally(() => { + compacting = null; + }); + return compacting; +} + +const isCompacting = () => compacting !== null; +const lastCompactResult = () => lastCompact; +const lastMaintenance = () => lastRun; + +if (!isMainThread && workerData?.agenthubCompactWorker) { + // Same driver order as server/db.js: better-sqlite3, else node:sqlite. + let conn; + try { + try { + const Database = require("better-sqlite3"); + conn = new Database(workerData.dbPath); + } catch { + const Database = require("../compat-sqlite"); + conn = new Database(workerData.dbPath); + } + conn.pragma("busy_timeout = 30000"); + conn.pragma("auto_vacuum = INCREMENTAL"); // stored by the VACUUM below + conn.exec("VACUUM"); + try { + conn.pragma("wal_checkpoint(TRUNCATE)"); + } catch { + /* a reader holds the WAL: the periodic checkpoint finishes it */ + } + parentPort.postMessage({ ok: true }); + } catch (err) { + parentPort.postMessage({ error: { message: err.message, code: err.code || "ECOMPACT" } }); + } finally { + conn?.close(); + } +} + +module.exports = { + KEYS, + MAX_DAYS, + retentionConfig, + purgeSessionIds, + countSessionIds, + purgeableBefore, + applyRetention, + reclaimFreePages, + runScheduledMaintenance, + spaceInfo, + runCompact, + isCompacting, + lastCompactResult, + lastMaintenance, +}; diff --git a/server/openapi-extra/misc.js b/server/openapi-extra/misc.js index d22bdd99..ab172934 100644 --- a/server/openapi-extra/misc.js +++ b/server/openapi-extra/misc.js @@ -11,6 +11,9 @@ * - GET /api/settings/backup (Settings) * - PUT /api/settings/backup (Settings) * - POST /api/settings/backup/run (Settings) + * - GET /api/settings/retention (Settings) + * - PUT /api/settings/retention (Settings) + * - POST /api/settings/compact (Settings) * - GET /api/workflows/runs (Workflows) * - GET /api/workflows/runs/{runId} (Workflows) * - GET /api/remote-sources (Remote Sources) @@ -1225,6 +1228,117 @@ const paths = { }, }, + "/api/settings/retention": (() => { + const state = { + description: "Automatic retention setting, database space and the last Compact", + content: { + "application/json": { + schema: { + type: "object", + required: ["days", "max_days", "env_owned", "compacting"], + properties: { + days: { + type: "integer", + minimum: 0, + maximum: 3650, + description: + "Finished sessions older than this many days are purged by the hourly maintenance; 0 = off (the default)", + }, + max_days: { type: "integer" }, + env_owned: { type: "object", properties: { days: { type: "boolean" } } }, + last_run: { + type: "object", + nullable: true, + description: + "This process's last hourly maintenance: {at, retention?: {purged_sessions, purged_events, purged_agents, error?}, reclaim: {reclaimed_pages, error?}, ms}", + }, + space: { + type: "object", + nullable: true, + properties: { + auto_vacuum: { type: "string", enum: ["none", "full", "incremental"] }, + page_size: { type: "integer" }, + page_count: { type: "integer" }, + freelist_pages: { type: "integer" }, + free_bytes: { type: "integer" }, + }, + }, + db_size: { type: "integer", description: "Database file size in bytes" }, + compacting: { type: "boolean" }, + last_compact: { + type: "object", + nullable: true, + description: + "This process's last Compact: {at, ok, size_before?, size_after?, ms?, error?}", + }, + }, + }, + }, + }, + }; + return { + get: { + tags: ["Settings"], + summary: "Automatic retention and database space", + operationId: "getRetention", + responses: { 200: state }, + }, + put: { + tags: ["Settings"], + summary: "Set the automatic retention", + description: + "Persisted to the settings file as AGENTHUB_RETENTION_DAYS. 0 turns it off. Same-origin only; in server mode an admin's.", + operationId: "setRetention", + requestBody: { + required: true, + content: { + "application/json": { + schema: { + type: "object", + required: ["days"], + properties: { days: { type: "integer", minimum: 0, maximum: 3650 } }, + }, + }, + }, + }, + responses: { + 200: { ...state, description: "The new state" }, + 400: { description: "INVALID_INPUT: days is not a whole number from 0 to 3650" }, + 403: { description: "EBADORIGIN: another site asked" }, + 409: { description: "EENVOWNED: the environment set AGENTHUB_RETENTION_DAYS" }, + 500: { description: "ESAVE: the settings file could not be written" }, + }, + }, + }; + })(), + + "/api/settings/compact": { + post: { + tags: ["Settings"], + summary: "Compact the database now (full VACUUM)", + description: + "Rewrites the database in a worker thread, switching it to incremental auto_vacuum, then truncates the WAL. Hook writes wait while it runs. Answers at once; poll GET /api/settings/retention while `compacting`. Same-origin only; in server mode an admin's.", + operationId: "compactDatabase", + requestBody: { + required: true, + content: { + "application/json": { + schema: { + type: "object", + required: ["confirm"], + properties: { confirm: { type: "boolean", enum: [true] } }, + }, + }, + }, + }, + responses: { + 202: { description: "Started (or already running); the retention state" }, + 400: { description: "ECONFIRM: confirm: true is missing" }, + 403: { description: "EBADORIGIN: another site asked" }, + }, + }, + }, + "/api/settings/codex-hooks": (() => { const status = { description: "Where AgentHub's Codex hooks stand", diff --git a/server/routes/settings.js b/server/routes/settings.js index ea67d61d..3c62e4bc 100644 --- a/server/routes/settings.js +++ b/server/routes/settings.js @@ -1,5 +1,5 @@ /** - * @file Express router for settings-related endpoints, providing system info, database statistics, hook status, and operations to clear data, re-import sessions, reinstall hooks, reset pricing, export data, and perform cleanup of stale sessions, plus the privacy controls (secret redaction, ignored folders), the push-ingest token and the daily database backup (server/lib/db-backup.js). This allows the frontend to manage and maintain the agent monitoring system effectively. + * @file Express router for settings-related endpoints, providing system info, database statistics, hook status, and operations to clear data, re-import sessions, reinstall hooks, reset pricing, export data, and perform cleanup of stale sessions, plus the privacy controls (secret redaction, ignored folders), the push-ingest token, the daily database backup (server/lib/db-backup.js) and the opt-in retention + Compact (server/lib/db-retention.js). This allows the frontend to manage and maintain the agent monitoring system effectively. * @author Nguyễn Ngọc Trí Vĩ */ @@ -28,6 +28,7 @@ const { getCodexHome, setCodexHome } = require("../lib/codex-home"); const { envOwned, applySetting } = require("../lib/settings-env"); const privacy = require("../lib/privacy"); const dbBackup = require("../lib/db-backup"); +const retention = require("../lib/db-retention"); const pushToken = require("../lib/push-token"); const pricingSync = require("../lib/pricing-sync"); const { @@ -678,10 +679,66 @@ function pushTokenWrite(fn) { router.post("/push-token", sameOriginGuard, pushTokenWrite(pushToken.rotatePushToken)); router.delete("/push-token", sameOriginGuard, pushTokenWrite(pushToken.revokePushToken)); +// GET/PUT /api/settings/retention, POST /api/settings/compact — database size +// upkeep (server/lib/db-retention.js). Retention is OFF by default: `days` 0 +// turns it off, 1–3650 makes the hourly maintenance purge finished sessions +// older than that, the manual purge's rule. It persists to the settings file; +// one the environment set is `env_owned` and refused. Compact (a full VACUUM +// in a worker) answers 202 and the page polls GET while `compacting`. +function retentionState() { + let space = null; + try { + space = retention.spaceInfo(db); + } catch { + /* pragma unavailable: the card shows no numbers */ + } + return { + ...retention.retentionConfig(), + max_days: retention.MAX_DAYS, + env_owned: { days: envOwned.has(retention.KEYS.days) }, + last_run: retention.lastMaintenance(), + space, + db_size: getDbSize(), + compacting: retention.isCompacting(), + last_compact: retention.lastCompactResult(), + }; +} +router.get("/retention", (_req, res) => res.json(retentionState())); +router.put("/retention", sameOriginGuard, (req, res) => { + const days = req.body?.days; + if (!Number.isInteger(days) || days < 0 || days > retention.MAX_DAYS) { + return res.status(400).json({ + error: { + code: "INVALID_INPUT", + message: `days must be a whole number from 0 (off) to ${retention.MAX_DAYS}`, + }, + }); + } + if (envOwned.has(retention.KEYS.days)) { + return res.status(409).json({ + error: { code: "EENVOWNED", message: `${retention.KEYS.days} is set by the environment` }, + }); + } + try { + applySetting(retention.KEYS.days, days === 0 ? "" : String(days)); + } catch (err) { + return res.status(500).json({ error: { code: "ESAVE", message: err.message } }); + } + res.json({ ok: true, ...retentionState() }); +}); +router.post("/compact", sameOriginGuard, (req, res) => { + if (req.body?.confirm !== true) { + return res.status(400).json({ + error: { code: "ECONFIRM", message: "compacting needs confirm: true" }, + }); + } + retention.runCompact().catch(() => {}); // the outcome is `last_compact` in GET + res.status(202).json({ ok: true, ...retentionState() }); +}); + // POST /api/settings/cleanup — abandon stale sessions, purge old data. // `purge_days` is a whole number ≥ 1 (0.1 days would purge almost everything); // `dry_run: true` answers the same counts and changes nothing. -const PURGE_CHUNK = 500; router.post("/cleanup", (req, res) => { const { abandon_hours, purge_days } = req.body; const dryRun = req.body.dry_run === true; @@ -739,11 +796,7 @@ router.post("/cleanup", (req, res) => { if (purge_days && typeof purge_days === "number" && purge_days > 0) { const cutoff = new Date(Date.now() - purge_days * 86400 * 1000).toISOString(); // Only purge completed/error/abandoned sessions, never active - const toDelete = db - .prepare( - "SELECT id FROM sessions WHERE status IN ('completed','error','abandoned') AND started_at < ?" - ) - .all(cutoff); + const toDelete = retention.purgeableBefore(db, cutoff); if (dryRun) { // Exactly the ids the real run deletes: those already purgeable, plus @@ -752,43 +805,15 @@ router.post("/cleanup", (req, res) => { ...toDelete.map((r) => r.id), ...wouldAbandon.filter((r) => r.started_at < cutoff).map((r) => r.id), ]; - for (let i = 0; i < ids.length; i += PURGE_CHUNK) { - const chunk = ids.slice(i, i + PURGE_CHUNK); - const placeholders = chunk.map(() => "?").join(","); - result.purged_events += db - .prepare(`SELECT COUNT(*) AS n FROM events WHERE session_id IN (${placeholders})`) - .get(...chunk).n; - result.purged_agents += db - .prepare(`SELECT COUNT(*) AS n FROM agents WHERE session_id IN (${placeholders})`) - .get(...chunk).n; - } - result.purged_sessions = ids.length; + Object.assign(result, retention.countSessionIds(db, ids)); return res.json({ ok: true, ...result }); } - // One transaction, in chunks: a single IN (...) list of every id would - // pass SQLite's bound-variable limit on a large history. - db.transaction(() => { - const ids = toDelete.map((r) => r.id); - for (let i = 0; i < ids.length; i += PURGE_CHUNK) { - const chunk = ids.slice(i, i + PURGE_CHUNK); - const placeholders = chunk.map(() => "?").join(","); - // Cascading deletes handle agents/events, but token_usage FK might not cascade on all setups - result.purged_events += db - .prepare(`DELETE FROM events WHERE session_id IN (${placeholders})`) - .run(...chunk).changes; - result.purged_agents += db - .prepare(`DELETE FROM agents WHERE session_id IN (${placeholders})`) - .run(...chunk).changes; - db.prepare(`DELETE FROM token_usage WHERE session_id IN (${placeholders})`).run(...chunk); - db.prepare(`DELETE FROM token_usage_hourly WHERE session_id IN (${placeholders})`).run( - ...chunk - ); - forgetHourly(chunk); - db.prepare(`DELETE FROM sessions WHERE id IN (${placeholders})`).run(...chunk); - } - result.purged_sessions = ids.length; - })(); + // One transaction; the deletion itself is shared with the automatic + // retention (server/lib/db-retention.js). + const ids = toDelete.map((r) => r.id); + const purged = db.transaction(() => retention.purgeSessionIds(db, ids))(); + Object.assign(result, purged, { purged_sessions: ids.length }); } res.json({ ok: true, ...result }); diff --git a/server/update-scheduler.js b/server/update-scheduler.js index 6a7a9ea6..eda5d0a3 100644 --- a/server/update-scheduler.js +++ b/server/update-scheduler.js @@ -1,5 +1,11 @@ /** - * @file Periodic git upstream check and WebSocket broadcast when update availability changes. + * @file Periodic release/upstream check and WebSocket broadcast when update + * availability changes. Hourly by default (`DASHBOARD_UPDATE_CHECK_INTERVAL_MS`, + * floor 60 s); a tick with no browser connected is skipped — a checkout install + * runs `git fetch` on every check, and nobody would see the answer — and the + * first browser to connect after a skipped or stale check gets one promptly. + * The manual "Check now" (`POST /api/updates/check`, `agenthub update-check`) + * is not routed through here and always runs. * @author Nguyễn Ngọc Trí Vĩ */ @@ -13,11 +19,39 @@ function isUpdateCheckDisabled() { function intervalMs() { const n = Number.parseInt(process.env.DASHBOARD_UPDATE_CHECK_INTERVAL_MS || "", 10); if (Number.isFinite(n) && n >= 60_000) return n; - return 5 * 60 * 1000; + return 60 * 60 * 1000; } -function startUpdateScheduler({ broadcast }) { +/** Delay before the catch-up check a reconnecting browser triggers. */ +const CONNECT_CHECK_DELAY_MS = 3_000; + +/** + * @param {object} opts + * @param {Function} opts.broadcast + * @param {() => number} [opts.getConnectionCount] browsers connected now + * (default: server/websocket.js's); 0 skips a scheduled tick. + * @param {(fn: Function) => Function} [opts.onConnection] subscribe to new + * browser connections; returns an unsubscribe. + * @param {Function} [opts.getStatus] the check itself (tests inject it). + * @param {number} [opts.initialDelayMs] first tick after start (default 8 s). + */ +function startUpdateScheduler({ + broadcast, + getConnectionCount, + onConnection, + getStatus = getUpdatesStatus, + initialDelayMs = 8_000, +}) { if (isUpdateCheckDisabled()) return { stop: () => {} }; + if (!getConnectionCount || !onConnection) { + const ws = require("./websocket"); + getConnectionCount = getConnectionCount || ws.getConnectionCount; + onConnection = onConnection || ws.onConnection; + } + const every = intervalMs(); + let lastCheckAt = 0; + let inFlight = false; + let connectTimer = null; let lastFingerprint = ""; let lastHadUpdate = false; @@ -25,10 +59,24 @@ function startUpdateScheduler({ broadcast }) { let initialTimer = null; let intervalTimer = null; - async function tick() { + // A scheduled tick: skipped while no browser is connected. + function scheduledTick() { if (stopped) return; + let clients = 1; try { - const status = await getUpdatesStatus(); + clients = getConnectionCount(); + } catch { + /* unknown: check as before */ + } + if (clients > 0) tick(); + } + + async function tick() { + if (stopped || inFlight) return; + inFlight = true; + lastCheckAt = Date.now(); + try { + const status = await getStatus(); // Fingerprint must change whenever anything user-visible changes, // including the *shape* of manual_command (e.g. user switches branch // or adds an upstream remote — same remote_sha / commits_behind, but @@ -72,15 +120,33 @@ function startUpdateScheduler({ broadcast }) { } } catch { // Non-fatal — never block the server on update checks + } finally { + inFlight = false; } } - initialTimer = setTimeout(() => { - tick(); - }, 8_000); + // A browser came back after the scheduled checks were skipped (or the last + // one is older than the interval): check once, shortly, so the banner is + // current without waiting up to an hour. + let unsubscribe = null; + try { + unsubscribe = onConnection(() => { + if (stopped || connectTimer || inFlight) return; + if (Date.now() - lastCheckAt < every) return; + connectTimer = setTimeout(() => { + connectTimer = null; + tick(); + }, CONNECT_CHECK_DELAY_MS); + if (typeof connectTimer.unref === "function") connectTimer.unref(); + }); + } catch { + /* no WebSocket server: scheduled ticks only */ + } + + initialTimer = setTimeout(scheduledTick, initialDelayMs); if (typeof initialTimer.unref === "function") initialTimer.unref(); - intervalTimer = setInterval(tick, intervalMs()); + intervalTimer = setInterval(scheduledTick, every); if (typeof intervalTimer.unref === "function") intervalTimer.unref(); return { @@ -88,8 +154,10 @@ function startUpdateScheduler({ broadcast }) { stopped = true; if (initialTimer) clearTimeout(initialTimer); if (intervalTimer) clearInterval(intervalTimer); + if (connectTimer) clearTimeout(connectTimer); + if (typeof unsubscribe === "function") unsubscribe(); }, }; } -module.exports = { startUpdateScheduler }; +module.exports = { startUpdateScheduler, intervalMs }; diff --git a/server/websocket.js b/server/websocket.js index 469f6582..9fdbd8bf 100644 --- a/server/websocket.js +++ b/server/websocket.js @@ -48,6 +48,13 @@ function initWebSocket(server) { }); wss.on("connection", (ws, req) => { + for (const fn of connectionListeners) { + try { + fn(ws); + } catch { + /* a listener never breaks a connection */ + } + } // Server mode: who this socket belongs to, for broadcast filtering, and // the cookie to re-check it by (a logout or a disabled user ends it). if (isServerMode()) { @@ -252,6 +259,16 @@ function broadcast(type, data) { }); } +/** + * Subscribe to new `/ws` browser connections (the update scheduler checks once + * when a browser returns after its ticks were skipped). Returns an unsubscribe. + */ +const connectionListeners = new Set(); +function onConnection(fn) { + connectionListeners.add(fn); + return () => connectionListeners.delete(fn); +} + function getConnectionCount() { if (!wss) return 0; let count = 0; @@ -306,6 +323,7 @@ module.exports = { initPtyWebSocket, broadcast, getConnectionCount, + onConnection, closeWebSocket, sizeFromQuery, };