Smartgift-AI/AgentHub
14
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases 86 Wiki Activity

fix(bootstrap): never stop the dashboard from inside its own systemd cgroup

Browse Source 
A plugin self-update took the dashboard down and left it down. The chain
ran correctly up to the point where the bootstrap SIGTERMs the server it is
replacing — and that server was systemd-managed, with the bootstrap running
as its descendant, i.e. inside its cgroup. Two failures fired at once:

  * the server exits 0, and `Restart=on-failure` reads a clean exit as
    "meant to stop", so systemd did not bring it back;
  * `KillMode=control-group` reaped the bootstrap itself, mid-run, before it
    could repoint the unit at the new plugin root or write state.json.

Evidence from the incident: bootstrap.log ends at "stopped the server
started from the previous plugin version" with no systemd line after it,
.bootstrap.lock survived with a dead pid, state.json still named the
previous version, and the unit's ExecStart still pointed at the old cache
dir. The same truncation is visible on three earlier updates — those only
recovered because a later SessionStart bootstrapped again.

systemd-service.js now splits install into writeUserService (repoints the
unit, starts nothing) and restartUserService (--no-block). The bootstrap
skips the manual stop whenever a unit of ours owns the server, and defers
the restart past writeState and releaseLock, so being killed by its own
request costs nothing. The unit is Restart=always: a graceful SIGTERM from
anything must no longer read as "leave it stopped".

installUserService keeps its old semantics for `agenthub service install`,
which runs from a user's shell and is not in the cgroup.
This commit is contained in:
nntrivi2001
2026-09-11 16:30:10 +07:00
parent 0335e0e6bc
commit de8d1079e0
6 changed files with 334 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/plugin-bootstrap.js
+54 -19
View File
@@ -465,6 +465,11 @@ function bootstrap(rt = runtimeDir(), { force = false } = {}) {
return "locked";
}
// Declared out here because the restart it guards has to happen in the
// `finally`, after the lock is released and the state is on disk: it is the
// step that may cost this process its own life.
let pendingSystemdRestart = false;
try {
const wantDeps = depsHash();
const state = readState(rt) || {};
@@ -504,34 +509,55 @@ function bootstrap(rt = runtimeDir(), { force = false } = {}) {
log(rt, "built the dashboard UI bundle");
}
const svc = require("./systemd-service");
// Who owns the running server decides how it may be retired. A unit of
// ours means systemd owns it — and this process is a descendant of that
// server, so it sits in the unit's cgroup. SIGTERM-ing the main process
// there does two fatal things at once: the clean exit(0) tells systemd
// "meant to stop", and `KillMode=control-group` reaps this bootstrap
// before it can write state or point the unit at the new cache dir. That
// is the bug that left a dashboard dead after every self-update until the
// next Claude Code session happened to bootstrap again.
const systemdOwned = svc.systemctlAvailable() && svc.unitIsOurs("agenthub");
// After a plugin update the running server executes code from a cache
// directory Claude Code has already discarded — retire it first.
// directory Claude Code has already discarded — retire it first. Only
// when nothing else owns its lifecycle; otherwise systemd retires it, in
// one restart job, once everything below is durable.
const movedRoot = force || (state.pluginRoot && state.pluginRoot !== PLUGIN_ROOT);
if (movedRoot && stopDashboard()) {
if (movedRoot && !systemdOwned && stopDashboard()) {
log(rt, "stopped the server started from the previous plugin version");
waitForPortsFree();
}
if (movedRoot || !serverIsLive()) {
const svc = require("./systemd-service");
const result = svc.systemctlAvailable()
? svc.installUserService({
name: "agenthub",
description: "AgentHub dashboard server (Claude Code plugin)",
workingDir: PLUGIN_ROOT,
execStart: `${process.execPath} ${path.join(PLUGIN_ROOT, "server", "index.js")}`,
env: {
NODE_ENV: "production",
NODE_PATH: path.join(rt, "node_modules"),
DASHBOARD_CLIENT_DIST: path.join(rt, "client-dist"),
// systemd --user units get systemd's own minimal PATH, not the
// shell's — forward it so the server can find `claude`/`codex`.
PATH: process.env.PATH || "",
},
})
: "unavailable";
const unit = {
name: "agenthub",
description: "AgentHub dashboard server (Claude Code plugin)",
workingDir: PLUGIN_ROOT,
execStart: `${process.execPath} ${path.join(PLUGIN_ROOT, "server", "index.js")}`,
env: {
NODE_ENV: "production",
NODE_PATH: path.join(rt, "node_modules"),
DASHBOARD_CLIENT_DIST: path.join(rt, "client-dist"),
// systemd --user units get systemd's own minimal PATH, not the
// shell's — forward it so the server can find `claude`/`codex`.
PATH: process.env.PATH || "",
},
};
// Two shapes of the same step. When systemd already owns the server the
// unit is only REWRITTEN here — pointing at the new plugin root — and
// the restart is deferred to `pendingSystemdRestart` below, because
// starting it now would kill this process mid-bootstrap. Otherwise
// there is no cgroup to be reaped by, and install-and-start is fine.
const result = systemdOwned
? svc.writeUserService(unit)
: svc.systemctlAvailable()
? svc.installUserService(unit)
: "unavailable";
if (result === "written" || result === "unchanged") {
if (result === "written") svc.enableLinger();
pendingSystemdRestart = systemdOwned;
log(
rt,
result === "written"
@@ -572,6 +598,15 @@ function bootstrap(rt = runtimeDir(), { force = false } = {}) {
return "ok";
} finally {
releaseLock(rt);
// LAST, and after the lock: systemd stops the old main process to start
// the new one, and this process shares its cgroup, so it may not outlive
// this call. Everything that had to be durable already is — the state
// file is written and the lock is gone, so even being killed here leaves
// no debris and no half-recorded bootstrap.
if (pendingSystemdRestart) {
log(rt, "handing the server over to systemd (restarting the unit)");
require("./systemd-service").restartUserService("agenthub");
}
}
}
scripts/systemd-service.js
+87 -15
View File
@@ -26,11 +26,43 @@ function unitPath(name) {
return path.join(unitDir(), `${name}.service`);
}
function systemctlAvailable() {
if (process.platform !== "linux") return false;
return spawnSync("systemctl", ["--user", "--version"], { stdio: "ignore" }).status === 0;
// Test seam: swap the systemctl implementation so unit tests never talk to a
// real service manager. Mirrors tmux.js's __setExecImpl/__reset style.
let execImpl = (args, opts) => spawnSync("systemctl", ["--user", ...args], opts);
function __setExecImpl(fn) {
execImpl = fn;
}
function __reset() {
execImpl = (args, opts) => spawnSync("systemctl", ["--user", ...args], opts);
}
function systemctlAvailable() {
if (process.platform !== "linux") return false;
return execImpl(["--version"], { stdio: "ignore" }).status === 0;
}
/**
* Is there a unit by this name, and did WE write it? The distinction decides
* whether the dashboard may be stopped by hand: a unit of ours means systemd
* owns the process, and SIGTERM-ing it out of band is the one thing that must
* never happen (see `restartUserService`).
*/
function unitIsOurs(name) {
try {
return fs.readFileSync(unitPath(name), "utf8").includes(MARKER);
} catch {
return false;
}
}
/**
* `Restart=always`, not `on-failure`: a graceful SIGTERM exits 0, which
* `on-failure` reads as "meant to stop". One stray kill — or an out-of-band
* stop during a plugin self-update — then left the dashboard down until some
* later Claude Code session happened to run the bootstrap. An explicit
* `systemctl stop` (what uninstall does) still stops it for good.
*/
function unitContent({ description, workingDir, execStart, env }) {
const envLines = Object.entries(env)
.map(([k, v]) => `Environment=${k}=${v}`)
@@ -44,7 +76,7 @@ After=network.target
Type=simple
WorkingDirectory=${workingDir}
ExecStart=${execStart}
Restart=on-failure
Restart=always
RestartSec=3
${envLines}
@@ -54,17 +86,18 @@ WantedBy=default.target
}
function systemctl(args) {
return spawnSync("systemctl", ["--user", ...args], { stdio: "inherit" });
return execImpl(args, { stdio: "inherit" });
}
/**
* Writes (or refreshes) the unit file and enables + starts it. Idempotent:
* an unchanged unit is left running as-is; a changed one (e.g. a plugin
* cache path after an update) is reloaded and restarted.
* Writes (or refreshes) the unit file and enables it for boot — but starts
* nothing. Split out of `installUserService` for the plugin self-update, which
* must be able to point the unit at the new cache directory *before* anything
* stops the running server, and defer the restart to its very last act.
*
* @returns {"written"|"unchanged"|"foreign"|"unavailable"|"failed"}
*/
function installUserService({ name, description, workingDir, execStart, env = {} }) {
function writeUserService({ name, description, workingDir, execStart, env = {} }) {
if (!systemctlAvailable()) return "unavailable";
const body = unitContent({ description, workingDir, execStart, env });
const target = unitPath(name);
@@ -72,22 +105,56 @@ function installUserService({ name, description, workingDir, execStart, env = {}
if (fs.existsSync(target)) {
const current = fs.readFileSync(target, "utf8");
if (!current.includes(MARKER)) return "foreign";
if (current === body) {
systemctl(["enable", "--now", `${name}.service`]);
return "unchanged";
}
if (current === body) return "unchanged";
}
fs.mkdirSync(unitDir(), { recursive: true });
fs.writeFileSync(target, body, "utf8");
systemctl(["daemon-reload"]);
systemctl(["enable", "--now", `${name}.service`]);
systemctl(["restart", `${name}.service`]);
systemctl(["enable", `${name}.service`]);
return "written";
} catch {
return "failed";
}
}
/**
* Ask systemd to restart the unit and do NOT wait for it.
*
* `--no-block` is the whole point, not a nicety. The caller is the bootstrap
* spawned by the running server, so it lives in that server's cgroup; the
* moment systemd stops the old main process, `KillMode=control-group` reaps
* the caller too. Blocking here means being killed mid-call, which is why the
* restart must be the last thing the bootstrap does — by then there is nothing
* left to lose.
*/
function restartUserService(name) {
if (!systemctlAvailable()) return "unavailable";
const res = systemctl(["restart", "--no-block", `${name}.service`]);
return res.status === 0 ? "restarted" : "failed";
}
/**
* Writes (or refreshes) the unit file and enables + starts it. Idempotent:
* an unchanged unit is left running as-is; a changed one (e.g. a plugin
* cache path after an update) is reloaded and restarted.
*
* Safe for `agenthub service install`, which is invoked from a user's own
* shell and is therefore not inside the unit's cgroup. The plugin bootstrap
* is, and uses `writeUserService` + `restartUserService` instead.
*
* @returns {"written"|"unchanged"|"foreign"|"unavailable"|"failed"}
*/
function installUserService(opts) {
const result = writeUserService(opts);
if (result === "unchanged") {
systemctl(["enable", "--now", `${opts.name}.service`]);
} else if (result === "written") {
systemctl(["enable", "--now", `${opts.name}.service`]);
systemctl(["restart", `${opts.name}.service`]);
}
return result;
}
/** @returns {"removed"|"absent"|"foreign"|"unavailable"} */
function uninstallUserService(name) {
if (!systemctlAvailable()) return "unavailable";
@@ -110,8 +177,13 @@ function enableLinger() {
module.exports = {
systemctlAvailable,
unitIsOurs,
writeUserService,
restartUserService,
installUserService,
uninstallUserService,
enableLinger,
unitPath,
__setExecImpl,
__reset,
};