fix(bootstrap): never stop the dashboard from inside its own systemd cgroup

A plugin self-update took the dashboard down and left it down. The chain
ran correctly up to the point where the bootstrap SIGTERMs the server it is
replacing — and that server was systemd-managed, with the bootstrap running
as its descendant, i.e. inside its cgroup. Two failures fired at once:

  * the server exits 0, and `Restart=on-failure` reads a clean exit as
    "meant to stop", so systemd did not bring it back;
  * `KillMode=control-group` reaped the bootstrap itself, mid-run, before it
    could repoint the unit at the new plugin root or write state.json.

Evidence from the incident: bootstrap.log ends at "stopped the server
started from the previous plugin version" with no systemd line after it,
.bootstrap.lock survived with a dead pid, state.json still named the
previous version, and the unit's ExecStart still pointed at the old cache
dir. The same truncation is visible on three earlier updates — those only
recovered because a later SessionStart bootstrapped again.

systemd-service.js now splits install into writeUserService (repoints the
unit, starts nothing) and restartUserService (--no-block). The bootstrap
skips the manual stop whenever a unit of ours owns the server, and defers
the restart past writeState and releaseLock, so being killed by its own
request costs nothing. The unit is Restart=always: a graceful SIGTERM from
anything must no longer read as "leave it stopped".

installUserService keeps its old semantics for `agenthub service install`,
which runs from a user's shell and is not in the cgroup.
This commit is contained in:
2026-09-11 16:30:10 +07:00
parent 0335e0e6bc
commit de8d1079e0
6 changed files with 334 additions and 38 deletions
+87 -15
View File
@@ -26,11 +26,43 @@ function unitPath(name) {
return path.join(unitDir(), `${name}.service`);
}
function systemctlAvailable() {
if (process.platform !== "linux") return false;
return spawnSync("systemctl", ["--user", "--version"], { stdio: "ignore" }).status === 0;
// Test seam: swap the systemctl implementation so unit tests never talk to a
// real service manager. Mirrors tmux.js's __setExecImpl/__reset style.
let execImpl = (args, opts) => spawnSync("systemctl", ["--user", ...args], opts);
function __setExecImpl(fn) {
execImpl = fn;
}
function __reset() {
execImpl = (args, opts) => spawnSync("systemctl", ["--user", ...args], opts);
}
function systemctlAvailable() {
if (process.platform !== "linux") return false;
return execImpl(["--version"], { stdio: "ignore" }).status === 0;
}
/**
* Is there a unit by this name, and did WE write it? The distinction decides
* whether the dashboard may be stopped by hand: a unit of ours means systemd
* owns the process, and SIGTERM-ing it out of band is the one thing that must
* never happen (see `restartUserService`).
*/
function unitIsOurs(name) {
try {
return fs.readFileSync(unitPath(name), "utf8").includes(MARKER);
} catch {
return false;
}
}
/**
* `Restart=always`, not `on-failure`: a graceful SIGTERM exits 0, which
* `on-failure` reads as "meant to stop". One stray kill — or an out-of-band
* stop during a plugin self-update — then left the dashboard down until some
* later Claude Code session happened to run the bootstrap. An explicit
* `systemctl stop` (what uninstall does) still stops it for good.
*/
function unitContent({ description, workingDir, execStart, env }) {
const envLines = Object.entries(env)
.map(([k, v]) => `Environment=${k}=${v}`)
@@ -44,7 +76,7 @@ After=network.target
Type=simple
WorkingDirectory=${workingDir}
ExecStart=${execStart}
Restart=on-failure
Restart=always
RestartSec=3
${envLines}
@@ -54,17 +86,18 @@ WantedBy=default.target
}
function systemctl(args) {
return spawnSync("systemctl", ["--user", ...args], { stdio: "inherit" });
return execImpl(args, { stdio: "inherit" });
}
/**
* Writes (or refreshes) the unit file and enables + starts it. Idempotent:
* an unchanged unit is left running as-is; a changed one (e.g. a plugin
* cache path after an update) is reloaded and restarted.
* Writes (or refreshes) the unit file and enables it for boot — but starts
* nothing. Split out of `installUserService` for the plugin self-update, which
* must be able to point the unit at the new cache directory *before* anything
* stops the running server, and defer the restart to its very last act.
*
* @returns {"written"|"unchanged"|"foreign"|"unavailable"|"failed"}
*/
function installUserService({ name, description, workingDir, execStart, env = {} }) {
function writeUserService({ name, description, workingDir, execStart, env = {} }) {
if (!systemctlAvailable()) return "unavailable";
const body = unitContent({ description, workingDir, execStart, env });
const target = unitPath(name);
@@ -72,22 +105,56 @@ function installUserService({ name, description, workingDir, execStart, env = {}
if (fs.existsSync(target)) {
const current = fs.readFileSync(target, "utf8");
if (!current.includes(MARKER)) return "foreign";
if (current === body) {
systemctl(["enable", "--now", `${name}.service`]);
return "unchanged";
}
if (current === body) return "unchanged";
}
fs.mkdirSync(unitDir(), { recursive: true });
fs.writeFileSync(target, body, "utf8");
systemctl(["daemon-reload"]);
systemctl(["enable", "--now", `${name}.service`]);
systemctl(["restart", `${name}.service`]);
systemctl(["enable", `${name}.service`]);
return "written";
} catch {
return "failed";
}
}
/**
* Ask systemd to restart the unit and do NOT wait for it.
*
* `--no-block` is the whole point, not a nicety. The caller is the bootstrap
* spawned by the running server, so it lives in that server's cgroup; the
* moment systemd stops the old main process, `KillMode=control-group` reaps
* the caller too. Blocking here means being killed mid-call, which is why the
* restart must be the last thing the bootstrap does — by then there is nothing
* left to lose.
*/
function restartUserService(name) {
if (!systemctlAvailable()) return "unavailable";
const res = systemctl(["restart", "--no-block", `${name}.service`]);
return res.status === 0 ? "restarted" : "failed";
}
/**
* Writes (or refreshes) the unit file and enables + starts it. Idempotent:
* an unchanged unit is left running as-is; a changed one (e.g. a plugin
* cache path after an update) is reloaded and restarted.
*
* Safe for `agenthub service install`, which is invoked from a user's own
* shell and is therefore not inside the unit's cgroup. The plugin bootstrap
* is, and uses `writeUserService` + `restartUserService` instead.
*
* @returns {"written"|"unchanged"|"foreign"|"unavailable"|"failed"}
*/
function installUserService(opts) {
const result = writeUserService(opts);
if (result === "unchanged") {
systemctl(["enable", "--now", `${opts.name}.service`]);
} else if (result === "written") {
systemctl(["enable", "--now", `${opts.name}.service`]);
systemctl(["restart", `${opts.name}.service`]);
}
return result;
}
/** @returns {"removed"|"absent"|"foreign"|"unavailable"} */
function uninstallUserService(name) {
if (!systemctlAvailable()) return "unavailable";
@@ -110,8 +177,13 @@ function enableLinger() {
module.exports = {
systemctlAvailable,
unitIsOurs,
writeUserService,
restartUserService,
installUserService,
uninstallUserService,
enableLinger,
unitPath,
__setExecImpl,
__reset,
};