fix(bootstrap): never stop the dashboard from inside its own systemd cgroup
A plugin self-update took the dashboard down and left it down. The chain
ran correctly up to the point where the bootstrap SIGTERMs the server it is
replacing — and that server was systemd-managed, with the bootstrap running
as its descendant, i.e. inside its cgroup. Two failures fired at once:
* the server exits 0, and `Restart=on-failure` reads a clean exit as
"meant to stop", so systemd did not bring it back;
* `KillMode=control-group` reaped the bootstrap itself, mid-run, before it
could repoint the unit at the new plugin root or write state.json.
Evidence from the incident: bootstrap.log ends at "stopped the server
started from the previous plugin version" with no systemd line after it,
.bootstrap.lock survived with a dead pid, state.json still named the
previous version, and the unit's ExecStart still pointed at the old cache
dir. The same truncation is visible on three earlier updates — those only
recovered because a later SessionStart bootstrapped again.
systemd-service.js now splits install into writeUserService (repoints the
unit, starts nothing) and restartUserService (--no-block). The bootstrap
skips the manual stop whenever a unit of ours owns the server, and defers
the restart past writeState and releaseLock, so being killed by its own
request costs nothing. The unit is Restart=always: a graceful SIGTERM from
anything must no longer read as "leave it stopped".
installUserService keeps its old semantics for `agenthub service install`,
which runs from a user's shell and is not in the cgroup.
This commit is contained in:
+87
-15
@@ -26,11 +26,43 @@ function unitPath(name) {
|
||||
return path.join(unitDir(), `${name}.service`);
|
||||
}
|
||||
|
||||
function systemctlAvailable() {
|
||||
if (process.platform !== "linux") return false;
|
||||
return spawnSync("systemctl", ["--user", "--version"], { stdio: "ignore" }).status === 0;
|
||||
// Test seam: swap the systemctl implementation so unit tests never talk to a
|
||||
// real service manager. Mirrors tmux.js's __setExecImpl/__reset style.
|
||||
let execImpl = (args, opts) => spawnSync("systemctl", ["--user", ...args], opts);
|
||||
|
||||
function __setExecImpl(fn) {
|
||||
execImpl = fn;
|
||||
}
|
||||
function __reset() {
|
||||
execImpl = (args, opts) => spawnSync("systemctl", ["--user", ...args], opts);
|
||||
}
|
||||
|
||||
function systemctlAvailable() {
|
||||
if (process.platform !== "linux") return false;
|
||||
return execImpl(["--version"], { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Is there a unit by this name, and did WE write it? The distinction decides
|
||||
* whether the dashboard may be stopped by hand: a unit of ours means systemd
|
||||
* owns the process, and SIGTERM-ing it out of band is the one thing that must
|
||||
* never happen (see `restartUserService`).
|
||||
*/
|
||||
function unitIsOurs(name) {
|
||||
try {
|
||||
return fs.readFileSync(unitPath(name), "utf8").includes(MARKER);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `Restart=always`, not `on-failure`: a graceful SIGTERM exits 0, which
|
||||
* `on-failure` reads as "meant to stop". One stray kill — or an out-of-band
|
||||
* stop during a plugin self-update — then left the dashboard down until some
|
||||
* later Claude Code session happened to run the bootstrap. An explicit
|
||||
* `systemctl stop` (what uninstall does) still stops it for good.
|
||||
*/
|
||||
function unitContent({ description, workingDir, execStart, env }) {
|
||||
const envLines = Object.entries(env)
|
||||
.map(([k, v]) => `Environment=${k}=${v}`)
|
||||
@@ -44,7 +76,7 @@ After=network.target
|
||||
Type=simple
|
||||
WorkingDirectory=${workingDir}
|
||||
ExecStart=${execStart}
|
||||
Restart=on-failure
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
${envLines}
|
||||
|
||||
@@ -54,17 +86,18 @@ WantedBy=default.target
|
||||
}
|
||||
|
||||
function systemctl(args) {
|
||||
return spawnSync("systemctl", ["--user", ...args], { stdio: "inherit" });
|
||||
return execImpl(args, { stdio: "inherit" });
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes (or refreshes) the unit file and enables + starts it. Idempotent:
|
||||
* an unchanged unit is left running as-is; a changed one (e.g. a plugin
|
||||
* cache path after an update) is reloaded and restarted.
|
||||
* Writes (or refreshes) the unit file and enables it for boot — but starts
|
||||
* nothing. Split out of `installUserService` for the plugin self-update, which
|
||||
* must be able to point the unit at the new cache directory *before* anything
|
||||
* stops the running server, and defer the restart to its very last act.
|
||||
*
|
||||
* @returns {"written"|"unchanged"|"foreign"|"unavailable"|"failed"}
|
||||
*/
|
||||
function installUserService({ name, description, workingDir, execStart, env = {} }) {
|
||||
function writeUserService({ name, description, workingDir, execStart, env = {} }) {
|
||||
if (!systemctlAvailable()) return "unavailable";
|
||||
const body = unitContent({ description, workingDir, execStart, env });
|
||||
const target = unitPath(name);
|
||||
@@ -72,22 +105,56 @@ function installUserService({ name, description, workingDir, execStart, env = {}
|
||||
if (fs.existsSync(target)) {
|
||||
const current = fs.readFileSync(target, "utf8");
|
||||
if (!current.includes(MARKER)) return "foreign";
|
||||
if (current === body) {
|
||||
systemctl(["enable", "--now", `${name}.service`]);
|
||||
return "unchanged";
|
||||
}
|
||||
if (current === body) return "unchanged";
|
||||
}
|
||||
fs.mkdirSync(unitDir(), { recursive: true });
|
||||
fs.writeFileSync(target, body, "utf8");
|
||||
systemctl(["daemon-reload"]);
|
||||
systemctl(["enable", "--now", `${name}.service`]);
|
||||
systemctl(["restart", `${name}.service`]);
|
||||
systemctl(["enable", `${name}.service`]);
|
||||
return "written";
|
||||
} catch {
|
||||
return "failed";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask systemd to restart the unit and do NOT wait for it.
|
||||
*
|
||||
* `--no-block` is the whole point, not a nicety. The caller is the bootstrap
|
||||
* spawned by the running server, so it lives in that server's cgroup; the
|
||||
* moment systemd stops the old main process, `KillMode=control-group` reaps
|
||||
* the caller too. Blocking here means being killed mid-call, which is why the
|
||||
* restart must be the last thing the bootstrap does — by then there is nothing
|
||||
* left to lose.
|
||||
*/
|
||||
function restartUserService(name) {
|
||||
if (!systemctlAvailable()) return "unavailable";
|
||||
const res = systemctl(["restart", "--no-block", `${name}.service`]);
|
||||
return res.status === 0 ? "restarted" : "failed";
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes (or refreshes) the unit file and enables + starts it. Idempotent:
|
||||
* an unchanged unit is left running as-is; a changed one (e.g. a plugin
|
||||
* cache path after an update) is reloaded and restarted.
|
||||
*
|
||||
* Safe for `agenthub service install`, which is invoked from a user's own
|
||||
* shell and is therefore not inside the unit's cgroup. The plugin bootstrap
|
||||
* is, and uses `writeUserService` + `restartUserService` instead.
|
||||
*
|
||||
* @returns {"written"|"unchanged"|"foreign"|"unavailable"|"failed"}
|
||||
*/
|
||||
function installUserService(opts) {
|
||||
const result = writeUserService(opts);
|
||||
if (result === "unchanged") {
|
||||
systemctl(["enable", "--now", `${opts.name}.service`]);
|
||||
} else if (result === "written") {
|
||||
systemctl(["enable", "--now", `${opts.name}.service`]);
|
||||
systemctl(["restart", `${opts.name}.service`]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** @returns {"removed"|"absent"|"foreign"|"unavailable"} */
|
||||
function uninstallUserService(name) {
|
||||
if (!systemctlAvailable()) return "unavailable";
|
||||
@@ -110,8 +177,13 @@ function enableLinger() {
|
||||
|
||||
module.exports = {
|
||||
systemctlAvailable,
|
||||
unitIsOurs,
|
||||
writeUserService,
|
||||
restartUserService,
|
||||
installUserService,
|
||||
uninstallUserService,
|
||||
enableLinger,
|
||||
unitPath,
|
||||
__setExecImpl,
|
||||
__reset,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user