fix: a run started a moment ago is never judged dead — another dashboard's tmux list may predate it

Only the process that starts a pane writes it into its own cached tmux list
(noteSession); a second dashboard on the same database (a dev server) holds
a member list up to 15 s old, so its heal beat ended a Codex pane seconds
after it started and the proxy gateway answered 401. reconcileOrphans and
healRunId skip a run younger than 20 s (dashboard-runs.isYoungRun).
Tests: dashboard-runs (young vs old).
This commit is contained in:
2026-10-12 14:15:22 +07:00
parent 943ba03b76
commit 64173a5428
4 changed files with 47 additions and 4 deletions
+18 -1
View File
@@ -22,8 +22,17 @@ before(() => {
});
after(() => fs.rmSync(ROOT, { recursive: true, force: true }));
// Long enough ago that no reconcile treats the run as just started.
const start = (id, extra = {}) =>
runs.recordRun({ id, mode: "interactive", cwd: "/r", status: "running", laneId: 4, ...extra });
runs.recordRun({
id,
mode: "interactive",
cwd: "/r",
status: "running",
laneId: 4,
startedAt: Date.parse("2026-01-01T00:00:00Z"),
...extra,
});
describe("dashboard-runs", () => {
it("keeps the previous run of a pane as history when the pane starts again", () => {
@@ -77,6 +86,14 @@ describe("dashboard-runs", () => {
assert.ok(runs.getRun("agenthub-lane-11"), "another tool's row stays");
});
it("never ends a run started a moment ago: another process's tmux list may predate it", () => {
start("agenthub-lane-30", { laneId: 30, startedAt: Date.now() - 2000 });
start("agenthub-lane-31", { laneId: 31, startedAt: Date.now() - 60_000 });
runs.reconcileOrphans(() => false);
assert.equal(runs.getRun("agenthub-lane-30").status, "running");
assert.equal(runs.getRun("agenthub-lane-31").status, "abandoned");
});
it("flags as abandoned only the running rows whose pane is gone", () => {
start("agenthub-lane-5", { laneId: 5 });
start("agenthub-lane-6", { laneId: 6 });
@@ -31,7 +31,15 @@ const { healAllRunIds } = require("../routes/lanes");
after(() => fs.rmSync(ROOT, { recursive: true, force: true }));
const start = (id, laneId, cwd) =>
runs.recordRun({ id, mode: "interactive", cwd, status: "running", laneId, sessionId: `s-${id}` });
runs.recordRun({
id,
mode: "interactive",
cwd,
status: "running",
laneId,
sessionId: `s-${id}`,
startedAt: Date.parse("2026-01-01T00:00:00Z"),
});
describe("a server without S3 on an S3 machine's database", () => {
it("never ends a member's run, and still ends its own", () => {
+16 -2
View File
@@ -258,8 +258,21 @@ function claimToolSessions(sessionId = null) {
}
const runningStmt = db.prepare(
"SELECT id, tool, session_id, cwd FROM dashboard_runs WHERE status IN ('running', 'spawning')"
"SELECT id, tool, session_id, cwd, started_at FROM dashboard_runs WHERE status IN ('running', 'spawning')"
);
/**
* How long a new run is never judged dead. A tmux list is a cached answer —
* a member's may be up to 15 s old (tmux.js MEMBER_LIST_MAX_AGE_MS) — and only
* the process that started a pane writes it into its own cache, so another
* dashboard on the same database (a dev server) ended a pane it had simply
* not listed yet: the proxy gateway then answered 401 to a live Codex.
*/
const YOUNG_RUN_MS = 20_000;
function isYoungRun(startedAt, now = Date.now()) {
const t = Date.parse(startedAt || "");
return Number.isFinite(t) && now - t < YOUNG_RUN_MS;
}
const endStmt = db.prepare(`
UPDATE dashboard_runs
SET status = @status,
@@ -300,7 +313,7 @@ function reconcileOrphans(isAlive = () => false) {
let changes = 0;
const { unseenMemberFolder } = require("./linux-accounts");
for (const row of runningStmt.all()) {
if (isAlive(row.id) || unseenMemberFolder(row.cwd)) continue;
if (isAlive(row.id) || unseenMemberFolder(row.cwd) || isYoungRun(row.started_at)) continue;
const interrupted =
row.tool !== "shell" &&
!!row.session_id &&
@@ -368,6 +381,7 @@ module.exports = {
getRun,
forgetShellRun,
reconcileOrphans,
isYoungRun,
listInterrupted,
dismissInterrupted,
proxyForSession,
+4
View File
@@ -128,6 +128,10 @@ function healRunId(lane, live = null) {
if (require("../lib/linux-accounts").unseenMemberFolder(lane.cwd)) return lane;
if (live) {
if (live.has(lane.run_id)) return lane;
// Started a moment ago: a cached list may not show it yet (dashboard-runs.js).
const dashboardRuns = require("../lib/dashboard-runs");
const row = dashboardRuns.getRun(lane.run_id);
if (row && row.status === "running" && dashboardRuns.isYoungRun(row.started_at)) return lane;
} else {
const run = runs.getRun(lane.run_id);
if (run && run.status === "running") return lane;