fix: a run started a moment ago is never judged dead — another dashboard's tmux list may predate it
Only the process that starts a pane writes it into its own cached tmux list (noteSession); a second dashboard on the same database (a dev server) holds a member list up to 15 s old, so its heal beat ended a Codex pane seconds after it started and the proxy gateway answered 401. reconcileOrphans and healRunId skip a run younger than 20 s (dashboard-runs.isYoungRun). Tests: dashboard-runs (young vs old).
This commit is contained in:
@@ -22,8 +22,17 @@ before(() => {
|
||||
});
|
||||
after(() => fs.rmSync(ROOT, { recursive: true, force: true }));
|
||||
|
||||
// Long enough ago that no reconcile treats the run as just started.
|
||||
const start = (id, extra = {}) =>
|
||||
runs.recordRun({ id, mode: "interactive", cwd: "/r", status: "running", laneId: 4, ...extra });
|
||||
runs.recordRun({
|
||||
id,
|
||||
mode: "interactive",
|
||||
cwd: "/r",
|
||||
status: "running",
|
||||
laneId: 4,
|
||||
startedAt: Date.parse("2026-01-01T00:00:00Z"),
|
||||
...extra,
|
||||
});
|
||||
|
||||
describe("dashboard-runs", () => {
|
||||
it("keeps the previous run of a pane as history when the pane starts again", () => {
|
||||
@@ -77,6 +86,14 @@ describe("dashboard-runs", () => {
|
||||
assert.ok(runs.getRun("agenthub-lane-11"), "another tool's row stays");
|
||||
});
|
||||
|
||||
it("never ends a run started a moment ago: another process's tmux list may predate it", () => {
|
||||
start("agenthub-lane-30", { laneId: 30, startedAt: Date.now() - 2000 });
|
||||
start("agenthub-lane-31", { laneId: 31, startedAt: Date.now() - 60_000 });
|
||||
runs.reconcileOrphans(() => false);
|
||||
assert.equal(runs.getRun("agenthub-lane-30").status, "running");
|
||||
assert.equal(runs.getRun("agenthub-lane-31").status, "abandoned");
|
||||
});
|
||||
|
||||
it("flags as abandoned only the running rows whose pane is gone", () => {
|
||||
start("agenthub-lane-5", { laneId: 5 });
|
||||
start("agenthub-lane-6", { laneId: 6 });
|
||||
|
||||
@@ -31,7 +31,15 @@ const { healAllRunIds } = require("../routes/lanes");
|
||||
after(() => fs.rmSync(ROOT, { recursive: true, force: true }));
|
||||
|
||||
const start = (id, laneId, cwd) =>
|
||||
runs.recordRun({ id, mode: "interactive", cwd, status: "running", laneId, sessionId: `s-${id}` });
|
||||
runs.recordRun({
|
||||
id,
|
||||
mode: "interactive",
|
||||
cwd,
|
||||
status: "running",
|
||||
laneId,
|
||||
sessionId: `s-${id}`,
|
||||
startedAt: Date.parse("2026-01-01T00:00:00Z"),
|
||||
});
|
||||
|
||||
describe("a server without S3 on an S3 machine's database", () => {
|
||||
it("never ends a member's run, and still ends its own", () => {
|
||||
|
||||
@@ -258,8 +258,21 @@ function claimToolSessions(sessionId = null) {
|
||||
}
|
||||
|
||||
const runningStmt = db.prepare(
|
||||
"SELECT id, tool, session_id, cwd FROM dashboard_runs WHERE status IN ('running', 'spawning')"
|
||||
"SELECT id, tool, session_id, cwd, started_at FROM dashboard_runs WHERE status IN ('running', 'spawning')"
|
||||
);
|
||||
|
||||
/**
|
||||
* How long a new run is never judged dead. A tmux list is a cached answer —
|
||||
* a member's may be up to 15 s old (tmux.js MEMBER_LIST_MAX_AGE_MS) — and only
|
||||
* the process that started a pane writes it into its own cache, so another
|
||||
* dashboard on the same database (a dev server) ended a pane it had simply
|
||||
* not listed yet: the proxy gateway then answered 401 to a live Codex.
|
||||
*/
|
||||
const YOUNG_RUN_MS = 20_000;
|
||||
function isYoungRun(startedAt, now = Date.now()) {
|
||||
const t = Date.parse(startedAt || "");
|
||||
return Number.isFinite(t) && now - t < YOUNG_RUN_MS;
|
||||
}
|
||||
const endStmt = db.prepare(`
|
||||
UPDATE dashboard_runs
|
||||
SET status = @status,
|
||||
@@ -300,7 +313,7 @@ function reconcileOrphans(isAlive = () => false) {
|
||||
let changes = 0;
|
||||
const { unseenMemberFolder } = require("./linux-accounts");
|
||||
for (const row of runningStmt.all()) {
|
||||
if (isAlive(row.id) || unseenMemberFolder(row.cwd)) continue;
|
||||
if (isAlive(row.id) || unseenMemberFolder(row.cwd) || isYoungRun(row.started_at)) continue;
|
||||
const interrupted =
|
||||
row.tool !== "shell" &&
|
||||
!!row.session_id &&
|
||||
@@ -368,6 +381,7 @@ module.exports = {
|
||||
getRun,
|
||||
forgetShellRun,
|
||||
reconcileOrphans,
|
||||
isYoungRun,
|
||||
listInterrupted,
|
||||
dismissInterrupted,
|
||||
proxyForSession,
|
||||
|
||||
@@ -128,6 +128,10 @@ function healRunId(lane, live = null) {
|
||||
if (require("../lib/linux-accounts").unseenMemberFolder(lane.cwd)) return lane;
|
||||
if (live) {
|
||||
if (live.has(lane.run_id)) return lane;
|
||||
// Started a moment ago: a cached list may not show it yet (dashboard-runs.js).
|
||||
const dashboardRuns = require("../lib/dashboard-runs");
|
||||
const row = dashboardRuns.getRun(lane.run_id);
|
||||
if (row && row.status === "running" && dashboardRuns.isYoungRun(row.started_at)) return lane;
|
||||
} else {
|
||||
const run = runs.getRun(lane.run_id);
|
||||
if (run && run.status === "running") return lane;
|
||||
|
||||
Reference in New Issue
Block a user