fix: a run started a moment ago is never judged dead — another dashboard's tmux list may predate it
Only the process that starts a pane writes it into its own cached tmux list (noteSession); a second dashboard on the same database (a dev server) holds a member list up to 15 s old, so its heal beat ended a Codex pane seconds after it started and the proxy gateway answered 401. reconcileOrphans and healRunId skip a run younger than 20 s (dashboard-runs.isYoungRun). Tests: dashboard-runs (young vs old).
This commit is contained in:
@@ -22,8 +22,17 @@ before(() => {
|
|||||||
});
|
});
|
||||||
after(() => fs.rmSync(ROOT, { recursive: true, force: true }));
|
after(() => fs.rmSync(ROOT, { recursive: true, force: true }));
|
||||||
|
|
||||||
|
// Long enough ago that no reconcile treats the run as just started.
|
||||||
const start = (id, extra = {}) =>
|
const start = (id, extra = {}) =>
|
||||||
runs.recordRun({ id, mode: "interactive", cwd: "/r", status: "running", laneId: 4, ...extra });
|
runs.recordRun({
|
||||||
|
id,
|
||||||
|
mode: "interactive",
|
||||||
|
cwd: "/r",
|
||||||
|
status: "running",
|
||||||
|
laneId: 4,
|
||||||
|
startedAt: Date.parse("2026-01-01T00:00:00Z"),
|
||||||
|
...extra,
|
||||||
|
});
|
||||||
|
|
||||||
describe("dashboard-runs", () => {
|
describe("dashboard-runs", () => {
|
||||||
it("keeps the previous run of a pane as history when the pane starts again", () => {
|
it("keeps the previous run of a pane as history when the pane starts again", () => {
|
||||||
@@ -77,6 +86,14 @@ describe("dashboard-runs", () => {
|
|||||||
assert.ok(runs.getRun("agenthub-lane-11"), "another tool's row stays");
|
assert.ok(runs.getRun("agenthub-lane-11"), "another tool's row stays");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("never ends a run started a moment ago: another process's tmux list may predate it", () => {
|
||||||
|
start("agenthub-lane-30", { laneId: 30, startedAt: Date.now() - 2000 });
|
||||||
|
start("agenthub-lane-31", { laneId: 31, startedAt: Date.now() - 60_000 });
|
||||||
|
runs.reconcileOrphans(() => false);
|
||||||
|
assert.equal(runs.getRun("agenthub-lane-30").status, "running");
|
||||||
|
assert.equal(runs.getRun("agenthub-lane-31").status, "abandoned");
|
||||||
|
});
|
||||||
|
|
||||||
it("flags as abandoned only the running rows whose pane is gone", () => {
|
it("flags as abandoned only the running rows whose pane is gone", () => {
|
||||||
start("agenthub-lane-5", { laneId: 5 });
|
start("agenthub-lane-5", { laneId: 5 });
|
||||||
start("agenthub-lane-6", { laneId: 6 });
|
start("agenthub-lane-6", { laneId: 6 });
|
||||||
|
|||||||
@@ -31,7 +31,15 @@ const { healAllRunIds } = require("../routes/lanes");
|
|||||||
after(() => fs.rmSync(ROOT, { recursive: true, force: true }));
|
after(() => fs.rmSync(ROOT, { recursive: true, force: true }));
|
||||||
|
|
||||||
const start = (id, laneId, cwd) =>
|
const start = (id, laneId, cwd) =>
|
||||||
runs.recordRun({ id, mode: "interactive", cwd, status: "running", laneId, sessionId: `s-${id}` });
|
runs.recordRun({
|
||||||
|
id,
|
||||||
|
mode: "interactive",
|
||||||
|
cwd,
|
||||||
|
status: "running",
|
||||||
|
laneId,
|
||||||
|
sessionId: `s-${id}`,
|
||||||
|
startedAt: Date.parse("2026-01-01T00:00:00Z"),
|
||||||
|
});
|
||||||
|
|
||||||
describe("a server without S3 on an S3 machine's database", () => {
|
describe("a server without S3 on an S3 machine's database", () => {
|
||||||
it("never ends a member's run, and still ends its own", () => {
|
it("never ends a member's run, and still ends its own", () => {
|
||||||
|
|||||||
@@ -258,8 +258,21 @@ function claimToolSessions(sessionId = null) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const runningStmt = db.prepare(
|
const runningStmt = db.prepare(
|
||||||
"SELECT id, tool, session_id, cwd FROM dashboard_runs WHERE status IN ('running', 'spawning')"
|
"SELECT id, tool, session_id, cwd, started_at FROM dashboard_runs WHERE status IN ('running', 'spawning')"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How long a new run is never judged dead. A tmux list is a cached answer —
|
||||||
|
* a member's may be up to 15 s old (tmux.js MEMBER_LIST_MAX_AGE_MS) — and only
|
||||||
|
* the process that started a pane writes it into its own cache, so another
|
||||||
|
* dashboard on the same database (a dev server) ended a pane it had simply
|
||||||
|
* not listed yet: the proxy gateway then answered 401 to a live Codex.
|
||||||
|
*/
|
||||||
|
const YOUNG_RUN_MS = 20_000;
|
||||||
|
function isYoungRun(startedAt, now = Date.now()) {
|
||||||
|
const t = Date.parse(startedAt || "");
|
||||||
|
return Number.isFinite(t) && now - t < YOUNG_RUN_MS;
|
||||||
|
}
|
||||||
const endStmt = db.prepare(`
|
const endStmt = db.prepare(`
|
||||||
UPDATE dashboard_runs
|
UPDATE dashboard_runs
|
||||||
SET status = @status,
|
SET status = @status,
|
||||||
@@ -300,7 +313,7 @@ function reconcileOrphans(isAlive = () => false) {
|
|||||||
let changes = 0;
|
let changes = 0;
|
||||||
const { unseenMemberFolder } = require("./linux-accounts");
|
const { unseenMemberFolder } = require("./linux-accounts");
|
||||||
for (const row of runningStmt.all()) {
|
for (const row of runningStmt.all()) {
|
||||||
if (isAlive(row.id) || unseenMemberFolder(row.cwd)) continue;
|
if (isAlive(row.id) || unseenMemberFolder(row.cwd) || isYoungRun(row.started_at)) continue;
|
||||||
const interrupted =
|
const interrupted =
|
||||||
row.tool !== "shell" &&
|
row.tool !== "shell" &&
|
||||||
!!row.session_id &&
|
!!row.session_id &&
|
||||||
@@ -368,6 +381,7 @@ module.exports = {
|
|||||||
getRun,
|
getRun,
|
||||||
forgetShellRun,
|
forgetShellRun,
|
||||||
reconcileOrphans,
|
reconcileOrphans,
|
||||||
|
isYoungRun,
|
||||||
listInterrupted,
|
listInterrupted,
|
||||||
dismissInterrupted,
|
dismissInterrupted,
|
||||||
proxyForSession,
|
proxyForSession,
|
||||||
|
|||||||
@@ -128,6 +128,10 @@ function healRunId(lane, live = null) {
|
|||||||
if (require("../lib/linux-accounts").unseenMemberFolder(lane.cwd)) return lane;
|
if (require("../lib/linux-accounts").unseenMemberFolder(lane.cwd)) return lane;
|
||||||
if (live) {
|
if (live) {
|
||||||
if (live.has(lane.run_id)) return lane;
|
if (live.has(lane.run_id)) return lane;
|
||||||
|
// Started a moment ago: a cached list may not show it yet (dashboard-runs.js).
|
||||||
|
const dashboardRuns = require("../lib/dashboard-runs");
|
||||||
|
const row = dashboardRuns.getRun(lane.run_id);
|
||||||
|
if (row && row.status === "running" && dashboardRuns.isYoungRun(row.started_at)) return lane;
|
||||||
} else {
|
} else {
|
||||||
const run = runs.getRun(lane.run_id);
|
const run = runs.getRun(lane.run_id);
|
||||||
if (run && run.status === "running") return lane;
|
if (run && run.status === "running") return lane;
|
||||||
|
|||||||
Reference in New Issue
Block a user