AgentHub/server/__tests__/lane-runtime.test.js

/**
 * @file Tests for lane runtime isolation: slot and port allocation, profile
 * parsing and hook execution, and the boot/down lifecycle.
 *
 * The lifecycle tests run a REAL profile whose boot hook starts a REAL server and
 * whose health hook polls it, because the properties worth proving are exactly
 * the ones a mock cannot show: that a service detached by `harness_spawn` keeps
 * listening after the hook exits, that runtime facts recomputed from scratch still
 * find it (the "survives a dashboard restart" claim), and that `downLane` reaches
 * a process it never spawned directly.
 * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
 */

const os = require("node:os");
const pathMod = require("node:path");
const fsMod = require("node:fs");

const SUITE_ROOT = fsMod.mkdtempSync(pathMod.join(os.tmpdir(), "agenthub-runtime-"));
process.env.DASHBOARD_DB_PATH = pathMod.join(SUITE_ROOT, "dashboard.db");
process.env.LANES_ROOT = pathMod.join(SUITE_ROOT, "lanes");

const { describe, it, after } = require("node:test");
const assert = require("node:assert/strict");
const net = require("node:net");

const lanesLib = require("../lib/lanes");
const slots = require("../lib/lane-slots");
const profileLib = require("../lib/lane-profile");
const runtime = require("../lib/lane-runtime");
const { isListening } = require("../lib/ports");

/**
 * Lanes whose boot hook actually spawned a detached process. `harness_spawn`
 * children outlive the hook by design, so a test that throws before its own
 * `downLane` — or never calls one — leaves a real server holding a real port
 * AFTER the suite exits. The port pool is only ten wide per name, so ten such
 * runs exhaust it and every later boot fails with EPORTBUSY: a leak in these
 * tests reads as a failure in whichever test runs next.
 */
const bootedLanes = new Set();

// One teardown, in this order on purpose: downLane reads the pid files under
// LANES_ROOT, which lives inside SUITE_ROOT — removing the tree first would
// leave nothing to kill the children with.
after(async () => {
  for (const id of bootedLanes) {
    try {
      const lane = lanesLib.getLane(id);
      if (lane) await runtime.downLane(lane);
    } catch {
      /* already stopped, or torn down by the test itself */
    }
  }
  fsMod.rmSync(SUITE_ROOT, { recursive: true, force: true });
});

let laneSeq = 0;
/** A lane row backed by a real directory, so profile lookup and mkdir work. */
function makeLane(over = {}) {
  laneSeq += 1;
  const cwd = pathMod.join(SUITE_ROOT, `lane-cwd-${laneSeq}`);
  fsMod.mkdirSync(cwd, { recursive: true });
  return lanesLib.createLane({ title: `lane ${laneSeq}`, cwd, kind: "managed", ...over });
}

/** Write a profile into a repo directory. `hooks` maps name -> script body. */
function writeProfile(root, envText, hooks = {}) {
  const dir = pathMod.join(root, ".agenthub", "profile");
  fsMod.mkdirSync(pathMod.join(dir, "hooks"), { recursive: true });
  fsMod.writeFileSync(pathMod.join(dir, "profile.env"), envText);
  for (const [name, body] of Object.entries(hooks)) {
    const file = pathMod.join(dir, "hooks", `${name}.sh`);
    fsMod.writeFileSync(file, body);
    fsMod.chmodSync(file, 0o755);
  }
  return dir;
}

/** Hold a port open so the allocator has to step aside. */
function occupy(port) {
  const server = net.createServer(() => {});
  return new Promise((resolve, reject) => {
    server.once("error", reject);
    server.listen(port, "127.0.0.1", () => resolve(server));
  });
}

describe("profile parsing", () => {
  it("keeps command substitution literal instead of executing it", () => {
    const parsed = profileLib.parseEnvFile(
      ['PORTS="api fe"', "export EVIL=$(id)", "TICK=`whoami`", "# comment", "", "BARE=plain"].join(
        "\n"
      )
    );
    assert.equal(parsed.PORTS, "api fe");
    assert.equal(parsed.EVIL, "$(id)");
    assert.equal(parsed.TICK, "`whoami`");
    assert.equal(parsed.BARE, "plain");
    assert.equal(parsed["# comment"], undefined);
  });

  it("fills defaults for omitted declarations", () => {
    const lane = makeLane();
    writeProfile(lane.cwd, "PORTS=web\n");
    const profile = profileLib.resolveProfile(lane);
    assert.deepEqual(profile.ports, ["web"]);
    assert.equal(profile.env.PORT_BASE_fe, "3000");
    assert.equal(profile.env.BACKEND_DIR, "backend");
  });

  it("returns null when no profile exists, without throwing", () => {
    const lane = makeLane();
    assert.equal(profileLib.resolveProfile(lane), null);
  });

  it("prefers the lane's own working copy over the source repo", () => {
    const source = pathMod.join(SUITE_ROOT, "source-repo");
    fsMod.mkdirSync(source, { recursive: true });
    writeProfile(source, "PORTS=fromsource\n");
    const lane = makeLane({ source_repo: source });
    writeProfile(lane.cwd, "PORTS=fromlane\n");
    assert.deepEqual(profileLib.resolveProfile(lane).ports, ["fromlane"]);
  });

  it("falls back to the source repo when the worktree has none", () => {
    const source = pathMod.join(SUITE_ROOT, "source-repo-2");
    fsMod.mkdirSync(source, { recursive: true });
    writeProfile(source, "PORTS=fromsource\n");
    const lane = makeLane({ source_repo: source });
    assert.deepEqual(profileLib.resolveProfile(lane).ports, ["fromsource"]);
  });
});

describe("slot allocation", () => {
  it("hands out the lowest free slot and reuses a released one", () => {
    const made = [];
    for (let i = 0; i < 4; i += 1) {
      const lane = makeLane();
      slots.allocateSlot(lane.id);
      made.push(lanesLib.getLane(lane.id));
    }
    assert.deepEqual(
      made.map((lane) => lane.slot),
      [1, 2, 3, 4]
    );

    slots.releaseSlot(made[1].id);
    assert.equal(lanesLib.getLane(made[1].id).slot, null);

    const next = makeLane();
    assert.equal(slots.allocateSlot(next.id), 2);

    for (const lane of [...made, next]) slots.releaseSlot(lane.id);
  });

  it("refuses to allocate past the configured ceiling", () => {
    const previous = process.env.LANE_MAX_SLOTS;
    process.env.LANE_MAX_SLOTS = "2";
    const made = [makeLane(), makeLane(), makeLane()];
    slots.allocateSlot(made[0].id);
    slots.allocateSlot(made[1].id);
    assert.throws(
      () => slots.allocateSlot(made[2].id),
      (err) => err.code === "ESLOTS"
    );
    for (const lane of made) slots.releaseSlot(lane.id);
    if (previous === undefined) delete process.env.LANE_MAX_SLOTS;
    else process.env.LANE_MAX_SLOTS = previous;
  });
});

describe("port allocation", () => {
  it("derives base + slot when the port is free", async () => {
    const lane = makeLane();
    writeProfile(lane.cwd, "PORTS=api\nPORT_BASE_api=18500\n");
    slots.allocateSlot(lane.id);
    const current = lanesLib.getLane(lane.id);
    const ports = await slots.resolvePorts(current, profileLib.resolveProfile(current));
    assert.equal(ports.api, 18500 + current.slot);
    slots.releaseSlot(lane.id);
  });

  it("steps aside by 100 when the preferred port is taken, keeping the slot digit", async () => {
    const lane = makeLane();
    writeProfile(lane.cwd, "PORTS=api\nPORT_BASE_api=18600\n");
    slots.allocateSlot(lane.id);
    const current = lanesLib.getLane(lane.id);
    const held = await occupy(18600 + current.slot);
    try {
      const ports = await slots.resolvePorts(current, profileLib.resolveProfile(current));
      assert.equal(ports.api, 18700 + current.slot);
      assert.equal(ports.api % 10, current.slot % 10);
    } finally {
      held.close();
      slots.releaseSlot(lane.id);
    }
  });

  it("never reuses a port another lane has recorded, even while that lane is down", async () => {
    const a = makeLane();
    const b = makeLane();
    writeProfile(a.cwd, "PORTS=api\nPORT_BASE_api=18800\n");
    // b's base is offset so its preferred number collides with a's recorded one.
    slots.allocateSlot(a.id);
    const laneA = lanesLib.getLane(a.id);
    lanesLib.setProvisioningFacts(laneA.id, { ports: { api: 18800 + laneA.slot } });

    slots.allocateSlot(b.id);
    const laneB = lanesLib.getLane(b.id);
    writeProfile(b.cwd, `PORTS=api\nPORT_BASE_api=${18800 + laneA.slot - laneB.slot}\n`);
    const ports = await slots.resolvePorts(laneB, profileLib.resolveProfile(laneB));
    assert.notEqual(ports.api, 18800 + laneA.slot);

    slots.releaseSlot(a.id);
    slots.releaseSlot(b.id);
  });
});

describe("hook execution", () => {
  it("exports the lane contract and scrubs inherited git variables", async () => {
    const lane = makeLane();
    writeProfile(lane.cwd, "PORTS=api\nPORT_BASE_api=18900\nCUSTOM=from-profile\n", {
      boot: '#!/usr/bin/env bash\necho "LANE=$LANE API_PORT=$API_PORT CUSTOM=$CUSTOM GITDIR=[${GIT_DIR:-unset}] ARG=$1"\n',
    });
    slots.allocateSlot(lane.id);
    lanesLib.setProvisioningFacts(lane.id, { ports: { api: 18999 } });
    const current = lanesLib.getLane(lane.id);

    process.env.GIT_DIR = "/somewhere/else/.git";
    try {
      const result = await profileLib.runHook(current, profileLib.resolveProfile(current), "boot", [
        "--no-build",
      ]);
      assert.equal(result.code, 0);
      assert.match(result.output, new RegExp(`LANE=${current.slot}\\b`));
      assert.match(result.output, /API_PORT=18999/);
      assert.match(result.output, /CUSTOM=from-profile/);
      assert.match(result.output, /GITDIR=\[unset\]/);
      assert.match(result.output, /ARG=--no-build/);
    } finally {
      delete process.env.GIT_DIR;
      slots.releaseSlot(lane.id);
    }
  });

  it("rejects a hook name outside the allowlist without spawning anything", async () => {
    const lane = makeLane();
    writeProfile(lane.cwd, "PORTS=api\n", { boot: "#!/usr/bin/env bash\ntrue\n" });
    slots.allocateSlot(lane.id);
    const current = lanesLib.getLane(lane.id);
    await assert.rejects(
      () => profileLib.runHook(current, profileLib.resolveProfile(current), "../../etc/passwd"),
      (err) => err.code === "ENOHOOK"
    );
    slots.releaseSlot(lane.id);
  });

  it("resolves with the exit code rather than throwing when a hook fails", async () => {
    const lane = makeLane();
    writeProfile(lane.cwd, "PORTS=api\n", {
      boot: '#!/usr/bin/env bash\necho "nope" >&2\nexit 3\n',
    });
    slots.allocateSlot(lane.id);
    const current = lanesLib.getLane(lane.id);
    const result = await profileLib.runHook(current, profileLib.resolveProfile(current), "boot");
    assert.equal(result.code, 3);
    assert.match(result.output, /nope/);
    assert.ok(fsMod.existsSync(result.logPath));
    slots.releaseSlot(lane.id);
  });
});

describe("lifecycle", () => {
  /** A profile whose boot hook starts a real HTTP listener and detaches it. */
  function serverProfile(lane, base) {
    writeProfile(lane.cwd, `PORTS=web\nPORT_BASE_web=${base}\nLANE_DIRS="uploads .cache"\n`, {
      boot: [
        "#!/usr/bin/env bash",
        "set -euo pipefail",
        'harness_spawn web "$LANE_DIR" python3 -m http.server "$WEB_PORT" --bind 127.0.0.1',
        "",
      ].join("\n"),
      health: [
        "#!/usr/bin/env bash",
        "set -euo pipefail",
        "for _ in $(seq 1 50); do",
        '  if curl -sf "http://127.0.0.1:$WEB_PORT/" >/dev/null; then exit 0; fi',
        "  sleep 0.2",
        "done",
        "exit 1",
        "",
      ].join("\n"),
    });
  }

  it("boots a detached stack, still sees it after a cold recompute, then stops it", async () => {
    const lane = makeLane();
    serverProfile(lane, 19100);

    bootedLanes.add(lane.id);
    const facts = await runtime.upLane(lanesLib.getLane(lane.id));
    assert.equal(facts.available, true);
    assert.equal(facts.up, true);
    assert.equal(facts.healthy, true);

    const booted = lanesLib.getLane(lane.id);
    const port = booted.ports.web;
    assert.ok(port, "a port was recorded");
    assert.equal(await isListening(port), true);

    // The declared per-lane directories exist inside the working copy.
    assert.ok(fsMod.existsSync(pathMod.join(booted.cwd, "uploads")));
    assert.ok(fsMod.existsSync(pathMod.join(booted.cwd, ".cache")));

    // "Survives a dashboard restart": nothing in memory is consulted — the facts
    // are rebuilt from the pid file on disk and a fresh port probe.
    const recomputed = await runtime.runtimeFacts(lanesLib.getLane(lane.id));
    assert.equal(recomputed.up, true);
    assert.equal(recomputed.healthy, true);
    assert.equal(recomputed.services.find((s) => s.name === "web").alive, true);

    await runtime.downLane(lanesLib.getLane(lane.id));
    assert.equal(await isListening(port), false);
    const afterDown = await runtime.runtimeFacts(lanesLib.getLane(lane.id));
    assert.equal(afterDown.up, false);
    assert.deepEqual(afterDown.services, []);

    bootedLanes.delete(lane.id);
    slots.releaseSlot(lane.id);
  });

  it("records a failed health check without touching the lane's agent fields", async () => {
    const lane = makeLane();
    writeProfile(lane.cwd, "PORTS=web\nPORT_BASE_web=19200\n", {
      boot: "#!/usr/bin/env bash\ntrue\n",
      health: "#!/usr/bin/env bash\nexit 1\n",
    });

    await assert.rejects(
      () => runtime.upLane(lanesLib.getLane(lane.id)),
      (err) => err.code === "EUNHEALTHY"
    );

    const after = lanesLib.getLane(lane.id);
    assert.equal(after.stage, "idle", "runtime must not write stage");
    assert.equal(after.status, "idle", "runtime must not write status");
    assert.equal(after.notes, null, "runtime must not write notes");

    const facts = await runtime.runtimeFacts(after);
    assert.equal(facts.lastError.code, "EUNHEALTHY");

    slots.releaseSlot(lane.id);
  });

  it("reports available:false for a lane with no profile", async () => {
    const lane = makeLane();
    const facts = await runtime.runtimeFacts(lane);
    assert.equal(facts.available, false);
    assert.ok(facts.searched.length > 0);
    await assert.rejects(
      () => runtime.upLane(lane),
      (err) => err.code === "ENOPROFILE"
    );
  });

  it("is a no-op when taking down a lane that was never up", async () => {
    const lane = makeLane();
    writeProfile(lane.cwd, "PORTS=web\n", { boot: "#!/usr/bin/env bash\ntrue\n" });
    assert.deepEqual(await runtime.downLane(lane), { killed: [] });
  });

  it("refuses a LANE_DIRS entry that escapes the working copy", () => {
    const lane = makeLane();
    writeProfile(lane.cwd, 'PORTS=web\nLANE_DIRS="../escaped"\n');
    assert.throws(
      () => runtime.makeLaneDirs(lane, profileLib.resolveProfile(lane)),
      (err) => err.code === "EBADLANEDIR"
    );
  });
});

describe("upLane qc option", () => {
  it("injects QC_BOOT_ENV into the boot hook's environment when qc:true", async () => {
    const lane = makeLane();
    writeProfile(lane.cwd, "PORTS=web\nPORT_BASE_web=19300\nQC_BOOT_ENV=SOME_QC_VAR=from-qc\n", {
      boot: [
        "#!/usr/bin/env bash",
        "set -euo pipefail",
        'echo "QC=$SOME_QC_VAR" > "$LANE_DIR/qc-marker.txt"',
        'harness_spawn web "$LANE_DIR" python3 -m http.server "$WEB_PORT" --bind 127.0.0.1',
        "",
      ].join("\n"),
      health: [
        "#!/usr/bin/env bash",
        "set -euo pipefail",
        "for _ in $(seq 1 50); do",
        '  if curl -sf "http://127.0.0.1:$WEB_PORT/" >/dev/null; then exit 0; fi',
        "  sleep 0.2",
        "done",
        "exit 1",
        "",
      ].join("\n"),
    });

    bootedLanes.add(lane.id);
    await runtime.upLane(lanesLib.getLane(lane.id), { qc: true });
    const booted = lanesLib.getLane(lane.id);
    const markerPath = pathMod.join(booted.cwd, "qc-marker.txt");
    const marker = fsMod.readFileSync(markerPath, "utf8");
    assert.match(marker, /QC=from-qc/);

    // Stop the stack BEFORE releasing the slot: downLane locates the pid file
    // through the lane's slot directory, so a released slot orphans a running
    // child with no way left to reach it. Asserting the port went quiet is what
    // makes a broken teardown fail HERE instead of leaking a live server into
    // the next run's port pool.
    const bootedPort = booted.ports.web;
    await runtime.downLane(lanesLib.getLane(lane.id));
    assert.equal(await isListening(bootedPort), false, "boot hook's child outlived downLane");
    bootedLanes.delete(lane.id);
    slots.releaseSlot(lane.id);
  });

  it("does not touch the environment when qc is omitted (default false)", async () => {
    const lane = makeLane();
    writeProfile(lane.cwd, "PORTS=web\nPORT_BASE_web=19400\nQC_BOOT_ENV=SOME_QC_VAR=from-qc\n", {
      boot: [
        "#!/usr/bin/env bash",
        "set -euo pipefail",
        'echo "QC=${SOME_QC_VAR:-not-set}" > "$LANE_DIR/qc-marker-no-qc.txt"',
        'harness_spawn web "$LANE_DIR" python3 -m http.server "$WEB_PORT" --bind 127.0.0.1',
        "",
      ].join("\n"),
      health: [
        "#!/usr/bin/env bash",
        "set -euo pipefail",
        "for _ in $(seq 1 50); do",
        '  if curl -sf "http://127.0.0.1:$WEB_PORT/" >/dev/null; then exit 0; fi',
        "  sleep 0.2",
        "done",
        "exit 1",
        "",
      ].join("\n"),
    });

    bootedLanes.add(lane.id);
    await runtime.upLane(lanesLib.getLane(lane.id));
    const booted = lanesLib.getLane(lane.id);
    const markerPath = pathMod.join(booted.cwd, "qc-marker-no-qc.txt");
    const marker = fsMod.readFileSync(markerPath, "utf8");
    assert.match(marker, /QC=not-set/);

    const bootedPort = booted.ports.web;
    await runtime.downLane(lanesLib.getLane(lane.id));
    assert.equal(await isListening(bootedPort), false, "boot hook's child outlived downLane");
    bootedLanes.delete(lane.id);
    slots.releaseSlot(lane.id);
  });
});

describe("allocation is not client-patchable", () => {
  it("ignores slot and ports coming through updateLane", () => {
    const lane = makeLane();
    slots.allocateSlot(lane.id);
    const before = lanesLib.getLane(lane.id);
    lanesLib.updateLane(lane.id, { slot: 99, ports: { api: 1 }, title: "renamed" });
    const after = lanesLib.getLane(lane.id);
    assert.equal(after.slot, before.slot);
    assert.deepEqual(after.ports, before.ports);
    assert.equal(after.title, "renamed", "patchable fields still apply");
    slots.releaseSlot(lane.id);
  });
});