/** * @file Lane stack lifecycle: bring a lane's services up through its profile's * hooks, take them down, and report what is actually running. Also owns the * data-isolation lifecycle (A2): `provisionLane`/`resetLaneData`/`removeLaneData` * seed `.env`, create/drop the lane's own database, and run `bootstrap`/ * `migrate`/`seed` at the points Shipyard's `lane-bootstrap.sh`/`lane-up.sh`/ * `lane-reset.sh`/`lane-remove.sh` do — CCAM's version of "ports came from A1, * everything else a lane needs to run its own stack comes from here." * * Two properties shape everything here. * * Services are FULLY DETACHED (see `harness_spawn` in lane-profile.js), so a * lane's stack outlives both the hook that started it and the dashboard itself — * restarting or updating CCAM must never kill work in progress. * * Liveness is COMPUTED, never stored. Whether a stack is up is not a fact CCAM * controls: a process dies to OOM, to a stray `kill`, to a reboot. A cached * "running" flag would be wrong from that moment until something noticed, so * `runtimeFacts` re-derives it from pid files and port probes on every read. The * payoff is that adopting a stack after a dashboard restart needs no code at all. * * This module writes `slot` and `ports` and nothing else on the lane row. It never * writes `stage`, `status` or `notes`: in CCAM those describe the AGENT's work, not * the stack's state, and conflating "a server is listening" with "an agent is * running" would corrupt lane liveness. Boot failures live in `last-error.json` * and surface through `runtimeFacts`. * @author Nguyễn Ngọc Trí Vĩ */ const fs = require("node:fs"); const path = require("node:path"); const { execFile } = require("node:child_process"); const { promisify } = require("node:util"); const lanesLib = require("./lanes"); const { allocateSlot, resolvePorts, slotDirs, portsSteppedAside, portBase, dbName, } = require("./lane-slots"); const { resolveProfile, profileSearchPaths, runHook } = require("./lane-profile"); const { isListening, listenerPids } = require("./ports"); const { seedEnv } = require("./lane-env"); const { ensureDatabase, dropDatabase } = require("./lane-services"); const { readSecrets } = require("./secrets"); const execFileAsync = promisify(execFile); /** A health hook that never returns is a failed boot, not an eternal wait. */ const HEALTH_TIMEOUT_MS = Number(process.env.LANE_HEALTH_TIMEOUT_MS) || 180_000; const BOOT_TIMEOUT_MS = Number(process.env.LANE_BOOT_TIMEOUT_MS) || 900_000; /** Resolve a lane's profile or throw the error a route turns into a 400. */ function requireProfile(lane) { const profile = resolveProfile(lane); if (!profile) { throw Object.assign( new Error(`lane has no .ccam/profile — looked in: ${profileSearchPaths(lane).join(", ")}`), { code: "ENOPROFILE", searched: profileSearchPaths(lane) } ); } return profile; } /** Direct children of a pid. Empty when pgrep is unavailable — the parent still dies. */ async function childPids(pid) { try { const { stdout } = await execFileAsync("pgrep", ["-P", String(pid)]); return stdout .split("\n") .map((line) => line.trim()) .filter((line) => /^\d+$/.test(line)) .map(Number); } catch { return []; } } /** * Kill a process and every descendant, deepest first. * * Bottom-up matters: killing the parent first reparents its children to init, * where nothing knows to look for them. This is the failure `lane-down.sh`'s * `kill_tree` was written for — a uvicorn reloader or a celery prefork pool whose * workers survived a kill aimed at the recorded pid, kept the port bound, and made * the next boot fail for a reason nothing reported. * * @param {number} pid - Root of the tree. * @returns {Promise} Pids signalled. */ async function killTree(pid) { const killed = []; for (const child of await childPids(pid)) { killed.push(...(await killTree(child))); } try { process.kill(pid, "SIGKILL"); killed.push(pid); } catch { /* already gone */ } return killed; } /** True when a pid exists and we may signal it. */ function isAlive(pid) { try { process.kill(pid, 0); return true; } catch (err) { // EPERM means it exists but belongs to someone else — still alive. return err.code === "EPERM"; } } /** Recorded services: every `.pid` the boot hook left in the run directory. */ function readPidFiles(runDir) { let entries = []; try { entries = fs.readdirSync(runDir); } catch { return []; } const out = []; for (const entry of entries) { if (!entry.endsWith(".pid")) continue; const raw = fs.readFileSync(path.join(runDir, entry), "utf8").trim(); if (!/^\d+$/.test(raw)) continue; out.push({ name: entry.slice(0, -4), pid: Number(raw), file: path.join(runDir, entry) }); } return out; } /** Persist why a boot failed, where runtimeFacts can find it. */ function recordError(lane, error) { const { stateDir, errorFile } = slotDirs(lane.slot); fs.mkdirSync(stateDir, { recursive: true }); fs.writeFileSync( errorFile, JSON.stringify( { at: new Date().toISOString(), code: error.code || null, message: error.message }, null, 2 ) ); } /** Drop a stale failure once a boot succeeds. */ function clearError(lane) { try { fs.rmSync(slotDirs(lane.slot).errorFile, { force: true }); } catch { /* nothing recorded */ } } function readError(slot) { try { return JSON.parse(fs.readFileSync(slotDirs(slot).errorFile, "utf8")); } catch { return null; } } /** * Resolve one `LANE_DIRS` entry against the lane's working copy, refusing * anything that escapes it (an absolute path, or one climbing out with `..`) * rather than silently touching a directory somewhere else on the machine. */ function resolveLaneDirEntry(lane, entry) { const target = path.resolve(lane.cwd, entry); const relative = path.relative(lane.cwd, target); if (relative.startsWith("..") || path.isAbsolute(relative)) { throw Object.assign(new Error(`LANE_DIRS entry escapes the lane: ${entry}`), { code: "EBADLANEDIR", entry, }); } return target; } /** Create the per-lane directories the profile declares. */ function makeLaneDirs(lane, profile) { for (const entry of profile.laneDirs) { fs.mkdirSync(resolveLaneDirEntry(lane, entry), { recursive: true }); } } /** * Empty and recreate the per-lane directories a `reset` should start fresh — * Shipyard's `rm -rf "$(lane_upload_dir "$N")"/*` generalized to every * declared `LANE_DIRS` entry, since a fresh feature should not inherit a * previous one's uploaded files. */ function clearLaneDirs(lane, profile) { for (const entry of profile.laneDirs) { const target = resolveLaneDirEntry(lane, entry); fs.rmSync(target, { recursive: true, force: true }); fs.mkdirSync(target, { recursive: true }); } } /** * Run a hook only when the profile declares it, throwing `errCode` on a * non-zero exit. Shared by every lifecycle step below (`bootstrap`, `migrate`, * `seed`) so "the profile never declared this" and "the hook failed" stay two * distinct, consistently-coded outcomes everywhere they're checked. */ async function runRequiredHook(lane, profile, name, options, errCode) { if (!profile.hooks.has(name)) return null; const result = await runHook(lane, profile, name, [], options); if (result.code !== 0) { throw Object.assign(new Error(`${name} hook exited ${result.code}`), { code: errCode, logPath: result.logPath, }); } return result; } /** * Provision a freshly-created managed lane's data isolation: seed its `.env`, * run `bootstrap`/`migrate`/`seed`, and create its database. Runs once, right * after the worktree itself exists — the other half of isolation A1 did not * cover (ports and directories came from A1; database, `.env` and uploads * come from here). Every step degrades to nothing when the profile never * declares the underlying feature: no `bootstrap` hook, no `DB_PREFIX`, no * `ENV_FILES` are all normal, not partial failures. * * @param {object} lane - Lane row, freshly worktree-provisioned (kind "managed"). * @param {{onLine?: Function}} [options] * @returns {Promise} The lane row after provisioning. */ async function provisionLane(lane, options = {}) { const profile = requireProfile(lane); const { onLine } = options; let current = lane; if (!current.slot) { allocateSlot(current.id); current = lanesLib.getLane(current.id); } seedEnv(current, profile, readSecrets()); await runRequiredHook( current, profile, "bootstrap", { onLine, timeoutMs: BOOT_TIMEOUT_MS }, "EBOOTSTRAPFAILED" ); await ensureDatabase(current, profile, { onLine }); await runRequiredHook(current, profile, "migrate", { onLine }, "EMIGRATEFAILED"); await runRequiredHook(current, profile, "seed", { onLine }, "ESEEDFAILED"); return lanesLib.getLane(current.id); } /** * Reset a lane's data isolation after `resetWorktree` has already put its * working copy back on the base branch: refresh `.env` (a `--force` refresh, * so it tracks whatever the base branch's source `.env` now says), re-run * `bootstrap` (a reset can land on a branch with different dependencies — * Shipyard's own comment: "deps move under node_modules/venv"), clear the * declared `LANE_DIRS`, and — unless `keepDb` — drop, recreate, migrate and * reseed the database. `keepDb` skips that whole block, not just the drop: * a caller who wants to keep their data wants it left alone, migrations * included. * * @param {object} lane - Lane row, after `resetWorktree`. * @param {object} profile - Resolved profile. * @param {{keepDb?: boolean, onLine?: Function}} [options] */ async function resetLaneData(lane, profile, options = {}) { const { keepDb = false, onLine } = options; seedEnv(lane, profile, readSecrets(), { force: true }); await runRequiredHook( lane, profile, "bootstrap", { onLine, timeoutMs: BOOT_TIMEOUT_MS }, "EBOOTSTRAPFAILED" ); clearLaneDirs(lane, profile); if (keepDb) return; if (!dbName(profile, lane.slot)) return; await dropDatabase(lane, profile, dbName(profile, lane.slot), { onLine }); await ensureDatabase(lane, profile, { onLine }); await runRequiredHook(lane, profile, "migrate", { onLine }, "EMIGRATEFAILED"); await runRequiredHook(lane, profile, "seed", { onLine }, "ESEEDFAILED"); } /** * Drop a managed lane's database and its `_test` sibling before the rest of * removal tears down the worktree and state directory. * * A no-op for an adopted lane — its data was never CCAM's to create, so it is * never CCAM's to destroy either, the same invariant `assertDestroyable` * enforces for the worktree itself. Best-effort per database: mirrors * Shipyard's `dropdb --if-exists ... || true` — a failed drop is logged, not * thrown, because it must never block removing the lane's dashboard record. * * @param {object} lane - Lane row. * @param {object} profile - Resolved profile. * @param {{onLine?: Function}} [options] */ async function removeLaneData(lane, profile, options = {}) { if (lane.kind !== "managed") return; const name = dbName(profile, lane.slot); if (!name) return; for (const target of [name, `${name}_test`]) { try { await dropDatabase(lane, profile, target, options); } catch (err) { console.warn(`[lane-runtime] lane ${lane.id}: db-drop ${target} failed: ${err.message}`); } } } /** * Boot a lane's stack. * * Caller MUST hold the lane lock: slot allocation reads-then-writes across the * `await` in port resolution, and two concurrent ups would otherwise both believe * they own the slot. * * Runs `boot` then `health`; deliberately NOT `bootstrap`, which belongs to * provisioning and reset (Shipyard's `lane-up.sh` draws the same line — installing * dependencies on every boot would make a routine restart minutes long). * * A failing health check leaves the processes running. They are the evidence: the * logs of a half-booted stack are what tells the user which service never came up, * and killing them to report a tidy failure destroys exactly that. * * @param {object} lane - Lane row. * @param {{build?: boolean, onLine?: Function}} [options] * @returns {Promise} Runtime facts after the attempt. */ async function upLane(lane, options = {}) { const profile = requireProfile(lane); const { build = true, onLine } = options; let current = lane; if (!current.slot) { allocateSlot(current.id); current = lanesLib.getLane(current.id); } const ports = await resolvePorts(current, profile); lanesLib.setProvisioningFacts(current.id, { ports }); current = lanesLib.getLane(current.id); const dirs = slotDirs(current.slot); fs.mkdirSync(dirs.runDir, { recursive: true }); fs.mkdirSync(dirs.logDir, { recursive: true }); makeLaneDirs(current, profile); // Defensive: a re-run on a live lane must not double-start services that then // fight over the same ports. await downLane(current, { profile }); try { // Repair .env on every boot (a lane whose file was hand-edited or never // seeded gets fixed here), then ensure the database exists — cheap when it // already does — migrate on every boot (schemas drift while a lane sits // idle), and seed only when this boot is the one that created the database. if (current.kind === "managed") seedEnv(current, profile, readSecrets()); const db = await ensureDatabase(current, profile, { onLine }); await runRequiredHook(current, profile, "migrate", { onLine }, "EMIGRATEFAILED"); if (db.created) await runRequiredHook(current, profile, "seed", { onLine }, "ESEEDFAILED"); const boot = await runHook(current, profile, "boot", build ? [] : ["--no-build"], { onLine, timeoutMs: BOOT_TIMEOUT_MS, }); if (boot.code !== 0) { throw Object.assign(new Error(`boot hook exited ${boot.code}`), { code: "EBOOTFAILED", logPath: boot.logPath, }); } if (profile.hooks.has("health")) { const health = await runHook(current, profile, "health", [], { onLine, timeoutMs: HEALTH_TIMEOUT_MS, }); if (health.code !== 0) { throw Object.assign(new Error(`health check failed (exit ${health.code})`), { code: "EUNHEALTHY", logPath: health.logPath, }); } } clearError(current); } catch (err) { recordError(current, err); throw err; } return runtimeFacts(lanesLib.getLane(current.id)); } /** * Stop a lane's stack. Idempotent, and safe on a lane that was never up. * * Kills each recorded pid tree, then — only when there WAS something recorded — * sweeps any listener still holding the lane's ports. That condition is a * deliberate departure from Shipyard, which always sweeps: a lane whose stack is * already down still owns its port numbers, and if the user has since started * their own server on one, an unconditional sweep would kill it. Requiring a pid * file keeps the backstop for the case it exists to cover — a detached child that * outlived the parent we recorded — without ever reaching a stranger's process. * * @param {object} lane - Lane row. * @param {{profile?: object}} [options] * @returns {Promise<{killed: number[]}>} */ async function downLane(lane, options = {}) { if (!lane.slot) return { killed: [] }; const { runDir } = slotDirs(lane.slot); const recorded = readPidFiles(runDir); const killed = []; for (const service of recorded) { killed.push(...(await killTree(service.pid))); fs.rmSync(service.file, { force: true }); } if (recorded.length) { for (const port of Object.values(lane.ports || {})) { for (const pid of await listenerPids(port)) { if (!killed.includes(pid)) killed.push(...(await killTree(pid))); } } } return { killed }; } /** * What is actually running for this lane, computed fresh on every call. * * Follows the contract of `GET /api/lanes/:id/git`: a lane with no profile is a * normal state, reported as `{available: false}` rather than an error. Callers * probe ports and stat pid files here, which is why this is its own endpoint and * not part of the polled lane list. * * @param {object} lane - Lane row. * @returns {Promise} */ async function runtimeFacts(lane) { const profile = resolveProfile(lane); if (!profile) return { available: false, searched: profileSearchPaths(lane) }; if (!lane.slot) { return { available: true, provisioned: false, hooks: [...profile.hooks], ports: {} }; } const dirs = slotDirs(lane.slot); const services = readPidFiles(dirs.runDir).map((service) => ({ name: service.name, pid: service.pid, alive: isAlive(service.pid), })); const ports = {}; for (const name of profile.ports) { const port = lane.ports?.[name] ?? null; ports[name] = { port, expected: portBase(profile, name) + lane.slot, listening: port ? await isListening(port) : false, }; } let logs = []; try { logs = fs.readdirSync(dirs.logDir).filter((entry) => entry.endsWith(".log")); } catch { /* never booted */ } // Names and an index, never a connection string: DATABASE_URL/REDIS_URL embed // the secrets.env password, and this object is exactly what GET /runtime // returns to a browser tab. const name = dbName(profile, lane.slot); const database = name ? { name, testName: `${name}_test` } : null; const redisIndex = profile.env.REDIS === "1" ? lane.slot : null; return { available: true, provisioned: true, slot: lane.slot, kind: lane.kind, hooks: [...profile.hooks], profileDir: profile.dir, services, ports, database, redisIndex, steppedAside: portsSteppedAside(lane, profile), up: services.some((service) => service.alive), healthy: Object.values(ports).length > 0 && Object.values(ports).every((entry) => entry.listening), logs, logDir: dirs.logDir, lastError: readError(lane.slot), }; } module.exports = { upLane, downLane, runtimeFacts, requireProfile, provisionLane, resetLaneData, removeLaneData, killTree, isAlive, readPidFiles, makeLaneDirs, clearLaneDirs, HEALTH_TIMEOUT_MS, BOOT_TIMEOUT_MS, };