feat(lanes): per-lane database, Redis, and .env isolation (A1+A2)
Gives each lane its own slot-derived runtime (ports, detached process lifecycle, profile-driven hooks) and its own database/Redis logical index/.env file, so two lanes running the same repo's stack at once no longer share state. Machine-level DB/Redis credentials live at ~/.ccam/secrets.env (mode 0600, never returned by any route); a hook's output is redacted of that password (raw and URL-encoded forms) before it reaches a log file or the lane_hook_output websocket broadcast. Wired into provision/up/reset/remove; reset accepts --keep-db to skip the drop/recreate/migrate/reseed block entirely.
This commit is contained in:
@@ -0,0 +1,397 @@
|
||||
/**
|
||||
* @file The stack seam. A profile is a repository's own description of how to
|
||||
* build, boot and check its stack: `<repo>/.ccam/profile/` holding a `profile.env`
|
||||
* of declarations plus a `hooks/` directory of shell scripts. CCAM stays
|
||||
* stack-agnostic and calls those hooks with a stable environment contract, which
|
||||
* is the same contract Shipyard's `run_hook` exports so its profiles port over
|
||||
* unchanged.
|
||||
*
|
||||
* Two rules this module exists to enforce. Config is PARSED, never sourced —
|
||||
* sourcing arbitrary shell from a repository into the dashboard process would be
|
||||
* a code-execution path; hooks are executed deliberately, config is only read.
|
||||
* And a hook is always spawned as `bash <hook> <args…>` with a fixed argument
|
||||
* array, never a command string.
|
||||
* @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
*/
|
||||
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const { spawn } = require("node:child_process");
|
||||
|
||||
const { slotDirs, dataFacts } = require("./lane-slots");
|
||||
|
||||
/** Directory, relative to a repository root, holding its profile. */
|
||||
const PROFILE_SUBDIR = path.join(".ccam", "profile");
|
||||
|
||||
/**
|
||||
* Hook names CCAM will run, ever.
|
||||
*
|
||||
* A fixed allowlist rather than "whatever is in hooks/": `:name` arrives from an
|
||||
* HTTP route, and a name taken from a request is a path taken from a request.
|
||||
* `bootstrap`/`migrate`/`seed`/`ci-gate`/`e2e`/`regen` are not called by A1's
|
||||
* lifecycle but are listed here because `POST /:id/hook/:name` can run them on a
|
||||
* session's behalf, and the driving skill needs that surface stable.
|
||||
*/
|
||||
const HOOKS = Object.freeze([
|
||||
"bootstrap",
|
||||
"boot",
|
||||
"health",
|
||||
"migrate",
|
||||
"seed",
|
||||
"ci-gate",
|
||||
"e2e",
|
||||
"regen",
|
||||
"db-create",
|
||||
"db-drop",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Defaults for every declaration a profile may omit, so a missing key degrades
|
||||
* to something usable rather than breaking a lane. Mirrors the defaults block in
|
||||
* Shipyard's `_common.sh` so a ported profile behaves identically.
|
||||
*/
|
||||
const DEFAULTS = Object.freeze({
|
||||
PORTS: "api fe",
|
||||
PORT_BASE_api: "8000",
|
||||
PORT_BASE_fe: "3000",
|
||||
LANE_DIRS: "",
|
||||
BACKEND_DIR: "backend",
|
||||
FRONTEND_DIR: "frontend",
|
||||
API_PATH: "/api",
|
||||
// A2 data isolation — every one of these empty/0 is "feature off", so an
|
||||
// adopted repo that never declares them gets no per-lane database, no Redis
|
||||
// index, and no .env rewriting: dead code never runs rather than running on
|
||||
// guessed values.
|
||||
DB_PREFIX: "",
|
||||
DB_KIND: "",
|
||||
DB_URL_SCHEME: "postgresql",
|
||||
REDIS: "0",
|
||||
ENV_FILES: "",
|
||||
ENV_SOURCE: "",
|
||||
ENV_REWRITE: "",
|
||||
ENV_PRESERVE: "",
|
||||
UPLOAD_SUBDIR: "",
|
||||
});
|
||||
|
||||
/**
|
||||
* Parse a `KEY=VALUE` declaration file.
|
||||
*
|
||||
* Not a shell parser and not trying to be: comments and blank lines are skipped,
|
||||
* an `export ` prefix is tolerated (profiles ported from Shipyard have it), and
|
||||
* one layer of matching quotes is stripped. Everything else is taken literally —
|
||||
* `$(id)`, backticks and `${VAR}` stay as written. That literalness IS the
|
||||
* security property; do not add expansion here.
|
||||
*
|
||||
* @param {string} text - File contents.
|
||||
* @returns {Record<string, string>}
|
||||
*/
|
||||
function parseEnvFile(text) {
|
||||
const out = {};
|
||||
for (const rawLine of text.split("\n")) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
const match = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
||||
if (!match) continue;
|
||||
let value = match[2].trim();
|
||||
if (
|
||||
value.length >= 2 &&
|
||||
((value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'")))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
out[match[1]] = value;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Split a space-separated declaration into a deduplicated list. */
|
||||
function splitList(value) {
|
||||
return [...new Set((value || "").split(/\s+/).filter(Boolean))];
|
||||
}
|
||||
|
||||
/**
|
||||
* Find and read a lane's profile.
|
||||
*
|
||||
* The lane's OWN working copy is searched first, its source repository second.
|
||||
* The profile lives in the repository, so a worktree already carries the version
|
||||
* belonging to its branch — and a branch that changes a boot command must boot
|
||||
* with the command it changed, not the one on the base branch. The source-repo
|
||||
* fallback covers a profile the user keeps gitignored, which never reaches a
|
||||
* worktree through git.
|
||||
*
|
||||
* @param {object} lane - Lane row (`cwd`, `source_repo`).
|
||||
* @returns {{dir: string, env: Record<string,string>, hooks: Set<string>, ports: string[], laneDirs: string[]}|null}
|
||||
* null when neither location has a profile — a normal state, not a fault.
|
||||
*/
|
||||
function resolveProfile(lane) {
|
||||
const candidates = [lane.cwd, lane.source_repo].filter(Boolean);
|
||||
for (const root of candidates) {
|
||||
const dir = path.join(root, PROFILE_SUBDIR);
|
||||
if (!fs.existsSync(path.join(dir, "profile.env"))) continue;
|
||||
|
||||
let declared = {};
|
||||
try {
|
||||
declared = parseEnvFile(fs.readFileSync(path.join(dir, "profile.env"), "utf8"));
|
||||
} catch {
|
||||
continue; // unreadable profile is the same as no profile
|
||||
}
|
||||
const env = { ...DEFAULTS, ...declared };
|
||||
|
||||
const hooks = new Set();
|
||||
for (const name of HOOKS) {
|
||||
if (fs.existsSync(path.join(dir, "hooks", `${name}.sh`))) hooks.add(name);
|
||||
}
|
||||
|
||||
return {
|
||||
dir,
|
||||
env,
|
||||
hooks,
|
||||
ports: splitList(env.PORTS),
|
||||
laneDirs: splitList(env.LANE_DIRS),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The profile paths searched for a lane, for an ENOPROFILE message. */
|
||||
function profileSearchPaths(lane) {
|
||||
return [lane.cwd, lane.source_repo]
|
||||
.filter(Boolean)
|
||||
.map((root) => path.join(root, PROFILE_SUBDIR));
|
||||
}
|
||||
|
||||
/**
|
||||
* A `harness_spawn` shell function, injected into every hook.
|
||||
*
|
||||
* Same name and signature as Shipyard's so ported hooks work untouched:
|
||||
* `harness_spawn <name> <workdir> <cmd…>` backgrounds a long-lived service and
|
||||
* records its pid where downLane looks.
|
||||
*
|
||||
* The stdio detachment is not optional. A child that inherits the caller's stdout
|
||||
* holds that pipe open, so a caller reading to EOF never returns — the failure
|
||||
* that stalled Shipyard's boot stage until it was fixed the same way. `nohup` plus
|
||||
* a closed stdin plus redirected output is what lets a lane's stack outlive both
|
||||
* the hook and the dashboard.
|
||||
*/
|
||||
const HARNESS_SPAWN = `
|
||||
harness_spawn() {
|
||||
local name="$1" wd="$2"; shift 2
|
||||
( cd "$wd" || exit 1
|
||||
nohup "$@" >"$LOG_DIR/$name.log" 2>&1 </dev/null &
|
||||
echo $! >"$RUN_DIR/$name.pid"
|
||||
) </dev/null >/dev/null 2>&1
|
||||
}
|
||||
die() { echo "profile: $*" >&2; exit 1; }
|
||||
`;
|
||||
|
||||
/**
|
||||
* The environment contract every hook can rely on.
|
||||
*
|
||||
* Deliberately identical to Shipyard's `run_hook` exports where the concept
|
||||
* survives the port, so a profile written for the harness runs here unchanged.
|
||||
* `LANE` is the SLOT, not the lane id — Shipyard hooks use it to derive per-lane
|
||||
* names, and the slot is what carries that meaning.
|
||||
*
|
||||
* Git variables are scrubbed for the same reason `worktree.js:git()` scrubs them:
|
||||
* a hook that shells out to git must not inherit a git context pointing at the
|
||||
* dashboard's own repository, and `GIT_CONFIG_*` can inject `core.hooksPath` into
|
||||
* every git call the hook makes.
|
||||
*
|
||||
* `require("./secrets")` is deferred to the function body rather than hoisted
|
||||
* to the top of the file: `secrets.js` itself requires this module for
|
||||
* `parseEnvFile`, and a top-level require here would complete the cycle while
|
||||
* this file's own `module.exports` is still empty, handing `secrets.js` an
|
||||
* `undefined` parser. Deferring past module-load time breaks the cycle.
|
||||
*/
|
||||
function hookEnv(lane, profile) {
|
||||
const { readSecrets } = require("./secrets");
|
||||
const dirs = slotDirs(lane.slot);
|
||||
const secrets = readSecrets();
|
||||
const facts = dataFacts(lane, profile, secrets);
|
||||
const env = { ...process.env };
|
||||
|
||||
delete env.GIT_DIR;
|
||||
delete env.GIT_WORK_TREE;
|
||||
delete env.GIT_INDEX_FILE;
|
||||
delete env.GIT_COMMON_DIR;
|
||||
delete env.GIT_OBJECT_DIRECTORY;
|
||||
delete env.GIT_ALTERNATE_OBJECT_DIRECTORIES;
|
||||
delete env.GIT_PREFIX;
|
||||
delete env.GIT_NAMESPACE;
|
||||
delete env.GIT_CONFIG_PARAMETERS;
|
||||
for (const name of Object.keys(env)) {
|
||||
if (/^GIT_CONFIG_(COUNT|KEY_\d+|VALUE_\d+|GLOBAL|SYSTEM)$/.test(name)) delete env[name];
|
||||
}
|
||||
env.GIT_TERMINAL_PROMPT = "0";
|
||||
|
||||
Object.assign(env, profile.env, {
|
||||
LANE: String(lane.slot),
|
||||
LANE_ID: String(lane.id),
|
||||
LANE_DIR: lane.cwd,
|
||||
SOURCE_REPO: lane.source_repo || lane.cwd,
|
||||
PROFILE_DIR: profile.dir,
|
||||
RUN_DIR: dirs.runDir,
|
||||
LOG_DIR: dirs.logDir,
|
||||
});
|
||||
|
||||
// A2 data-isolation facts, present only when their owning declaration is —
|
||||
// a profile with no DB_PREFIX sees no DB_NAME/DATABASE_URL at all, so a
|
||||
// db-create.sh hook that forgot to check DB_PREFIX fails loudly (unset var
|
||||
// under `set -u`) instead of quietly touching a database named "undefined".
|
||||
if (facts.dbName) {
|
||||
env.DB_NAME = facts.dbName;
|
||||
env.DATABASE_URL = facts.databaseUrl;
|
||||
env.TEST_DATABASE_URL = facts.testDatabaseUrl;
|
||||
// Raw connection settings, not just the assembled URL: Shipyard's own
|
||||
// db-create/db-drop hooks call `createdb -U "$PG_USER"` directly (trust
|
||||
// auth inside the compose network, no password needed), and a ported
|
||||
// profile expects these names verbatim. PG_PASS is deliberately withheld —
|
||||
// nothing in the ported hooks needs it, and every value that DOES reach a
|
||||
// hook's environment is a value that could end up in an echoed debug line.
|
||||
env.PG_HOST = secrets.PG_HOST;
|
||||
env.PG_PORT = secrets.PG_PORT;
|
||||
env.PG_USER = secrets.PG_USER;
|
||||
}
|
||||
if (facts.redisUrl) {
|
||||
env.REDIS_URL = facts.redisUrl;
|
||||
env.REDIS_HOST = secrets.REDIS_HOST;
|
||||
env.REDIS_PORT = secrets.REDIS_PORT;
|
||||
}
|
||||
if (facts.uploadDir) env.UPLOAD_DIR = facts.uploadDir;
|
||||
|
||||
// <NAME>_PORT for every declared port, upper-cased: PORTS="api fe" -> API_PORT, FE_PORT.
|
||||
for (const [name, port] of Object.entries(lane.ports || {})) {
|
||||
env[`${name.toUpperCase()}_PORT`] = String(port);
|
||||
}
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one of the profile's hooks.
|
||||
*
|
||||
* Spawned through a one-line bash wrapper that defines the shell helpers,
|
||||
* `export -f`s them (exactly as Shipyard's `run_hook` does) and then `exec`s the
|
||||
* hook, so the hook runs as its own script with its own `set -e` while still
|
||||
* seeing `harness_spawn`. The hook path and its arguments travel as an argument
|
||||
* ARRAY appended after the wrapper — never interpolated into the script text, so
|
||||
* a lane directory containing a quote or a space is a path, not a command.
|
||||
*
|
||||
* Output is streamed line by line to `onLine` (the caller broadcasts it) and
|
||||
* appended to `$LOG_DIR/<name>.log`, so a boot is watchable live and readable
|
||||
* afterwards. Any value that came from `secrets.js` (currently `PG_PASS`, and
|
||||
* therefore the password segment of `DATABASE_URL`/`TEST_DATABASE_URL`) is
|
||||
* redacted from that stream first: `runHook`'s output reaches a browser tab
|
||||
* over the `lane_hook_output` websocket, and a hook that echoes its own
|
||||
* environment (common while debugging a failing migration) must not publish a
|
||||
* database password to everyone watching.
|
||||
*
|
||||
* @param {object} lane - Lane row with an allocated slot and resolved ports.
|
||||
* @param {object} profile - From resolveProfile().
|
||||
* @param {string} name - Hook name; must be in HOOKS.
|
||||
* @param {string[]} [args] - Extra arguments passed to the hook.
|
||||
* @param {{onLine?: (line: string, stream: "stdout"|"stderr") => void, timeoutMs?: number}} [options]
|
||||
* @returns {Promise<{code: number, output: string}>} Resolves even on a non-zero exit.
|
||||
*/
|
||||
function runHook(lane, profile, name, args = [], options = {}) {
|
||||
if (!HOOKS.includes(name)) {
|
||||
return Promise.reject(
|
||||
Object.assign(new Error(`unknown hook: ${name}`), { code: "ENOHOOK", hook: name })
|
||||
);
|
||||
}
|
||||
const hookPath = path.join(profile.dir, "hooks", `${name}.sh`);
|
||||
if (!fs.existsSync(hookPath)) {
|
||||
return Promise.reject(
|
||||
Object.assign(new Error(`profile has no hook "${name}.sh"`), {
|
||||
code: "ENOHOOK",
|
||||
hook: name,
|
||||
path: hookPath,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
const dirs = slotDirs(lane.slot);
|
||||
fs.mkdirSync(dirs.runDir, { recursive: true });
|
||||
fs.mkdirSync(dirs.logDir, { recursive: true });
|
||||
const logPath = path.join(dirs.logDir, `${name}.log`);
|
||||
const logStream = fs.createWriteStream(logPath, { flags: "a" });
|
||||
|
||||
// Deferred for the same reason as inside hookEnv(): secrets.js requires this
|
||||
// module, so a top-level require here would complete the load cycle early.
|
||||
// Only the password is redacted — host/port/user are not secret on their
|
||||
// own, and treating them as such would mangle unrelated numbers in output.
|
||||
// Both forms: DATABASE_URL embeds the URL-encoded password, so a raw echo
|
||||
// of the password and an echo of DATABASE_URL need separate substrings.
|
||||
const pgPass = require("./secrets").readSecrets().PG_PASS;
|
||||
const secretValues = [pgPass, pgPass && encodeURIComponent(pgPass)].filter(Boolean);
|
||||
const redact = (text) => secretValues.reduce((s, v) => s.split(v).join("[REDACTED]"), text);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(
|
||||
"bash",
|
||||
[
|
||||
"-c",
|
||||
`${HARNESS_SPAWN}\nexport -f harness_spawn die\nexec bash "$@"`,
|
||||
"bash",
|
||||
hookPath,
|
||||
...args.map(String),
|
||||
],
|
||||
{
|
||||
cwd: lane.cwd,
|
||||
env: hookEnv(lane, profile),
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
}
|
||||
);
|
||||
|
||||
let output = "";
|
||||
let settled = false;
|
||||
const timer = options.timeoutMs
|
||||
? setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
}, options.timeoutMs)
|
||||
: null;
|
||||
|
||||
const consume = (stream, which) => {
|
||||
let buffer = "";
|
||||
stream.setEncoding("utf8");
|
||||
stream.on("data", (raw) => {
|
||||
const chunk = secretValues.length ? redact(raw) : raw;
|
||||
output += chunk;
|
||||
logStream.write(chunk);
|
||||
buffer += chunk;
|
||||
const lines = buffer.split("\n");
|
||||
buffer = lines.pop();
|
||||
for (const line of lines) options.onLine?.(line, which);
|
||||
});
|
||||
stream.on("end", () => {
|
||||
if (buffer) options.onLine?.(buffer, which);
|
||||
});
|
||||
};
|
||||
consume(child.stdout, "stdout");
|
||||
consume(child.stderr, "stderr");
|
||||
|
||||
const finish = (fn, value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
logStream.end();
|
||||
fn(value);
|
||||
};
|
||||
|
||||
child.on("error", (err) => finish(reject, err));
|
||||
child.on("close", (code) => finish(resolve, { code: code ?? 1, output, logPath }));
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
HOOKS,
|
||||
DEFAULTS,
|
||||
PROFILE_SUBDIR,
|
||||
parseEnvFile,
|
||||
splitList,
|
||||
resolveProfile,
|
||||
profileSearchPaths,
|
||||
hookEnv,
|
||||
runHook,
|
||||
};
|
||||
Reference in New Issue
Block a user