AgentHub/server/lib/host-stats.js

/**
 * @file host-stats.js
 * @description The Administration page's Server tab (`GET /api/host`, staff
 * only): the machine's hardware and load, and how much of its CPU and memory
 * each AgentHub user is using right now. Read-only — nothing here signals,
 * renices or kills a process.
 *
 * Hardware comes from `os` plus `/proc` (meminfo, stat, mounts) and `statfs`
 * per real mounted file system (HDD/SSD from `/sys/block/<dev>/queue/rotational`);
 * a GPU only when `nvidia-smi` answers within 3 s. Usage is two `/proc`
 * samples `SAMPLE_MS` apart: a process's CPU % is its utime+stime delta over
 * the whole machine's jiffies delta (100 % = every core busy), its memory its
 * RSS. A process is attributed, in order, to: the AgentHub user of its uid
 * (`ah-<name>` under S3, `memberNameForUid`); for a process of the host
 * account, the OWNER of the workspace named by its inherited
 * `AGENTHUB_LANE_ID` (every pane gets it through tmux `-e`, so an agent's
 * children carry it too); else `host` (the dashboard and everything else the
 * host account runs) or `system` (root and every other account). Only the
 * kernel's `comm` name is reported for a process — never its argv or env,
 * which can hold a prompt or a key. Every read is async, and one sample is
 * shared by concurrent callers and cached for `FRESH_MS`. Linux only: other
 * platforms get the `os` facts with `usage: null`.
 * @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
 */

const fs = require("node:fs");
const fsp = require("node:fs/promises");
const os = require("node:os");
const path = require("node:path");
const { execFile } = require("node:child_process");

const SAMPLE_MS = 1000;
const FRESH_MS = 3000;
const TOP_PER_GROUP = 5;
// File systems that hold data; pseudo and loop mounts are left out.
const REAL_FS = new Set([
  "ext2",
  "ext3",
  "ext4",
  "xfs",
  "btrfs",
  "zfs",
  "f2fs",
  "vfat",
  "exfat",
  "ntfs",
  "ntfs3",
  "fuseblk",
  "nfs",
  "nfs4",
  "cifs",
]);

const readText = (file) => fsp.readFile(file, "utf8").catch(() => null);

/** `/proc/meminfo` in bytes: total, available, swap. */
async function memory() {
  const text = await readText("/proc/meminfo");
  const kb = (key) => {
    const m = text && new RegExp(`^${key}:\\s+(\\d+)`, "m").exec(text);
    return m ? Number(m[1]) * 1024 : null;
  };
  const total = kb("MemTotal") ?? os.totalmem();
  const available = kb("MemAvailable") ?? os.freemem();
  return {
    total,
    available,
    used: total - available,
    swapTotal: kb("SwapTotal") ?? 0,
    swapUsed: (kb("SwapTotal") ?? 0) - (kb("SwapFree") ?? 0),
  };
}

/** The machine's jiffies from `/proc/stat`'s `cpu` line: `{ total, idle }`. */
async function cpuTimes() {
  const text = await readText("/proc/stat");
  const line = text && text.split("\n").find((l) => l.startsWith("cpu "));
  if (!line) return null;
  const n = line.trim().split(/\s+/).slice(1).map(Number);
  return { total: n.reduce((a, b) => a + b, 0), idle: n[3] + (n[4] || 0) };
}

/** `HDD`/`SSD` for a `/dev/...` device, or null when /sys cannot tell. */
function diskKind(device) {
  if (!device.startsWith("/dev/")) return null;
  let name = path.basename(device);
  try {
    name = path.basename(fs.realpathSync(device));
  } catch {
    /* a mapper name that does not resolve: try it as is */
  }
  // A partition's queue lives on its parent: sda1 → sda, nvme0n1p2 → nvme0n1.
  const candidates = [name, name.replace(/p?\d+$/, "")];
  for (const c of candidates) {
    try {
      const v = fs.readFileSync(`/sys/block/${c}/queue/rotational`, "utf8").trim();
      return v === "1" ? "HDD" : "SSD";
    } catch {
      /* next */
    }
  }
  return null;
}

/** Every real mounted file system, one row per device. */
async function disks() {
  const text = await readText("/proc/mounts");
  if (!text) return [];
  const seen = new Set();
  const out = [];
  for (const line of text.split("\n")) {
    const [device, rawMount, type] = line.split(" ");
    if (!device || !REAL_FS.has(type) || seen.has(device)) continue;
    if (device.startsWith("/dev/loop")) continue;
    const mount = rawMount.replace(/\\040/g, " ");
    if (mount.startsWith("/snap/") || mount.startsWith("/boot/efi")) continue;
    seen.add(device);
    try {
      const st = await fsp.statfs(mount);
      const total = st.blocks * st.bsize;
      if (!total) continue;
      const free = st.bavail * st.bsize;
      out.push({
        device,
        mount,
        type,
        kind: diskKind(device),
        total,
        used: total - st.bfree * st.bsize,
        free,
      });
    } catch {
      /* unreadable mount: skip */
    }
  }
  return out;
}

/** The mount (from `list`) holding `p`: the longest mount point that prefixes it. */
function mountOf(list, p) {
  if (!p) return null;
  let best = null;
  for (const d of list) {
    const m = d.mount === "/" ? "/" : `${d.mount}/`;
    if ((p === d.mount || p.startsWith(m)) && (!best || d.mount.length > best.mount.length))
      best = d;
  }
  return best ? best.mount : null;
}

/** `nvidia-smi` GPUs, or [] when there is none (or it does not answer in 3 s). */
function gpus() {
  return new Promise((resolve) => {
    execFile(
      "nvidia-smi",
      [
        "--query-gpu=name,utilization.gpu,memory.used,memory.total",
        "--format=csv,noheader,nounits",
      ],
      { timeout: 3000, windowsHide: true },
      (err, stdout) => {
        if (err) return resolve([]);
        resolve(
          String(stdout)
            .trim()
            .split("\n")
            .filter(Boolean)
            .map((l) => {
              const [name, util, used, total] = l.split(",").map((s) => s.trim());
              return {
                name,
                utilization: Number(util) || 0,
                memoryUsed: (Number(used) || 0) * 1024 * 1024,
                memoryTotal: (Number(total) || 0) * 1024 * 1024,
              };
            })
        );
      }
    );
  });
}

function distro() {
  try {
    const m = /^PRETTY_NAME="?([^"\n]*)"?/m.exec(fs.readFileSync("/etc/os-release", "utf8"));
    return m ? m[1] : null;
  } catch {
    return null;
  }
}

const PAGE = 4096;

/**
 * One pass over `/proc`: `Map<pid, { uid, comm, ticks, rss }>`. `stat`'s comm
 * is in parentheses and may itself hold spaces or `)`, so the fields are read
 * after the LAST `)`.
 */
async function processes() {
  const out = new Map();
  let names;
  try {
    names = await fsp.readdir("/proc");
  } catch {
    return out;
  }
  await Promise.all(
    names
      .filter((n) => /^\d+$/.test(n))
      .map(async (pid) => {
        try {
          const [stat, st] = await Promise.all([
            fsp.readFile(`/proc/${pid}/stat`, "utf8"),
            fsp.stat(`/proc/${pid}`),
          ]);
          const close = stat.lastIndexOf(")");
          const comm = stat.slice(stat.indexOf("(") + 1, close);
          const f = stat.slice(close + 2).split(" ");
          // After comm: state(0) ppid(1) … utime(11) stime(12) … rss(21).
          out.set(Number(pid), {
            uid: st.uid,
            comm,
            ticks: Number(f[11]) + Number(f[12]),
            rss: Number(f[21]) * PAGE,
          });
        } catch {
          /* exited meanwhile */
        }
      })
  );
  return out;
}

/** The `AGENTHUB_LANE_ID` a host process inherited, or null. */
async function laneIdOf(pid) {
  try {
    const env = await fsp.readFile(`/proc/${pid}/environ`, "utf8");
    const m = /(?:^|\0)AGENTHUB_LANE_ID=(\d+)(?:\0|$)/.exec(env);
    return m ? Number(m[1]) : null;
  } catch {
    return null;
  }
}

/** Defaults wired to the real modules; a test swaps them (`__setDeps`). */
let deps = {
  memberNameForUid: (uid) => require("./run-as").memberNameForUid(uid),
  laneOwner: (laneId) => {
    const { db } = require("../db");
    const row = db.prepare("SELECT owner_id FROM lanes WHERE id = ?").get(laneId);
    return row ? row.owner_id : null;
  },
  users: () => {
    try {
      return require("./auth").listUsers();
    } catch {
      return [];
    }
  },
  homesRoot: () => {
    try {
      const linux = require("./linux-accounts");
      if (linux.s3Enabled()) return linux.serverConf().usersRoot;
    } catch {
      /* no S3 */
    }
    return process.env.AGENTHUB_USERS_ROOT || os.homedir();
  },
  wait: (ms) => new Promise((r) => setTimeout(r, ms)),
};

/** Per-user CPU and memory from two samples. */
async function usage() {
  const t0 = await cpuTimes();
  const p0 = await processes();
  await deps.wait(SAMPLE_MS);
  const t1 = await cpuTimes();
  const p1 = await processes();
  const mem = await memory();
  if (!t0 || !t1) return null;
  const totalDelta = Math.max(1, t1.total - t0.total);
  const hostUid = typeof process.getuid === "function" ? process.getuid() : -1;

  const users = deps.users();
  const byUsername = new Map(users.map((u) => [u.username, u]));
  const byId = new Map(users.map((u) => [u.id, u]));
  const ownerCache = new Map();
  const groups = new Map();
  const group = (key, extra) => {
    if (!groups.has(key))
      groups.set(key, { key, cpu: 0, memory: 0, processes: 0, top: [], ...extra });
    return groups.get(key);
  };

  await Promise.all(
    [...p1].map(async ([pid, p]) => {
      const before = p0.get(pid);
      // A process born between the samples: all its ticks are in the window.
      const ticks = Math.max(0, p.ticks - (before ? before.ticks : 0));
      let g = null;
      const member = deps.memberNameForUid(p.uid);
      if (member && byUsername.has(member)) {
        const u = byUsername.get(member);
        g = group(`user:${u.id}`, { kind: "user", userId: u.id, username: u.username });
      } else if (p.uid === hostUid) {
        const laneId = await laneIdOf(pid);
        let ownerId = null;
        if (laneId != null) {
          if (!ownerCache.has(laneId)) {
            let o = null;
            try {
              o = deps.laneOwner(laneId);
            } catch {
              o = null;
            }
            ownerCache.set(laneId, o);
          }
          ownerId = ownerCache.get(laneId);
        }
        const u = ownerId != null ? byId.get(ownerId) : null;
        g = u
          ? group(`user:${u.id}`, { kind: "user", userId: u.id, username: u.username })
          : group("host", { kind: "host" });
      } else {
        g = group("system", { kind: "system" });
      }
      const cpu = (ticks / totalDelta) * 100;
      g.cpu += cpu;
      g.memory += p.rss;
      g.processes += 1;
      g.top.push({ pid, name: p.comm, cpu, memory: p.rss });
    })
  );

  const list = [...groups.values()].map((g) => {
    g.top.sort((a, b) => b.cpu - a.cpu || b.memory - a.memory);
    g.top = g.top.slice(0, TOP_PER_GROUP).map((t) => ({ ...t, cpu: round(t.cpu) }));
    return { ...g, cpu: round(g.cpu), memoryPercent: round((g.memory / mem.total) * 100) };
  });
  // Every user appears, idle ones with zeros, so the table is the whole roster.
  for (const u of users) {
    if (!groups.has(`user:${u.id}`))
      list.push({
        key: `user:${u.id}`,
        kind: "user",
        userId: u.id,
        username: u.username,
        cpu: 0,
        memory: 0,
        memoryPercent: 0,
        processes: 0,
        top: [],
      });
  }
  list.sort((a, b) => b.cpu - a.cpu || b.memory - a.memory);
  return {
    cpu: round(((totalDelta - (t1.idle - t0.idle)) / totalDelta) * 100),
    sampleMs: SAMPLE_MS,
    groups: list,
  };
}

const round = (n) => Math.round(n * 10) / 10;

async function collect() {
  const linux = process.platform === "linux";
  const cpus = os.cpus();
  const [mem, diskList, gpuList, use] = await Promise.all([
    memory(),
    linux ? disks() : [],
    linux ? gpus() : [],
    linux ? usage() : null,
  ]);
  return {
    at: new Date().toISOString(),
    hostname: os.hostname(),
    os: {
      platform: process.platform,
      release: os.release(),
      distro: linux ? distro() : null,
      arch: os.arch(),
    },
    uptime: os.uptime(),
    cpu: { model: cpus[0]?.model?.trim() || null, cores: cpus.length, load: os.loadavg() },
    memory: mem,
    disks: diskList,
    homesMount: mountOf(diskList, deps.homesRoot()),
    gpus: gpuList,
    usage: use,
  };
}

let cache = null; // { at, value }
let inflight = null;

/** The current snapshot: cached for `FRESH_MS`, one sample shared by every caller. */
async function hostStats() {
  if (cache && Date.now() - cache.at < FRESH_MS) return cache.value;
  if (!inflight) {
    inflight = collect()
      .then((value) => {
        cache = { at: Date.now(), value };
        return value;
      })
      .finally(() => {
        inflight = null;
      });
  }
  return inflight;
}

function __setDeps(over) {
  deps = { ...deps, ...over };
  cache = null;
}

module.exports = { hostStats, mountOf, __setDeps };