fix(server): a member's test run can no longer freeze the box
On 11-02 open sessions plus test runs filled RAM and the 4 GB swap on a slow disk; the box hung until a hard reset. Member panes now start at nice 10, IO priority 7 and oom_score_adj 500 with Vitest capped at 2 workers by default, and the system unit caps its cgroup (server + every pane) at 85% of RAM with 1G swap and OOMPolicy=continue, so the kernel kills the biggest pane process instead of hanging the machine. Needs `agenthub server install-system` once.
This commit is contained in:
@@ -20,7 +20,8 @@
|
||||
# /etc/sudoers.d/agenthub checked with visudo before it is installed
|
||||
# /etc/claude-code/managed-settings.json the member hooks (never over a foreign file)
|
||||
# /proc hidepid=invisible,gid=agenthub-proc (now and in /etc/fstab)
|
||||
# /etc/systemd/system/agenthub.service reads the host-owned env file
|
||||
# /etc/systemd/system/agenthub.service reads the host-owned env file; caps
|
||||
# the cgroup (server + every pane) at 85% of RAM, 1G swap
|
||||
# @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
set -euo pipefail
|
||||
|
||||
@@ -203,6 +204,14 @@ ExecStart=/usr/bin/env \${AGENTHUB_NODE} \${AGENTHUB_ROOT}/server/index.js
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
KillMode=process
|
||||
# Every member pane lives in this cgroup. Capped below RAM with little swap
|
||||
# (swap on a slow disk froze the whole box on 11-02), a runaway test run is
|
||||
# OOM-killed inside the unit — pane-launch makes panes the first pick — and
|
||||
# "continue" keeps the dashboard up when that happens.
|
||||
MemoryMax=85%
|
||||
MemorySwapMax=1G
|
||||
OOMPolicy=continue
|
||||
OOMScoreAdjust=-100
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -10,7 +10,8 @@
|
||||
# the environment and execs the command; no shell ever parses a value.
|
||||
#
|
||||
# The file is NUL-separated: the number of environment entries, the
|
||||
# NAME=value entries, then the argv (none = a login shell).
|
||||
# NAME=value entries, then the argv (none = a login shell). Every pane starts
|
||||
# at a lower CPU/IO priority and as the kernel's first OOM pick (see below).
|
||||
# @author Nguyễn Ngọc Trí Vĩ <vinnt@smartgift.vn>
|
||||
set -eu
|
||||
f=${1:?usage: pane-launch <launch file>}
|
||||
@@ -22,6 +23,20 @@ for kv in "${parts[@]:1:n}"; do
|
||||
[[ "$kv" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]] || { echo "pane-launch: bad variable" >&2; exit 2; }
|
||||
export "$kv"
|
||||
done
|
||||
# A member's pane shares the 4-core box with the dashboard and every other
|
||||
# member: its test run, build or dev server must slow down before anything
|
||||
# else does, and die first if memory runs out (11-02: 36 sessions plus test
|
||||
# runs filled RAM and swap, and the box froze until a hard reset). A member
|
||||
# cannot take the nice back (that needs root); oom_score_adj they can lower
|
||||
# only to the unit's own -100, and even then the biggest process dies. The unit
|
||||
# caps the whole cgroup below RAM (install-system.sh), so the kernel's pick
|
||||
# inside it is the biggest pane process, never the dashboard or tmux.
|
||||
renice -n 10 -p $$ >/dev/null 2>&1 || true
|
||||
ionice -c 2 -n 7 -p $$ >/dev/null 2>&1 || true
|
||||
echo 500 >/proc/self/oom_score_adj 2>/dev/null || true
|
||||
# Vitest ≤ 3 starts one worker per core by default and these override the
|
||||
# config; a member may set their own in the pane.
|
||||
export VITEST_MAX_THREADS="${VITEST_MAX_THREADS:-2}" VITEST_MAX_FORKS="${VITEST_MAX_FORKS:-2}"
|
||||
argv=("${parts[@]:n+1}")
|
||||
if [ ${#argv[@]} -eq 0 ]; then exec -l "${SHELL:-/bin/bash}"; fi
|
||||
exec "${argv[@]}"
|
||||
|
||||
Reference in New Issue
Block a user