#!/bin/sh # SPDX-License-Identifier: MIT # NumericalOS init - shell PID-1 path. # Walks BootPhase records by ordinal, resolves the unit DAG, runs units. set -u NUMOS_STATE_FILE="" numos_die() { echo "numos: HALT: $*" >&2 exit 1 } numos_load_state() { NUMOS_STATE_FILE="$1" [ -f "$NUMOS_STATE_FILE" ] || numos_die "state not found: $1" } numos_phase_ordinals() { grep '^P ' "$NUMOS_STATE_FILE" | cut -d' ' -f2 | sort -n } numos_phase_field() { # numos_phase_field line="$(grep "^P $1 " "$NUMOS_STATE_FILE" | head -n 1)" [ -n "$line" ] || numos_die "no phase with ordinal $1" case "$2" in name) echo "$line" | cut -d' ' -f3 ;; on_failure) echo "$line" | cut -d' ' -f4 ;; required_units) echo "$line" | cut -d' ' -f5 | tr ',' ' ' | sed 's/^-$//' ;; *) numos_die "unknown phase field: $2" ;; esac } numos_unit_field() { # numos_unit_field . exec is last on the line, so it keeps spaces. line="$(grep "^U $1 " "$NUMOS_STATE_FILE" | head -n 1)" [ -n "$line" ] || numos_die "unknown unit: $1" case "$2" in kind) echo "$line" | cut -d' ' -f3 ;; restart) echo "$line" | cut -d' ' -f4 ;; backoff_ms) echo "$line" | cut -d' ' -f5 ;; backoff_max_ms) echo "$line" | cut -d' ' -f6 ;; requires) echo "$line" | cut -d' ' -f9 | tr ',' ' ' | sed 's/^-$//' ;; after) echo "$line" | cut -d' ' -f10 | tr ',' ' ' | sed 's/^-$//' ;; exec) echo "$line" | cut -d' ' -f11- ;; *) numos_die "unknown unit field: $2" ;; esac } # Emit units in dependency order (Kahn's algorithm). # # Deliberately iterative, not recursive: POSIX sh has no `local`, so a # recursive helper's variables are global and the recursive call clobbers the # caller's loop variable. Two passes instead: expand the dependency closure, # then repeatedly emit whichever units have all their deps already emitted. numos_unit_deps() { # numos_unit_deps . Fails (nonzero) if the unit is unknown; caller # must check the status, since exit inside a $(...) only kills the subshell. reqs="$(numos_unit_field "$1" requires)" || return 1 afters="$(numos_unit_field "$1" after)" || return 1 echo "$reqs $afters" } numos_resolve_order() { closure="" pending="$*" while [ -n "$(echo $pending)" ]; do nextwave="" for node in $pending; do case " $closure " in *" $node "*) continue ;; esac closure="$closure $node" deps="$(numos_unit_deps "$node")" || numos_die "unit $node: dependency lookup failed" nextwave="$nextwave $deps" done pending="$nextwave" done emitted="" remaining="$closure" while [ -n "$(echo $remaining)" ]; do progress=0 stillwaiting="" for node in $remaining; do ready=1 deps="$(numos_unit_deps "$node")" || numos_die "unit $node: dependency lookup failed" for dep in $deps; do case " $emitted " in *" $dep "*) ;; *) ready=0 ;; esac done if [ "$ready" = "1" ]; then echo "$node" emitted="$emitted $node" progress=1 else stillwaiting="$stillwaiting $node" fi done remaining="$stillwaiting" [ "$progress" = "1" ] || numos_die "unresolvable dependency order in: $remaining" done } # Execute a unit's exec line synchronously and return its exit status. # The NUMOS_DRY_RUN seam lives here, so every path that would execute # something -- the phase walk and the supervisor alike -- reports instead. # A unit's resource envelope, from its R record. Absent record or absent # field yields the empty string, meaning unlimited. # # R numos_resource_field() { line="$(grep "^R $1 " "$NUMOS_STATE_FILE" 2>/dev/null | head -n 1)" [ -n "$line" ] || { echo ""; return 0; } case "$2" in cpu_pct) value="$(echo "$line" | cut -d' ' -f3)" ;; ram_mb) value="$(echo "$line" | cut -d' ' -f4)" ;; timeout_s) value="$(echo "$line" | cut -d' ' -f5)" ;; *) numos_die "unknown resource field: $2" ;; esac [ "$value" = "-" ] && value="" echo "$value" } # Enforce what the shell floor can enforce, and REFUSE what it cannot. # # Declaring a limit that nothing applies is a lie told to whoever wrote the # state: they set cpu_pct expecting protection and got none. So an envelope # this floor cannot honour halts with a named reason rather than running the # unit unbounded (docs/PRINCIPLES.md stewardship, and truthfulness/CCC 2469). # Setting the field to `-` is how an operator opts out. # # ram_mb enforced with ulimit -v, which is per-process address space # timeout_s enforced with timeout(1); busybox provides it # cpu_pct NOT enforceable here. A percentage needs cgroups or a # scheduler knob; ulimit -t is cumulative CPU seconds, which is # a different quantity. Refused rather than approximated. # Resolve a unit's envelope into globals, and halt on anything this floor # cannot honour. MUST run in the parent shell, before a longrun unit is # backgrounded, for two reasons: # # 1. numos_die inside a background job kills only that job. A cpu_pct halt # would print and the boot would carry on - the swallowed-exit class # this codebase has hit repeatedly. # 2. The lookups are several greps. Doing them inside the backgrounded job # keeps that process alive long enough that numos_longrun_still_running # mistakes a unit which died instantly for a healthy one. NUMOS_ENV_FOR="" NUMOS_ENV_RAM="" NUMOS_ENV_TIMEOUT="" numos_resolve_envelope() { name="$1" cpu_pct="$(numos_resource_field "$name" cpu_pct)" NUMOS_ENV_RAM="$(numos_resource_field "$name" ram_mb)" NUMOS_ENV_TIMEOUT="$(numos_resource_field "$name" timeout_s)" if [ -n "$cpu_pct" ]; then numos_die "unit $name: resource_envelope.cpu_pct=$cpu_pct cannot be enforced by the shell floor (needs cgroups); set it to - to run without a CPU limit" fi if [ -n "$NUMOS_ENV_TIMEOUT" ]; then command -v timeout >/dev/null 2>&1 || numos_die "unit $name: resource_envelope.timeout_s=$NUMOS_ENV_TIMEOUT declared but no timeout(1) is available to enforce it" fi NUMOS_ENV_FOR="$name" } numos_exec_unit() { name="$1" if [ "${NUMOS_DRY_RUN:-0}" = "1" ]; then echo "RUN $name" return 0 fi cmd="$(numos_unit_field "$name" exec)" || numos_die "unit $name: exec lookup failed" # Reuse what the parent already resolved; resolve here only for callers # that run synchronously and did not pre-resolve. [ "$NUMOS_ENV_FOR" = "$name" ] || numos_resolve_envelope "$name" prelude="" [ -n "$NUMOS_ENV_RAM" ] && prelude="ulimit -v $((NUMOS_ENV_RAM * 1024)) || exit 1; " if [ -n "$NUMOS_ENV_TIMEOUT" ]; then timeout "$NUMOS_ENV_TIMEOUT" sh -c "$prelude$cmd" else sh -c "$prelude$cmd" fi } # Confirm a just-backgrounded longrun is still running. # # Reporting success the instant a daemon is forked made a missing binary # (or `false`, or any crash-on-start) look like a healthy phase: the phase's # on_failure never ran, and only steady-state restart policy noticed later. # That is a lie about the phase. A short settle is required because some # hosts (notably Git Bash on Windows) still report the child as alive for # a beat after it has already decided to exit. # # This settle is NOT throttling and is NOT skipped by NUMOS_NO_SLEEP: that # variable suppresses restart backoff and tick delays, not observability. numos_longrun_still_running() { pid="$1" # Poll rather than a single snapshot: Git Bash has been observed to keep # kill -0 succeeding for one beat after `false` (and missing binaries) # have already decided to exit. Two or three sub-second samples catch # that without turning a healthy daemon into a oneshot wait. # Prefer fractional sleep; one whole-second sample is the floor where # sleep(1) rejects fractions, rather than skipping the check. n=0 while [ "$n" -lt 4 ]; do if ! kill -0 "$pid" 2>/dev/null; then return 1 fi n=$((n + 1)) sleep 0.05 2>/dev/null || { # One whole-second sample is enough on strict POSIX sleep; do not # multiply it by the loop or a crash-loop host waits for seconds. [ "$n" -eq 1 ] && sleep 1 break } done kill -0 "$pid" 2>/dev/null } # Start a unit according to its kind. # # oneshot and target run to completion: the phase walk needs their exit # status to apply the phase's on_failure policy. A longrun unit is a daemon # -- it does not return -- so PID 1 must not wait on one. It is started in # the background and reported successful once it is observed still running; # the seeded `join` unit is exactly this shape and would otherwise block # the phase walk at phase 50 forever. Its PID is recorded so steady state # can notice when it dies later and apply its restart policy -- see # numos_supervise_tick. An immediately-exiting longrun is a phase failure. numos_run_unit() { name="$1" kind="$(numos_unit_field "$name" kind)" || numos_die "unit $name: kind lookup failed" case "$kind" in oneshot|target) numos_exec_unit "$name" ;; longrun) if [ "${NUMOS_DRY_RUN:-0}" = "1" ]; then # Nothing is executed in dry run, so nothing can block: report # synchronously to keep RUN lines in dependency order. numos_exec_unit "$name" return 0 fi # Resolve in the parent: a halt inside the background job would be # swallowed, and the lookups would keep that job alive long enough to # defeat the immediate-death check below. numos_resolve_envelope "$name" numos_exec_unit "$name" & pid=$! numos_track "$name" "$pid" if numos_longrun_still_running "$pid"; then return 0 fi # Leave the pidfile in place: phase on_failure must see the failure, # and steady state (if the phase policy lets boot continue) still has # to reap and apply the unit's restart policy. Reaping here would hide # the unit from the supervisor and turn restart=always into a no-op. echo "numos: longrun $name exited immediately" >&2 return 1 ;; *) numos_die "unit $name: unknown kind: $kind" ;; esac } # Delays double from initial up to the ceiling, then hold there. numos_backoff_sequence() { delay="$1" ceiling="$2" count="$3" [ "$delay" -le "$ceiling" ] || delay="$ceiling" n=0 while [ "$n" -lt "$count" ]; do echo "$delay" delay=$((delay * 2)) [ "$delay" -le "$ceiling" ] || delay="$ceiling" n=$((n + 1)) done } # Compute the sleep(1) argument for a millisecond delay, without sleeping. # Split out from numos_sleep_ms so tests can assert on the computed value # without waiting on a real sleep. A nonzero delay must never compute to an # argument that means "no wait" -- the fractional form below always carries # a nonzero fractional part when ms is not an exact multiple of 1000, and # numos_sleep_ms's whole-second fallback (below) separately guards the case # where the fractional form is rejected by the running sleep(1). numos_sleep_arg() { ms="$1" [ "$ms" -gt 0 ] || { echo 0; return 0; } secs=$((ms / 1000)) rem=$((ms % 1000)) frac="$(printf '%03d' "$rem")" echo "$secs.$frac" } numos_sleep_ms() { [ "${NUMOS_NO_SLEEP:-0}" = "1" ] && return 0 ms="$1" [ "$ms" -gt 0 ] || return 0 arg="$(numos_sleep_arg "$ms")" # Prefer the fractional form (GNU coreutils and busybox both accept it). # POSIX only guarantees whole-second sleep, so fall back rather than # assume -- and round the fallback UP so a sub-second delay never # collapses to "sleep 0" (a crash-looping unit would otherwise spin at # full speed with zero throttling). sleep "$arg" 2>/dev/null && return 0 whole=$(( (ms + 999) / 1000 )) [ "$whole" -gt 0 ] || whole=1 sleep "$whole" } numos_run_phase() { ordinal="$1" required="$(numos_phase_field "$ordinal" required_units)" || numos_die "phase $ordinal: required_units lookup failed" [ -n "$required" ] || return 0 order="$(numos_resolve_order $required)" || numos_die "phase $ordinal: dependency resolution failed" for name in $order; do numos_run_unit "$name" || return 1 done return 0 } NUMOS_DEGRADED=0 numos_degraded() { echo "$NUMOS_DEGRADED" } numos_mark_degraded() { NUMOS_DEGRADED=1 # Persist it. Degraded was an in-process shell variable, so nothing outside # numinit -- including numctl, which builds the capability document -- could # tell a degraded node from a healthy one. A node that cannot report its own # degradation advertises a capability it does not have. [ -d "$NUMOS_RUNDIR" ] && echo 1 > "$NUMOS_RUNDIR/degraded" } # A phase failed. What happens next is the phase's own on_failure policy. numos_handle_phase_failure() { ordinal="$1" name="$(numos_phase_field "$ordinal" name)" || numos_die "phase $ordinal: name lookup failed" policy="$(numos_phase_field "$ordinal" on_failure)" || numos_die "phase $ordinal: on_failure lookup failed" case "$policy" in halt) numos_die "phase $name failed and is on_failure=halt" ;; degrade) numos_mark_degraded echo "numos: DEGRADED: $name" ;; continue) echo "numos: WARN: phase $name failed, continuing" ;; *) numos_die "phase $name has an unknown on_failure policy" ;; esac } # ------------------------------------------------------------------ steady state # # The phase walk brings a machine up; steady state is what keeps it up. It is # also what makes this a valid PID 1 at all: a PID 1 that returns panics the # kernel, so in a real boot this loop never exits. NUMOS_MAX_TICKS bounds it # for tests and for nothing else. # # Supervision here is tick-driven over tracked PIDs rather than a blocking # per-unit supervisor. A loop watching N units cannot afford to block on any # one of them, which is why numos_supervise's shape does not fit and the # policy logic lives here instead. NUMOS_RUNDIR="${NUMOS_RUNDIR:-/run/numos}" numos_rundir_init() { mkdir -p "$NUMOS_RUNDIR/units" "$NUMOS_RUNDIR/health" || numos_die "cannot create runtime directory: $NUMOS_RUNDIR" } numos_track() { echo "$2" > "$NUMOS_RUNDIR/units/$1.pid" } numos_alive() { pidfile="$NUMOS_RUNDIR/units/$1.pid" [ -f "$pidfile" ] || return 1 kill -0 "$(cat "$pidfile")" 2>/dev/null } # Collect a dead child's status. This is also the zombie reaping duty: the # shell holds a terminated child's status until something waits for it. numos_reap() { pidfile="$NUMOS_RUNDIR/units/$1.pid" pid="$(cat "$pidfile")" wait "$pid" 2>/dev/null status=$? rm -f "$pidfile" return "$status" } # Give up on a unit that will never come up. # # Backoff bounded the DELAY but nothing bounded the ATTEMPTS, so a unit that # crash-loops from boot retried forever at the ceiling. That is not # self-healing; it is an infinite retry with no exit condition, and it was # invisible: no health predicate fired, no capability was retracted, and # nothing prompted an operator to look. A permanently failing unit must reach # a terminal state that something can observe. # # Quarantine is deliberately NOT a halt. One dead unit should not take a # machine down; the node keeps running, marks itself degraded so a # coordinator would stop scheduling onto it, and leaves a marker naming what # gave up and after how many attempts. NUMOS_MAX_ATTEMPTS="${NUMOS_MAX_ATTEMPTS:-10}" numos_quarantined() { [ -f "$NUMOS_RUNDIR/units/$1.quarantined" ] } numos_quarantine_unit() { name="$1" attempts="$2" echo "$attempts" > "$NUMOS_RUNDIR/units/$name.quarantined" rm -f "$NUMOS_RUNDIR/units/$name.pid" numos_mark_degraded echo "numos: QUARANTINED $name after $attempts attempts; node degraded" } numos_restart_unit() { name="$1" status="$2" attemptfile="$NUMOS_RUNDIR/units/$name.attempt" attempt=0 [ -f "$attemptfile" ] && attempt="$(cat "$attemptfile")" attempt=$((attempt + 1)) echo "$attempt" > "$attemptfile" if [ "$attempt" -gt "$NUMOS_MAX_ATTEMPTS" ]; then numos_quarantine_unit "$name" "$((attempt - 1))" return 0 fi init_backoff="$(numos_unit_field "$name" backoff_ms)" || numos_die "unit $name: backoff_ms lookup failed" ceiling="$(numos_unit_field "$name" backoff_max_ms)" || numos_die "unit $name: backoff_max_ms lookup failed" # Validate before any arithmetic: $(( )) on a non-numeric value from the # state file produces a raw shell error instead of a named halt. case "$init_backoff" in ""|*[!0-9]*) numos_die "unit $name: backoff_ms is not a non-negative integer: $init_backoff" ;; esac case "$ceiling" in ""|*[!0-9]*) numos_die "unit $name: backoff_max_ms is not a non-negative integer: $ceiling" ;; esac delays="$(numos_backoff_sequence "$init_backoff" "$ceiling" "$attempt")" || numos_die "unit $name: backoff sequence generation failed" delay="$(echo "$delays" | tail -n 1)" echo "numos: restart $name attempt=$attempt delay=${delay}ms (exit $status)" numos_sleep_ms "$delay" # Resolve in the parent, exactly as numos_run_unit does before ITS # background start. Without this the envelope check -- and its numos_die -- # runs inside the backgrounded job on every RESTART: an unenforceable # cpu_pct would print a HALT that halts nothing, numos_track would record # an already-dying pid, and the unit would crash-loop to quarantine while # the boot carried on. Same defect the sibling call site documents; this # was the one path the original fix did not cover. numos_resolve_envelope "$name" numos_exec_unit "$name" & numos_track "$name" "$!" } numos_supervise_tick() { for pidfile in "$NUMOS_RUNDIR"/units/*.pid; do [ -e "$pidfile" ] || continue name="$(basename "$pidfile" .pid)" numos_alive "$name" && continue # A quarantined unit is terminal: reap it once, then leave it alone. if numos_quarantined "$name"; then rm -f "$pidfile" continue fi numos_reap "$name" status=$? policy="$(numos_unit_field "$name" restart)" || numos_die "unit $name: restart lookup failed" case "$policy" in never) echo "numos: unit $name exited ($status), restart=never" ;; on-failure) if [ "$status" = "0" ]; then echo "numos: unit $name completed, restart=on-failure" else numos_restart_unit "$name" "$status" fi ;; always) numos_restart_unit "$name" "$status" ;; *) numos_die "unit $name: unknown restart policy: $policy" ;; esac done } numos_health_names() { grep '^X ' "$NUMOS_STATE_FILE" 2>/dev/null | cut -d' ' -f2 } numos_health_field() { line="$(grep "^X $1 " "$NUMOS_STATE_FILE" | head -n 1)" [ -n "$line" ] || numos_die "unknown health predicate: $1" case "$2" in interval_s) echo "$line" | cut -d' ' -f3 ;; threshold) echo "$line" | cut -d' ' -f4 ;; local_action) echo "$line" | cut -d' ' -f5 ;; on_fire) echo "$line" | cut -d' ' -f6 | sed 's/^-$//' ;; probe) echo "$line" | cut -d' ' -f7- ;; *) numos_die "unknown health field: $2" ;; esac } # Which unit declared this predicate as its health_probe (field 8 of a U # record). Used by the restart-unit action, which otherwise has no way to # know what it is supposed to restart. numos_units_with_probe() { grep '^U ' "$NUMOS_STATE_FILE" 2>/dev/null | while read -r _tag uname _kind _restart _bms _bmax _mask probe _rest; do [ "$probe" = "$1" ] && echo "$uname" done } numos_health_fire() { name="$1" action="$(numos_health_field "$name" local_action)" echo "numos: HEALTH-FIRE $name action=$action" case "$action" in none) ;; degrade-node) numos_mark_degraded echo "numos: DEGRADED: health $name" ;; restart-unit) for owner in $(numos_units_with_probe "$name"); do echo "numos: health $name restarting $owner" numos_alive "$owner" && kill "$(cat "$NUMOS_RUNDIR/units/$owner.pid")" 2>/dev/null done ;; *) numos_die "health $name: unknown local_action: $action" ;; esac # Escalation needs the network. Queue it unconditionally rather than # blocking the loop on reachability: an offline machine must keep # supervising itself, and the queue is what a later reconcile drains. on_fire="$(numos_health_field "$name" on_fire)" if [ -n "$on_fire" ]; then echo "$name $on_fire" >> "$NUMOS_RUNDIR/escalations.queue" echo "numos: queued escalation $on_fire for $name" fi } # interval_s is counted in ticks, and a tick is NUMOS_TICK_S seconds (default # 1), so the two agree by construction at the default and stay proportional # if the tick is retuned. numos_health_tick() { for name in $(numos_health_names); do countfile="$NUMOS_RUNDIR/health/$name.count" failfile="$NUMOS_RUNDIR/health/$name.fails" count=0 [ -f "$countfile" ] && count="$(cat "$countfile")" count=$((count + 1)) interval="$(numos_health_field "$name" interval_s)" if [ "$count" -lt "$interval" ]; then echo "$count" > "$countfile" continue fi echo 0 > "$countfile" fails=0 [ -f "$failfile" ] && fails="$(cat "$failfile")" probe="$(numos_health_field "$name" probe)" if sh -c "$probe" >/dev/null 2>&1; then [ "$fails" = "0" ] || echo "numos: health $name recovered" echo 0 > "$failfile" else fails=$((fails + 1)) echo "$fails" > "$failfile" threshold="$(numos_health_field "$name" threshold)" echo "numos: health $name failed ($fails/$threshold)" [ "$fails" -lt "$threshold" ] || numos_health_fire "$name" fi done } numos_steady_state() { ticks=0 max="${NUMOS_MAX_TICKS:-0}" echo "numos: steady state entered" while : ; do numos_supervise_tick numos_health_tick ticks=$((ticks + 1)) if [ "$max" != "0" ] && [ "$ticks" -ge "$max" ]; then echo "numos: steady state stopped after $ticks ticks degraded=$(numos_degraded)" return 0 fi numos_sleep_ms "$(( ${NUMOS_TICK_S:-1} * 1000 ))" done } numos_main() { [ -n "${NUMOS_STATE:-}" ] || numos_die "NUMOS_STATE is unset" numos_load_state "$NUMOS_STATE" numos_rundir_init for ordinal in $(numos_phase_ordinals); do if ! numos_run_phase "$ordinal"; then numos_handle_phase_failure "$ordinal" fi done echo "numos: boot complete degraded=$(numos_degraded)" numos_steady_state } # Shell-floor signal handling. TERM/INT exit with a named halt so a supervisor # or human does not get a silent death. This is NOT static-binary SIGCHLD # discipline and does not close the PID-1 residual on real hardware — that # still needs QEMU/metal (and Spec 3 Zig for proper signal quality). numos_on_term() { echo "numos: HALT: received SIGTERM" >&2 exit 143 } numos_on_int() { echo "numos: HALT: received SIGINT" >&2 exit 130 } # Terminate every unit this init started, before this init goes away. # # Longrun units are backgrounded with `&`, and nothing killed them when # numinit exited. In a real boot numinit never returns, so this only fires on # a halt or a signal -- exactly the moments when leaving orphaned daemons # behind is wrong: PID 1 is going away and the processes it started would be # reparented and abandoned. # # It also repairs the test instrument. Bounded runs (NUMOS_MAX_TICKS) leaked # real sleeping children into the next test module, and those leaks starved # later wall-clock assertions into spurious failures -- a suite that # intermittently reported failures no targeted run could reproduce. An # evidence oracle with a false-positive channel is a defect in the evidence. numos_kill_tracked() { [ -d "${NUMOS_RUNDIR:-}/units" ] || return 0 for pidfile in "$NUMOS_RUNDIR"/units/*.pid; do [ -e "$pidfile" ] || continue pid="$(cat "$pidfile" 2>/dev/null)" || continue [ -n "$pid" ] || continue kill "$pid" 2>/dev/null || : done return 0 } if [ "${NUMOS_SOURCE_ONLY:-0}" != "1" ]; then # EXIT only. Trapping TERM/INT here as well would override the named-halt # handlers below; the default disposition terminates and that fires EXIT, # so cleanup runs on every path without changing signal semantics. trap numos_kill_tracked EXIT trap numos_on_term TERM trap numos_on_int INT numos_main "$@" fi