From 78ac3864039e066354181cb6b53c92e96d86ff3e Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sat, 17 Oct 2026 17:51:19 -0400 Subject: vgsp: m177 pin-free self-sufficiency wave + tool hardening (1) svc_session_flush keeps svc.direct_base (per-boot constant; the pin-free targeted scanner's gate/input) and a new gsp_sema_service_guest_reset() hook wipes boot-scoped discovery state on guest RM reload; (2) counter-page derive now takes the first READABLE source (vaspaces -> discovered -> targeted) instead of failing -EIO on a walked-but-garbage page, with one-shot diagnostics; (3) new m177c UVM ring-space directory discovery: content-verified (push_va -> push_pa) samples from the GR content discovery and CE push pairing feed a second discovered-directory slot (walks the 0x2xxxxxxx family), consulted by svc_walk_discovered; (4) GR channels now deliver withheld opdef sets at entry consume (normal tail and the stale-skip path) instead of waiting for the 10 s sweep; (5) host tools hardened: semfix_uvmt QMP reconnect + pid-generation dedupe + stale-sem eviction + post-success pin latch + heartbeats; pin_gr_stream accept timeout + guest-scanner reinstall backoff + restart loop + subprocess timeouts; new pin_supervisor.sh (heartbeat watchdog, capped restarts) wired into start_vm_keep.sh. Evidence: r194-r196 cycles; pool resolves src=trusted pin-free; ring dir 0x10801c000 found 4/4; row: 'rel-apply ... written' with tools up. --- tools/guest-diag/pin_supervisor.sh | 138 +++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100755 tools/guest-diag/pin_supervisor.sh (limited to 'tools/guest-diag/pin_supervisor.sh') diff --git a/tools/guest-diag/pin_supervisor.sh b/tools/guest-diag/pin_supervisor.sh new file mode 100755 index 0000000..5bbab0c --- /dev/null +++ b/tools/guest-diag/pin_supervisor.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Open-IOV contributors +# pin_supervisor.sh - supervise the host-side pin tools (pin_gr_stream.py and +# semfix_uvmt.py) so a silent death cannot strand every later CUDA client at +# first cudaMalloc. +# +# Both tools write a heartbeat file; this script checks, every 5 s, that each +# process is alive (exact script path) and that its heartbeat mtime is < 30 s. +# On failure it kills the tool (exact path match) and restarts it, appending +# RESTART lines to the supervisor log. After 10 restarts in one hour for a +# tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever +# started (QEMU's QMP socket accepts a single client). +# +# Usage: pin_supervisor.sh LOGDIR TAG +# Logs: LOGDIR/12-pin-supervisor-TAG.log (supervisor) +# LOGDIR/12-pin_gr_stream-TAG.log, LOGDIR/12-semfix-TAG.log (tools) +set -uo pipefail + +LOGDIR="${1:?usage: pin_supervisor.sh LOGDIR TAG}" +TAG="${2:?usage: pin_supervisor.sh LOGDIR TAG}" + +HERE=$(cd "$(dirname "$0")" && pwd) +PIN="$HERE/pin_gr_stream.py" +SEMFIX="$HERE/semfix_uvmt.py" +PIN_HB="${PIN_GR_HEARTBEAT:-/tmp/opencode/pin_gr_heartbeat}" +SEMFIX_HB="${SEMFIX_HEARTBEAT_FILE:-/tmp/opencode/semfix_heartbeat}" +SUPLOG="$LOGDIR/12-pin-supervisor-$TAG.log" +PIN_LOG="$LOGDIR/12-pin_gr_stream-$TAG.log" +SEMFIX_LOG="$LOGDIR/12-semfix-$TAG.log" +WINDOW=3600 +CAP=10 +mkdir -p "$LOGDIR" /tmp/opencode + +log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$SUPLOG"; } + +tool_alive() { pgrep -f "python3 $1" >/dev/null 2>&1; } + +hb_age() { # age in seconds of the heartbeat file, 9999 if missing + local f="$1" m + m=$(stat -c %Y "$f" 2>/dev/null) || { echo 9999; return; } + echo $(( $(date +%s) - m )) +} + +kill_tool() { + pkill -f "python3 $1" 2>/dev/null + sleep 1 + if pgrep -f "python3 $1" >/dev/null 2>&1; then + pkill -9 -f "python3 $1" 2>/dev/null + sleep 1 + fi + return 0 +} + +start_pin() { + if tool_alive "$PIN"; then return 0; fi + touch "$PIN_HB" 2>/dev/null || true + ( setsid nohup env SEMFIX_COUNTER_RETRY=10 \ + python3 "$PIN" >> "$PIN_LOG" 2>&1 & ) + log "START pin_gr_stream" +} + +start_semfix() { + # pgrep exact-count check before start: if an instance already exists + # (ours or a leftover), never start a second one. + if tool_alive "$SEMFIX"; then + log "semfix already running - not starting another" + return 0 + fi + touch "$SEMFIX_HB" 2>/dev/null || true + ( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 \ + python3 "$SEMFIX" 0.5 >> "$SEMFIX_LOG" 2>&1 & ) + log "START semfix_uvmt" +} + +prune() { # $1 now, $2 list -> list with entries older than WINDOW dropped + local now="$1" t out="" + for t in $2; do + [ $(( now - t )) -lt "$WINDOW" ] && out="$out $t" + done + printf '%s' "$out" +} + +count_list() { # $1 list -> element count + local n=0 t + for t in $1; do n=$(( n + 1 )); done + echo "$n" +} + +restarts_pin="" +restarts_semfix="" + +restart() { # $1 label, $2 script path, $3 start function + local now list n + now=$(date +%s) + if [ "$1" = pin_gr_stream ]; then + restarts_pin=$(prune "$now" "$restarts_pin") + n=$(count_list "$restarts_pin") + else + restarts_semfix=$(prune "$now" "$restarts_semfix") + n=$(count_list "$restarts_semfix") + fi + if [ "$n" -ge "$CAP" ]; then + log "GIVEUP $1: $n restarts within ${WINDOW}s" + exit 1 + fi + kill_tool "$2" + if [ "$1" = pin_gr_stream ]; then + restarts_pin="$restarts_pin $now" + else + restarts_semfix="$restarts_semfix $now" + fi + log "RESTART $(( n + 1 )) $1 $(date +%Y-%m-%dT%H:%M:%S)" + "$3" +} + +check_tool() { # $1 label, $2 path, $3 heartbeat file, $4 start function + local age + age=$(hb_age "$3") + if tool_alive "$2" && [ "$age" -le 30 ]; then + return 0 + fi + if ! tool_alive "$2"; then + log "$1 not running (heartbeat age ${age}s) - restarting" + else + log "$1 heartbeat stale (${age}s) - restarting" + fi + restart "$1" "$2" "$4" +} + +log "pin_supervisor up (LOGDIR=$LOGDIR TAG=$TAG)" +start_pin +start_semfix +while true; do + sleep 5 + check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin + check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix +done -- cgit v1.2.3