1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
#!/bin/bash
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2026 Open-IOV contributors
# pin_supervisor.sh - supervise the host-side pin tools (pin_gr_stream.py and
# semfix_uvmt.py) so a silent death cannot strand every later CUDA client at
# first cudaMalloc.
#
# Both tools write a heartbeat file; this script checks, every 5 s, that each
# process is alive (exact script path) and that its heartbeat mtime is < 30 s.
# On failure it kills the tool (exact path match) and restarts it, appending
# RESTART lines to the supervisor log. After 10 restarts in one hour for a
# tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever
# started (QEMU's QMP socket accepts a single client).
#
# Usage: pin_supervisor.sh LOGDIR TAG
# Logs: LOGDIR/12-pin-supervisor-TAG.log (supervisor)
# LOGDIR/12-pin_gr_stream-TAG.log, LOGDIR/12-semfix-TAG.log (tools)
set -uo pipefail
LOGDIR="${1:?usage: pin_supervisor.sh LOGDIR TAG}"
TAG="${2:?usage: pin_supervisor.sh LOGDIR TAG}"
HERE=$(cd "$(dirname "$0")" && pwd)
PIN="$HERE/pin_gr_stream.py"
SEMFIX="$HERE/semfix_uvmt.py"
PIN_HB="${PIN_GR_HEARTBEAT:-/tmp/opencode/pin_gr_heartbeat}"
SEMFIX_HB="${SEMFIX_HEARTBEAT_FILE:-/tmp/opencode/semfix_heartbeat}"
SUPLOG="$LOGDIR/12-pin-supervisor-$TAG.log"
PIN_LOG="$LOGDIR/12-pin_gr_stream-$TAG.log"
SEMFIX_LOG="$LOGDIR/12-semfix-$TAG.log"
WINDOW=3600
CAP=10
mkdir -p "$LOGDIR" /tmp/opencode
log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$SUPLOG"; }
tool_alive() { pgrep -f "python3 $1" >/dev/null 2>&1; }
hb_age() { # age in seconds of the heartbeat file, 9999 if missing
local f="$1" m
m=$(stat -c %Y "$f" 2>/dev/null) || { echo 9999; return; }
echo $(( $(date +%s) - m ))
}
kill_tool() {
pkill -f "python3 $1" 2>/dev/null
sleep 1
if pgrep -f "python3 $1" >/dev/null 2>&1; then
pkill -9 -f "python3 $1" 2>/dev/null
sleep 1
fi
return 0
}
start_pin() {
if tool_alive "$PIN"; then return 0; fi
touch "$PIN_HB" 2>/dev/null || true
( setsid nohup env SEMFIX_COUNTER_RETRY=10 \
python3 "$PIN" >> "$PIN_LOG" 2>&1 & )
log "START pin_gr_stream"
}
start_semfix() {
# pgrep exact-count check before start: if an instance already exists
# (ours or a leftover), never start a second one.
if tool_alive "$SEMFIX"; then
log "semfix already running - not starting another"
return 0
fi
touch "$SEMFIX_HB" 2>/dev/null || true
( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 \
python3 "$SEMFIX" 0.5 >> "$SEMFIX_LOG" 2>&1 & )
log "START semfix_uvmt"
}
prune() { # $1 now, $2 list -> list with entries older than WINDOW dropped
local now="$1" t out=""
for t in $2; do
[ $(( now - t )) -lt "$WINDOW" ] && out="$out $t"
done
printf '%s' "$out"
}
count_list() { # $1 list -> element count
local n=0 t
for t in $1; do n=$(( n + 1 )); done
echo "$n"
}
restarts_pin=""
restarts_semfix=""
restart() { # $1 label, $2 script path, $3 start function
local now list n
now=$(date +%s)
if [ "$1" = pin_gr_stream ]; then
restarts_pin=$(prune "$now" "$restarts_pin")
n=$(count_list "$restarts_pin")
else
restarts_semfix=$(prune "$now" "$restarts_semfix")
n=$(count_list "$restarts_semfix")
fi
if [ "$n" -ge "$CAP" ]; then
log "GIVEUP $1: $n restarts within ${WINDOW}s"
exit 1
fi
kill_tool "$2"
if [ "$1" = pin_gr_stream ]; then
restarts_pin="$restarts_pin $now"
else
restarts_semfix="$restarts_semfix $now"
fi
log "RESTART $(( n + 1 )) $1 $(date +%Y-%m-%dT%H:%M:%S)"
"$3"
}
check_tool() { # $1 label, $2 path, $3 heartbeat file, $4 start function
local age
age=$(hb_age "$3")
if tool_alive "$2" && [ "$age" -le 30 ]; then
return 0
fi
if ! tool_alive "$2"; then
log "$1 not running (heartbeat age ${age}s) - restarting"
else
log "$1 heartbeat stale (${age}s) - restarting"
fi
restart "$1" "$2" "$4"
}
log "pin_supervisor up (LOGDIR=$LOGDIR TAG=$TAG)"
start_pin
start_semfix
while true; do
sleep 5
check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin
check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix
done
|
