diff options
Diffstat (limited to 'tools/guest-diag/pin_supervisor.sh')
| -rwxr-xr-x | tools/guest-diag/pin_supervisor.sh | 44 |
1 files changed, 32 insertions, 12 deletions
diff --git a/tools/guest-diag/pin_supervisor.sh b/tools/guest-diag/pin_supervisor.sh index 5bbab0c..f1f4eae 100755 --- a/tools/guest-diag/pin_supervisor.sh +++ b/tools/guest-diag/pin_supervisor.sh @@ -6,10 +6,12 @@ # first cudaMalloc. # # Both tools write a heartbeat file; this script checks, every 5 s, that each -# process is alive (exact script path) and that its heartbeat mtime is < 30 s. -# On failure it kills the tool (exact path match) and restarts it, appending -# RESTART lines to the supervisor log. After 10 restarts in one hour for a -# tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever +# process is alive (exact script path) and that its heartbeat mtime is < 75 s. +# A startup grace (120 s since the tool started, skipped early when the tool +# log already shows first successful work) keeps a slow cold start from being +# restarted. On failure it kills the tool (exact path match) and restarts it, +# appending RESTART lines to the supervisor log. After 10 restarts in one hour +# for a tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever # started (QEMU's QMP socket accepts a single client). # # Usage: pin_supervisor.sh LOGDIR TAG @@ -30,6 +32,8 @@ PIN_LOG="$LOGDIR/12-pin_gr_stream-$TAG.log" SEMFIX_LOG="$LOGDIR/12-semfix-$TAG.log" WINDOW=3600 CAP=10 +STALE=75 +GRACE=120 mkdir -p "$LOGDIR" /tmp/opencode log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$SUPLOG"; } @@ -57,6 +61,7 @@ start_pin() { touch "$PIN_HB" 2>/dev/null || true ( setsid nohup env SEMFIX_COUNTER_RETRY=10 \ python3 "$PIN" >> "$PIN_LOG" 2>&1 & ) + pin_started=$(date +%s) log "START pin_gr_stream" } @@ -70,6 +75,7 @@ start_semfix() { touch "$SEMFIX_HB" 2>/dev/null || true ( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 \ python3 "$SEMFIX" 0.5 >> "$SEMFIX_LOG" 2>&1 & ) + semfix_started=$(date +%s) log "START semfix_uvmt" } @@ -89,6 +95,8 @@ count_list() { # $1 list -> element count restarts_pin="" restarts_semfix="" +pin_started=$(date +%s) +semfix_started=$(date +%s) restart() { # $1 label, $2 script path, $3 start function local now list n @@ -114,16 +122,26 @@ restart() { # $1 label, $2 script path, $3 start function "$3" } -check_tool() { # $1 label, $2 path, $3 heartbeat file, $4 start function - local age +check_tool() { # $1 label, $2 path, $3 heartbeat file, $4 start function, + # $5 tool start TS, $6 tool log file, $7 first-work pattern + local age runtime age=$(hb_age "$3") - if tool_alive "$2" && [ "$age" -le 30 ]; then + if tool_alive "$2" && [ "$age" -le "$STALE" ]; then return 0 fi - if ! tool_alive "$2"; then - log "$1 not running (heartbeat age ${age}s) - restarting" - else + if tool_alive "$2"; then + # Startup grace: a cold tool can spend its first ~2 min installing the + # guest scanner or connecting QMP with a lagging heartbeat; only + # enforce staleness once the grace has passed or the log shows the + # tool has already done successful work. + runtime=$(( $(date +%s) - $5 )) + if [ "$runtime" -lt "$GRACE" ] && \ + ! grep -q -- "$7" "$6" 2>/dev/null; then + return 0 + fi log "$1 heartbeat stale (${age}s) - restarting" + else + log "$1 not running (heartbeat age ${age}s) - restarting" fi restart "$1" "$2" "$4" } @@ -133,6 +151,8 @@ start_pin start_semfix while true; do sleep 5 - check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin - check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix + check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin "$pin_started" \ + "$PIN_LOG" "listening on vsock" + check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix \ + "$semfix_started" "$SEMFIX_LOG" "semfix_uvmt: watching" done |