diff options
Diffstat (limited to 'tools/guest-diag/pin_supervisor.sh')
| -rwxr-xr-x | tools/guest-diag/pin_supervisor.sh | 44 |
1 files changed, 12 insertions, 32 deletions
diff --git a/tools/guest-diag/pin_supervisor.sh b/tools/guest-diag/pin_supervisor.sh index f1f4eae..5bbab0c 100755 --- a/tools/guest-diag/pin_supervisor.sh +++ b/tools/guest-diag/pin_supervisor.sh @@ -6,12 +6,10 @@ # first cudaMalloc. # # Both tools write a heartbeat file; this script checks, every 5 s, that each -# process is alive (exact script path) and that its heartbeat mtime is < 75 s. -# A startup grace (120 s since the tool started, skipped early when the tool -# log already shows first successful work) keeps a slow cold start from being -# restarted. On failure it kills the tool (exact path match) and restarts it, -# appending RESTART lines to the supervisor log. After 10 restarts in one hour -# for a tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever +# process is alive (exact script path) and that its heartbeat mtime is < 30 s. +# On failure it kills the tool (exact path match) and restarts it, appending +# RESTART lines to the supervisor log. After 10 restarts in one hour for a +# tool it logs GIVEUP and exits 1. Only one semfix_uvmt instance is ever # started (QEMU's QMP socket accepts a single client). # # Usage: pin_supervisor.sh LOGDIR TAG @@ -32,8 +30,6 @@ PIN_LOG="$LOGDIR/12-pin_gr_stream-$TAG.log" SEMFIX_LOG="$LOGDIR/12-semfix-$TAG.log" WINDOW=3600 CAP=10 -STALE=75 -GRACE=120 mkdir -p "$LOGDIR" /tmp/opencode log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$SUPLOG"; } @@ -61,7 +57,6 @@ start_pin() { touch "$PIN_HB" 2>/dev/null || true ( setsid nohup env SEMFIX_COUNTER_RETRY=10 \ python3 "$PIN" >> "$PIN_LOG" 2>&1 & ) - pin_started=$(date +%s) log "START pin_gr_stream" } @@ -75,7 +70,6 @@ start_semfix() { touch "$SEMFIX_HB" 2>/dev/null || true ( setsid nohup env SEMFIX_SKIP_PB=0 SEMFIX_COUNTER_RETRY=10 \ python3 "$SEMFIX" 0.5 >> "$SEMFIX_LOG" 2>&1 & ) - semfix_started=$(date +%s) log "START semfix_uvmt" } @@ -95,8 +89,6 @@ count_list() { # $1 list -> element count restarts_pin="" restarts_semfix="" -pin_started=$(date +%s) -semfix_started=$(date +%s) restart() { # $1 label, $2 script path, $3 start function local now list n @@ -122,26 +114,16 @@ restart() { # $1 label, $2 script path, $3 start function "$3" } -check_tool() { # $1 label, $2 path, $3 heartbeat file, $4 start function, - # $5 tool start TS, $6 tool log file, $7 first-work pattern - local age runtime +check_tool() { # $1 label, $2 path, $3 heartbeat file, $4 start function + local age age=$(hb_age "$3") - if tool_alive "$2" && [ "$age" -le "$STALE" ]; then + if tool_alive "$2" && [ "$age" -le 30 ]; then return 0 fi - if tool_alive "$2"; then - # Startup grace: a cold tool can spend its first ~2 min installing the - # guest scanner or connecting QMP with a lagging heartbeat; only - # enforce staleness once the grace has passed or the log shows the - # tool has already done successful work. - runtime=$(( $(date +%s) - $5 )) - if [ "$runtime" -lt "$GRACE" ] && \ - ! grep -q -- "$7" "$6" 2>/dev/null; then - return 0 - fi - log "$1 heartbeat stale (${age}s) - restarting" - else + if ! tool_alive "$2"; then log "$1 not running (heartbeat age ${age}s) - restarting" + else + log "$1 heartbeat stale (${age}s) - restarting" fi restart "$1" "$2" "$4" } @@ -151,8 +133,6 @@ start_pin start_semfix while true; do sleep 5 - check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin "$pin_started" \ - "$PIN_LOG" "listening on vsock" - check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix \ - "$semfix_started" "$SEMFIX_LOG" "semfix_uvmt: watching" + check_tool pin_gr_stream "$PIN" "$PIN_HB" start_pin + check_tool semfix_uvmt "$SEMFIX" "$SEMFIX_HB" start_semfix done |