#!/bin/bash
# battery_resume2.sh - resume a battery after a partial stop, with a
# zombie-tolerant completion check: a client is DONE when its .bat.out holds
# a verdict line (STATUS OK|FAIL|CUDA_ERROR) even if a zombie with the same
# comm lingers (the pgrep-x poll in battery_smileylast falsely hangs on
# those). A client with an alive non-zombie process and no verdict after
# HANG_S is a real hang.
# Usage: battery_resume2.sh <outfile> <client...>
set -u
cd /home/rmcore/Git/integdev_gpu_drv/open-iov
OUT="${1:?usage: battery_resume2.sh <out> <client...>}"; shift
CLIENTS="$*"
HANG_S=600
: >> "$OUT"
pass=0; fail=0; hang=0
for c in $CLIENTS; do
    echo "=== $c ===" | tee -a "$OUT"
    timeout 40 scripts/ssh_guest.sh "test -x /tmp/$c || exit 9; cd /tmp && rm -f /tmp/$c.bat.out && setsid nohup ./$c > /tmp/$c.bat.out 2>&1 < /dev/null & echo LAUNCHED" >/dev/null 2>&1
    t0=$(date +%s); done_ok=0
    while :; do
        sleep 5
        block=$(timeout 25 scripts/ssh_guest.sh "cat /tmp/$c.bat.out 2>/dev/null" 2>/dev/null | grep -v "Permanently added")
        if printf '%s\n' "$block" | grep -qE "STATUS OK|STATUS FAIL|CUDA_ERROR"; then done_ok=1; break; fi
        # alive non-zombie process with no verdict?
        alive=$(printf '%s\n' "$block"; timeout 25 scripts/ssh_guest.sh "ps -eo stat=,comm= | awk -v n='$c' '\$2==n && \$1 !~ /^Z/' | wc -l" 2>/dev/null | tail -1)
        now=$(date +%s)
        if [ $((now - t0)) -ge $HANG_S ]; then
            echo "HANG: $c no verdict after ${HANG_S}s" | tee -a "$OUT"
            hang=$((hang+1))
            echo "BATTERY RESUME STOPPED at $c (hang; boot preserved, do not kill)" | tee -a "$OUT"
            echo "BATTERY RESUME pass=$pass fail=$fail hang=$hang" | tee -a "$OUT"
            exit 0
        fi
    done
    printf '%s\n' "$block" | tail -8 | tee -a "$OUT"
    if printf '%s\n' "$block" | grep -q "STATUS OK"; then pass=$((pass+1)); else fail=$((fail+1)); fi
    echo "--- done ---" | tee -a "$OUT"
done
echo "BATTERY RESUME pass=$pass fail=$fail hang=$hang" | tee -a "$OUT"