#!/bin/bash
# cuda_cycle.sh - one full CUDA bring-up attempt: restart the VM, push the
# guest tools, start cdt holding on failure, wait for the 20 GR channels,
# register/pin the completion counters, and arm the RM-error scanner so the
# next forced 999 names its record.
#
# Usage: tools/guest-diag/cuda_cycle.sh [hold-seconds]
# Output: guest /tmp/cdt.out, /tmp/scanerr.out; host sema-svc log in dmesg.
set -uo pipefail

HERE="$(cd "$(dirname "$0")" && pwd)"
REPO="$HERE/../.."
VMEXEC="$REPO/scripts/vgpu_vm.py"
HOLD="${1:-600}"

echo "[*] vmrestart"
timeout 150 "$HERE/vmrestart.sh" || { echo "vmrestart failed"; exit 1; }

echo "[*] waiting for guest"
for i in $(seq 1 20); do
    if timeout 10 "$VMEXEC" exec "echo UP" 2>/dev/null | grep -q UP; then
        break
    fi
    sleep 5
done

echo "[*] pushing guest tools"
timeout 240 "$HERE/push_guest_tools.sh" >/dev/null 2>&1 || { echo "push failed"; exit 1; }

echo "[*] starting cdt (CDT_HOLD=$HOLD)"
timeout 30 "$VMEXEC" exec \
    "rm -f /tmp/cdt.out /tmp/scanerr.out; CDT_HOLD=$HOLD nohup /tmp/cdt >/tmp/cdt.out 2>&1 & echo STARTED" \
    >/dev/null 2>&1

echo "[*] waiting for 20 GR channels"
# The client times out its context-init wait in ~30 s, so detection must be
# cheap: count the per-channel first-pump log lines straight from dmesg
# instead of round-tripping through the chan_dump parameter every second.
for i in $(seq 1 240); do
    n=$(sudo dmesg -T | grep -c "sema-chan: h=0x5c" || true)
    if [ "$n" -ge 20 ]; then
        echo "    GR channels up (poll $i)"
        break
    fi
    sleep 0.5
done

echo "[*] injecting counters"
timeout 150 "$HERE/inject_counters.sh" 2>/dev/null | grep -c registered

PID=$(timeout 20 "$VMEXEC" exec "pgrep -x cdt" 2>/dev/null | grep -E '^[0-9]+$' | head -1)
echo "[*] cdt pid=$PID; arming scanerr"
timeout 30 "$VMEXEC" exec \
    "nohup gdb -p $PID -batch -x /tmp/scanerr.py >/tmp/scanerr.out 2>&1 & echo ARMED" \
    >/dev/null 2>&1

echo "[*] cycle armed: watch /tmp/cdt.out and /tmp/scanerr.out in the guest"