#!/bin/bash
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2026 Open-IOV contributors
# cycle-r1780.sh - durable vGPU cycle launcher for the r1780 campaign
# (12 GiB file-backed guest RAM, kvmfr/kvmsr ivshmem, V31-class module).
#
# Usage: tools/guest-diag/cycle-r1780.sh [cycle-tag] [module.ko]
# Env:   INSMOD_PARAMS override args file (default params-r1780.txt)
set -euo pipefail

HERE="$(cd "$(dirname "$0")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
TAG="${1:-c27}"
MODULE="${2:-$REPO/kernel/vgsp.ko}"
ARGV_FILE="${INSMOD_PARAMS:-$HERE/params-r1780.txt}"
BACKUP="$REPO/backups/evidence-20261002-victory/r1780"
DISK="${GUEST_DISK:-/home/rmcore/.local/libvf.io/kernel/linux.qcow2}"
MDEV_PARENT="0000:01:00.0"
MDEV_TYPE="${MDEV_TYPE:-}"
RUNDIR="$HERE/run"
QEMU_SCRIPT="$HERE/launch_qemu_r1780.run.sh"
mkdir -p "$RUNDIR" "$BACKUP"

echo "[cycle $TAG] stop QEMU"
# Flush the guest before the forced teardown: a killed GL/X workload has
# corrupted the ext4 root twice (c38 panic, c43 EIO), costing an offline fsck
# cycle each time.
if [ -S /tmp/qemu-guest-agent.sock ]; then
    timeout 8 python3 - <<'PYEOF' >/dev/null 2>&1 || true
import json, socket
s = socket.socket(socket.AF_UNIX)
s.settimeout(6)
s.connect('/tmp/qemu-guest-agent.sock')
for cmd in (["sync"], ["/bin/sh", "-c", "sync; echo 1 > /proc/sys/kernel/sysrq; echo o > /proc/sysrq-trigger"]):
    try:
        s.sendall(json.dumps({"execute": "guest-exec", "arguments": {"path": cmd[0], "arg": cmd[1:], "capture-output": True}}).encode() + b'\n')
        s.recv(65536)
    except Exception:
        break
s.close()
PYEOF
    sleep 2
fi
sudo pkill -f qemu-system-x86_64 2>/dev/null || true
for i in $(seq 1 20); do pgrep -f qemu-system-x86_64 >/dev/null || break; sleep 0.5; done
sudo pkill -9 -f qemu-system-x86_64 2>/dev/null || true
sleep 1

# Always repair the guest root after a forced teardown (cheap when clean,
# prevents the recurring "guest-agent exec EIO / kill init" corruption class).
DISK="/home/rmcore/.local/libvf.io/kernel/linux.qcow2"
if [ -f "$DISK" ]; then
    sudo modprobe nbd max_part=8 2>/dev/null || true
    sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true
    if sudo qemu-nbd --connect=/dev/nbd0 "$DISK" -f qcow2 2>/dev/null; then
        sleep 1
        sudo e2fsck -fy /dev/nbd0p1 > "$RUNDIR/fsck-$TAG.log" 2>&1 || true
        sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true
    fi
fi

echo "[cycle $TAG] remove mdevs"
for m in /sys/bus/mdev/devices/*; do
    [ -d "$m" ] && echo 1 | sudo tee "$m/remove" >/dev/null || true
done

echo "[cycle $TAG] unload vgsp"
sudo rmmod vgsp 2>/dev/null || true
for i in $(seq 1 20); do [ -d /sys/module/vgsp ] || break; sleep 0.5; done
if [ -d /sys/module/vgsp ]; then
    echo "FATAL: vgsp stuck loaded (see /proc/modules)"; exit 1
fi

echo "[cycle $TAG] load VFIO/mdev prerequisites"
for m in vfio vfio_iommu_type1 vfio_pci mdev; do sudo modprobe "$m" 2>/dev/null || true; done

echo "[cycle $TAG] stop nvidia-persistenced, power/control=on"
sudo systemctl stop nvidia-persistenced 2>/dev/null || true
echo on | sudo tee /sys/bus/pci/devices/0000:01:00.0/power/control >/dev/null
echo on | sudo tee /sys/bus/pci/devices/0000:00:01.0/power/control >/dev/null 2>&1 || true

echo "[cycle $TAG] insmod $(basename "$MODULE")"
# shellcheck disable=SC2046
sudo insmod "$MODULE" $(cat "$ARGV_FILE")

for i in $(seq 1 40); do
    [ -d "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types" ] && break
    sleep 0.5
done
if [ -z "$MDEV_TYPE" ]; then
    MDEV_TYPE=$(ls "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types" 2>/dev/null | grep -m1 'vgpu-turing-2g$' || true)
fi
if [ -z "$MDEV_TYPE" ]; then
    MDEV_TYPE=$(ls "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types" 2>/dev/null | grep -m1 'vgpu-turing$' || true)
fi
if [ -z "$MDEV_TYPE" ]; then echo "FATAL: no vgpu-turing mdev type under $MDEV_PARENT"; ls "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types" 2>/dev/null; exit 1; fi
echo "[cycle $TAG] mdev type $MDEV_TYPE"
U=$(uuidgen)
echo "$U" | sudo tee "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types/$MDEV_TYPE/create" >/dev/null
if [ ! -d "/sys/bus/mdev/devices/$U" ]; then echo "FATAL: mdev create failed"; exit 1; fi

mkdir -p "/tmp/sockets/$U"
sudo rm -f /tmp/qemu-qmp.sock /tmp/qemu-guest-agent.sock /tmp/qemu-harness.pid
rm -f /dev/shm/vgpu-guest-ram 2>/dev/null || true
touch /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true
chmod 660 /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true

QLOG="$BACKUP/${TAG}-qemu-session.txt"
SL="$BACKUP/${TAG}-serial.log"
cat > "$QEMU_SCRIPT" <<EOF
#!/bin/bash
exec /bin/qemu-system-x86_64 -D $QLOG -no-hpet \
  -qmp unix:/tmp/qemu-qmp.sock,server,nowait \
  -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 \
  -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \
  -serial file:$SL -gdb tcp::1234 -display none \
  -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr \
  -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-$U,size=128M,share=yes \
  -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr \
  -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-$U,size=2M,share=yes \
  -object memory-backend-file,id=pc.ram,size=12288M,mem-path=/dev/shm/vgpu-guest-ram,share=yes,prealloc=on \
  -uuid $U -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram \
  -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on \
  -rtc clock=host,base=localtime -m 12288 -smp cores=4,threads=1,sockets=1 \
  -hda $DISK --enable-kvm \
  -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 \
  -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/$U,display=off \
  -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 \
  -net user -net nic -device vhost-vsock-pci,guest-cid=85770 \
  -daemonize -pidfile /tmp/qemu-harness.pid
EOF
chmod +x "$QEMU_SCRIPT"

echo "[cycle $TAG] launch QEMU (mdev $U)"
sudo bash "$QEMU_SCRIPT" > "$RUNDIR/qemu-$TAG.log" 2>&1
sleep 3
if ! pgrep -f qemu-system-x86_64 >/dev/null; then
    echo "FATAL: QEMU failed"; tail -5 "$RUNDIR/qemu-$TAG.log"; exit 1
fi

echo "[cycle $TAG] mdev $U"
echo "$U" > "$RUNDIR/current-mdev"
echo "$TAG" > "$RUNDIR/current-tag"

for i in $(seq 1 90); do
    if timeout 5 sudo python3 "$REPO/scripts/vgpu_vm.py" exec "echo CYCLE_UP" 2>/dev/null | grep -q CYCLE_UP; then
        echo "[cycle $TAG] guest agent up"
        # 2026-11-13 (eng9, c198-c203 RCA): arm the GR forward lane. Without
        # this block the pump's GR request gate refuses EVERY captured launch
        # with "backend not ready" (gsp_phys_backend_gr_ready() is false until
        # the rm-call ladder latches boot_stage.rmcall_gr_exec_ok) and the
        # backend submit then refuses with -EAGAIN until the guest RAM
        # backing is registered: captured GL sections never become
        # `GR LAUNCH submitted`, gl_frames reads black (px0=0), work_gr shows
        # refused==fallback==req with submitted==0. Trigger 4 alone on a
        # fresh session is EXPECTED to fail at stage 1 (NV01_ROOT 0x23):
        # trigger 5 adopts the static handles first (docs/eng-ops.md Step 4).
        LADDER_LOG="$BACKUP/$TAG-ladder.txt"
        : > "$LADDER_LOG"
        echo "[cycle $TAG] arm GR lane (rm-call ladder 5->3->4 + guest-ram backing)"
        for t in 5 3 4; do
            sudo bash -c "echo $t > /sys/kernel/debug/vgsp/gsp-phys-rm-call" 2>>"$LADDER_LOG" || true
            sleep 4
        done
        sudo dmesg -T 2>/dev/null | grep -E "rm-call: (stage|ladder|REAL GR)|ladder at stage" | tail -10 >> "$LADDER_LOG" || true
        sudo bash -c "echo '$U /dev/shm/vgpu-guest-ram 0x80000000 0x0 0x300000000' > /sys/kernel/debug/vgsp/gsp-phys-guest-ram" 2>>"$LADDER_LOG" || true
        sleep 1
        sudo cat /sys/kernel/debug/vgsp/gsp-phys-state 2>/dev/null | grep -E "work_gr|guest_ram:" | cut -c1-220 >> "$LADDER_LOG" || true
        if grep -q "ladder at stage 4" "$LADDER_LOG"; then
            echo "[cycle $TAG] GR lane armed (ladder stage 4; guest-ram configured; log: $LADDER_LOG)"
        else
            echo "[cycle $TAG] WARN: GR lane arm incomplete (see $LADDER_LOG); cycle still usable"
        fi
        exit 0
    fi
    sleep 2
done
echo "WARN: guest agent not reachable yet (serial: $SL)"
exit 0