vgsp: direct-GL ring/push rescue, VAS/pdb scanners and PTIMER tick; r1780 cycle tooling

1 files changed, 142 insertions, 0 deletions

diff --git a/tools/guest-diag/cycle-r1780.sh b/tools/guest-diag/cycle-r1780.sh
new file mode 100755
index 0000000..23cf0d1
--- /dev/null
+++ b/tools/guest-diag/cycle-r1780.sh
@@ -0,0 +1,142 @@
+#!/bin/bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2026 Open-IOV contributors
+# cycle-r1780.sh - durable vGPU cycle launcher for the r1780 campaign
+# (12 GiB file-backed guest RAM, kvmfr/kvmsr ivshmem, V31-class module).
+#
+# Usage: tools/guest-diag/cycle-r1780.sh [cycle-tag] [module.ko]
+# Env:   INSOMOD_PARAMS override args file (default params-r1780.txt)
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+REPO="$(cd "$HERE/../.." && pwd)"
+TAG="${1:-c27}"
+MODULE="${2:-$REPO/kernel/vgsp.ko}"
+ARGV_FILE="${INSMOD_PARAMS:-$HERE/params-r1780.txt}"
+BACKUP="$REPO/backups/evidence-20261002-victory/r1780"
+DISK="${GUEST_DISK:-/home/rmcore/.local/libvf.io/kernel/linux.qcow2}"
+MDEV_PARENT="0000:01:00.0"
+MDEV_TYPE="nvidia-nvidia-vgpu-turing"
+RUNDIR="$HERE/run"
+QEMU_SCRIPT="$HERE/launch_qemu_r1780.run.sh"
+mkdir -p "$RUNDIR" "$BACKUP"
+
+echo "[cycle $TAG] stop QEMU"
+# Flush the guest before the forced teardown: a killed GL/X workload has
+# corrupted the ext4 root twice (c38 panic, c43 EIO), costing an offline fsck
+# cycle each time.
+if [ -S /tmp/qemu-guest-agent.sock ]; then
+    timeout 8 python3 - <<'PYEOF' >/dev/null 2>&1 || true
+import json, socket
+s = socket.socket(socket.AF_UNIX)
+s.settimeout(6)
+s.connect('/tmp/qemu-guest-agent.sock')
+for cmd in (["sync"], ["/bin/sh", "-c", "sync; echo 1 > /proc/sys/kernel/sysrq; echo o > /proc/sysrq-trigger"]):
+    try:
+        s.sendall(json.dumps({"execute": "guest-exec", "arguments": {"path": cmd[0], "arg": cmd[1:], "capture-output": True}}).encode() + b'\n')
+        s.recv(65536)
+    except Exception:
+        break
+s.close()
+PYEOF
+    sleep 2
+fi
+sudo pkill -f qemu-system-x86_64 2>/dev/null || true
+for i in $(seq 1 20); do pgrep -f qemu-system-x86_64 >/dev/null || break; sleep 0.5; done
+sudo pkill -9 -f qemu-system-x86_64 2>/dev/null || true
+sleep 1
+
+# Always repair the guest root after a forced teardown (cheap when clean,
+# prevents the recurring "guest-agent exec EIO / kill init" corruption class).
+DISK="/home/rmcore/.local/libvf.io/kernel/linux.qcow2"
+if [ -f "$DISK" ]; then
+    sudo modprobe nbd max_part=8 2>/dev/null || true
+    sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true
+    if sudo qemu-nbd --connect=/dev/nbd0 "$DISK" -f qcow2 2>/dev/null; then
+        sleep 1
+        sudo e2fsck -fy /dev/nbd0p1 > "$RUNDIR/fsck-$TAG.log" 2>&1 || true
+        sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true
+    fi
+fi
+
+echo "[cycle $TAG] remove mdevs"
+for m in /sys/bus/mdev/devices/*; do
+    [ -d "$m" ] && echo 1 | sudo tee "$m/remove" >/dev/null || true
+done
+
+echo "[cycle $TAG] unload vgsp"
+sudo rmmod vgsp 2>/dev/null || true
+for i in $(seq 1 20); do [ -d /sys/module/vgsp ] || break; sleep 0.5; done
+if [ -d /sys/module/vgsp ]; then
+    echo "FATAL: vgsp stuck loaded (see /proc/modules)"; exit 1
+fi
+
+echo "[cycle $TAG] load VFIO/mdev prerequisites"
+sudo modprobe vfio vfio_iommu_type1 vfio_pci mdev 2>/dev/null || true
+sudo modprobe vfio_iommu_type1 2>/dev/null || true
+
+echo "[cycle $TAG] stop nvidia-persistenced, power/control=on"
+sudo systemctl stop nvidia-persistenced 2>/dev/null || true
+echo on | sudo tee /sys/bus/pci/devices/0000:01:00.0/power/control >/dev/null
+echo on | sudo tee /sys/bus/pci/devices/0000:00:01.0/power/control >/dev/null 2>&1 || true
+
+echo "[cycle $TAG] insmod $(basename "$MODULE")"
+# shellcheck disable=SC2046
+sudo insmod "$MODULE" $(cat "$ARGV_FILE")
+
+U=$(uuidgen)
+echo "$U" | sudo tee "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types/$MDEV_TYPE/create" >/dev/null
+if [ ! -d "/sys/bus/mdev/devices/$U" ]; then echo "FATAL: mdev create failed"; exit 1; fi
+
+mkdir -p "/tmp/sockets/$U"
+sudo rm -f /tmp/qemu-qmp.sock /tmp/qemu-guest-agent.sock /tmp/qemu-harness.pid
+rm -f /dev/shm/vgpu-guest-ram 2>/dev/null || true
+touch /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true
+chmod 660 /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true
+
+QLOG="$BACKUP/${TAG}-qemu-session.txt"
+SL="$BACKUP/${TAG}-serial.log"
+cat > "$QEMU_SCRIPT" <<EOF
+#!/bin/bash
+exec /bin/qemu-system-x86_64 -D $QLOG -no-hpet \
+  -qmp unix:/tmp/qemu-qmp.sock,server,nowait \
+  -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 \
+  -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \
+  -serial file:$SL -gdb tcp::1234 -display none \
+  -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr \
+  -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-$U,size=128M,share=yes \
+  -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr \
+  -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-$U,size=2M,share=yes \
+  -object memory-backend-file,id=pc.ram,size=12288M,mem-path=/dev/shm/vgpu-guest-ram,share=yes,prealloc=on \
+  -uuid $U -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram \
+  -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on \
+  -rtc clock=host,base=localtime -m 12288 -smp cores=4,threads=1,sockets=1 \
+  -hda $DISK --enable-kvm \
+  -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 \
+  -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/$U,display=off \
+  -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 \
+  -net user -net nic -device vhost-vsock-pci,guest-cid=85770 \
+  -daemonize -pidfile /tmp/qemu-harness.pid
+EOF
+chmod +x "$QEMU_SCRIPT"
+
+echo "[cycle $TAG] launch QEMU (mdev $U)"
+sudo bash "$QEMU_SCRIPT" > "$RUNDIR/qemu-$TAG.log" 2>&1
+sleep 3
+if ! pgrep -f qemu-system-x86_64 >/dev/null; then
+    echo "FATAL: QEMU failed"; tail -5 "$RUNDIR/qemu-$TAG.log"; exit 1
+fi
+
+echo "[cycle $TAG] mdev $U"
+echo "$U" > "$RUNDIR/current-mdev"
+echo "$TAG" > "$RUNDIR/current-tag"
+
+for i in $(seq 1 90); do
+    if timeout 5 sudo python3 "$REPO/scripts/vgpu_vm.py" exec "echo CYCLE_UP" 2>/dev/null | grep -q CYCLE_UP; then
+        echo "[cycle $TAG] guest agent up"
+        exit 0
+    fi
+    sleep 2
+done
+echo "WARN: guest agent not reachable yet (serial: $SL)"
+exit 0