1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
|
#!/bin/bash
# SPDX-License-Identifier: BSD-3-Clause
# Copyright (c) 2026 Open-IOV contributors
# cycle-r1780.sh - durable vGPU cycle launcher for the r1780 campaign
# (12 GiB file-backed guest RAM, kvmfr/kvmsr ivshmem, V31-class module).
#
# Usage: tools/guest-diag/cycle-r1780.sh [cycle-tag] [module.ko]
# Env: INSMOD_PARAMS override args file (default params-r1780.txt)
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
REPO="$(cd "$HERE/../.." && pwd)"
TAG="${1:-c27}"
MODULE="${2:-$REPO/kernel/vgsp.ko}"
ARGV_FILE="${INSMOD_PARAMS:-$HERE/params-r1780.txt}"
BACKUP="$REPO/backups/evidence-20261002-victory/r1780"
DISK="${GUEST_DISK:-/home/rmcore/.local/libvf.io/kernel/linux.qcow2}"
MDEV_PARENT="0000:01:00.0"
MDEV_TYPE="${MDEV_TYPE:-}"
RUNDIR="$HERE/run"
QEMU_SCRIPT="$HERE/launch_qemu_r1780.run.sh"
mkdir -p "$RUNDIR" "$BACKUP"
echo "[cycle $TAG] stop QEMU"
# Flush the guest before the forced teardown: a killed GL/X workload has
# corrupted the ext4 root twice (c38 panic, c43 EIO), costing an offline fsck
# cycle each time.
if [ -S /tmp/qemu-guest-agent.sock ]; then
timeout 8 python3 - <<'PYEOF' >/dev/null 2>&1 || true
import json, socket
s = socket.socket(socket.AF_UNIX)
s.settimeout(6)
s.connect('/tmp/qemu-guest-agent.sock')
for cmd in (["sync"], ["/bin/sh", "-c", "sync; echo 1 > /proc/sys/kernel/sysrq; echo o > /proc/sysrq-trigger"]):
try:
s.sendall(json.dumps({"execute": "guest-exec", "arguments": {"path": cmd[0], "arg": cmd[1:], "capture-output": True}}).encode() + b'\n')
s.recv(65536)
except Exception:
break
s.close()
PYEOF
sleep 2
fi
sudo pkill -f qemu-system-x86_64 2>/dev/null || true
for i in $(seq 1 20); do pgrep -f qemu-system-x86_64 >/dev/null || break; sleep 0.5; done
sudo pkill -9 -f qemu-system-x86_64 2>/dev/null || true
sleep 1
# Always repair the guest root after a forced teardown (cheap when clean,
# prevents the recurring "guest-agent exec EIO / kill init" corruption class).
DISK="/home/rmcore/.local/libvf.io/kernel/linux.qcow2"
if [ -f "$DISK" ]; then
sudo modprobe nbd max_part=8 2>/dev/null || true
sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true
if sudo qemu-nbd --connect=/dev/nbd0 "$DISK" -f qcow2 2>/dev/null; then
sleep 1
sudo e2fsck -fy /dev/nbd0p1 > "$RUNDIR/fsck-$TAG.log" 2>&1 || true
sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true
fi
fi
echo "[cycle $TAG] remove mdevs"
for m in /sys/bus/mdev/devices/*; do
[ -d "$m" ] && echo 1 | sudo tee "$m/remove" >/dev/null || true
done
echo "[cycle $TAG] unload vgsp"
sudo rmmod vgsp 2>/dev/null || true
for i in $(seq 1 20); do [ -d /sys/module/vgsp ] || break; sleep 0.5; done
if [ -d /sys/module/vgsp ]; then
echo "FATAL: vgsp stuck loaded (see /proc/modules)"; exit 1
fi
echo "[cycle $TAG] load VFIO/mdev prerequisites"
for m in vfio vfio_iommu_type1 vfio_pci mdev; do sudo modprobe "$m" 2>/dev/null || true; done
echo "[cycle $TAG] stop nvidia-persistenced, power/control=on"
sudo systemctl stop nvidia-persistenced 2>/dev/null || true
echo on | sudo tee /sys/bus/pci/devices/0000:01:00.0/power/control >/dev/null
echo on | sudo tee /sys/bus/pci/devices/0000:00:01.0/power/control >/dev/null 2>&1 || true
echo "[cycle $TAG] insmod $(basename "$MODULE")"
# shellcheck disable=SC2046
sudo insmod "$MODULE" $(cat "$ARGV_FILE")
for i in $(seq 1 40); do
[ -d "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types" ] && break
sleep 0.5
done
if [ -z "$MDEV_TYPE" ]; then
MDEV_TYPE=$(ls "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types" 2>/dev/null | grep -m1 'vgpu-turing-2g$' || true)
fi
if [ -z "$MDEV_TYPE" ]; then
MDEV_TYPE=$(ls "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types" 2>/dev/null | grep -m1 'vgpu-turing$' || true)
fi
if [ -z "$MDEV_TYPE" ]; then echo "FATAL: no vgpu-turing mdev type under $MDEV_PARENT"; ls "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types" 2>/dev/null; exit 1; fi
echo "[cycle $TAG] mdev type $MDEV_TYPE"
U=$(uuidgen)
echo "$U" | sudo tee "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types/$MDEV_TYPE/create" >/dev/null
if [ ! -d "/sys/bus/mdev/devices/$U" ]; then echo "FATAL: mdev create failed"; exit 1; fi
mkdir -p "/tmp/sockets/$U"
sudo rm -f /tmp/qemu-qmp.sock /tmp/qemu-guest-agent.sock /tmp/qemu-harness.pid
rm -f /dev/shm/vgpu-guest-ram 2>/dev/null || true
touch /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true
chmod 660 /dev/shm/kvmfr-$U /dev/shm/kvmsr-$U 2>/dev/null || true
QLOG="$BACKUP/${TAG}-qemu-session.txt"
SL="$BACKUP/${TAG}-serial.log"
cat > "$QEMU_SCRIPT" <<EOF
#!/bin/bash
exec /bin/qemu-system-x86_64 -D $QLOG -no-hpet \
-qmp unix:/tmp/qemu-qmp.sock,server,nowait \
-chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 \
-device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \
-serial file:$SL -gdb tcp::1234 -display none \
-device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr \
-object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-$U,size=128M,share=yes \
-device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr \
-object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-$U,size=2M,share=yes \
-object memory-backend-file,id=pc.ram,size=12288M,mem-path=/dev/shm/vgpu-guest-ram,share=yes,prealloc=on \
-uuid $U -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off,memory-backend=pc.ram \
-cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on \
-rtc clock=host,base=localtime -m 12288 -smp cores=4,threads=1,sockets=1 \
-hda $DISK --enable-kvm \
-device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 \
-device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/$U,display=off \
-device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 \
-net user -net nic -device vhost-vsock-pci,guest-cid=85770 \
-daemonize -pidfile /tmp/qemu-harness.pid
EOF
chmod +x "$QEMU_SCRIPT"
echo "[cycle $TAG] launch QEMU (mdev $U)"
sudo bash "$QEMU_SCRIPT" > "$RUNDIR/qemu-$TAG.log" 2>&1
sleep 3
if ! pgrep -f qemu-system-x86_64 >/dev/null; then
echo "FATAL: QEMU failed"; tail -5 "$RUNDIR/qemu-$TAG.log"; exit 1
fi
echo "[cycle $TAG] mdev $U"
echo "$U" > "$RUNDIR/current-mdev"
echo "$TAG" > "$RUNDIR/current-tag"
for i in $(seq 1 90); do
if timeout 5 sudo python3 "$REPO/scripts/vgpu_vm.py" exec "echo CYCLE_UP" 2>/dev/null | grep -q CYCLE_UP; then
echo "[cycle $TAG] guest agent up"
# 2026-11-13 (eng9, c198-c203 RCA): arm the GR forward lane. Without
# this block the pump's GR request gate refuses EVERY captured launch
# with "backend not ready" (gsp_phys_backend_gr_ready() is false until
# the rm-call ladder latches boot_stage.rmcall_gr_exec_ok) and the
# backend submit then refuses with -EAGAIN until the guest RAM
# backing is registered: captured GL sections never become
# `GR LAUNCH submitted`, gl_frames reads black (px0=0), work_gr shows
# refused==fallback==req with submitted==0. Trigger 4 alone on a
# fresh session is EXPECTED to fail at stage 1 (NV01_ROOT 0x23):
# trigger 5 adopts the static handles first (docs/eng-ops.md Step 4).
LADDER_LOG="$BACKUP/$TAG-ladder.txt"
: > "$LADDER_LOG"
echo "[cycle $TAG] arm GR lane (rm-call ladder 5->3->4 + guest-ram backing)"
for t in 5 3 4; do
sudo bash -c "echo $t > /sys/kernel/debug/vgsp/gsp-phys-rm-call" 2>>"$LADDER_LOG" || true
sleep 4
done
sudo dmesg -T 2>/dev/null | grep -E "rm-call: (stage|ladder|REAL GR)|ladder at stage" | tail -10 >> "$LADDER_LOG" || true
sudo bash -c "echo '$U /dev/shm/vgpu-guest-ram 0x80000000 0x0 0x300000000' > /sys/kernel/debug/vgsp/gsp-phys-guest-ram" 2>>"$LADDER_LOG" || true
sleep 1
sudo cat /sys/kernel/debug/vgsp/gsp-phys-state 2>/dev/null | grep -E "work_gr|guest_ram:" | cut -c1-220 >> "$LADDER_LOG" || true
if grep -q "ladder at stage 4" "$LADDER_LOG"; then
echo "[cycle $TAG] GR lane armed (ladder stage 4; guest-ram configured; log: $LADDER_LOG)"
else
echo "[cycle $TAG] WARN: GR lane arm incomplete (see $LADDER_LOG); cycle still usable"
fi
exit 0
fi
sleep 2
done
echo "WARN: guest agent not reachable yet (serial: $SL)"
exit 0
|
