tools: keep all guest-diag artifacts in-tree (push helper, runtime launch script, README)

8 files changed, 256 insertions, 7 deletions

diff --git a/tools/guest-diag/README.md b/tools/guest-diag/README.md
index 5c32788..618663b 100644
--- a/tools/guest-diag/README.md
+++ b/tools/guest-diag/README.md
@@ -65,3 +65,39 @@ Tries to kretprobe the nvGpuOps*/nvUvmInterface* chain to read return values.
 Blocked on this kernel: "trace_kprobe: Could not probe notrace function
 <name> [nvidia]" - kprobes are not available for nvidia module text.
 Kept for documentation.
+
+## push_guest_tools.sh - rebuild the guest-side binaries from these sources
+
+The guest's /tmp is wiped on boot; this pushes+compiles ptio/ge/rmqp/cdt into
+the guest from the sources kept here. Run it after every VM start.
+
+## guest_engines.c - client RM introspection (used with the ptrace tracer)
+
+Queries the guest client RM for exactly the data UVM's register path uses:
+virtualization mode (0x800289), subdevice count (0x800280), CLASS LIST V2
+(0x800292), ENGINE LIST V2 (0x20800170), CE caps (0x20802a03) and CE->PCE
+masks (0x20802a02). Needs the NVOS64 alloc path; see rmq_probe.c note.
+
+## Guest-side UVM logging (critical for diagnosis!)
+
+UVM's messages are silent until enabled:
+
+    echo 1 | sudo tee /sys/module/nvidia_uvm/parameters/uvm_debug_prints
+
+Then run /tmp/cdt and `dmesg | grep nvidia-uvm`. This produced the current
+blocker's exact message:
+`nvidia-uvm: uvm_gpu.c:1574 init_gpu[pid:..] Failed to initialize the
+channel manager: Call not supported [NV_ERR_NOT_SUPPORTED]`
+(also useful: uvm_enable_debug_procfs, uvm_debug_enable_push_desc,
+uvm_release_asserts, uvm_release_asserts_dump_stack)
+
+## launch_qemu_rootport.sh / .run.sh - QEMU command line
+
+Template (root-port fix) + runtime copy with the live mdev UUID, patched by
+vmcycle.sh. Sockets still live under /tmp because scripts/vgpu_vm.py talks to
+/tmp/qemu-guest-agent.sock by design.
+
+## dbg_seq.sh / dbg_seq2.sh - control-sequence helpers (guest, needs ptio)
+
+Print the RM control sequence immediately before/after the failing
+UVM_REGISTER_GPU ioctl.
diff --git a/tools/guest-diag/dbg_seq.sh b/tools/guest-diag/dbg_seq.sh
new file mode 100755
index 0000000..247bc91
--- /dev/null
+++ b/tools/guest-diag/dbg_seq.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+/tmp/ptio /tmp/cdt > /dev/null 2>/tmp/tr.log
+n=$(grep -n 'req=0x25' /tmp/tr.log | head -1 | cut -d: -f1)
+echo "marker line: $n"
+sed -n "$((n-28)),$((n+2))p" /tmp/tr.log
diff --git a/tools/guest-diag/dbg_seq2.sh b/tools/guest-diag/dbg_seq2.sh
new file mode 100755
index 0000000..8710fd9
--- /dev/null
+++ b/tools/guest-diag/dbg_seq2.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+/tmp/ptio /tmp/cdt > /dev/null 2>/tmp/tr.log
+n=$(grep -n 'req=0x25' /tmp/tr.log | head -1 | cut -d: -f1)
+echo "=== controls AFTER register (line $n), first 40: ==="
+sed -n "$((n+1)),$((n+45))p" /tmp/tr.log
diff --git a/tools/guest-diag/guest_engines.c b/tools/guest-diag/guest_engines.c
new file mode 100644
index 0000000..066eec6
--- /dev/null
+++ b/tools/guest-diag/guest_engines.c
@@ -0,0 +1,160 @@
+/* guest_engines.c - query the guest client RM for the data UVM's register
+ * path depends on: virtualization mode, subdevice count, ENGINE LIST,
+ * CLASS LIST, CE caps and CE->PCE masks. Uses the NVOS64 serialized alloc
+ * path (same as tools/rmq/rmq.c) so device allocs succeed.
+ *
+ * Build in guest: gcc -O1 -o /tmp/ge guest_engines.c ; run /tmp/ge
+ */
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+typedef uint32_t NvU32;
+typedef uint32_t NvHandle;
+typedef uint64_t NvP64;
+
+#define NV_IOCTL_MAGIC      'F'
+#define NV_ESC_RM_ALLOC     0x2B
+#define NV_ESC_RM_CONTROL   0x2A
+#define NV01_ROOT           0x0
+#define NV01_DEVICE_0       0x80
+#define NV20_SUBDEVICE_0    0x2080
+
+typedef struct {
+	NvHandle hRoot, hObjectParent, hObjectNew;
+	NvU32 hClass;
+	NvP64 pAllocParms;
+	NvU32 paramsSize, status;
+} NVOS21_PARAMETERS;
+
+typedef struct {
+	NvHandle hRoot, hObjectParent, hObjectNew;
+	NvU32 hClass;
+	NvP64 pAllocParms;
+	NvP64 pRightsRequested;
+	NvU32 paramsSize, flags, status;
+} NVOS64_PARAMETERS;
+
+typedef struct {
+	NvHandle hClient, hObject;
+	NvU32 cmd, flags;
+	NvP64 params;
+	NvU32 paramsSize, status;
+} NVOS54_PARAMETERS;
+
+static int fd;
+
+static NvU32 rm_alloc(NvHandle hRoot, NvHandle hParent, NvU32 hClass,
+		      NvHandle *pOut)
+{
+	NVOS21_PARAMETERS p;
+	memset(&p, 0, sizeof(p));
+	p.hRoot = hRoot; p.hObjectParent = hParent; p.hClass = hClass;
+	if (ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS21_PARAMETERS), &p) < 0) {
+		perror("alloc ioctl"); return 0xffffffff;
+	}
+	*pOut = p.hObjectNew;
+	return p.status;
+}
+
+static NvU32 rm_alloc64(NvHandle hRoot, NvHandle hParent, NvU32 hClass,
+			NvHandle *pOut)
+{
+	NVOS64_PARAMETERS p;
+	memset(&p, 0, sizeof(p));
+	p.hRoot = hRoot; p.hObjectParent = hParent; p.hClass = hClass;
+	p.flags = 1; /* FINN_SERIALIZED */
+	if (ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS64_PARAMETERS), &p) < 0) {
+		perror("alloc64 ioctl"); return 0xffffffff;
+	}
+	*pOut = p.hObjectNew;
+	return p.status;
+}
+
+static NvU32 rm_control(NvHandle hClient, NvHandle hObject, NvU32 cmd,
+			void *params, NvU32 size)
+{
+	NVOS54_PARAMETERS p;
+	memset(&p, 0, sizeof(p));
+	p.hClient = hClient; p.hObject = hObject; p.cmd = cmd;
+	p.params = (NvP64)params; p.paramsSize = size;
+	if (ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS), &p) < 0) {
+		perror("control ioctl"); return 0xffffffff;
+	}
+	return p.status;
+}
+
+#define MAX_CLASSES 256
+
+int main(void)
+{
+	NvHandle hClient = 0, hDevice = 0, hSubdevice = 0;
+	NvU32 st;
+
+	fd = open("/dev/nvidiactl", O_RDWR | O_CLOEXEC);
+	if (fd < 0) { perror("open nvidiactl"); return 1; }
+
+	st = rm_alloc(0, 0, NV01_ROOT, &hClient);
+	printf("alloc client: st=0x%x hClient=0x%x\n", st, hClient);
+
+	st = rm_alloc64(hClient, hClient, NV01_DEVICE_0, &hDevice);
+	printf("alloc device: st=0x%x hDevice=0x%x\n", st, hDevice);
+	if (st) return 2;
+
+	st = rm_alloc64(hClient, hDevice, NV20_SUBDEVICE_0, &hSubdevice);
+	printf("alloc subdevice: st=0x%x hSub=0x%x\n", st, hSubdevice);
+
+	{
+		struct { NvU32 mode; NvU32 isGridBuild; } v = {0xdead, 0xdead};
+		st = rm_control(hClient, hDevice, 0x800289, &v, sizeof(v));
+		printf("GET_VIRTUALIZATION_MODE: st=0x%x mode=%u (0=NONE,1=NMOS,2=VGX) gridBuild=%u\n",
+		       st, v.mode, v.isGridBuild);
+	}
+	{
+		struct { NvU32 n; } s = {0xdead};
+		st = rm_control(hClient, hDevice, 0x800280, &s, sizeof(s));
+		printf("GET_NUM_SUBDEVICES: st=0x%x n=%u\n", st, s.n);
+	}
+	{
+		struct { NvU32 numClasses; NvU32 classList[MAX_CLASSES]; } c;
+		int i;
+		memset(&c, 0, sizeof(c));
+		st = rm_control(hClient, hDevice, 0x800292, &c, sizeof(c));
+		printf("GET_CLASSLIST_V2: st=0x%x numClasses=%u\n", st, c.numClasses);
+		if (!st && c.numClasses) {
+			printf("  classes:");
+			for (i = 0; i < (int)c.numClasses && i < MAX_CLASSES; i++)
+				printf(" 0x%x", c.classList[i]);
+			printf("\n");
+		}
+	}
+	if (hSubdevice) {
+		struct { NvU32 engineCount; NvU32 engineList[0x54]; } e;
+		int i;
+		memset(&e, 0, sizeof(e));
+		st = rm_control(hClient, hSubdevice, 0x20800170, &e, sizeof(e));
+		printf("GPU_GET_ENGINES_V2: st=0x%x engineCount=%u\n", st, e.engineCount);
+		if (!st && e.engineCount) {
+			printf("  engines:");
+			for (i = 0; i < (int)e.engineCount && i < 0x54; i++)
+				printf(" 0x%x", e.engineList[i]);
+			printf("\n");
+		}
+		for (i = 0; i < (int)e.engineCount && i < 0x54; i++) {
+			struct { NvU32 t, mask; } m;
+			struct { NvU32 t; unsigned char caps[2]; } cc;
+			NvU32 t = e.engineList[i];
+			memset(&m, 0, sizeof(m));
+			st = rm_control(hClient, hSubdevice, 0x20802a02, &m, sizeof(m));
+			printf("  CE_GET_CE_PCE_MASK(type=0x%x): st=0x%x mask=0x%x\n", t, st, m.mask);
+			memset(&cc, 0, sizeof(cc));
+			cc.t = t;
+			st = rm_control(hClient, hSubdevice, 0x20802a03, &cc, sizeof(cc));
+			printf("  CE_GET_CAPS_V2(type=0x%x): st=0x%x caps=%02x %02x\n", t, st, cc.caps[0], cc.caps[1]);
+		}
+	}
+	return 0;
+}
diff --git a/tools/guest-diag/kret.sh b/tools/guest-diag/kret.sh
index 9ed1220..9ed1220 100644..100755
--- a/tools/guest-diag/kret.sh
+++ b/tools/guest-diag/kret.sh
diff --git a/tools/guest-diag/launch_qemu_rootport.run.sh b/tools/guest-diag/launch_qemu_rootport.run.sh
new file mode 100755
index 0000000..69663f7
--- /dev/null
+++ b/tools/guest-diag/launch_qemu_rootport.run.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+exec /bin/qemu-system-x86_64 -D /home/rmcore/.local/libvf.io/logs/qemu/e726ba0b-204f-4ec9-91da-983ed7f41d06-session.txt -no-hpet -mem-prealloc -qmp unix:/tmp/qemu-qmp.sock,server,nowait -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 -serial file:/tmp/vgpu-guest-serial.log -gdb tcp::1234 -display none -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=128M,share=yes -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=2M,share=yes -uuid e726ba0b-204f-4ec9-91da-983ed7f41d06 -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on -rtc clock=host,base=localtime -m 4096 -smp cores=4,threads=1,sockets=1 -hda /home/rmcore/.local/libvf.io/kernel/linux.qcow2 --enable-kvm -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/b2e19ccf-87bb-4761-b634-9a8e8d218c89,display=off -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 -net user -net nic -device vhost-vsock-pci,guest-cid=50718 -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/main.sock,server,nowait -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/master.sock,server,nowait
diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh
new file mode 100755
index 0000000..a2a3dbb
--- /dev/null
+++ b/tools/guest-diag/push_guest_tools.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+# push_guest_tools.sh - (re)build the guest-side diagnostic binaries from the
+# sources kept in this directory. The guest's /tmp is wiped on boot, so run
+# this after every VM start; the sources here are the persistent copies.
+#
+# Usage: tools/guest-diag/push_guest_tools.sh
+#        (env: VM="vgpu-linux" is not needed; scripts/vgpu_vm.py talks to the
+#         running VM's guest agent socket)
+set -e
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+VMEXEC="$HERE/../../scripts/vgpu_vm.py"
+REPO="$HERE/../.."
+
+push_b64() {   # push_b64 <local file> <remote file>
+    local b64
+    b64=$(base64 -w0 "$1")
+    "$VMEXEC" exec "echo '$b64' | base64 -d > '$2'" >/dev/null 2>&1
+}
+
+echo "[*] pushing sources..."
+push_b64 "$HERE/ptrace_ioctl.c"  /tmp/ptio.c
+push_b64 "$HERE/guest_engines.c" /tmp/ge.c
+push_b64 "$HERE/rmq_probe.c"     /tmp/rmqp.c
+push_b64 "$REPO/tools/test/cuda_drv_test.c" /tmp/cdt.c
+
+echo "[*] compiling in guest..."
+"$VMEXEC" exec "gcc -O1 -w -o /tmp/ptio /tmp/ptio.c && \
+gcc -O1 -w -o /tmp/ge /tmp/ge.c && \
+gcc -O1 -w -o /tmp/rmqp /tmp/rmqp.c && \
+gcc -O1 -o /tmp/cdt /tmp/cdt.c -ldl && echo BUILD_OK"
diff --git a/tools/guest-diag/vmcycle.sh b/tools/guest-diag/vmcycle.sh
index 08eb4ea..6faaa75 100755
--- a/tools/guest-diag/vmcycle.sh
+++ b/tools/guest-diag/vmcycle.sh
@@ -1,19 +1,29 @@
 #!/bin/bash
 # vmcycle.sh - reload cycle for the emulated vGPU stack:
 #   stop QEMU -> remove mdevs -> rmmod/insmod open-iov-vfio.ko -> create a
-#   fresh mdev -> patch its UUID into the QEMU launch script -> relaunch.
+#   fresh mdev -> patch its UUID into the runtime launch script -> relaunch.
+#
+# ALL artifacts stay inside this directory (tools/guest-diag/) so nothing is
+# lost on reboot:
+#   launch_qemu_rootport.sh     - template QEMU command line (root-port fix)
+#   launch_qemu_rootport.run.sh - runtime copy with the current mdev UUID
+#   run/qemu.log                - QEMU stderr/stdout of the last launch
 #
 # Usage:  tools/guest-diag/vmcycle.sh
-# Env:    QEMU_SCRIPT (default: ./launch_qemu_rootport.sh next to this script)
-#         MDEV_PARENT (default: 0000:01:00.0)
+# Env:    MDEV_PARENT (default: 0000:01:00.0)
 #         MDEV_TYPE   (default: nvidia-nvidia-vgpu-turing)
 set -e
 
 HERE="$(cd "$(dirname "$0")" && pwd)"
-QEMU_SCRIPT="${QEMU_SCRIPT:-$HERE/launch_qemu_rootport.sh}"
+TEMPLATE="$HERE/launch_qemu_rootport.sh"
+QEMU_SCRIPT="${QEMU_SCRIPT:-$HERE/launch_qemu_rootport.run.sh}"
 MODULE="${MODULE:-$HERE/../../kernel/open-iov-vfio.ko}"
 MDEV_PARENT="${MDEV_PARENT:-0000:01:00.0}"
 MDEV_TYPE="${MDEV_TYPE:-nvidia-nvidia-vgpu-turing}"
+RUNDIR="$HERE/run"
+
+mkdir -p "$RUNDIR"
+[ -f "$QEMU_SCRIPT" ] || cp "$TEMPLATE" "$QEMU_SCRIPT"
 
 pkill -f qemu-system-x86_64 2>/dev/null || true
 sleep 4
@@ -27,10 +37,10 @@ echo "$U" | sudo tee "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types/$MDE
 sed -i "s|/sys/bus/mdev/devices/[0-9a-f-]*|/sys/bus/mdev/devices/$U|" "$QEMU_SCRIPT"
 mkdir -p /tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06
 rm -f /tmp/qemu-guest-agent.sock /tmp/qemu-qmp.sock
-(setsid nohup "$QEMU_SCRIPT" > /tmp/opencode-qemu.log 2>&1 &)
+(setsid nohup "$QEMU_SCRIPT" > "$RUNDIR/qemu.log" 2>&1 &)
 sleep 6
 if pgrep -f qemu-system-x86_64 >/dev/null; then
-    echo "VM up (mdev $U)"
+    echo "VM up (mdev $U)  [qemu log: $RUNDIR/qemu.log]"
 else
-    echo "VM FAILED"; tail -3 /tmp/opencode-qemu.log; exit 1
+    echo "VM FAILED"; tail -3 "$RUNDIR/qemu.log"; exit 1
 fi