tools: add guest-diag (ptrace ioctl tracer, vmcycle, root-port QEMU cmdline)

The ptrace tracer resolves the current CUDA blocker: libcuda issues raw
syscall instructions, so LD_PRELOAD shims and ltrace see nothing; following
the child with PTRACE_SYSCALL + PTRACE_GET_SYSCALL_INFO does. It found
UVM_REGISTER_GPU (ioctl 0x25) returning rmStatus=0x56 NV_ERR_NOT_SUPPORTED,
with inputs identical to the working host run (rmCtrlFd=-1, hClient=0,
hSmcPartRef=0). Also dumps NV_ESC_RM_CONTROL cmd/status and RM alloc/free
statuses (all zero) and the UVM ioctl set (guest stops after
0x27,0x1,0x25,0x2 vs 141 ioctls on the host).

launch_qemu_rootport.sh is the working QEMU command line with the
pcie-root-port fix (see its header + the handoff PART 4); vmcycle.sh drives
the module/mdev/VM reload cycle around it.

5 files changed, 397 insertions, 0 deletions

diff --git a/tools/guest-diag/README.md b/tools/guest-diag/README.md
new file mode 100644
index 0000000..e113ea5
--- /dev/null
+++ b/tools/guest-diag/README.md
@@ -0,0 +1,50 @@
+# guest-diag - tracing tools for the emulated vGPU guest
+
+Tools used while driving the guest's stock NVIDIA driver through CUDA bring-up.
+All are tiny and dependency-free (libc only).
+
+## ptrace_ioctl.c - ioctl/result tracer (the important one)
+
+libcuda issues **raw `syscall` instructions**, so LD_PRELOAD shims and ltrace
+never see its ioctls. This ptrace tracer follows a child with
+PTRACE_SYSCALL + PTRACE_GET_SYSCALL_INFO and dumps:
+
+- `req=0x25` (UVM_REGISTER_GPU) buffer on entry and exit
+  (`rmStatus` is the high dword of the 5th qword) - this is how the current
+  failure was found: `rmStatus = 0x56 NV_ERR_NOT_SUPPORTED`
+- `req=0xc020462a` (NV_ESC_RM_CONTROL) cmd + status
+- `req=0xc020462b` / `0x...4629` (RM alloc/free) status when nonzero
+- UVM ioctls 0x1/0x2/0x17/0x19/0x25/0x27
+
+Build/run **in the guest**:
+
+    gcc -O1 -w -o /tmp/ptio ptrace_ioctl.c
+    /tmp/ptio /tmp/cdt 2>&1 | grep -E 'ptio|rmctl|rmio|\^'
+
+Note: `op` in `struct ptrace_syscall_info` is ENTRY=1 / EXIT=2, and the struct
+has `pad[3], arch, instruction_pointer, stack_pointer` before the union.
+
+## sysc_shim.c - known dead end
+
+An `syscall()` LD_PRELOAD interceptor: libcuda bypasses it entirely (inline
+syscall instruction). Kept only as documentation of that fact.
+
+## vmcycle.sh - reload cycle
+
+pkill QEMU -> remove mdevs -> rmmod/insmod open-iov-vfio.ko -> create a fresh
+`nvidia-nvidia-vgpu-turing` mdev -> patch its UUID into
+launch_qemu_rootport.sh -> relaunch. Guest boots in ~60-75 s.
+
+## launch_qemu_rootport.sh - QEMU command line with pcie-root-port
+
+Snapshot of the working launch command (derived from 's argv of a stopped
+VM) plus the root-port fix. **Requires** the mdev UUID to be substituted and:
+
+    mkdir -p /tmp/sockets/<domain-uuid>      # else QMP bind fails
+    rm -f /tmp/qemu-guest-agent.sock /tmp/qemu-qmp.sock
+
+Why the root port matters: on the Q35 root bus (`pcie.0`) QEMU's
+`vfio_setup_pcie_cap()` (hw/vfio/pci.c) relabels the mdev as "Root Complex
+Integrated Endpoint" and clears LNK/LNK2 + DeviceCaps, so the guest reads
+`LnkCap = 0` and `getPCIELinkRateMBps()` fails with "Unknown PCIe speed".
+Behind a `pcie-root-port` QEMU leaves the capability alone.
diff --git a/tools/guest-diag/launch_qemu_rootport.sh b/tools/guest-diag/launch_qemu_rootport.sh
new file mode 100755
index 0000000..52728bd
--- /dev/null
+++ b/tools/guest-diag/launch_qemu_rootport.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+exec /bin/qemu-system-x86_64 -D /home/rmcore/.local/libvf.io/logs/qemu/e726ba0b-204f-4ec9-91da-983ed7f41d06-session.txt -no-hpet -mem-prealloc -qmp unix:/tmp/qemu-qmp.sock,server,nowait -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 -serial file:/tmp/vgpu-guest-serial.log -gdb tcp::1234 -display none -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=128M,share=yes -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=2M,share=yes -uuid e726ba0b-204f-4ec9-91da-983ed7f41d06 -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on -rtc clock=host,base=localtime -m 4096 -smp cores=4,threads=1,sockets=1 -hda /home/rmcore/.local/libvf.io/kernel/linux.qcow2 --enable-kvm -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/<MDEV-UUID>,display=off -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 -net user -net nic -device vhost-vsock-pci,guest-cid=50718 -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/main.sock,server,nowait -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/master.sock,server,nowait
diff --git a/tools/guest-diag/ptrace_ioctl.c b/tools/guest-diag/ptrace_ioctl.c
new file mode 100644
index 0000000..5f96b31
--- /dev/null
+++ b/tools/guest-diag/ptrace_ioctl.c
@@ -0,0 +1,236 @@
+/* ptrace_ioctl.c - follow a child with PTRACE_SYSCALL and dump the buffer
+ * results of selected UVM ioctls. Needed because libcuda issues raw syscall
+ * instructions (LD_PRELOAD/ltrace never see these).
+ *
+ * Build (guest): gcc -O1 -w -o /tmp/ptio ptrace_ioctl.c
+ * Run:           /tmp/ptio /tmp/cdt
+ */
+#define _GNU_SOURCE
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ptrace.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#define SYS_IOCTL_X86_64 16
+
+struct syscall_info {
+	unsigned char op;
+	unsigned char pad[3];
+	unsigned int arch;
+	unsigned long long instruction_pointer;
+	unsigned long long stack_pointer;
+	union {
+		struct {
+			unsigned long long nr;
+			unsigned long long args[6];
+		} entry;
+		struct {
+			long long rval;
+			unsigned char is_error;
+		} exit;
+	} u;
+};
+
+static int interesting(unsigned long req)
+{
+	switch (req) {
+	case 0x1:  /* UVM_RESERVE_VA */
+	case 0x2:  /* UVM_RELEASE_VA */
+	case 0x25: /* UVM_REGISTER_GPU */
+	case 0x27: /* UVM_PAGEABLE_MEM_ACCESS */
+	case 0x17: /* UVM_EVENT_CTRL */
+	case 0x19: /* UVM_REGISTER_MPS_CLIENT */
+		return 1;
+	default:
+		return 0;
+	}
+}
+
+static void dump_words(pid_t pid, unsigned long addr, int nwords)
+{
+	int i;
+
+	fprintf(stderr, "  buffer @%#lx:", addr);
+	for (i = 0; i < nwords; i++) {
+		errno = 0;
+		long v = ptrace(PTRACE_PEEKDATA, pid, addr + i * 8, NULL);
+		if (errno)
+			break;
+		fprintf(stderr, " %016lx", (unsigned long)v);
+	}
+	fprintf(stderr, "\n");
+}
+
+struct saved {
+	pid_t pid;
+	int used;
+	unsigned long req, arg;
+} saved[64];
+
+static struct saved *slot(pid_t pid)
+{
+	int i, free_i = -1;
+
+	for (i = 0; i < 64; i++) {
+		if (saved[i].used && saved[i].pid == pid)
+			return &saved[i];
+		if (!saved[i].used && free_i < 0)
+			free_i = i;
+	}
+	if (free_i >= 0) {
+		saved[free_i].used = 1;
+		saved[free_i].pid = pid;
+		saved[free_i].req = 0;
+		return &saved[free_i];
+	}
+	return NULL;
+}
+
+int main(int argc, char **argv)
+{
+	pid_t pid;
+	int nprint = 0;
+
+	if (argc < 2) {
+		fprintf(stderr, "usage: %s prog [args...]\n", argv[0]);
+		return 2;
+	}
+
+	pid = fork();
+	if (pid == 0) {
+		ptrace(PTRACE_TRACEME, 0, 0, 0);
+		raise(SIGSTOP);
+		execvp(argv[1], &argv[1]);
+		perror("execvp");
+		_exit(127);
+	}
+
+	waitpid(pid, NULL, WUNTRACED);
+	ptrace(PTRACE_SETOPTIONS, pid, 0,
+	       (void *)(PTRACE_O_TRACESYSGOOD | PTRACE_O_TRACECLONE |
+			PTRACE_O_TRACEFORK | PTRACE_O_TRACEVFORK | PTRACE_O_EXITKILL));
+	ptrace(PTRACE_SYSCALL, pid, 0, 0);
+
+	fprintf(stderr, "[ptio] attached to pid %d\n", pid);
+	for (;;) {
+		int status;
+		pid_t p = waitpid(-1, &status, __WALL);
+
+		if (p < 0) {
+			if (errno == EINTR)
+				continue;
+			break;
+		}
+		if (WIFEXITED(status) || WIFSIGNALED(status))
+			continue;
+		if (!WIFSTOPPED(status))
+			continue;
+
+		{
+			int sig = WSTOPSIG(status);
+			struct syscall_info si;
+
+			memset(&si, 0, sizeof(si));
+			if ((sig == (SIGTRAP | 0x80)) &&
+			    ptrace(PTRACE_GET_SYSCALL_INFO, p, sizeof(si), &si) > 0) {
+				struct saved *sv = slot(p);
+
+				if (si.op == 1 /* entry */) {
+					if (sv && si.u.entry.nr == SYS_IOCTL_X86_64) {
+						sv->req = si.u.entry.args[1];
+						sv->arg = si.u.entry.args[2];
+						if (sv->req == 0x25) {
+							fprintf(stderr, "[ptio] ENTER UVM_REGISTER_GPU pid=%d\n", p);
+							dump_words(p, sv->arg, 6);
+							fprintf(stderr, "  ^ IN: uuid(2q) | numaEnabled numaNodeId | rmCtrlFd hClient | hSmcPartRef rmStatus\n");
+						}
+					} else if (sv) {
+						sv->req = 0;
+					}
+				} else if (si.op == 2 /* exit */ && sv &&
+					   ((sv->req & 0xff) == 0x4629ul ||
+					    ((sv->req & 0xffff00fful) == 0x0046002bul) ||
+					    ((sv->req & 0xffff00fful) == 0x0046002aul))) {
+					unsigned int own = (unsigned int)(sv->req & 0xff);
+					long q0 = 0, q1 = 0, q2 = 0, q3 = 0;
+					unsigned long st = 0;
+
+					errno = 0; q0 = ptrace(PTRACE_PEEKDATA, p, sv->arg, NULL);
+					errno = 0; q1 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 8, NULL);
+					errno = 0; q2 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 16, NULL);
+					errno = 0; q3 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 24, NULL);
+					if (own == 0x29)
+						st = (unsigned long)((q1 >> 32) & 0xfffffffful);
+					else
+						st = (unsigned long)((q3 >> 32) & 0xfffffffful);
+					if (st != 0)
+						fprintf(stderr,
+							"[rmio] pid=%d nr=0x%x status=0x%lx hClient=0x%lx hObj=0x%lx cmd/class=0x%lx size=%lu\n",
+							p, own,
+							st,
+							(unsigned long)(q0 & 0xfffffffful),
+							(unsigned long)((q0 >> 32) & 0xfffffffful),
+							(unsigned long)(q1 & 0xfffffffful),
+							(unsigned long)((q3 >> 32) & 0xfffffffful));
+				} else if (si.op == 2 /* exit */ && sv &&
+					   sv->req == 0xc020462aul) {
+					long q0, q1, q2, q3;
+					errno = 0;
+					q1 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 8, NULL);
+					errno = 0;
+					q2 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 16, NULL);
+					errno = 0;
+					q3 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 24, NULL);
+					q0 = ptrace(PTRACE_PEEKDATA, p, sv->arg, NULL);
+					(void)q0;
+					fprintf(stderr,
+						"[rmctl] pid=%d cmd=0x%lx hClient=0x%lx params=%#lx size=%lu status=0x%lx\n",
+						p, (unsigned long)(q1 & 0xfffffffful),
+						(unsigned long)(q0 & 0xfffffffful),
+						(unsigned long)(q2 & 0xfffffffffffful),
+						(unsigned long)((q2 >> 32) & 0xfffffffful),
+						(unsigned long)((q3 >> 32) & 0xfffffffful));
+					if (((q3 >> 32) & 0xfffffffful) != 0 &&
+					    (q2 & 0xfffffffffffful) > 0x10000) {
+						fprintf(stderr, "   params:");
+						{
+							int i2;
+							for (i2 = 0; i2 < 6; i2++) {
+								errno = 0;
+								long v = ptrace(PTRACE_PEEKDATA, p,
+										(unsigned long)(q2 & 0xfffffffffffful) + i2 * 8, NULL);
+								if (errno)
+									break;
+								fprintf(stderr, " %016lx",
+									(unsigned long)v);
+							}
+						}
+						fprintf(stderr, "\n");
+					}
+				} else if (si.op == 2 /* exit */ && sv &&
+					   interesting(sv->req)) {
+					fprintf(stderr,
+						"[ptio] pid=%d ioctl req=0x%lx arg=%#lx ret=%ld%s\n",
+						p, sv->req, sv->arg,
+						si.u.exit.rval,
+						interesting(sv->req) ? " *" : "");
+					if (sv->req == 0x25) {
+						dump_words(p, sv->arg, 6);
+						fprintf(stderr,
+							"  ^ UVM_REGISTER_GPU: uuid(2q) | numaEnabled numaNodeId rmCtrlFd hClient | hSmcPartRef rmStatus\n");
+					} else {
+						dump_words(p, sv->arg, 4);
+					}
+				}
+			}
+			ptrace(PTRACE_SYSCALL, p, 0, sig == (SIGTRAP | 0x80) ? 0 :
+			       (sig == SIGTRAP ? 0 : sig));
+		}
+	}
+	return 0;
+}
diff --git a/tools/guest-diag/sysc_shim.c b/tools/guest-diag/sysc_shim.c
new file mode 100644
index 0000000..9b07e4c
--- /dev/null
+++ b/tools/guest-diag/sysc_shim.c
@@ -0,0 +1,73 @@
+/* sysc_shim.c - intercept syscall() so we can dump ioctl buffer *results*
+ * for UVM ioctls (libcuda calls syscall(SYS_ioctl, ...) directly, so an
+ * ioctl() wrapper is bypassed but syscall() is not).
+ * Build in guest: gcc -shared -fPIC -O1 -o /tmp/sysc.so sysc_shim.c -ldl
+ * Run: LD_PRELOAD=/tmp/sysc.so /tmp/cdt
+ */
+#define _GNU_SOURCE
+#include <dlfcn.h>
+#include <stdarg.h>
+#include <stddef.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+static long (*real_syscall)(long, ...);
+
+static void dump(const char *tag, unsigned long req, void *arg, long ret)
+{
+	unsigned int *p = (unsigned int *)arg;
+	char buf[512];
+	int n, i;
+	long *vp = (long *)arg;
+
+	if (!p)
+		return;
+	n = 0;
+	n += __builtin_snprintf(buf + n, sizeof(buf) - n,
+				"SYSC[%s] req=0x%lx ret=%ld dwords: ", tag, req,
+				ret);
+	for (i = 0; i < 16 && n < (int)sizeof(buf) - 16; i++)
+		n += __builtin_snprintf(buf + n, sizeof(buf) - n, "%08x ",
+					p[i]);
+	(void)vp;
+	write(2, buf, n);
+	write(2, "\n", 1);
+}
+
+long syscall(long nr, ...)
+{
+	va_list ap;
+	long a1, a2, a3, a4, a5, a6;
+	long ret;
+
+	if (!real_syscall)
+		real_syscall = dlsym(RTLD_NEXT, "syscall");
+
+	va_start(ap, nr);
+	a1 = va_arg(ap, long);
+	a2 = va_arg(ap, long);
+	a3 = va_arg(ap, long);
+	a4 = va_arg(ap, long);
+	a5 = va_arg(ap, long);
+	a6 = va_arg(ap, long);
+	va_end(ap);
+
+	ret = real_syscall(nr, a1, a2, a3, a4, a5, a6);
+
+	if (nr == SYS_ioctl) {
+		unsigned long req = (unsigned long)a2;
+		switch (req) {
+		case 0x1:  /* UVM_RESERVE_VA */
+		case 0x2:  /* UVM_RELEASE_VA */
+		case 0x25: /* UVM_REGISTER_GPU */
+		case 0x27: /* UVM_PAGEABLE_MEM_ACCESS */
+		case 0x17: /* UVM_EVENT_CTRL */
+		case 0x19: /* UVM_REGISTER_MPS_CLIENT */
+			dump("uvmioctl", req, (void *)a3, ret);
+			break;
+		default:
+			break;
+		}
+	}
+	return ret;
+}
diff --git a/tools/guest-diag/vmcycle.sh b/tools/guest-diag/vmcycle.sh
new file mode 100755
index 0000000..08eb4ea
--- /dev/null
+++ b/tools/guest-diag/vmcycle.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# vmcycle.sh - reload cycle for the emulated vGPU stack:
+#   stop QEMU -> remove mdevs -> rmmod/insmod open-iov-vfio.ko -> create a
+#   fresh mdev -> patch its UUID into the QEMU launch script -> relaunch.
+#
+# Usage:  tools/guest-diag/vmcycle.sh
+# Env:    QEMU_SCRIPT (default: ./launch_qemu_rootport.sh next to this script)
+#         MDEV_PARENT (default: 0000:01:00.0)
+#         MDEV_TYPE   (default: nvidia-nvidia-vgpu-turing)
+set -e
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+QEMU_SCRIPT="${QEMU_SCRIPT:-$HERE/launch_qemu_rootport.sh}"
+MODULE="${MODULE:-$HERE/../../kernel/open-iov-vfio.ko}"
+MDEV_PARENT="${MDEV_PARENT:-0000:01:00.0}"
+MDEV_TYPE="${MDEV_TYPE:-nvidia-nvidia-vgpu-turing}"
+
+pkill -f qemu-system-x86_64 2>/dev/null || true
+sleep 4
+for m in /sys/bus/mdev/devices/*; do
+    [ -d "$m" ] && echo 1 | sudo tee "$m/remove" >/dev/null
+done
+sudo rmmod open_iov_vfio 2>/dev/null || true
+sudo insmod "$MODULE"
+U=$(uuidgen)
+echo "$U" | sudo tee "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types/$MDEV_TYPE/create" >/dev/null
+sed -i "s|/sys/bus/mdev/devices/[0-9a-f-]*|/sys/bus/mdev/devices/$U|" "$QEMU_SCRIPT"
+mkdir -p /tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06
+rm -f /tmp/qemu-guest-agent.sock /tmp/qemu-qmp.sock
+(setsid nohup "$QEMU_SCRIPT" > /tmp/opencode-qemu.log 2>&1 &)
+sleep 6
+if pgrep -f qemu-system-x86_64 >/dev/null; then
+    echo "VM up (mdev $U)"
+else
+    echo "VM FAILED"; tail -3 /tmp/opencode-qemu.log; exit 1
+fi