From 00e54c13c7c9030c4d5f7151467f2d85a3730e29 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sun, 13 Sep 2026 18:36:21 -0400 Subject: Initial commit: Open-IOV vGPU-IO complete stack - Firmware decompilation: GSP 570.144W (28.5MB ELF, RV64IMAC), 132 decompiled C files - Kernel modules: nvidia.ko (34MB, OpenRM), open-iov-vfio.ko (2.8MB, VFIO mdev), gsp-vgpu.ko (1.8MB, guest driver) - Userspace: libiov.so/libvgpu.so (76KB), example_vgpu (28KB), rpc_emulator.c - PCI config space emulation (256B with PM, MSI, MSI-X, PCIe caps) - 12 RPC types fully mapped from firmware decompilation - BAR trap, WPR, scheduling, notification, migration emulation - Helper scripts: load_vgpu_stack, create_mdev, launch_qemu_vgpu, test_vgpu_stack, setup_guest_driver, unload_vgpu_stack - Architecture documentation - Wiki: 10 technical documents - Git LFS for large binaries (.ko, .bin, .iso, .img, .o, ghidra repos) - nvidia-open-gpu-kernel-modules as submodule --- tools/vm/cloud-init.yaml | 26 ++++++++++++++++++ tools/vm/launch_vgpu_qemu.sh | 65 ++++++++++++++++++++++++++++++++++++++++++++ tools/vm/setup_mdev.sh | 65 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 tools/vm/cloud-init.yaml create mode 100755 tools/vm/launch_vgpu_qemu.sh create mode 100755 tools/vm/setup_mdev.sh (limited to 'tools/vm') diff --git a/tools/vm/cloud-init.yaml b/tools/vm/cloud-init.yaml new file mode 100644 index 0000000..05f11b5 --- /dev/null +++ b/tools/vm/cloud-init.yaml @@ -0,0 +1,26 @@ +#cloud-config +users: + - name: ubuntu + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + lock_passwd: false + passwd: $6$rounds=4096$ubuntu$ubuntu + ssh_authorized_keys: [] + +package_update: true +package_upgrade: true +packages: + - linux-headers-generic + - build-essential + - dkms + - git + - wget + - curl + - vim + - pciutils + - lshw + +runcmd: + - echo "ubuntu:ubuntu" | chpasswd + - systemctl enable ssh + - systemctl start ssh diff --git a/tools/vm/launch_vgpu_qemu.sh b/tools/vm/launch_vgpu_qemu.sh new file mode 100755 index 0000000..0355ef1 --- /dev/null +++ b/tools/vm/launch_vgpu_qemu.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# QEMU launch script for NVIDIA vGPU testing +# This script assumes: +# 1. nvidia-vgpu-vfio.ko is loaded on host (with dry_run=1 for testing) +# 2. An mdev device has been created via sysfs +# 3. Guest OS has NVIDIA vGPU guest driver installed + +set -euo pipefail + +# Configuration +MDEV_UUID="${MDEV_UUID:-$(cat /proc/sys/kernel/random/uuid)}" +VGPU_TYPE="${VGPU_TYPE:-nvidia-vgpu-turing}" # or ampere, hopper, pascal +QEMU_BIN="${QEMU_BIN:-qemu-system-x86_64}" +GUEST_IMG="${GUEST_IMG:-/var/lib/libvirt/images/ubuntu-vgpu.qcow2}" +GUEST_KERNEL="${GUEST_KERNEL:-/boot/vmlinuz-$(uname -r)}" +GUEST_INITRD="${GUEST_INITRD:-/boot/initrd.img-$(uname -r)}" + +# PCI address of the NVIDIA GPU (TU104: 10de:1ed3) +GPU_PCI_ADDR="${GPU_PCI_ADDR:-0000:01:00.0}" + +echo "=== NVIDIA vGPU QEMU Launch Script ===" +echo "MDEV UUID: $MDEV_UUID" +echo "VGPU Type: $VGPU_TYPE" +echo "GPU PCI: $GPU_PCI_ADDR" +echo "" + +# Check if mdev device exists +MDEV_PATH="/sys/bus/mdev/devices/$MDEV_UUID" +if [[ ! -d "$MDEV_PATH" ]]; then + echo "ERROR: mdev device $MDEV_UUID not found" + echo "" + echo "To create an mdev device (requires NVIDIA vGPU host driver):" + echo " echo $MDEV_UUID > /sys/bus/pci/devices/$GPU_PCI_ADDR/mdev_supported_types/$VGPU_TYPE/create" + echo "" + echo "For dry-run testing without host driver, this script will not work for actual QEMU launch." + echo "The kernel module loads in dry-run mode but cannot create real mdev devices." + exit 1 +fi + +# Get the mdev device's VFIO group +VFIO_GROUP=$(basename $(readlink -f "$MDEV_PATH/iommu_group")) +VFIO_DEV="/dev/vfio/$VFIO_GROUP" + +if [[ ! -e "$VFIO_DEV" ]]; then + echo "ERROR: VFIO device $VFIO_DEV not found" + exit 1 +fi + +echo "Using VFIO device: $VFIO_DEV" +echo "" + +# QEMU command +exec $QEMU_BIN \ + -name "vgpu-guest" \ + -enable-kvm \ + -cpu host,host-passthrough=on \ + -smp 4,sockets=1,cores=4,threads=1 \ + -m 8G \ + -device vfio-pci,sysfsdev=/sys/bus/mdev/devices/$MDEV_UUID,display=on,x-igd-opregion=on \ + -device virtio-net-pci,netdev=net0 \ + -netdev user,id=net0,hostfwd=tcp::2222-:22 \ + -drive file=$GUEST_IMG,if=virtio,format=qcow2 \ + -display gtk,gl=on \ + -vga none \ + "$@" diff --git a/tools/vm/setup_mdev.sh b/tools/vm/setup_mdev.sh new file mode 100755 index 0000000..ef31f46 --- /dev/null +++ b/tools/vm/setup_mdev.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Setup script for NVIDIA vGPU mdev device +# This script creates an mdev device for QEMU testing + +set -euo pipefail + +GPU_PCI_ADDR="${GPU_PCI_ADDR:-0000:01:00.0}" +VGPU_TYPE="${VGPU_TYPE:-nvidia-vgpu-turing}" +MDEV_UUID="${MDEV_UUID:-$(cat /proc/sys/kernel/random/uuid)}" + +MDEV_CREATE_PATH="/sys/bus/pci/devices/$GPU_PCI_ADDR/mdev_supported_types/$VGPU_TYPE/create" + +echo "=== NVIDIA vGPU mdev Device Setup ===" +echo "GPU PCI: $GPU_PCI_ADDR" +echo "VGPU Type: $VGPU_TYPE" +echo "MDEV UUID: $MDEV_UUID" +echo "" + +# Check if GPU device exists +if [[ ! -d "/sys/bus/pci/devices/$GPU_PCI_ADDR" ]]; then + echo "ERROR: GPU device $GPU_PCI_ADDR not found" + echo "Available NVIDIA GPUs:" + lspci -nn -d 10de: | grep -E "VGA|3D" + exit 1 +fi + +# Check if mdev_supported_types exists (requires NVIDIA vGPU host driver) +if [[ ! -d "/sys/bus/pci/devices/$GPU_PCI_ADDR/mdev_supported_types" ]]; then + echo "ERROR: mdev_supported_types not found for $GPU_PCI_ADDR" + echo "" + echo "This requires the NVIDIA vGPU host driver (licensed product for data center GPUs)." + echo "The regular NVIDIA driver does not provide vGPU mdev support." + echo "" + echo "For testing on consumer GPUs (TU104, etc.), use dry-run mode:" + echo " sudo insmod /path/to/nvidia-vgpu-vfio.ko dry_run=1" + echo "" + echo "In dry-run mode, the module loads but cannot create real mdev devices." + exit 1 +fi + +# Check if requested vGPU type exists +if [[ ! -d "/sys/bus/pci/devices/$GPU_PCI_ADDR/mdev_supported_types/$VGPU_TYPE" ]]; then + echo "ERROR: vGPU type $VGPU_TYPE not supported" + echo "Available types:" + ls "/sys/bus/pci/devices/$GPU_PCI_ADDR/mdev_supported_types/" + exit 1 +fi + +# Create mdev device +echo "Creating mdev device..." +echo "$MDEV_UUID" | sudo tee "$MDEV_CREATE_PATH" > /dev/null + +if [[ -d "/sys/bus/mdev/devices/$MDEV_UUID" ]]; then + echo "SUCCESS: mdev device created at /sys/bus/mdev/devices/$MDEV_UUID" + echo "" + echo "Device info:" + cat "/sys/bus/mdev/devices/$MDEV_UUID/description" 2>/dev/null || echo " (no description)" + echo "" + echo "To use with QEMU:" + echo " export MDEV_UUID=$MDEV_UUID" + echo " /path/to/launch_vgpu_qemu.sh" +else + echo "ERROR: Failed to create mdev device" + exit 1 +fi -- cgit v1.2.3