From 68cdd7728a7cc6715bbbff7decfe2b11c3bdf558 Mon Sep 17 00:00:00 2001 From: Open-IOV Date: Wed, 23 Sep 2026 05:44:08 -0400 Subject: Open-IOV: initial public release Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with real GPU functionality - CUDA first, 3D and video to follow - on unmodified hosts, without a proprietary vGPU license and without patching any driver, firmware or hypervisor. Core principle: emulate the interface, pass through the work. kernel/ open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device (PCI config space, BAR shadows, MSI-X), GSP boot emulation, GSP-RM message-queue/RPC emulation, memory model (GPU-VA to guest-PA discovery) and the semaphore-completion service that advances guest UVM progress. userspace/ libiov.so management API and examples. tools/ guest diagnostics, firmware analysis wrappers, RM probes. scripts/ stack manager, VM harness, build/load/test automation. config/ hardware/version profiles (the portability surface). docs/ documentation tree - start at docs/README.md. wiki/ long-form firmware and protocol deep dives. Status: initialization works end to end for a stock 570.144 Linux guest on Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery, object creation and the early CUDA/UVM start-up path. GPU work forwarding - the step that makes CUDA kernels execute on real hardware - is the active work item; see docs/04-implementation/01-known-knowns-and-unknowns.md. License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md. --- tools/module_ops/dry_run.sh | 47 +++++++++++++++++++ tools/module_ops/load_converged.sh | 95 ++++++++++++++++++++++++++++++++++++++ tools/module_ops/unload_all.sh | 42 +++++++++++++++++ 3 files changed, 184 insertions(+) create mode 100755 tools/module_ops/dry_run.sh create mode 100755 tools/module_ops/load_converged.sh create mode 100755 tools/module_ops/unload_all.sh (limited to 'tools/module_ops') diff --git a/tools/module_ops/dry_run.sh b/tools/module_ops/dry_run.sh new file mode 100755 index 0000000..e6bb6f2 --- /dev/null +++ b/tools/module_ops/dry_run.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Run vGPU dry-run validation +# Usage: sudo ./dry_run.sh + +set -euo pipefail + +VGPU_KERNEL="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}/kernel" + +echo "=== Open-IOV vGPU Dry-Run Validation ===" +echo "" + +# Check if host nvidia.ko is loaded +if ! lsmod | grep -q "^nvidia "; then + echo "ERROR: Host nvidia.ko not loaded. Load it first with:" + echo " rzsh nvidia vgpu load" + exit 1 +fi + +echo "Host nvidia.ko: LOADED ✓" +echo "" + +# Load vGPU module in dry-run mode +echo "Loading nvidia-vgpu-vfio.ko in dry-run mode..." +sudo insmod "$VGPU_KERNEL/nvidia-vgpu-vfio.ko" dyndbg='+p' dry_run=1 + +# Check result +sleep 2 +if dmesg -T | grep -q "DRY-RUN MODE: validating init without hardware changes"; then + echo "" + echo "✓ Dry-run validation PASSED" + echo " Firmware, BAR layout, IRQ caps, IOMMU validated" +else + echo "" + echo "✗ Dry-run validation FAILED" + echo "Check dmesg for errors:" + dmesg -T | grep -i -e nvidia-vgpu -e vgpu | tail -20 +fi + +# Unload +echo "" +echo "Unloading dry-run module..." +sudo rmmod nvidia-vgpu-vfio + +echo "" +echo "=== Dry-Run Complete ===" +echo "If validation passed, load for real with:" +echo " sudo insmod $VGPU_KERNEL/nvidia-vgpu-vfio.ko dyndbg='+p'" \ No newline at end of file diff --git a/tools/module_ops/load_converged.sh b/tools/module_ops/load_converged.sh new file mode 100755 index 0000000..dbd090e --- /dev/null +++ b/tools/module_ops/load_converged.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Load vGPU stack in converged mode +# Usage: sudo ./load_converged.sh [dry_run] + +set -euo pipefail + +OPEN_IOV_ROOT="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +NVIDIA_OPEN_KERNEL="$OPEN_IOV_ROOT/nvidia-open-gpu-kernel-modules/kernel-open" +VGPU_KERNEL="$OPEN_IOV_ROOT/kernel" + +DRY_RUN="${1:-false}" + +echo "=== Open-IOV Converged Mode Module Load ===" +echo "Open-IOV Root: $OPEN_IOV_ROOT" +echo "Dry-run mode: $DRY_RUN" +echo "" + +# Function to check if module is loaded +is_loaded() { + lsmod | grep -q "^$1 " +} + +# Function to load module with error checking +load_module() { + local module_path=$1 + local module_name=$(basename $module_path .ko) + local params="${2:-}" + + if is_loaded "$module_name"; then + echo "Module $module_name already loaded" + return 0 + fi + + echo "Loading $module_name..." + if sudo insmod "$module_path" $params; then + echo " ✓ $module_name loaded successfully" + return 0 + else + echo " ✗ Failed to load $module_name" + return 1 + fi +} + +# 1. Load mdev core +echo "Step 1: Loading mdev core module..." +sudo modprobe mdev +echo " ✓ mdev loaded" +echo "" + +# 2. Load host nvidia.ko with vGPU support +echo "Step 2: Loading host nvidia.ko (patched open-gpu-kernel-modules)..." +load_module "$NVIDIA_OPEN_KERNEL/nvidia.ko" "NVreg_RegisterPCIDriver=1 NVreg_RmMsg=:" +echo "" + +# 3. Load dependent modules +echo "Step 3: Loading dependent modules..." +load_module "$NVIDIA_OPEN_KERNEL/nvidia-modeset.ko" +load_module "$NVIDIA_OPEN_KERNEL/nvidia-uvm.ko" +load_module "$NVIDIA_OPEN_KERNEL/nvidia-drm.ko" +load_module "$NVIDIA_OPEN_KERNEL/nvidia-peermem.ko" +echo "" + +# 4. Load nvidia-vgpu-vfio.ko +echo "Step 4: Loading nvidia-vgpu-vfio.ko..." +if [ "$DRY_RUN" = "dry_run" ] || [ "$DRY_RUN" = "true" ]; then + echo " Running in DRY-RUN mode first..." + load_module "$VGPU_KERNEL/nvidia-vgpu-vfio.ko" "dyndbg='+p' dry_run=1" + echo " Dry-run successful, unloading..." + sudo rmmod nvidia-vgpu-vfio + echo " Loading for real..." + load_module "$VGPU_KERNEL/nvidia-vgpu-vfio.ko" "dyndbg='+p'" +else + load_module "$VGPU_KERNEL/nvidia-vgpu-vfio.ko" "dyndbg='+p'" +fi +echo "" + +# 5. Verify +echo "Step 5: Verifying vGPU registration..." +sleep 2 +if [ -d "/sys/bus/pci/devices/0000:01:00.0/mdev_supported_types" ]; then + echo " ✓ mdev_supported_types directory exists" + ls -la /sys/bus/pci/devices/0000:01:00.0/mdev_supported_types/ +else + echo " ✗ mdev_supported_types NOT found" + echo " Check dmesg for errors: dmesg -T | grep -i vgpu" + exit 1 +fi + +echo "" +echo "=== Converged Mode Load Complete ===" +echo "" +echo "Next steps:" +echo " - Create mdev device: echo | sudo tee /sys/bus/pci/devices/0000:01:00.0/mdev_supported_types/nvidia-vgpu-turing/create" +echo " - Run QEMU with vGPU passthrough" +echo " - Run tests: $OPEN_IOV_ROOT/tools/test/run_integration_tests.sh" \ No newline at end of file diff --git a/tools/module_ops/unload_all.sh b/tools/module_ops/unload_all.sh new file mode 100755 index 0000000..4c73a0b --- /dev/null +++ b/tools/module_ops/unload_all.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Unload all vGPU modules cleanly +# Usage: sudo ./unload_all.sh + +set -euo pipefail + +echo "=== Unloading Open-IOV vGPU Stack ===" + +# Unload in reverse order of loading +modules=( + "nvidia-vgpu-vfio" + "nvidia-drm" + "nvidia-modeset" + "nvidia-uvm" + "nvidia-peermem" + "nvidia" + "mdev" +) + +for module in "${modules[@]}"; do + if lsmod | grep -q "^$module "; then + echo "Unloading $module..." + if sudo rmmod "$module" 2>/dev/null; then + echo " ✓ $module unloaded" + else + echo " ⚠ $module could not be unloaded (may be in use)" + # Check refcnt + if [[ -f /sys/module/$module/refcnt ]]; then + echo " refcnt: $(cat /sys/module/$module/refcnt)" + fi + if [[ -f /sys/module/$module/initstate ]]; then + echo " initstate: $(cat /sys/module/$module/initstate)" + fi + fi + else + echo " - $module not loaded" + fi +done + +echo "" +echo "=== Unload Complete ===" +lsmod | grep -E 'nvidia|mdev' || echo "No NVIDIA/mdev modules loaded" \ No newline at end of file -- cgit v1.2.3