From 68cdd7728a7cc6715bbbff7decfe2b11c3bdf558 Mon Sep 17 00:00:00 2001 From: Open-IOV Date: Wed, 23 Sep 2026 05:44:08 -0400 Subject: Open-IOV: initial public release Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with real GPU functionality - CUDA first, 3D and video to follow - on unmodified hosts, without a proprietary vGPU license and without patching any driver, firmware or hypervisor. Core principle: emulate the interface, pass through the work. kernel/ open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device (PCI config space, BAR shadows, MSI-X), GSP boot emulation, GSP-RM message-queue/RPC emulation, memory model (GPU-VA to guest-PA discovery) and the semaphore-completion service that advances guest UVM progress. userspace/ libiov.so management API and examples. tools/ guest diagnostics, firmware analysis wrappers, RM probes. scripts/ stack manager, VM harness, build/load/test automation. config/ hardware/version profiles (the portability surface). docs/ documentation tree - start at docs/README.md. wiki/ long-form firmware and protocol deep dives. Status: initialization works end to end for a stock 570.144 Linux guest on Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery, object creation and the early CUDA/UVM start-up path. GPU work forwarding - the step that makes CUDA kernels execute on real hardware - is the active work item; see docs/04-implementation/01-known-knowns-and-unknowns.md. License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md. --- tools/test/cuda_drv_test.c | 87 +++++++++++++++++ tools/test/run_integration_tests.sh | 182 ++++++++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 tools/test/cuda_drv_test.c create mode 100755 tools/test/run_integration_tests.sh (limited to 'tools/test') diff --git a/tools/test/cuda_drv_test.c b/tools/test/cuda_drv_test.c new file mode 100644 index 0000000..6868a2e --- /dev/null +++ b/tools/test/cuda_drv_test.c @@ -0,0 +1,87 @@ +/* cuda_drv_test.c - minimal CUDA driver-API smoke test (dlopen, no toolkit). + * + * Run inside the guest. Exercises the paths CUDA needs beyond NVML: + * cuInit -> cuDeviceGetCount/Get -> cuCtxCreate -> cuMemAlloc -> + * cuMemcpyHtoD/DtoH -> cuMemFree -> cuCtxDestroy + * + * Expected output on success: "ALL OK". + * + * Status (2026-09-23): cuInit returns 3 (CUDA_ERROR_NOT_INITIALIZED) on the + * emulated GPU. strace shows every nvidia.ko ioctl succeeding (RM status 0), + * the driver completing RmInitAdapter and then closing the device, and + * libcuda opening /dev/nvidia-uvm before /dev/nvidia0 - i.e. the next blocker + * is the UVM driver (nvidia-uvm.ko), which needs the GPU MMU/VAS model that + * the GSP-RM emulation does not provide yet. Build in the guest with: + * gcc -O1 -o /tmp/cdt cuda_drv_test.c -ldl + */ +#include +#include +#include +#include + +typedef int CUresult; +typedef int CUdevice; +typedef void *CUcontext; +typedef uint64_t CUdeviceptr; + +#define CHECK(expr) do { \ + CUresult r = (expr); \ + if (r) { printf("%-24s FAILED rc=%d\n", #expr, r); return 1; } \ + printf("%-24s OK\n", #expr); \ +} while (0) + +int main(void) +{ + void *h = dlopen("libcuda.so.1", RTLD_NOW); + if (!h) { printf("dlopen libcuda.so.1 failed: %s\n", dlerror()); return 1; } + + CUresult (*cuInit)(unsigned int) = dlsym(h, "cuInit"); + CUresult (*cuDeviceGetCount)(int *) = dlsym(h, "cuDeviceGetCount"); + CUresult (*cuDeviceGet)(CUdevice *, int) = dlsym(h, "cuDeviceGet"); + CUresult (*cuCtxCreate)(CUcontext *, unsigned int, CUdevice) = dlsym(h, "cuCtxCreate_v2"); + CUresult (*cuMemAlloc)(CUdeviceptr *, size_t) = dlsym(h, "cuMemAlloc_v2"); + CUresult (*cuMemcpyHtoD)(CUdeviceptr, const void *, size_t) = dlsym(h, "cuMemcpyHtoD_v2"); + CUresult (*cuMemcpyDtoH)(void *, CUdeviceptr, size_t) = dlsym(h, "cuMemcpyDtoH_v2"); + CUresult (*cuMemFree)(CUdeviceptr) = dlsym(h, "cuMemFree_v2"); + CUresult (*cuCtxDestroy)(CUcontext) = dlsym(h, "cuCtxDestroy_v2"); + + if (!cuInit || !cuDeviceGetCount || !cuDeviceGet || !cuCtxCreate || + !cuMemAlloc || !cuMemcpyHtoD || !cuMemcpyDtoH || !cuMemFree || !cuCtxDestroy) { + printf("dlsym failed\n"); + return 1; + } + + int count = 0; + CUdevice dev; + CUcontext ctx = NULL; + CUdeviceptr dptr = 0; + unsigned int buf[64]; + unsigned int back[64]; + size_t i; + + CHECK(cuInit(0)); + CHECK(cuDeviceGetCount(&count)); + printf("device count = %d\n", count); + if (count < 1) return 1; + + CHECK(cuDeviceGet(&dev, 0)); + printf("device 0 = %d\n", dev); + + CHECK(cuCtxCreate(&ctx, 0, dev)); + printf("context = %p\n", ctx); + + CHECK(cuMemAlloc(&dptr, sizeof(buf))); + printf("device ptr = 0x%llx (%zu bytes)\n", (unsigned long long)dptr, sizeof(buf)); + + for (i = 0; i < 64; i++) buf[i] = (unsigned int)(i * 3 + 1); + CHECK(cuMemcpyHtoD(dptr, buf, sizeof(buf))); + memset(back, 0, sizeof(back)); + CHECK(cuMemcpyDtoH(back, dptr, sizeof(buf))); + printf("roundtrip %s (back[7]=%u expected 22)\n", + memcmp(buf, back, sizeof(buf)) ? "MISMATCH" : "match", back[7]); + + CHECK(cuMemFree(dptr)); + CHECK(cuCtxDestroy(ctx)); + printf("ALL OK\n"); + return 0; +} diff --git a/tools/test/run_integration_tests.sh b/tools/test/run_integration_tests.sh new file mode 100755 index 0000000..06e309c --- /dev/null +++ b/tools/test/run_integration_tests.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# GSP vGPU-IO Integration Test Runner + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +KERNEL_TEST_DIR="$PROJECT_ROOT/kernel/tests" +DRIVER_DIR="$PROJECT_ROOT/driver" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# Check if running as root +check_root() { + if [[ $EUID -ne 0 ]]; then + log_error "This script must be run as root" + exit 1 + fi +} + +# Build kernel test module +build_test_module() { + log_info "Building kernel test module..." + cd "$KERNEL_TEST_DIR" + make clean + make + if [[ $? -ne 0 ]]; then + log_error "Failed to build test module" + exit 1 + fi + log_info "Test module built successfully" +} + +# Load test module and capture output +run_test_module() { + log_info "Loading test module..." + cd "$KERNEL_TEST_DIR" + + # Clear dmesg + dmesg -C + + # Load module + insmod gsp_vgpu_comprehensive_test.ko 2>&1 || { + log_error "Failed to load test module" + dmesg -T | tail -30 + return 1 + } + + # Wait for tests to complete + sleep 2 + + # Get test results from dmesg + dmesg -T | grep -E "(PASS|FAIL|Test Results|TEST SUITE)" | tail -100 + + # Check if tests passed + if dmesg | grep -q "TEST SUITE PASSED"; then + log_info "All tests PASSED!" + return 0 + elif dmesg | grep -q "TEST SUITE FAILED"; then + log_error "Tests FAILED!" + return 1 + else + log_warn "Could not determine test result" + return 1 + fi +} + +# Unload test module +unload_test_module() { + log_info "Unloading test module..." + rmmod gsp_vgpu_comprehensive_test 2>/dev/null || true + sleep 1 +} + +# Build driver +build_driver() { + log_info "Building vGPU-IO driver..." + cd "$DRIVER_DIR" + make clean + make + if [[ $? -ne 0 ]]; then + log_error "Failed to build driver" + exit 1 + fi + log_info "Driver built successfully" +} + +# Load driver +load_driver() { + log_info "Loading vGPU-IO driver..." + cd "$DRIVER_DIR" + insmod gsp-vgpu.ko firmware_name=gsp-570.144.fwimage.bin 2>&1 || { + log_error "Failed to load driver" + dmesg -T | tail -30 + return 1 + } + + # Check if device created + sleep 2 + if ls /dev/gsp-vgpu* 2>/dev/null; then + log_info "Driver loaded, device created" + return 0 + else + log_warn "Driver loaded but no device node found" + dmesg -T | tail -20 + fi +} + +# Unload driver +unload_driver() { + log_info "Unloading vGPU-IO driver..." + rmmod gsp-vgpu 2>/dev/null || true + sleep 1 +} + +# Run VF creation test +test_vf_creation() { + log_info "Testing VF creation via debugfs..." + + # Check debugfs + if [[ -d /sys/kernel/debug/gsp-vgpu ]]; then + log_info "Debugfs interface available" + cat /sys/kernel/debug/gsp-vgpu/vfs + else + log_warn "Debugfs not mounted or driver not loaded" + return 1 + fi +} + +# Run RPC test via debugfs +test_rpc_stats() { + log_info "Checking RPC statistics..." + if [[ -f /sys/kernel/debug/gsp-vgpu/rpc ]]; then + cat /sys/kernel/debug/gsp-vgpu/rpc + fi +} + +# Main test runner +main() { + local test_type="${1:-all}" + + check_root + + case "$test_type" in + "unit") + build_test_module + run_test_module + unload_test_module + ;; + "driver") + build_driver + load_driver + test_vf_creation + test_rpc_stats + unload_driver + ;; + "all"|*) + log_info "Running ALL tests..." + build_test_module + run_test_module + unload_test_module + + build_driver + load_driver + test_vf_creation + test_rpc_stats + unload_driver + ;; + esac + + log_info "Test run completed" +} + +main "$@" \ No newline at end of file -- cgit v1.2.3