diff options
Diffstat (limited to 'tools')
48 files changed, 5647 insertions, 0 deletions
diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..f9e0018 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,158 @@ +# Open-IOV Development Tools + +Collection of scripts and utilities for vGPU development, debugging, and testing. + +## Directory Structure + +``` +tools/ +├── module_ops/ # Kernel module load/unload/reload utilities +├── debug/ # Debugging and diagnostic tools +├── test/ # Test runners and validation scripts +├── vm/ # VM/QEMU management for vGPU testing +├── firmware/ # GSP firmware management +└── monitoring/ # Real-time monitoring utilities +``` + +## Quick Start + +```bash +# Load modules in converged mode (host + vGPU) +sudo ./module_ops/load_converged.sh + +# Unload all modules cleanly +sudo ./module_ops/unload_all.sh + +# Run dry-run validation +sudo ./module_ops/dry_run.sh + +# Monitor kernel logs in real-time +./monitoring/watch_dmesg.sh + +# Verify vGPU sysfs interface +./debug/verify_vgpu_sysfs.sh +``` + +## Module Operations + +### `module_ops/load_converged.sh` +Loads the complete vGPU stack in converged mode: +1. `mdev` core module +2. `nvidia.ko` (patched open-gpu-kernel-modules with NV_VGPU_KVM_BUILD) +3. `nvidia-modeset.ko`, `nvidia-uvm.ko`, `nvidia-drm.ko` +4. `nvidia-vgpu-vfio.ko` (open-iov kernel module) + +### `module_ops/unload_all.sh` +Cleanly unloads all modules in reverse order with proper cleanup. + +### `module_ops/dry_run.sh` +Runs nvidia-vgpu-vfio.ko in dry-run mode to validate firmware, BAR layout, IRQ caps, IOMMU without hardware changes. + +### `module_ops/reload_vgpu.sh` +Reloads only the vGPU kernel module (nvidia-vgpu-vfio.ko) while keeping host nvidia.ko loaded. + +## Debug Tools + +### `debug/verify_vgpu_sysfs.sh` +Verifies mdev parent registration and vGPU types in sysfs. + +### `debug/check_module_state.sh` +Checks module initstate, refcnt, and holder information. + +### `debug/dump_vgpu_regs.sh` +Dumps vGPU device registers via debugfs (if available). + +### `debug/capture_crash_logs.sh` +Captures kernel logs, dmesg, and module state after a crash. + +## Test Scripts + +### `test/run_integration_tests.sh` +Runs the integration test suite (host_init, guest_init). + +### `test/test_mdev_create.sh` +Creates an mdev device and verifies basic functionality. + +### `test/benchmark_vgpu.sh` +Runs performance benchmarks on vGPU. + +## VM Management + +### `vm/create_test_vm.sh` +Creates a test VM with vGPU passthrough using QEMU. + +### `vm/launch_guest_driver.sh` +Launches guest with NVIDIA driver installed for validation. + +## Firmware Tools + +### `firmware/verify_gsp_firmware.sh` +Verifies GSP firmware files exist and are correct version. + +### `firmware/install_tu104_firmware.sh` +Installs TU104 GSP firmware to /lib/firmware/nvidia/tu104/gsp/ + +## Monitoring + +### `monitoring/watch_dmesg.sh` +Real-time kernel log monitoring with filtering for vGPU/NVIDIA messages. + +### `monitoring/watch_module_params.sh` +Monitors module parameters and sysfs attributes. + +## Environment Variables + +```bash +export OPEN_IOV_ROOT=$OPEN_IOV_ROOT +export NVIDIA_OPEN_KERNEL=$OPEN_IOV_ROOT/nvidia-open-gpu-kernel-modules/kernel-open +export VGPU_KERNEL=$OPEN_IOV_ROOT/kernel +export VGPU_USERSPACE=$OPEN_IOV_ROOT/userspace +``` + +## Module Parameters for Debugging + +| Parameter | Description | Values | +|-----------|-------------|--------| +| `NVreg_RmMsg` | RM debug logging | `":"` (verbose), `"0"` (none) | +| `NVreg_RegisterPCIDriver` | PCI driver registration | `0` (disable), `1` (enable) | +| `NVreg_GpuInitOnProbe` | GPU init at probe | `0` (defer), `1` (immediate) | +| `NVreg_PreserveVideoMemoryAllocations` | Video memory preservation | `0` (disable), `1` (enable), `2` (auto) | +| `NVreg_UseKernelSuspendNotifiers` | Suspend notifiers | `0` (disable), `1` (enable) | +| `dyndbg` | Dynamic debug | `'+p'` (all), `'+pf'` (probe+fail) | + +## Converged Mode Architecture + +``` +Host nvidia.ko (patched) nvidia-vgpu-vfio.ko (open-iov) +├── mdev_register_parent() ←── mdev_register_driver() only +├── EXPORT nvidia_vgpu_vfio_set_ops +├── EXPORT nvidia_vgpu_vfio_get_ops +└── Implements RM vGPU ops Calls RM via get_ops, provides VFIO ops via set_ops +``` + +## Troubleshooting + +### Module stuck in "going" state +```bash +cat /sys/module/nvidia-vgpu-vfio/initstate +cat /sys/module/nvidia-vgpu-vfio/refcnt +# Requires reboot to clear +``` + +### mdev parent not registered +```bash +ls /sys/bus/pci/devices/0000:01:00.0/mdev_supported_types/ +# Should show nvidia-vgpu-* directories with create/available_instances +``` + +### Symbol version mismatch +```bash +# Rebuild open-gpu-kernel-modules with VGX_KVM_BUILD=1 +cd $NVIDIA_OPEN_KERNEL && make clean && make -j$(nproc) VGX_KVM_BUILD=1 +``` + +### IOMMU/VT-d not enabled +```bash +cat /proc/cmdline | grep -E "iommu|intel_iommu" +# Add: intel_iommu=on iommu=pt to kernel cmdline +```
\ No newline at end of file diff --git a/tools/benchmark_infrastructure.sh b/tools/benchmark_infrastructure.sh new file mode 100755 index 0000000..606aa10 --- /dev/null +++ b/tools/benchmark_infrastructure.sh @@ -0,0 +1,217 @@ +#!/bin/bash +# benchmark_infrastructure.sh - Benchmark runner for VM vs bare metal comparison + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BASE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +RESULTS_DIR="/tmp/gsp_emu_benchmarks/$(date +%Y%m%d_%H%M%S)" +mkdir -p "$RESULTS_DIR" + +log_info() { echo -e "\033[0;34m[INFO]\033[0m $1"; } +log_ok() { echo -e "\033[0;32m[OK]\033[0m $1"; } +log_warn() { echo -e "\033[1;33m[WARN]\033[0m $1"; } + +run_bare_metal() { + local test_name=$1 + shift + log_info "Running $test_name on bare metal..." + local output="$RESULTS_DIR/bare_metal_${test_name}.json" + "$@" --benchmark --output "$output" 2>&1 | tee "$RESULTS_DIR/bare_metal_${test_name}.log" +} + +run_in_vm() { + local test_name=$1 + shift + log_info "Running $test_name in VM..." + local output="$RESULTS_DIR/vm_${test_name}.json" + # This would run inside the guest via guest_test_agent + # For now, placeholder + echo '{"test": "'$test_name'", "status": "not_implemented"}' > "$output" +} + +compare_results() { + local test_name=$1 + local bare_file="$RESULTS_DIR/bare_metal_${test_name}.json" + local vm_file="$RESULTS_DIR/vm_${test_name}.json" + + if [[ -f "$bare_file" && -f "$vm_file" ]]; then + python3 -c " +import json, sys +with open('$bare_file') as f: bare = json.load(f) +with open('$vm_file') as f: vm = json.load(f) +print(f'Test: $test_name') +for key in bare: + if key in vm and isinstance(bare[key], (int, float)) and isinstance(vm[key], (int, float)): + ratio = vm[key] / bare[key] * 100 + print(f' {key}: bare={bare[key]:.2f} vm={vm[key]:.2f} ratio={ratio:.1f}%') +" + fi +} + +# CUDA benchmarks +benchmark_cuda() { + log_info "=== CUDA Benchmarks ===" + + # vectorAdd + run_bare_metal "vectorAdd" /usr/local/cuda/samples/1_Utilities/vectorAdd/vectorAdd + run_in_vm "vectorAdd" + compare_results "vectorAdd" + + # matrixMul + run_bare_metal "matrixMul" /usr/local/cuda/samples/0_Simple/matrixMul/matrixMul + run_in_vm "matrixMul" + compare_results "matrixMul" + + # deviceQuery (bandwidth) + run_bare_metal "deviceQuery" /usr/local/cuda/samples/1_Utilities/deviceQuery/deviceQuery + run_in_vm "deviceQuery" + compare_results "deviceQuery" + + # bandwidthTest + run_bare_metal "bandwidthTest" /usr/local/cuda/samples/1_Utilities/bandwidthTest/bandwidthTest + run_in_vm "bandwidthTest" + compare_results "bandwidthTest" +} + +# cuBLAS benchmarks +benchmark_cublas() { + log_info "=== cuBLAS Benchmarks ===" + run_bare_metal "cublas_sgemm" /usr/local/cuda/samples/7_CUDALibraries/simpleCUBLAS/simpleCUBLAS + run_in_vm "cublas_sgemm" + compare_results "cublas_sgemm" +} + +# Graphics benchmarks +benchmark_graphics() { + log_info "=== Graphics Benchmarks ===" + + # vkcube (frame time) + run_bare_metal "vkcube" vkcube --duration 10 --benchmark + run_in_vm "vkcube" + compare_results "vkcube" + + # glmark2 + run_bare_metal "glmark2" glmark2 --benchmark + run_in_vm "glmark2" + compare_results "glmark2" +} + +# Video benchmarks +benchmark_video() { + log_info "=== Video Benchmarks ===" + + # NVENC encode + run_bare_metal "nvenc_encode" ffmpeg -i test_1080p.yuv -c:v h264_nvenc -f null - + run_in_vm "nvenc_encode" + compare_results "nvenc_encode" + + # NVDEC decode + run_bare_metal "nvdec_decode" ffmpeg -i test_1080p.h264 -c:v h264_nvdec -f null - + run_in_vm "nvdec_decode" + compare_results "nvdec_decode" +} + +# PyTorch benchmarks +benchmark_pytorch() { + log_info "=== PyTorch Benchmarks ===" + + cat > /tmp/bench_pytorch.py <<'EOF' +import torch, time, json, sys +device = 'cuda' if torch.cuda.is_available() else 'cpu' +results = {} + +# Matmul benchmark +for size in [512, 1024, 2048, 4096]: + x = torch.randn(size, size, device=device) + y = torch.randn(size, size, device=device) + torch.cuda.synchronize() if device == 'cuda' else None + start = time.time() + for _ in range(10): + z = torch.mm(x, y) + torch.cuda.synchronize() if device == 'cuda' else None + elapsed = (time.time() - start) / 10 + gflops = 2 * size**3 / elapsed / 1e9 + results[f'matmul_{size}'] = {'time': elapsed, 'gflops': gflops} + print(f"Matmul {size}x{size}: {elapsed:.4f}s ({gflops:.1f} GFLOPS)") + +# ResNet-50 forward +import torchvision.models as models +model = models.resnet50().to(device) +model.eval() +x = torch.randn(32, 3, 224, 224).to(device) +torch.cuda.synchronize() if device == 'cuda' else None +start = time.time() +for _ in range(10): + with torch.no_grad(): + _ = model(x) +torch.cuda.synchronize() if device == 'cuda' else None +elapsed = (time.time() - start) / 10 +results['resnet50_forward'] = {'time': elapsed, 'fps': 32/elapsed} +print(f"ResNet-50 forward: {elapsed:.4f}s ({32/elapsed:.1f} FPS)") + +with open(sys.argv[1], 'w') as f: + json.dump(results, f, indent=2) +EOF + + run_bare_metal "pytorch" python3 /tmp/bench_pytorch.py + run_in_vm "pytorch" + compare_results "pytorch" +} + +# Generate summary report +generate_summary() { + log_info "Generating summary report..." + cat > "$RESULTS_DIR/summary.md" <<EOF +# GSP Emulation Benchmark Summary +Date: $(date) +Host: $(hostname) +Kernel: $(uname -r) + +## Results + +| Test | Bare Metal | VM | Ratio | Status | +|------|------------|-----|-------|--------| +EOF + + for f in "$RESULTS_DIR"/bare_metal_*.json; do + test_name=$(basename "$f" .json | sed 's/bare_metal_//') + vm_f="$RESULTS_DIR/vm_${test_name}.json" + if [[ -f "$vm_f" ]]; then + python3 -c " +import json +with open('$f') as bf: bare = json.load(bf) +with open('$vm_f') as vf: vm = json.load(vf) +for k in bare: + if k in vm and isinstance(bare[k], (int, float)): + ratio = vm[k] / bare[k] * 100 + status = '✅' if ratio > 85 else '⚠️' if ratio > 70 else '❌' + print(f'| {test_name} / {k} | {bare[k]:.2f} | {vm[k]:.2f} | {ratio:.1f}% | {status} |') +" >> "$RESULTS_DIR/summary.md" + fi + done + + log_ok "Summary: $RESULTS_DIR/summary.md" +} + +main() { + case "${1:-all}" in + cuda) benchmark_cuda ;; + cublas) benchmark_cublas ;; + graphics) benchmark_graphics ;; + video) benchmark_video ;; + pytorch) benchmark_pytorch ;; + all) + benchmark_cuda + benchmark_cublas + benchmark_graphics + benchmark_video + benchmark_pytorch + ;; + *) echo "Usage: $0 {cuda|cublas|graphics|video|pytorch|all}"; exit 1 ;; + esac + generate_summary +} + +main "$@"
\ No newline at end of file diff --git a/tools/debug/nv2080_ctrl.c b/tools/debug/nv2080_ctrl.c new file mode 100644 index 0000000..46ade02 --- /dev/null +++ b/tools/debug/nv2080_ctrl.c @@ -0,0 +1,531 @@ +/* + * NV2080 Control Command Sender + * C program to send NV2080 RM control commands via ioctl + * Compile: gcc -o nv2080_ctrl nv2080_ctrl.c + * Usage: sudo ./nv2080_ctrl /dev/nvidia0 <command> [args...] + */ + +#define _GNU_SOURCE +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <string.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/ioctl.h> +#include <errno.h> + +// NV_IOCTL definitions from nv-ioctl.h +#define NV_IOCTL_MAGIC 0x46 +#define NV_ESC_RM_CONTROL 0x54 +#define NV_ESC_CHECK_VERSION_STR 0x32 +#define NV_ESC_RM_ALLOC 0x21 +#define NV_ESC_RM_FREE 0x00 +#define NV_ESC_ALLOC_OS_EVENT 0x24 + +// IOCTL macros +#define _IOC_NRBITS 8 +#define _IOC_TYPEBITS 8 +#define _IOC_SIZEBITS 14 +#define _IOC_DIRBITS 2 + +#define _IOC_NRSHIFT 0 +#define _IOC_TYPESHIFT (_IOC_NRSHIFT + _IOC_NRBITS) +#define _IOC_SIZESHIFT (_IOC_TYPESHIFT + _IOC_TYPEBITS) +#define _IOC_DIRSHIFT (_IOC_SIZESHIFT + _IOC_SIZEBITS) + +#define _IOC_NONE 0U +#define _IOC_WRITE 1U +#define _IOC_READ 2U + +#define _IOC(dir, type, nr, size) \ + (((dir) << _IOC_DIRSHIFT) | \ + ((type) << _IOC_TYPESHIFT) | \ + ((nr) << _IOC_NRSHIFT) | \ + ((size) << _IOC_SIZESHIFT)) + +#define _IOWR(type, nr, size) _IOC(_IOC_READ | _IOC_WRITE, type, nr, size) + +// NV_CTRL_RES = _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS) +#define NV_CTRL_RES _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, 128) + +// NV2080 Control Commands (from ctrl2080unix.h and ctrl2080vgpumgrinternal.h) +#define NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT 0x20803d01 +#define NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF 0x20803d02 +#define NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER 0x20803d03 +#define NV2080_CTRL_CMD_OS_UNIX_INSTALL_PROFILER_HOOKS 0x20803d04 +#define NV2080_CTRL_CMD_OS_UNIX_FLUSH_SNAPSHOT_BUFFER 0x20803d05 +#define NV2080_CTRL_CMD_OS_UNIX_STOP_PROFILER 0x20803d06 +#define NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS 0x20803d07 +#define NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS 0x20803d08 + +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK 0x20804001 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK 0x20804002 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE 0x20804003 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU 0x20804004 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO 0x20804005 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE 0x20804006 +#define NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY 0x20804007 +#define NV2080_CTRL_CMD_INTERNAL_VGPU_PLUGIN_CLEANUP 0x20804008 + +// Action values +#define NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC 1 +#define NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC 2 +#define NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW 1 +#define NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW 2 + +// NVOS54_PARAMETERS structure (from nv-ioctl.h) +typedef struct { + uint32_t hClient; + uint32_t hObject; + uint32_t cmd; + uint32_t flags; + void *params; + uint32_t paramsSize; + uint32_t status; +} NVOS54_PARAMETERS; + +// NVOS21_PARAMETERS for RM_ALLOC +typedef struct { + uint32_t hRoot; + uint32_t hObjectParent; + uint32_t hObjectNew; + uint32_t hClass; + void *pAllocParms; + uint32_t status; +} NVOS21_PARAMETERS; + +// NVOS00_PARAMETERS for RM_FREE +typedef struct { + uint32_t hRoot; + uint32_t hObjectParent; + uint32_t hObjectOld; + uint32_t status; +} NVOS00_PARAMETERS; + +// Version check structure +typedef struct { + uint32_t cmd; + char versionString[256]; + uint32_t reply; +} nv_ioctl_rm_api_version_t; + +// OS Event allocation +typedef struct { + uint32_t hClient; + uint32_t hDevice; + int fd; + uint32_t Status; +} nv_ioctl_alloc_os_event_t; + +// GC6 Blocker Refcnt Params +typedef struct { + uint32_t action; +} NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS; + +// Allow/Disallow GCOFF Params +typedef struct { + uint32_t action; +} NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS; + +// Audio Dynamic Power Params +typedef struct { + uint32_t bEnter; +} NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS; + +// Vidmem Persistence Status Params +typedef struct { + uint32_t bVidmemPersistent; +} NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS; + +// TGP Status Params +typedef struct { + uint32_t bUpdateTGP; +} NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS; + +// VGPU Bootload Params (simplified) +typedef struct { + uint32_t dbdf; + uint32_t gfid; + uint32_t vgpuType; + uint32_t vmPid; + uint32_t swizzId; + uint32_t numChannels; + uint32_t numPluginChannels; + uint32_t bDisableDefaultSmcExecPartRestore; + uint32_t numGuestFbSegments; + uint64_t guestFbPhysAddrList[384]; + uint64_t guestFbLengthList[384]; + uint64_t pluginHeapMemoryPhysAddr; + uint64_t pluginHeapMemoryLength; + uint64_t ctrlBuffOffset; +} NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS; + +// VGPU Shutdown Params +typedef struct { + uint32_t gfid; +} NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS; + +// VGPU Add Type Params +typedef struct { + uint32_t discardVgpuTypes; + uint64_t vgpuInfo; // NVA081_CTRL_VGPU_INFO +} NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS; + +// VGPU Enumerate Params +typedef struct { + uint32_t numVgpu; + uint64_t vgpuGuest[32]; // NV2080_VGPU_GUEST array +} NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS; + +// VGPU Clear Guest Params +typedef struct { + uint32_t gfid; +} NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS; + +// VGPU FB Usage Params +typedef struct { + uint32_t vgpuCount; + struct { + uint32_t gfid; + uint64_t fbUsed; + } vgpuFbUsage[32]; +} NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS; + +// VGPU Encoder Capacity Params +typedef struct { + uint32_t gfid; + uint32_t encoderCapacity; +} NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS; + +// VGPU Plugin Cleanup Params +typedef struct { + uint32_t gfid; +} NV2080_CTRL_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS; + +// Send RM Control command +int send_rm_control(int fd, uint32_t client, uint32_t object, uint32_t cmd, void *params, uint32_t params_size) { + NVOS54_PARAMETERS ctrl = { + .hClient = client, + .hObject = object, + .cmd = cmd, + .flags = 0, + .params = params, + .paramsSize = params_size, + .status = 0 + }; + + int ret = ioctl(fd, NV_CTRL_RES, &ctrl); + if (ret == -1) { + perror("ioctl NV_CTRL_RES failed"); + return -1; + } + + if (ctrl.status != 0) { + fprintf(stderr, "RM Control failed: cmd=0x%x, status=0x%x\n", cmd, ctrl.status); + return ctrl.status; + } + + return 0; +} + +// Version check +int version_check(int fd, const char *version) { + nv_ioctl_rm_api_version_t vc = {0}; + vc.cmd = 0x32; // NV_ESC_CHECK_VERSION_STR + if (version) { + strncpy(vc.versionString, version, sizeof(vc.versionString) - 1); + } + + int ret = ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_CHECK_VERSION_STR, sizeof(vc)), &vc); + if (ret == -1) { + perror("ioctl version check failed"); + return -1; + } + + printf("Version check: reply=%u, version=%s\n", vc.reply, vc.versionString); + return vc.reply == 1 ? 0 : -1; +} + +// Allocate RM resource +int rm_alloc(int fd, uint32_t root, uint32_t parent, uint32_t object, uint32_t class, void *data) { + NVOS21_PARAMETERS alloc = { + .hRoot = root, + .hObjectParent = parent, + .hObjectNew = object, + .hClass = class, + .pAllocParms = data, + .status = 0 + }; + + int ret = ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, sizeof(alloc)), &alloc); + if (ret == -1) { + perror("ioctl RM_ALLOC failed"); + return -1; + } + + if (alloc.status != 0) { + fprintf(stderr, "RM_ALLOC failed: status=0x%x\n", alloc.status); + return alloc.status; + } + + printf("Allocated object 0x%x (class 0x%x)\n", alloc.hObjectNew, class); + return alloc.hObjectNew; +} + +// Free RM resource +int rm_free(int fd, uint32_t root, uint32_t parent, uint32_t object) { + NVOS00_PARAMETERS free_params = { + .hRoot = root, + .hObjectParent = parent, + .hObjectOld = object, + .status = 0 + }; + + int ret = ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_FREE, sizeof(free_params)), &free_params); + if (ret == -1) { + perror("ioctl RM_FREE failed"); + return -1; + } + + return free_params.status == 0 ? 0 : free_params.status; +} + +// Print usage +void print_usage(const char *prog) { + fprintf(stderr, "Usage: %s <device> <command> [args...]\n", prog); + fprintf(stderr, "\nDevice: /dev/nvidia0, /dev/nvidiactl, etc.\n"); + fprintf(stderr, "\nCommands:\n"); + fprintf(stderr, " version-check [version] - Check RM API version\n"); + fprintf(stderr, " gc6-inc <client> <object> - Increment GC6 blocker refcount\n"); + fprintf(stderr, " gc6-dec <client> <object> - Decrement GC6 blocker refcount\n"); + fprintf(stderr, " gcoff-allow <client> <object> - Allow GCOFF\n"); + fprintf(stderr, " gcoff-disallow <client> <object> - Disallow GCOFF\n"); + fprintf(stderr, " audio-power <client> <object> <enter> - Audio dynamic power (1=enter, 0=exit)\n"); + fprintf(stderr, " vidmem-persistence <client> <object> - Query vidmem persistence\n"); + fprintf(stderr, " tgp-status <client> <object> <update> - Update TGP status (1=update, 0=don't)\n"); + fprintf(stderr, " vgpu-bootload <client> <object> <dbdf> <gfid> <vgpuType> <vmPid> ... - Bootload GSP VGPU plugin\n"); + fprintf(stderr, " vgpu-shutdown <client> <object> <gfid> - Shutdown GSP VGPU plugin\n"); + fprintf(stderr, " vgpu-add-type <client> <object> <discard> <vgpuInfo> - Add vGPU type\n"); + fprintf(stderr, " vgpu-enumerate <client> <object> - Enumerate vGPUs per pGPU\n"); + fprintf(stderr, " vgpu-clear-guest <client> <object> <gfid> - Clear guest VM info\n"); + fprintf(stderr, " vgpu-fb-usage <client> <object> - Get vGPU FB usage\n"); + fprintf(stderr, " vgpu-encoder-cap <client> <object> <gfid> <capacity> - Set encoder capacity (0-100)\n"); + fprintf(stderr, " vgpu-plugin-cleanup <client> <object> <gfid> - Cleanup VGPU plugin\n"); + fprintf(stderr, " alloc <root> <parent> <object> <class> - Allocate RM resource\n"); + fprintf(stderr, " free <root> <parent> <object> - Free RM resource\n"); +} + +int main(int argc, char *argv[]) { + if (argc < 3) { + print_usage(argv[0]); + return 1; + } + + const char *device = argv[1]; + const char *command = argv[2]; + + int fd = open(device, O_RDWR); + if (fd < 0) { + perror("Failed to open device"); + return 1; + } + + int ret = 0; + + if (strcmp(command, "version-check") == 0) { + const char *version = argc > 3 ? argv[3] : NULL; + ret = version_check(fd, version); + } + else if (strcmp(command, "gc6-inc") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "gc6-dec") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "gcoff-allow") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "gcoff-disallow") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_PARAMS params = { .action = NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "audio-power") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t enter = strtoul(argv[5], NULL, 0); + NV2080_CTRL_OS_UNIX_AUDIO_DYNAMIC_POWER_PARAMS params = { .bEnter = enter }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vidmem-persistence") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_OS_UNIX_VIDMEM_PERSISTENCE_STATUS_PARAMS params = {0}; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS, ¶ms, sizeof(params)); + if (ret == 0) { + printf("Vidmem persistent: %s\n", params.bVidmemPersistent ? "yes" : "no"); + } + } + } + else if (strcmp(command, "tgp-status") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t update = strtoul(argv[5], NULL, 0); + NV2080_CTRL_OS_UNIX_UPDATE_TGP_STATUS_PARAMS params = { .bUpdateTGP = update }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-bootload") == 0) { + if (argc < 12) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK_PARAMS params = {0}; + params.dbdf = strtoul(argv[5], NULL, 0); + params.gfid = strtoul(argv[6], NULL, 0); + params.vgpuType = strtoul(argv[7], NULL, 0); + params.vmPid = strtoul(argv[8], NULL, 0); + params.swizzId = strtoul(argv[9], NULL, 0); + params.numChannels = strtoul(argv[10], NULL, 0); + params.numPluginChannels = strtoul(argv[11], NULL, 0); + // Additional params would need more args + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-shutdown") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t gfid = strtoul(argv[5], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK_PARAMS params = { .gfid = gfid }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-add-type") == 0) { + if (argc < 7) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t discard = strtoul(argv[5], NULL, 0); + uint64_t vgpuInfo = strtoull(argv[6], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE_PARAMS params = { .discardVgpuTypes = discard, .vgpuInfo = vgpuInfo }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-enumerate") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU_PARAMS params = {0}; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU, ¶ms, sizeof(params)); + if (ret == 0) { + printf("Enumerated %u vGPUs\n", params.numVgpu); + } + } + } + else if (strcmp(command, "vgpu-clear-guest") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t gfid = strtoul(argv[5], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO_PARAMS params = { .gfid = gfid }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-fb-usage") == 0) { + if (argc < 5) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE_PARAMS params = {0}; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE, ¶ms, sizeof(params)); + if (ret == 0) { + printf("vGPU count: %u\n", params.vgpuCount); + for (uint32_t i = 0; i < params.vgpuCount; i++) { + printf(" GFID 0x%x: FB used %lu bytes\n", params.vgpuFbUsage[i].gfid, params.vgpuFbUsage[i].fbUsed); + } + } + } + } + else if (strcmp(command, "vgpu-encoder-cap") == 0) { + if (argc < 7) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t gfid = strtoul(argv[5], NULL, 0); + uint32_t capacity = strtoul(argv[6], NULL, 0); + NV2080_CTRL_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY_PARAMS params = { .gfid = gfid, .encoderCapacity = capacity }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "vgpu-plugin-cleanup") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t client = strtoul(argv[3], NULL, 0); + uint32_t object = strtoul(argv[4], NULL, 0); + uint32_t gfid = strtoul(argv[5], NULL, 0); + NV2080_CTRL_INTERNAL_VGPU_PLUGIN_CLEANUP_PARAMS params = { .gfid = gfid }; + ret = send_rm_control(fd, client, object, NV2080_CTRL_CMD_INTERNAL_VGPU_PLUGIN_CLEANUP, ¶ms, sizeof(params)); + } + } + else if (strcmp(command, "alloc") == 0) { + if (argc < 7) { print_usage(argv[0]); ret = 1; } + else { + uint32_t root = strtoul(argv[3], NULL, 0); + uint32_t parent = strtoul(argv[4], NULL, 0); + uint32_t object = strtoul(argv[5], NULL, 0); + uint32_t class = strtoul(argv[6], NULL, 0); + ret = rm_alloc(fd, root, parent, object, class, NULL); + } + } + else if (strcmp(command, "free") == 0) { + if (argc < 6) { print_usage(argv[0]); ret = 1; } + else { + uint32_t root = strtoul(argv[3], NULL, 0); + uint32_t parent = strtoul(argv[4], NULL, 0); + uint32_t object = strtoul(argv[5], NULL, 0); + ret = rm_free(fd, root, parent, object); + } + } + else { + fprintf(stderr, "Unknown command: %s\n", command); + print_usage(argv[0]); + ret = 1; + } + + close(fd); + return ret; +}
\ No newline at end of file diff --git a/tools/debug/nv2080_ctrl.sh b/tools/debug/nv2080_ctrl.sh new file mode 100755 index 0000000..22c7117 --- /dev/null +++ b/tools/debug/nv2080_ctrl.sh @@ -0,0 +1,168 @@ +#!/bin/bash +# NV2080 Control Command Tool +# Sends NV2080 RM control commands to the NVIDIA kernel module via ioctl +# Usage: ./nv2080_ctrl.sh <device_fd> <command> [params...] + +set -euo pipefail + +# NV2080 Control Command Definitions (from ctrl2080unix.h and ctrl2080vgpumgrinternal.h) +# These are the same commands used by GVM userspace via NV_CTRL_RES ioctl + +# NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT (0x20803d01) +# Params: action (INC=1, DEC=2) +NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT=0x20803d01 +NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC=1 +NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC=2 + +# NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF (0x20803d02) +# Params: action (ALLOW=1, DISALLOW=2) +NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF=0x20803d02 +NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW=1 +NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW=2 + +# NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER (0x20803d03) +# Params: bEnter (true/false) +NV2080_CTRL_CMD_OS_UNIX_AUDIO_DYNAMIC_POWER=0x20803d03 + +# NV2080_CTRL_CMD_OS_UNIX_INSTALL_PROFILER_HOOKS (0x20803d04) +NV2080_CTRL_CMD_OS_UNIX_INSTALL_PROFILER_HOOKS=0x20803d04 + +# NV2080_CTRL_CMD_OS_UNIX_FLUSH_SNAPSHOT_BUFFER (0x20803d05) +NV2080_CTRL_CMD_OS_UNIX_FLUSH_SNAPSHOT_BUFFER=0x20803d05 + +# NV2080_CTRL_CMD_OS_UNIX_STOP_PROFILER (0x20803d06) +NV2080_CTRL_CMD_OS_UNIX_STOP_PROFILER=0x20803d06 + +# NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS (0x20803d07) +NV2080_CTRL_CMD_OS_UNIX_VIDMEM_PERSISTENCE_STATUS=0x20803d07 + +# NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS (0x20803d08) +NV2080_CTRL_CMD_OS_UNIX_UPDATE_TGP_STATUS=0x20803d08 + +# VGPU Manager Internal Commands +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_BOOTLOAD_GSP_VGPU_PLUGIN_TASK=0x20804001 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SHUTDOWN_GSP_VGPU_PLUGIN_TASK=0x20804002 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_PGPU_ADD_VGPU_TYPE=0x20804003 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_ENUMERATE_VGPU_PER_PGPU=0x20804004 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_CLEAR_GUEST_VM_INFO=0x20804005 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_GET_VGPU_FB_USAGE=0x20804006 +NV2080_CTRL_CMD_VGPU_MGR_INTERNAL_SET_VGPU_ENCODER_CAPACITY=0x20804007 +NV2080_CTRL_CMD_INTERNAL_VGPU_PLUGIN_CLEANUP=0x20804008 + +# NV_IOCTL_MAGIC and NV_ESC_RM_CONTROL +NV_IOCTL_MAGIC=0x46 +NV_ESC_RM_CONTROL=0x54 + +# NV_CTRL_RES = _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS) +NV_CTRL_RES=$(printf "0x%x" $(( (2 << 30) | (NV_IOCTL_MAGIC << 8) | (NV_ESC_RM_CONTROL) | (128 << 16) ))) + +# Helper to send ioctl +send_ioctl() { + local fd=$1 + local cmd=$2 + local data=$3 + local size=$4 + + # This would need a C program to actually send the ioctl + # For now, we document the commands + echo "Command: 0x$(printf "%x" $cmd)" + echo "Data size: $size" +} + +# Main command dispatcher +case "${1:-help}" in + gc6-inc) + echo "Increment GC6 blocker refcount" + echo "ioctl(fd, NV_CTRL_RES, {cmd: $NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT, action: $NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_INC})" + ;; + gc6-dec) + echo "Decrement GC6 blocker refcount" + echo "ioctl(fd, NV_CTRL_RES, {cmd: $NV2080_CTRL_CMD_OS_UNIX_GC6_BLOCKER_REFCNT, action: $NV2080_CTRL_OS_UNIX_GC6_BLOCKER_REFCNT_DEC})" + ;; + gcoff-allow) + echo "Allow GCOFF" + echo "ioctl(fd, NV_CTRL_RES, {cmd: $NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF, action: $NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_ALLOW})" + ;; + gcoff-disallow) + echo "Disallow GCOFF" + echo "ioctl(fd, NV_CTRL_RES, {cmd: $NV2080_CTRL_CMD_OS_UNIX_ALLOW_DISALLOW_GCOFF, action: $NV2080_CTRL_OS_UNIX_ALLOW_DISALLOW_GCOFF_DISALLOW})" + ;; + audio-power) + echo "Audio dynamic power: $2" + ;; + profiler-install) + echo "Install profiler hooks" + ;; + profiler-flush) + echo "Flush snapshot buffer" + ;; + profiler-stop) + echo "Stop profiler" + ;; + vidmem-persistence) + echo "Query vidmem persistence status" + ;; + tgp-status) + echo "Update TGP status: $2" + ;; + vgpu-bootload) + echo "Bootload GSP VGPU plugin task" + echo "Params: dbdf, gfid, vgpuType, vmPid, swizzId, numChannels, numPluginChannels, bDisableSmcPartitionRestore, guestFbPhysAddrList, guestFbLengthList, pluginHeapMemoryPhysAddr, pluginHeapMemoryLength, ctrlBuffOffset" + ;; + vgpu-shutdown) + echo "Shutdown GSP VGPU plugin task" + echo "Params: gfid" + ;; + vgpu-add-type) + echo "Add vGPU type to pGPU" + echo "Params: discardVgpuTypes, vgpuInfo" + ;; + vgpu-enumerate) + echo "Enumerate vGPUs per pGPU" + ;; + vgpu-clear-guest) + echo "Clear guest VM info" + echo "Params: gfid" + ;; + vgpu-fb-usage) + echo "Get vGPU FB usage" + ;; + vgpu-encoder-cap) + echo "Set vGPU encoder capacity" + echo "Params: gfid, encoderCapacity (0-100)" + ;; + vgpu-plugin-cleanup) + echo "Cleanup VGPU plugin" + echo "Params: gfid" + ;; + list) + echo "Available NV2080 Control Commands:" + echo " OS Unix Commands:" + echo " gc6-inc - Increment GC6 blocker refcount" + echo " gc6-dec - Decrement GC6 blocker refcount" + echo " gcoff-allow - Allow GCOFF" + echo " gcoff-disallow - Disallow GCOFF" + echo " audio-power - Audio dynamic power (enter/exit)" + echo " profiler-install - Install profiler hooks (HWPM)" + echo " profiler-flush - Flush snapshot buffer" + echo " profiler-stop - Stop profiler" + echo " vidmem-persistence- Query vidmem persistence status" + echo " tgp-status - Update TGP status" + echo "" + echo " VGPU Manager Internal Commands:" + echo " vgpu-bootload - Bootload GSP VGPU plugin task" + echo " vgpu-shutdown - Shutdown GSP VGPU plugin task" + echo " vgpu-add-type - Add vGPU type to pGPU" + echo " vgpu-enumerate - Enumerate vGPUs per pGPU" + echo " vgpu-clear-guest - Clear guest VM info" + echo " vgpu-fb-usage - Get vGPU FB usage" + echo " vgpu-encoder-cap - Set vGPU encoder capacity" + echo " vgpu-plugin-cleanup - Cleanup VGPU plugin" + ;; + help|*) + echo "NV2080 Control Command Tool" + echo "Usage: $0 <command> [args...]" + echo "" + echo "Run '$0 list' to see all available commands" + ;; +esac
\ No newline at end of file diff --git a/tools/failure_analyzer.py b/tools/failure_analyzer.py new file mode 100644 index 0000000..7ffb776 --- /dev/null +++ b/tools/failure_analyzer.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +""" +failure_analyzer.py - Analyze and classify guest initialization failures +Correlates host tracepoints, guest logs, and QEMU state +""" + +import sys +import os +import re +import json +import argparse +from datetime import datetime +from pathlib import Path +from collections import defaultdict +from typing import Dict, List, Optional, Any + +class FailureAnalyzer: + def __init__(self): + self.failure_signatures = self._load_signatures() + self.recommendations = self._load_recommendations() + + def _load_signatures(self) -> Dict: + return { + "PCI_PROBE_FAILURE": { + "guest_patterns": [ + r"nvidia: probe of .* failed with error -?\d+", + r"pci .*: can't claim BAR \d+", + r"pci .*: BAR \d+: no space for \[mem size", + r"pci .*: BAR \d+: failed to assign", + ], + "host_trace_patterns": [ + r"gsp_emu_rpc_received.*rpc_type=0x1.*status=-?\d+", + r"gsp_emu_fallback_used.*rpc_type=0x1", + ], + "severity": "critical", + "category": "pci_config" + }, + "GSP_FIRMWARE_LOAD_FAILURE": { + "guest_patterns": [ + r"GSP.*firmware.*load.*failed", + r"GSP.*bootstrap.*failed", + r"nv_gsp_firmware_load.*error", + r"GSP.*RPC.*timeout", + r"GSP.*communication.*failed", + ], + "host_trace_patterns": [ + r"gsp_emu_firmware_analysis.*matched=0", + r"gsp_emu_dispatcher_select.*emulation=1", + r"gsp_emu_rpc_received.*rpc_type=0x1.*seq_id=1", + r"gsp_emu_rpc_processed.*status=-110", + ], + "severity": "critical", + "category": "gsp_firmware" + }, + "MSIX_SETUP_FAILURE": { + "guest_patterns": [ + r"MSI-X.*failed", + r"pci_enable_msix.*failed", + r"request_threaded_irq.*failed", + r"irq.*allocation.*failed", + ], + "host_trace_patterns": [ + r"gsp_emu_msix_injected.*vector=0", + r"gsp_emu_rpc_processed.*rpc_type=0x7.*status=-?\d+", + ], + "severity": "high", + "category": "interrupts" + }, + "BAR_MAPPING_FAILURE": { + "guest_patterns": [ + r"ioremap.*failed", + r"pci_iomap.*failed", + r"BAR.*mapping.*failed", + r"VFIO.*region.*mmap.*failed", + ], + "host_trace_patterns": [ + r"gsp_emu_shared_mem_access.*is_write=1", + r"gsp_emu_bar_emulation.*", + ], + "severity": "high", + "category": "memory_mapping" + }, + "DMA_MAPPING_FAILURE": { + "guest_patterns": [ + r"dma_map.*failed", + r"IOMMU.*fault", + r"DMAR.*error", + r"iommu.*domain.*failed", + ], + "host_trace_patterns": [ + r"gsp_emu_rpc_processed.*rpc_type=0x5.*status=-?\d+", + r"gsp_emu_fallback_used.*rpc_type=0x5", + ], + "severity": "high", + "category": "dma" + }, + "VGPU_VFIO_FAILURE": { + "guest_patterns": [ + r"vGPU VFIO.*failed", + r"nvidia_vgpu_vfio_get_ops.*failed", + r"mdev.*register.*failed", + r"VFIO.*device.*not found", + ], + "host_trace_patterns": [ + r"gsp_emu_thread_state.*state=0", + ], + "severity": "critical", + "category": "vfio" + }, + "GSP_RPC_TIMEOUT": { + "guest_patterns": [ + r"GSP.*RPC.*timeout", + r"doorbell.*timeout", + r"GSP.*response.*timeout", + ], + "host_trace_patterns": [ + r"gsp_emu_doorbell_poll.*", + r"gsp_emu_rpc_received.*", + r"gsp_emu_rpc_processed.*latency_ns=\d{9,}", + ], + "severity": "medium", + "category": "rpc_timeout" + }, + } + + def _load_recommendations(self) -> Dict: + return { + "PCI_PROBE_FAILURE": [ + "Check PCI config space emulation in open-iov-vfio.ko (main.c)", + "Verify BAR sizes match physical GPU (16MB/256MB/32MB/32MB)", + "Check MSI-X capability pointer (0x70) and table/PBA offsets", + "Ensure mdev parent registered with NVIDIA host driver", + ], + "GSP_FIRMWARE_LOAD_FAILURE": [ + "Verify firmware file exists at /lib/firmware/nvidia/tu102/gsp/", + "Check firmware CRC32/SHA256 matches known signature", + "Ensure GSP emulation thread started (gsp_emu_thread_state)", + "Check doorbell register offset (BAR0+0x110C00)", + "Verify shared memory mapping (BAR0+0x1000, 4KB)", + ], + "MSIX_SETUP_FAILURE": [ + "Verify MSI-X table at BAR0+0x2000 (32 entries × 16 bytes)", + "Verify MSI-X PBA at BAR0+0x3000 (64-bit)", + "Check eventfd signaling for interrupt injection", + "Ensure guest MSI-X enable bit set in Message Control", + ], + "BAR_MAPPING_FAILURE": [ + "Check VFIO region mmap handlers (bar_emulation.c)", + "Verify BAR0 includes doorbell (0x110C00) and shared mem (0x1000)", + "Check MSI-X region (VFIO_PCI_MSIX_REGION_INDEX)", + "Ensure WC (write-combine) mapping for BAR0", + ], + "DMA_MAPPING_FAILURE": [ + "Verify IOMMU domain attached to VFIO group", + "Check DMA mapping in memory.c (nv_vgpu_dma_map)", + "Ensure physical pages pinned before mapping", + "Check for IOMMU faults in dmesg", + ], + "VGPU_VFIO_FAILURE": [ + "Verify nvidia.ko loaded before open-iov-vfio.ko", + "Check KBUILD_EXTRA_SYMBOLS points to nvidia.ko Module.symvers", + "Verify nvidia_vgpu_vfio_get_ops exported by nvidia.ko", + "Check mdev parent registration callback", + ], + "GSP_RPC_TIMEOUT": [ + "Check GSP emulation thread CPU usage (should be polling)", + "Verify doorbell polling adaptive (busy-wait 100us then usleep)", + "Check shared memory coherence (cache flush/invalidate)", + "Trace RPC sequence IDs for gaps/duplicates", + ], + } + + def analyze(self, guest_log_file: str, host_trace_file: str, qemu_log_file: str = None) -> Dict: + """Analyze failure from log files""" + + guest_logs = self._read_logs(guest_log_file) + host_traces = self._read_logs(host_trace_file) + qemu_logs = self._read_logs(qemu_log_file) if qemu_log_file else [] + + # Classify failure + failure_type = self._classify_failure(guest_logs, host_traces) + + # Get matching evidence + evidence = self._collect_evidence(failure_type, guest_logs, host_traces) + + # Generate report + report = { + "timestamp": datetime.now().isoformat(), + "failure_type": failure_type, + "severity": self.failure_signatures.get(failure_type, {}).get("severity", "unknown"), + "category": self.failure_signatures.get(failure_type, {}).get("category", "unknown"), + "evidence": evidence, + "recommendations": self.recommendations.get(failure_type, []), + "guest_log_summary": self._summarize_logs(guest_logs), + "host_trace_summary": self._summarize_logs(host_traces), + } + + return report + + def _read_logs(self, filepath: str) -> List[str]: + if not filepath or not os.path.exists(filepath): + return [] + with open(filepath, 'r') as f: + return [line.strip() for line in f.readlines()] + + def _classify_failure(self, guest_logs: List[str], host_traces: List[str]) -> str: + """Classify failure based on patterns""" + scores = defaultdict(int) + + for failure_type, sig in self.failure_signatures.items(): + # Check guest patterns + for pattern in sig.get("guest_patterns", []): + for line in guest_logs: + if re.search(pattern, line, re.IGNORECASE): + scores[failure_type] += 3 + + # Check host trace patterns + for pattern in sig.get("host_trace_patterns", []): + for line in host_traces: + if re.search(pattern, line): + scores[failure_type] += 2 + + if not scores: + return "UNKNOWN" + + # Return highest scoring failure type + return max(scores, key=scores.get) + + def _collect_evidence(self, failure_type: str, guest_logs: List[str], host_traces: List[str]) -> Dict: + """Collect matching evidence for failure type""" + sig = self.failure_signatures.get(failure_type, {}) + evidence = {"guest": [], "host": []} + + for pattern in sig.get("guest_patterns", []): + for line in guest_logs: + if re.search(pattern, line, re.IGNORECASE): + evidence["guest"].append({"pattern": pattern, "match": line}) + + for pattern in sig.get("host_trace_patterns", []): + for line in host_traces: + if re.search(pattern, line): + evidence["host"].append({"pattern": pattern, "match": line}) + + return evidence + + def _summarize_logs(self, logs: List[str]) -> Dict: + """Summarize log content""" + return { + "total_lines": len(logs), + "error_lines": sum(1 for l in logs if "error" in l.lower() or "fail" in l.lower()), + "nvidia_lines": sum(1 for l in logs if "nvidia" in l.lower()), + "gsp_lines": sum(1 for l in logs if "gsp" in l.lower()), + "last_20": logs[-20:] if logs else [], + } + + def print_report(self, report: Dict): + """Print formatted failure report""" + print("\n" + "="*70) + print(f"FAILURE ANALYSIS REPORT") + print("="*70) + print(f"Timestamp: {report['timestamp']}") + print(f"Failure Type: {report['failure_type']}") + print(f"Severity: {report['severity']}") + print(f"Category: {report['category']}") + print() + + print("EVIDENCE:") + print("-"*70) + for src in ["guest", "host"]: + ev = report['evidence'].get(src, []) + if ev: + print(f" {src.upper()} ({len(ev)} matches):") + for e in ev[:5]: + print(f" Pattern: {e['pattern']}") + print(f" Match: {e['match'][:120]}") + print() + + print("RECOMMENDATIONS:") + print("-"*70) + for i, rec in enumerate(report['recommendations'], 1): + print(f" {i}. {rec}") + print() + + print("LOG SUMMARIES:") + print("-"*70) + for src in ["guest", "host"]: + summ = report.get(f'{src}_log_summary', {}) + print(f" {src.upper()}: {summ.get('total_lines', 0)} lines, " + f"{summ.get('error_lines', 0)} errors, " + f"{summ.get('nvidia_lines', 0)} nvidia, " + f"{summ.get('gsp_lines', 0)} gsp") + print("="*70) + +def main(): + parser = argparse.ArgumentParser(description="Analyze guest NVIDIA driver initialization failures") + parser.add_argument("--guest-log", required=True, help="Guest serial log file") + parser.add_argument("--host-trace", required=True, help="Host trace_pipe log file") + parser.add_argument("--qemu-log", help="QEMU log file (optional)") + parser.add_argument("--output", help="Output JSON report file") + args = parser.parse_args() + + analyzer = FailureAnalyzer() + report = analyzer.analyze(args.guest_log, args.host_trace, args.qemu_log) + analyzer.print_report(report) + + if args.output: + with open(args.output, 'w') as f: + json.dump(report, f, indent=2) + print(f"\nReport saved to: {args.output}") + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/tools/firmware/verify_gsp_firmware.sh b/tools/firmware/verify_gsp_firmware.sh new file mode 100755 index 0000000..79f474c --- /dev/null +++ b/tools/firmware/verify_gsp_firmware.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Verify GSP firmware for TU104 +# Usage: ./verify_gsp_firmware.sh + +set -euo pipefail + +FIRMWARE_DIR="/lib/firmware/nvidia/tu104/gsp" + +echo "=== GSP Firmware Verification (TU104) ===" +echo "" + +if [[ ! -d "$FIRMWARE_DIR" ]]; then + echo "ERROR: Firmware directory not found: $FIRMWARE_DIR" + echo "Install firmware with: rzsh nvidia vgpu firmware install" + exit 1 +fi + +echo "Firmware directory: $FIRMWARE_DIR" +echo "" + +# Check required files +required_files=( + "gsp-535.113.01.bin" + "gsp-570.144.bin" + "booter_load-535.113.01.bin" + "booter_load-570.144.bin" + "booter_unload-535.113.01.bin" + "booter_unload-570.144.bin" + "bootloader-535.113.01.bin" + "bootloader-570.144.bin" +) + +all_ok=true +for file in "${required_files[@]}"; do + if [[ -f "$FIRMWARE_DIR/$file" ]]; then + size=$(stat -c%s "$FIRMWARE_DIR/$file") + echo " ✓ $file ($size bytes)" + else + echo " ✗ $file MISSING" + all_ok=false + fi +done + +echo "" +if $all_ok; then + echo "✓ All required firmware files present" +else + echo "✗ Some firmware files missing" + echo "" + echo "To install firmware:" + echo " 1. Download from NVIDIA or copy from nvidia-linux-firmware repo" + echo " 2. sudo cp -r nvidia/tu104/gsp/* /lib/firmware/nvidia/tu104/gsp/" + echo " 3. sudo update-initramfs -u" +fi + +echo "" +echo "Current kernel module version:" +if lsmod | grep -q nvidia; then + cat /proc/driver/nvidia/version 2>/dev/null | head -1 +fi
\ No newline at end of file diff --git a/tools/gdb_scripts/nvidia_init.gdb b/tools/gdb_scripts/nvidia_init.gdb new file mode 100644 index 0000000..c217419 --- /dev/null +++ b/tools/gdb_scripts/nvidia_init.gdb @@ -0,0 +1,179 @@ +# nvidia_init.gdb - GDB script for debugging NVIDIA driver initialization in guest +# Usage: gdb -x nvidia_init.gdb ./vmlinux +# Then: (gdb) target remote localhost:1234 + +# ============================================================ +# CONFIGURATION +# ============================================================ +# Path to guest kernel vmlinux with debug symbols +# set pagination off +# set confirm off + +# ============================================================ +# BREAKPOINTS - NVIDIA Driver Probe/Init +# ============================================================ + +# Core probe entry points +break nvidia_probe +break nvidia_init_module +break nvidia_exit_module + +# GSP/RM initialization +break nv_init_gsp +break nv_gsp_init +break nv_gsp_firmware_load +break nv_gsp_bootstrap +break nv_gsp_rpc_init + +# VFIO/mdev integration +break nvidia_vgpu_vfio_get_ops +break nvidia_vgpu_vfio_set_ops +break nv_vgpu_mdev_register_parent +break nv_vgpu_mdev_unregister_parent +break nv_vgpu_create_callback +break nv_vgpu_delete_callback +break nv_vgpu_bar_info_callback +break nv_vgpu_start_callback + +# PCI config space +break nvidia_pci_probe +break nvidia_pci_remove +break pci_enable_device +break pci_request_regions + +# MSI/MSI-X +break pci_enable_msi_block +break pci_enable_msix_range +break nvidia_setup_msi +break nvidia_setup_msix + +# BAR mapping +break pci_iomap +break pci_iounmap +break nvidia_map_bars + +# Interrupt handling +break request_threaded_irq +break free_irq +break nvidia_isr +break nvidia_msi_isr +break nvidia_msix_isr + +# DMA mapping +break dma_map_page +break dma_unmap_page +break dma_map_sg +break dma_unmap_sg + +# GSP RPC communication (OpenRM internal) +break nv_gsp_rpc_send +break nv_gsp_rpc_recv +break nv_gsp_doorbell_ring +break nv_gsp_doorbell_poll + +# ============================================================ +# WATCHPOINTS - Key Data Structures +# ============================================================ + +# Watch for GSP firmware load completion +# watch *(int*)nv_gsp_fw_loaded + +# Watch for MSI-X table setup +# watch *(int*)nv_msix_table_configured + +# ============================================================ +# COMMANDS - Helper Functions +# ============================================================ + +define nvidia_backtrace + bt + printf "\n=== NVIDIA Driver Stack ===\n" + info registers +end +document nvidia_backtrace + Show backtrace with register state for NVIDIA driver +end + +define nvidia_pci_state + printf "=== PCI Config Space ===\n" + # Requires access to pci_dev structure + # p/x ((struct pci_dev*)$rdi)->vendor + # p/x ((struct pci_dev*)$rdi)->device + # p/x ((struct pci_dev*)$rdi)->command + # p/x ((struct pci_dev*)$rdi)->status +end +document nvidia_pci_state + Print PCI device state +end + +define nvidia_gsp_state + printf "=== GSP State ===\n" + # p/x nv_gsp_fw_version + # p/x nv_gsp_fw_loaded + # p/x nv_gsp_rpc_initialized +end +document nvidia_gsp_state + Print GSP firmware state +end + +define nvidia_vfio_state + printf "=== VFIO/mdev State ===\n" + # p/x nv_vgpu_mdev_registered + # p/x nv_vgpu_vfio_ops +end +document nvidia_vfio_state + Print VFIO/mdev registration state +end + +# ============================================================ +# TRACEPOINTS (if kernel has tracepoints enabled) +# ============================================================ + +# trace nvidia:nvidia_probe_entry +# trace nvidia:nvidia_probe_exit +# trace nvidia:gsp_firmware_load +# trace nvidia:gsp_rpc_send +# trace nvidia:gsp_rpc_recv + +# ============================================================ +# AUTOMATION - Continue and Log +# ============================================================ + +# Log all breakpoint hits to file +set logging file nvidia_init_debug.log +set logging on + +# Auto-continue after logging (uncomment for automated runs) +# set confirm off +# commands 1 +# nvidia_backtrace +# continue +# end + +# ============================================================ +# CONVENIENCE ALIASES +# ============================================================ + +define hook-stop + # Print current function on every stop + printf "\n=== STOPPED at %s ===\n", __func__ + # Print key state + # nvidia_gsp_state +end + +# ============================================================ +# USAGE NOTES +# ============================================================ +# 1. Compile guest kernel with CONFIG_DEBUG_INFO=y CONFIG_GDB_SCRIPTS=y +# 2. Copy vmlinux from guest to host +# 3. Launch QEMU: ./scripts/qemu_gdb_attach.sh +# 4. In another terminal: gdb -x tools/gdb_scripts/nvidia_init.gdb ./vmlinux +# 5. (gdb) target remote localhost:1234 +# 6. (gdb) continue +# +# For specific debugging: +# - PCI probe issues: break nvidia_probe +# - GSP firmware load: break nv_gsp_firmware_load +# - MSI-X setup: break pci_enable_msix_range +# - BAR mapping: break pci_iomap +# - RPC communication: break nv_gsp_rpc_send
\ No newline at end of file diff --git a/tools/ghidra/analyze_fwimage.py b/tools/ghidra/analyze_fwimage.py new file mode 100644 index 0000000..5d98504 --- /dev/null +++ b/tools/ghidra/analyze_fwimage.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Analyze GSP firmware .fwimage binary in Ghidra. +Usage: python3 analyze_fwimage.py <fwimage_bin> <project_dir> <project_name> +""" + +import sys +import os +import subprocess + +def analyze_fwimage(fwimage_path, project_dir, project_name): + """Analyze the .fwimage binary in Ghidra.""" + + ghidra_path = "/snap/bin/ghidra" + analyze_headless = "/snap/ghidra/47/ghidra/support/analyzeHeadless" + + if not os.path.exists(analyze_headless): + analyze_headless = "/opt/ghidra/support/analyzeHeadless" + + if not os.path.exists(analyze_headless): + print("ERROR: Ghidra analyzeHeadless not found") + return False + + # Create project directory + os.makedirs(project_dir, exist_ok=True) + + # Import the .fwimage binary + cmd = [ + analyze_headless, + project_dir, + project_name, + "-import", fwimage_path, + "-processor", "RISCV:LE:64:default", + "-cspec", "gcc", + "-overwrite", + "-scriptPath", os.path.join(project_dir, "scripts"), + "-postScript", "analyze_gsp_fwimage.py" + ] + + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True, timeout=3600) + + print(result.stdout) + if result.stderr: + print(result.stderr, file=sys.stderr) + + return result.returncode == 0 + +def main(): + if len(sys.argv) < 4: + print("Usage: analyze_fwimage.py <fwimage_bin> <project_dir> <project_name>") + sys.exit(1) + + fwimage_path = sys.argv[1] + project_dir = sys.argv[2] + project_name = sys.argv[3] + + if not os.path.exists(fwimage_path): + print(f"ERROR: Firmware image not found: {fwimage_path}") + sys.exit(1) + + success = analyze_fwimage(fwimage_path, project_dir, project_name) + sys.exit(0 if success else 1) + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/tools/ghidra/analyze_gsp_firmware.sh b/tools/ghidra/analyze_gsp_firmware.sh new file mode 100755 index 0000000..b2e40d5 --- /dev/null +++ b/tools/ghidra/analyze_gsp_firmware.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# Full GSP firmware analysis pipeline +# Usage: ./analyze_gsp_firmware.sh <firmware_bin> [output_dir] + +set -euo pipefail + +FIRMWARE_BIN="${1:-~/Git/nvidia-linux-firmware/nvidia/tu102/gsp/gsp-570.144.bin}" +OUTPUT_DIR="${2:-/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP}" +PROJECT_DIR="/home/rmcore/Git/integdev_gpu_drv/open-iov/tools/ghidra/gsp-tu104-570.144" +PROJECT_NAME="gsp-tu104-570.144" + +FIRMWARE_BIN=$(eval echo $FIRMWARE_BIN) + +echo "=== GSP Firmware Analysis Pipeline ===" +echo "Firmware: $FIRMWARE_BIN" +echo "Output: $OUTPUT_DIR" +echo "Project: $PROJECT_DIR/$PROJECT_NAME" +echo "" + +# Step 1: Extract .fwimage section +echo "Step 1: Extracting .fwimage section..." +FWIMAGE_BIN="$OUTPUT_DIR/gsp-570.144.fwimage.bin" +objcopy -O binary -j .fwimage "$FIRMWARE_BIN" "$FWIMAGE_BIN" +echo " Saved to $FWIMAGE_BIN ($(stat -c%s "$FWIMAGE_BIN") bytes)" + +# Step 2: Extract version +echo "Step 2: Extracting version..." +VERSION=$(readelf -x .fwversion "$FIRMWARE_BIN" | grep -A1 "Hex dump" | tail -1 | awk '{print $2$3$4$5}' | xxd -r -p) +echo " Version: $VERSION" +echo "$VERSION" > "$OUTPUT_DIR/gsp_version.txt" + +# Step 3: Extract signatures +echo "Step 3: Extracting signatures..." +for i in 0 1 2; do + objcopy -O binary -j ".fwsignature$i" "$FIRMWARE_BIN" "$OUTPUT_DIR/gsp-570.144.fwsignature$i.bin" 2>/dev/null || true +done + +# Step 4: Create Ghidra project for ELF container +echo "Step 4: Creating Ghidra project for ELF container..." +/snap/ghidra/47/ghidra/support/analyzeHeadless "$PROJECT_DIR" "$PROJECT_NAME" \ + -import "$FIRMWARE_BIN" \ + -processor "RISCV:LE:64:default" \ + -cspec "gcc" \ + -overwrite \ + -scriptPath "$PROJECT_DIR/scripts" \ + -postScript "extract_fwimage.py" + +# Step 5: Create Ghidra project for .fwimage binary +echo "Step 5: Creating Ghidra project for .fwimage binary..." +FWIMAGE_PROJECT_DIR="/home/rmcore/Git/integdev_gpu_drv/open-iov/tools/ghidra/gsp-fwimage-570.144" +FWIMAGE_PROJECT_NAME="gsp-fwimage-570.144" +mkdir -p "$FWIMAGE_PROJECT_DIR/scripts" +cp "$PROJECT_DIR/scripts/analyze_gsp_fwimage.py" "$FWIMAGE_PROJECT_DIR/scripts/" + +/snap/ghidra/47/ghidra/support/analyzeHeadless "$FWIMAGE_PROJECT_DIR" "$FWIMAGE_PROJECT_NAME" \ + -import "$FWIMAGE_BIN" \ + -processor "RISCV:LE:64:default" \ + -cspec "gcc" \ + -overwrite \ + -scriptPath "$FWIMAGE_PROJECT_DIR/scripts" \ + -postScript "analyze_gsp_fwimage.py" + +# Step 6: Also analyze bootloader and booter +echo "Step 6: Analyzing bootloader and booter..." +BOOTLOADER_BIN="$OUTPUT_DIR/bootloader-570.144.bin" +BOOTER_LOAD_BIN="$OUTPUT_DIR/booter_load-570.144.bin" +BOOTER_UNLOAD_BIN="$OUTPUT_DIR/booter_unload-570.144.bin" + +cp ~/Git/nvidia-linux-firmware/nvidia/tu102/gsp/bootloader-570.144.bin "$BOOTLOADER_BIN" +cp ~/Git/nvidia-linux-firmware/nvidia/tu102/gsp/booter_load-570.144.bin "$BOOTER_LOAD_BIN" +cp ~/Git/nvidia-linux-firmware/nvidia/tu102/gsp/booter_unload-570.144.bin "$BOOTER_UNLOAD_BIN" + +for name in bootloader booter_load booter_unload; do + bin_file="$OUTPUT_DIR/${name}-570.144.bin" + proj_dir="/home/rmcore/Git/integdev_gpu_drv/open-iov/tools/ghidra/gsp-${name}-570.144" + proj_name="gsp-${name}-570.144" + mkdir -p "$proj_dir/scripts" + cp "$PROJECT_DIR/scripts/analyze_gsp_fwimage.py" "$proj_dir/scripts/analyze_${name}.py" + + # Modify script for raw binary + sed -i 's/program = getCurrentProgram()/program = getCurrentProgram()\n# Raw binary - set base address to 0/' "$proj_dir/scripts/analyze_${name}.py" + + /snap/ghidra/47/ghidra/support/analyzeHeadless "$proj_dir" "$proj_name" \ + -import "$bin_file" \ + -processor "RISCV:LE:64:default" \ + -cspec "gcc" \ + -overwrite \ + -scriptPath "$proj_dir/scripts" \ + -postScript "analyze_${name}.py" 2>&1 | tail -20 +done + +echo "" +echo "=== Analysis Complete ===" +echo "Results in: $OUTPUT_DIR" +echo "" +echo "Key files:" +ls -la "$OUTPUT_DIR"/*.json "$OUTPUT_DIR"/*.txt "$OUTPUT_DIR"/*.c 2>/dev/null | head -20
\ No newline at end of file diff --git a/tools/ghidra/create_gsp_project.py b/tools/ghidra/create_gsp_project.py new file mode 100644 index 0000000..3f505f1 --- /dev/null +++ b/tools/ghidra/create_gsp_project.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +""" +Create Ghidra project for GSP firmware analysis. +Usage: python3 create_gsp_project.py <firmware_bin> <project_dir> <project_name> +""" + +import sys +import os +import subprocess +import shutil + +def create_ghidra_project(firmware_path, project_dir, project_name): + """Create a Ghidra project and import the GSP firmware.""" + + ghidra_path = "/snap/bin/ghidra" + analyze_headless = "/snap/ghidra/47/ghidra/support/analyzeHeadless" + + if not os.path.exists(analyze_headless): + analyze_headless = "/opt/ghidra/support/analyzeHeadless" + + if not os.path.exists(analyze_headless): + print("ERROR: Ghidra analyzeHeadless not found") + sys.exit(1) + + # Create project directory + os.makedirs(project_dir, exist_ok=True) + + # Import the firmware binary + cmd = [ + analyze_headless, + project_dir, + project_name, + "-import", firmware_path, + "-processor", "RISCV:LE:64:default", + "-cspec", "gcc", + "-overwrite", + "-scriptPath", os.path.join(project_dir, "scripts"), + "-postScript", "extract_fwimage.py" + ] + + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + + print(result.stdout) + if result.stderr: + print(result.stderr, file=sys.stderr) + + return result.returncode == 0 + +def main(): + if len(sys.argv) < 4: + print("Usage: create_gsp_project.py <firmware_bin> <project_dir> <project_name>") + sys.exit(1) + + firmware_path = sys.argv[1] + project_dir = sys.argv[2] + project_name = sys.argv[3] + + if not os.path.exists(firmware_path): + print(f"ERROR: Firmware not found: {firmware_path}") + sys.exit(1) + + success = create_ghidra_project(firmware_path, project_dir, project_name) + sys.exit(0 if success else 1) + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/tools/ghidra/diff_gsp_firmware.py b/tools/ghidra/diff_gsp_firmware.py new file mode 100644 index 0000000..8145982 --- /dev/null +++ b/tools/ghidra/diff_gsp_firmware.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +""" +Diff two GSP firmware versions. +Usage: python3 diff_gsp_firmware.py <old_firmware> <new_firmware> <output_report> +""" + +import sys +import os +import subprocess +import json +import hashlib + +def extract_fwimage(firmware_path, output_path): + """Extract .fwimage section from GSP firmware ELF.""" + cmd = ["readelf", "-x", ".fwimage", firmware_path] + result = subprocess.run(cmd, capture_output=True, text=True) + # This is a hex dump, we need to parse it + # Better to use objcopy + cmd = ["objcopy", "-O", "binary", "-j", ".fwimage", firmware_path, output_path] + result = subprocess.run(cmd, capture_output=True) + return result.returncode == 0 + +def extract_version(firmware_path): + """Extract version from .fwversion section.""" + cmd = ["readelf", "-x", ".fwversion", firmware_path] + result = subprocess.run(cmd, capture_output=True, text=True) + # Parse hex dump + for line in result.stdout.split('\n'): + if '0x' in line and len(line.strip()) > 10: + parts = line.split() + if len(parts) >= 5: + hex_str = ''.join(parts[1:5]) + try: + version = bytes.fromhex(hex_str).decode('ascii').strip('\x00') + return version + except: + pass + return "unknown" + +def run_ghidra_analysis(fwimage_path, project_dir, project_name): + """Run Ghidra analysis on fwimage.""" + analyze_headless = "/snap/ghidra/47/ghidra/support/analyzeHeadless" + if not os.path.exists(analyze_headless): + analyze_headless = "/opt/ghidra/support/analyzeHeadless" + + script_path = "/home/rmcore/Git/integdev_gpu_drv/open-iov/tools/ghidra/gsp-tu104-570.144/scripts" + + cmd = [ + analyze_headless, + project_dir, + project_name, + "-import", fwimage_path, + "-processor", "RISCV:LE:64:default", + "-cspec", "gcc", + "-overwrite", + "-scriptPath", script_path, + "-postScript", "analyze_gsp_fwimage.py" + ] + + result = subprocess.run(cmd, capture_output=True, text=True, timeout=7200) + return result.returncode == 0 + +def load_analysis_results(output_dir): + """Load analysis results from JSON files.""" + results = {} + for fname in ["functions.json", "rpc_functions.json", "vgpu_strings.json", + "wpr_functions.json", "mem_functions.json", "irq_functions.json", + "analysis_summary.json"]: + path = os.path.join(output_dir, fname) + if os.path.exists(path): + with open(path, 'r') as f: + results[fname.replace('.json', '')] = json.load(f) + return results + +def diff_functions(old_funcs, new_funcs): + """Compare function lists.""" + old_names = {f['name']: f for f in old_funcs} + new_names = {f['name']: f for f in new_funcs} + + added = [f for name, f in new_names.items() if name not in old_names] + removed = [f for name, f in old_names.items() if name not in new_names] + modified = [] + + for name in set(old_names.keys()) & set(new_names.keys()): + if old_names[name]['size'] != new_names[name]['size'] or \ + old_names[name]['entry'] != new_names[name]['entry']: + modified.append({ + 'name': name, + 'old': old_names[name], + 'new': new_names[name] + }) + + return added, removed, modified + +def generate_diff_report(old_firmware, new_firmware, old_results, new_results, output_path): + """Generate markdown diff report.""" + old_version = extract_version(old_firmware) + new_version = extract_version(new_firmware) + + old_funcs = old_results.get('functions', []) + new_funcs = new_results.get('functions', []) + + added, removed, modified = diff_functions(old_funcs, new_funcs) + + old_rpc = {f['name'] for f in old_results.get('rpc_functions', [])} + new_rpc = {f['name'] for f in new_results.get('rpc_functions', [])} + + rpc_added = new_rpc - old_rpc + rpc_removed = old_rpc - new_rpc + + report = f"""# GSP Firmware Diff Report + +## Versions +- **Old**: {old_version} ({os.path.basename(old_firmware)}) +- **New**: {new_version} ({os.path.basename(new_firmware)}) + +## Function Changes +- **Total Functions**: {len(old_funcs)} → {len(new_funcs)} ({len(new_funcs) - len(old_funcs):+d}) +- **Added**: {len(added)} +- **Removed**: {len(removed)} +- **Modified**: {len(modified)} + +## RPC Handler Changes +- **Added RPC Functions**: {len(rpc_added)} +- **Removed RPC Functions**: {len(rpc_removed)} + +## Added Functions +""" + for f in added[:50]: + report += f"- `{f['name']}` at {f['entry']} (size: {f['size']})\n" + + if len(added) > 50: + report += f"\n... and {len(added) - 50} more\n" + + report += "\n## Removed Functions\n" + for f in removed[:50]: + report += f"- `{f['name']}` at {f['entry']} (size: {f['size']})\n" + + if len(removed) > 50: + report += f"\n... and {len(removed) - 50} more\n" + + report += "\n## Modified Functions\n" + for f in modified[:50]: + report += f"- `{f['name']}`: {f['old']['entry']}→{f['new']['entry']}, size {f['old']['size']}→{f['new']['size']}\n" + + if len(modified) > 50: + report += f"\n... and {len(modified) - 50} more\n" + + report += "\n## Added RPC Functions\n" + for name in sorted(rpc_added): + report += f"- `{name}`\n" + + report += "\n## Removed RPC Functions\n" + for name in sorted(rpc_removed): + report += f"- `{name}`\n" + + # vGPU strings comparison + old_strings = {s['value'] for s in old_results.get('vgpu_strings', [])} + new_strings = {s['value'] for s in new_results.get('vgpu_strings', [])} + + report += "\n## vGPU String Changes\n" + report += f"- **Added**: {len(new_strings - old_strings)}\n" + report += f"- **Removed**: {len(old_strings - new_strings)}\n" + + for s in sorted(new_strings - old_strings)[:20]: + report += f" + `{s}`\n" + for s in sorted(old_strings - new_strings)[:20]: + report += f" - `{s}`\n" + + with open(output_path, 'w') as f: + f.write(report) + + print(f"Diff report saved to {output_path}") + +def main(): + if len(sys.argv) < 4: + print("Usage: diff_gsp_firmware.py <old_firmware> <new_firmware> <output_report>") + sys.exit(1) + + old_firmware = sys.argv[1] + new_firmware = sys.argv[2] + output_report = sys.argv[3] + + # Extract .fwimage sections + old_fwimage = "/tmp/old_fwimage.bin" + new_fwimage = "/tmp/new_fwimage.bin" + + print("Extracting .fwimage sections...") + if not extract_fwimage(old_firmware, old_fwimage): + print("Failed to extract old .fwimage") + sys.exit(1) + if not extract_fwimage(new_firmware, new_fwimage): + print("Failed to extract new .fwimage") + sys.exit(1) + + # Run Ghidra analysis on both + print("Analyzing old firmware...") + old_project_dir = "/tmp/ghidra_old" + if not run_ghidra_analysis(old_fwimage, old_project_dir, "gsp_old"): + print("Failed to analyze old firmware") + sys.exit(1) + + print("Analyzing new firmware...") + new_project_dir = "/tmp/ghidra_new" + if not run_ghidra_analysis(new_fwimage, new_project_dir, "gsp_new"): + print("Failed to analyze new firmware") + sys.exit(1) + + # Load results + old_results = load_analysis_results("/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP") + # For new firmware, we'd need a separate output dir + new_results = load_analysis_results("/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/NEW-GSP") + + # Generate diff report + generate_diff_report(old_firmware, new_firmware, old_results, new_results, output_report) + + print("Done!") + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/tools/ghidra/gsp-fwimage-570.144/scripts/BasicAnalysis.java b/tools/ghidra/gsp-fwimage-570.144/scripts/BasicAnalysis.java new file mode 100644 index 0000000..1776ea2 --- /dev/null +++ b/tools/ghidra/gsp-fwimage-570.144/scripts/BasicAnalysis.java @@ -0,0 +1,156 @@ +// Ghidra script to do basic disassembly analysis of GSP .fwimage binary +// Usage: analyzeHeadless ... -postScript BasicAnalysis.java + +import ghidra.app.script.GhidraScript; +import ghidra.program.model.listing.*; +import ghidra.program.model.mem.*; +import ghidra.program.model.symbol.*; +import ghidra.program.model.address.*; +import ghidra.util.task.TaskMonitor; +import java.io.*; +import java.util.*; +import java.nio.file.*; + +public class BasicAnalysis extends GhidraScript { + + @Override + protected void run() throws Exception { + Program program = getCurrentProgram(); + Listing listing = program.getListing(); + Memory memory = program.getMemory(); + FunctionManager functionManager = program.getFunctionManager(); + + println("=== GSP Firmware .fwimage Basic Analysis ==="); + println("Program: " + program.getName()); + println("Architecture: " + program.getLanguage().toString()); + + // Create output directory + String outputDir = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP"; + Files.createDirectories(Paths.get(outputDir)); + + // 1. Check memory blocks + println("\n--- Memory Blocks ---"); + List<Map<String, Object>> memBlocks = new ArrayList<>(); + for (MemoryBlock block : memory.getBlocks()) { + Map<String, Object> blockInfo = new HashMap<>(); + blockInfo.put("name", block.getName()); + blockInfo.put("start", block.getStart().toString()); + blockInfo.put("end", block.getEnd().toString()); + blockInfo.put("size", block.getSize()); + blockInfo.put("execute", block.isExecute()); + blockInfo.put("read", block.isRead()); + blockInfo.put("write", block.isWrite()); + memBlocks.add(blockInfo); + println(" " + block.getName() + ": " + block.getStart() + " - " + block.getEnd() + + " (" + block.getSize() + " bytes) r=" + block.isRead() + " w=" + block.isWrite() + " x=" + block.isExecute()); + } + writeJson(outputDir + "/memory_blocks.json", memBlocks); + + // 2. Find all functions + println("\n--- Finding Functions ---"); + List<Map<String, Object>> functions = new ArrayList<>(); + FunctionIterator funcIter = functionManager.getFunctions(true); + int count = 0; + while (funcIter.hasNext()) { + Function func = funcIter.next(); + Map<String, Object> funcInfo = new HashMap<>(); + funcInfo.put("name", func.getName()); + funcInfo.put("entry", func.getEntryPoint().toString()); + funcInfo.put("size", func.getBody().getNumAddresses()); + functions.add(funcInfo); + count++; + if (count % 1000 == 0) { + println(" Found " + count + " functions..."); + } + } + println("Total functions: " + functions.size()); + writeJson(outputDir + "/functions.json", functions); + + // 3. Find strings + println("\n--- Finding Strings ---"); + List<Map<String, String>> strings = new ArrayList<>(); + DataIterator dataIter = listing.getDefinedData(true); + while (dataIter.hasNext()) { + Data data = dataIter.next(); + if (data.getDataType().getName().equals("string")) { + Object value = data.getValue(); + if (value != null && value.toString().length() > 4) { + Map<String, String> strInfo = new HashMap<>(); + strInfo.put("address", data.getAddress().toString()); + strInfo.put("value", value.toString()); + strings.add(strInfo); + } + } + } + println("Found " + strings.size() + " strings"); + + // Filter for interesting strings + String[] interestingKeywords = {"vgpu", "vmiop", "rpc", "hal", "bar", "migrate", "init", "boot", "kgsp", "wpr", "firmware", "version", "error", "fail", "assert", "panic", "nvidia", "rmapi", "gsp", "falcon", "mailbox", "interrupt", "dma", "iommu", "page", "mmu", "pde", "pte"}; + List<Map<String, String>> interestingStrings = new ArrayList<>(); + for (Map<String, String> s : strings) { + String val = s.get("value").toLowerCase(); + for (String kw : interestingKeywords) { + if (val.contains(kw)) { + interestingStrings.add(s); + break; + } + } + } + println("Interesting strings: " + interestingStrings.size()); + writeJson(outputDir + "/interesting_strings.json", interestingStrings); + + // 4. Find entry point + println("\n--- Entry Point Analysis ---"); + Address entryAddr = program.getMinAddress(); + println("Program entry (min address): " + entryAddr); + Function funcAtZero = functionManager.getFunctionAt(entryAddr); + if (funcAtZero != null) { + println("Function at entry: " + funcAtZero.getName()); + } else { + println("No function at entry point - creating one..."); + Function func = functionManager.createFunction("entry_point", entryAddr, null, null, monitor); + if (func != null) { + println("Created function: " + func.getName()); + } + } + + println("\n--- Analysis Complete ---"); + println("Results saved to " + outputDir); + } + + private void writeJson(String path, Object data) throws IOException { + StringBuilder json = new StringBuilder(); + json.append("[\n"); + if (data instanceof List) { + List<?> list = (List<?>) data; + for (int i = 0; i < list.size(); i++) { + Object item = list.get(i); + if (item instanceof Map) { + Map<?, ?> map = (Map<?, ?>) item; + json.append(" {"); + int j = 0; + for (Map.Entry<?, ?> entry : map.entrySet()) { + json.append("\"").append(entry.getKey()).append("\": "); + Object val = entry.getValue(); + if (val instanceof String) { + json.append("\"").append(escapeJson(val.toString())).append("\""); + } else { + json.append(val); + } + if (j < map.size() - 1) json.append(", "); + j++; + } + json.append("}"); + } + if (i < list.size() - 1) json.append(","); + json.append("\n"); + } + } + json.append("]"); + Files.writeString(Paths.get(path), json.toString()); + } + + private String escapeJson(String s) { + return s.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t"); + } +}
\ No newline at end of file diff --git a/tools/ghidra/gsp-fwimage-570.144/scripts/DecompileFunctions.java b/tools/ghidra/gsp-fwimage-570.144/scripts/DecompileFunctions.java new file mode 100644 index 0000000..da522cd --- /dev/null +++ b/tools/ghidra/gsp-fwimage-570.144/scripts/DecompileFunctions.java @@ -0,0 +1,88 @@ +import ghidra.app.script.GhidraScript; +import ghidra.program.model.address.Address; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionManager; +import ghidra.app.decompiler.DecompInterface; +import ghidra.app.decompiler.DecompileResults; +import ghidra.util.task.TaskMonitor; +import java.io.File; +import java.io.FileWriter; +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class DecompileFunctions extends GhidraScript { + private static final String OUTPUT_DIR = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP/decompiled/"; + + @Override + protected void run() throws Exception { + File outDir = new File(OUTPUT_DIR); + if (!outDir.exists()) { + outDir.mkdirs(); + } + + String[] args = getScriptArgs(); + if (args.length == 0) { + println("No addresses provided. Usage: <addr1>,<addr2>,..."); + return; + } + + Set<Address> targetAddresses = new HashSet<>(); + for (String arg : args) { + String[] addrs = arg.split(","); + for (String addrStr : addrs) { + addrStr = addrStr.trim(); + if (addrStr.startsWith("0x")) { + addrStr = addrStr.substring(2); + } + Address addr = toAddr(addrStr); + if (addr != null) { + targetAddresses.add(addr); + println("Target: " + addr); + } + } + } + + FunctionManager fm = currentProgram.getFunctionManager(); + DecompInterface decompiler = new DecompInterface(); + decompiler.openProgram(currentProgram); + + for (Address addr : targetAddresses) { + Function func = fm.getFunctionAt(addr); + if (func == null) { + println("No function at " + addr + ", creating..."); + func = createFunction(addr, null); + } + if (func != null) { + decompileAndSave(func, decompiler); + } else { + println("Failed to create function at " + addr); + } + } + + decompiler.dispose(); + } + + private void decompileAndSave(Function func, DecompInterface decompiler) throws Exception { + String name = func.getName(); + Address entry = func.getEntryPoint(); + String safeName = name.replaceAll("[^a-zA-Z0-9_]", "_"); + File outFile = new File(OUTPUT_DIR, safeName + "_" + entry.toString().replace(":", "_") + ".c"); + + DecompileResults results = decompiler.decompileFunction(func, 60, TaskMonitor.DUMMY); + if (results.decompileCompleted()) { + String code = results.getDecompiledFunction().getC(); + try (PrintWriter pw = new PrintWriter(new FileWriter(outFile))) { + pw.println("// Function: " + name); + pw.println("// Address: " + entry); + pw.println("// Size: " + func.getBody().getNumAddresses() + " bytes"); + pw.println(); + pw.print(code); + } + println("Decompiled: " + name + " @ " + entry + " -> " + outFile.getName()); + } else { + println("Decompilation failed for " + name + " @ " + entry + ": " + results.getErrorMessage()); + } + } +}
\ No newline at end of file diff --git a/tools/ghidra/gsp-fwimage-570.144/scripts/ExtractFunctions.java b/tools/ghidra/gsp-fwimage-570.144/scripts/ExtractFunctions.java new file mode 100644 index 0000000..2296a42 --- /dev/null +++ b/tools/ghidra/gsp-fwimage-570.144/scripts/ExtractFunctions.java @@ -0,0 +1,126 @@ +import ghidra.app.script.GhidraScript; +import ghidra.program.model.listing.FunctionManager; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionIterator; +import ghidra.program.model.address.Address; +import ghidra.program.model.listing.Program; +import ghidra.program.model.mem.Memory; +import ghidra.program.model.mem.MemoryBlock; +import ghidra.util.task.TaskMonitor; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class ExtractFunctions extends GhidraScript { + + @Override + protected void run() throws Exception { + Program program = currentProgram; + FunctionManager functionManager = program.getFunctionManager(); + Memory memory = program.getMemory(); + + String outputDir = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP"; + + // 1. Extract memory blocks + List<MemoryBlockInfo> memBlocks = new ArrayList<>(); + for (MemoryBlock block : memory.getBlocks()) { + memBlocks.add(new MemoryBlockInfo( + block.getName(), + block.getStart().toString(), + block.getEnd().toString(), + block.getSize(), + block.isExecute(), + block.isRead(), + block.isWrite() + )); + } + writeJson(outputDir + "/memory_blocks.json", memBlocks); + + // 2. Extract all functions (analysis must have run first) + List<FunctionInfo> functions = new ArrayList<>(); + FunctionIterator funcIter = functionManager.getFunctions(true); + while (funcIter.hasNext()) { + Function func = funcIter.next(); + functions.add(new FunctionInfo( + func.getName(), + func.getEntryPoint().toString(), + func.getBody().getNumAddresses() + )); + } + writeJson(outputDir + "/functions.json", functions); + + // 3. Extract interesting strings + String[] keywords = {"vgpu", "vmiop", "rpc", "hal", "bar", "migrate", "init", "boot", "kgsp", "wpr", "firmware", "version", "error", "fail", "assert", "panic", "nvidia", "rmapi", "gsp", "falcon", "mailbox", "interrupt", "dma", "iommu", "page", "mmu", "pde", "pte"}; + List<StringInfo> strings = new ArrayList<>(); + var listing = program.getListing(); + var dataIter = listing.getDefinedData(true); + while (dataIter.hasNext()) { + var data = dataIter.next(); + if (data.getDataType().getName().equals("string")) { + String value = data.getValue().toString(); + if (value.length() > 4) { + String lower = value.toLowerCase(); + for (String kw : keywords) { + if (lower.contains(kw)) { + strings.add(new StringInfo(data.getAddress().toString(), value)); + break; + } + } + } + } + } + writeJson(outputDir + "/interesting_strings.json", strings); + + println("Extracted " + functions.size() + " functions, " + memBlocks.size() + " memory blocks, " + strings.size() + " interesting strings"); + } + + private void writeJson(String path, List<?> list) throws IOException { + try (FileWriter fw = new FileWriter(path)) { + fw.write("[\n"); + for (int i = 0; i < list.size(); i++) { + fw.write(" " + toJson(list.get(i))); + if (i < list.size() - 1) fw.write(","); + fw.write("\n"); + } + fw.write("]\n"); + } + } + + private String toJson(Object obj) { + if (obj instanceof MemoryBlockInfo) { + MemoryBlockInfo m = (MemoryBlockInfo) obj; + return String.format("{\"name\":\"%s\",\"start\":\"%s\",\"end\":\"%s\",\"size\":%d,\"execute\":%b,\"read\":%b,\"write\":%b}", + m.name, m.start, m.end, m.size, m.execute, m.read, m.write); + } else if (obj instanceof FunctionInfo) { + FunctionInfo f = (FunctionInfo) obj; + return String.format("{\"name\":\"%s\",\"entry\":\"%s\",\"size\":%d}", + f.name, f.entry, f.size); + } else if (obj instanceof StringInfo) { + StringInfo s = (StringInfo) obj; + return String.format("{\"address\":\"%s\",\"value\":\"%s\"}", + s.address, s.value.replace("\\", "\\\\").replace("\"", "\\\"")); + } + return "{}"; + } + + static class MemoryBlockInfo { + String name, start, end; + long size; + boolean execute, read, write; + MemoryBlockInfo(String n, String s, String e, long sz, boolean ex, boolean rd, boolean wr) { + name = n; start = s; end = e; size = sz; execute = ex; read = rd; write = wr; + } + } + + static class FunctionInfo { + String name, entry; + long size; + FunctionInfo(String n, String e, long sz) { name = n; entry = e; size = sz; } + } + + static class StringInfo { + String address, value; + StringInfo(String a, String v) { address = a; value = v; } + } +} diff --git a/tools/ghidra/gsp-fwimage-570.144/scripts/basic_analysis.py b/tools/ghidra/gsp-fwimage-570.144/scripts/basic_analysis.py new file mode 100644 index 0000000..4992106 --- /dev/null +++ b/tools/ghidra/gsp-fwimage-570.144/scripts/basic_analysis.py @@ -0,0 +1,108 @@ +# Ghidra script to do basic disassembly analysis of GSP .fwimage binary +# Usage: analyzeHeadless ... -postScript basic_analysis.py + +import sys +import os +import json + +# Get the current program +program = getCurrentProgram() +listing = program.getListing() +memory = program.getMemory() +function_manager = program.getFunctionManager() + +print("=== GSP Firmware .fwimage Basic Analysis ===") +print(f"Program: {program.getName()}") +print(f"Architecture: {program.getLanguage().toString()}") + +# Create output directory +output_dir = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP" +os.makedirs(output_dir, exist_ok=True) + +# 1. Check memory blocks +print("\n--- Memory Blocks ---") +mem_blocks = [] +for block in memory.getBlocks(): + mem_blocks.append({ + "name": block.getName(), + "start": str(block.getStart()), + "end": str(block.getEnd()), + "size": block.getSize(), + "execute": block.isExecute(), + "read": block.isRead(), + "write": block.isWrite() + }) + print(f" {block.getName()}: {block.getStart()} - {block.getEnd()} ({block.getSize()} bytes) r={block.isRead()} w={block.isWrite()} x={block.isExecute()}") + +with open(os.path.join(output_dir, "memory_blocks.json"), "w") as f: + json.dump(mem_blocks, f, indent=2) + +# 2. Find all functions (but don't decompile yet) +print("\n--- Finding Functions ---") +functions = [] +func_iter = function_manager.getFunctions(True) +count = 0 +while func_iter.hasNext(): + func = func_iter.next() + functions.append({ + "name": func.getName(), + "entry": str(func.getEntryPoint()), + "size": func.getBody().getNumAddresses() + }) + count += 1 + if count % 1000 == 0: + print(f" Found {count} functions...") + +print(f"Total functions: {len(functions)}") + +with open(os.path.join(output_dir, "functions.json"), "w") as f: + json.dump(functions, f, indent=2) + +# 3. Find strings +print("\n--- Finding Strings ---") +strings = [] +string_iter = listing.getDefinedData(True) +while string_iter.hasNext(): + data = string_iter.next() + if data.getDataType().getName() == "string": + value = data.getValue() + if value and len(str(value)) > 4: + strings.append({ + "address": str(data.getAddress()), + "value": str(value) + }) + +print(f"Found {len(strings)} strings") + +# Filter for interesting strings +interesting_keywords = ["vgpu", "vmiop", "rpc", "hal", "bar", "migrate", "init", "boot", "kgsp", "wpr", "firmware", "version", "error", "fail", "assert", "panic", "nvidia", "rmapi", "gsp", "falcon", "mailbox", "interrupt", "dma", "iommu", "page", "mmu", "pde", "pte"] +interesting_strings = [s for s in strings if any(kw in s['value'].lower() for kw in interesting_keywords)] +print(f"Interesting strings: {len(interesting_strings)}") + +with open(os.path.join(output_dir, "interesting_strings.json"), "w") as f: + json.dump(interesting_strings, f, indent=2) + +# 4. Find entry point (offset 0) +print("\n--- Entry Point Analysis ---") +entry_addr = program.getMinAddress() +print(f"Program entry (min address): {entry_addr}") + +# Check what's at address 0 +func_at_zero = function_manager.getFunctionAt(entry_addr) +if func_at_zero: + print(f"Function at entry: {func_at_zero.getName()}") +else: + print("No function at entry point - creating one...") + # Create function at entry point + func = function_manager.createFunction("entry_point", entry_addr, None, None) + if func: + print(f"Created function: {func.getName()}") + +# 5. Look for RISC-V standard function prologues +print("\n--- Looking for Function Prologues ---") +# RISC-V typical prologue: addi sp, sp, -X; sd ra, X-8(sp); sd s0, X-16(sp); addi s0, sp, X +# Let's search for common patterns + +# 6. Find cross-references to interesting addresses +print("\n--- Analysis Complete ---") +print(f"Results saved to {output_dir}")
\ No newline at end of file diff --git a/tools/ghidra/gsp-tu104-570.144/scripts/DecompileFunctions.java b/tools/ghidra/gsp-tu104-570.144/scripts/DecompileFunctions.java new file mode 100644 index 0000000..e1d78c3 --- /dev/null +++ b/tools/ghidra/gsp-tu104-570.144/scripts/DecompileFunctions.java @@ -0,0 +1,143 @@ +import ghidra.app.script.GhidraScript; +import ghidra.program.model.address.Address; +import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionManager; +import ghidra.program.model.listing.Instruction; +import ghidra.program.model.listing.Program; +import ghidra.program.model.mem.Memory; +import ghidra.app.decompiler.DecompInterface; +import ghidra.app.decompiler.DecompileResults; +import ghidra.util.task.TaskMonitor; +import java.io.*; + +public class DecompileFunctions extends GhidraScript { + + private DecompInterface decompiler; + + @Override + protected void run() throws Exception { + decompiler = new DecompInterface(); + decompiler.openProgram(currentProgram); + + String[] args = getScriptArgs(); + if (args.length == 0) { + println("Usage: DecompileFunctions.java <comma-separated-addresses>"); + return; + } + + String[] addresses = args[0].split(","); + + println("=== Targeted Decompilation ==="); + for (String addrStr : addresses) { + addrStr = addrStr.trim(); + if (addrStr.isEmpty()) continue; + + long addrVal; + try { + addrVal = Long.parseUnsignedLong(addrStr.startsWith("0x") ? addrStr.substring(2) : addrStr, 16); + } catch (NumberFormatException e) { + println("Invalid address: " + addrStr); + continue; + } + + Address addr = currentProgram.getAddressFactory().getAddress("0x" + Long.toHexString(addrVal)); + if (addr == null) { + println("Invalid address: " + addrStr); + continue; + } + + decompileFunction(addr); + } + + println("\n=== Decompilation Complete ==="); + } + + private void decompileFunction(Address entryAddr) throws Exception { + FunctionManager fm = currentProgram.getFunctionManager(); + Memory mem = currentProgram.getMemory(); + + // Manually disassemble from entry point by creating instructions + Address addr = entryAddr; + int maxBytes = 2000; + int bytes = 0; + + while (bytes < maxBytes && !monitor.isCancelled()) { + Instruction instr = currentProgram.getListing().getInstructionAt(addr); + if (instr != null) { + addr = instr.getMaxAddress().next(); + bytes += instr.getLength(); + continue; + } + + // Try to create instruction from bytes + try { + byte[] buf = new byte[4]; + int read = mem.getBytes(addr, buf); + if (read <= 0) break; + + // Create instruction prototype and try to disassemble + instr = currentProgram.getListing().createInstruction(addr, + currentProgram.getLanguage().getInstructionPrototype(buf), + new ghidra.program.model.mem.ByteMemBuffer(buf, 0, read), + currentProgram.getProgramContext(), 0); + + if (instr != null) { + addr = instr.getMaxAddress().next(); + bytes += instr.getLength(); + } else { + addr = addr.next().next(); + bytes += 2; + } + } catch (Exception e) { + addr = addr.next().next(); + bytes += 2; + } + } + println(" Disassembled " + bytes + " bytes from " + entryAddr); + + // Get or create function + Function func = fm.getFunctionAt(entryAddr); + if (func == null) { + func = fm.createFunction("FUNC_" + Long.toHexString(entryAddr.getOffset()).toUpperCase(), + entryAddr, null, ghidra.program.model.symbol.SourceType.ANALYSIS); + if (func == null) { + println("Failed to create function at " + entryAddr); + return; + } + println("Created function: " + func.getName() + " @ " + func.getEntryPoint()); + } else { + println("Existing function: " + func.getName() + " @ " + func.getEntryPoint()); + } + + // Decompile + println("\n--- Decompiling " + func.getName() + " @ " + func.getEntryPoint() + " ---"); + DecompileResults results = decompiler.decompileFunction(func, 120, monitor); + + if (results.decompileCompleted()) { + String cCode = results.getDecompiledFunction().getC(); + + // Save to file + String filename = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP/decompiled_" + + func.getName() + ".c"; + try (PrintWriter pw = new PrintWriter(new FileWriter(filename))) { + pw.println("// Function: " + func.getName()); + pw.println("// Address: " + func.getEntryPoint()); + pw.println("// Stack frame: " + func.getStackFrame().getFrameSize() + " bytes"); + pw.println(); + pw.print(cCode); + } + println(" Saved to: " + filename); + + // Print first 100 lines + String[] lines = cCode.split("\n"); + for (int i = 0; i < Math.min(100, lines.length); i++) { + println(" " + lines[i]); + } + if (lines.length > 100) { + println(" ... (" + (lines.length - 100) + " more lines)"); + } + } else { + println(" Decompilation failed: " + results.getErrorMessage()); + } + } +} diff --git a/tools/ghidra/gsp-tu104-570.144/scripts/analyze_gsp_fwimage.py b/tools/ghidra/gsp-tu104-570.144/scripts/analyze_gsp_fwimage.py new file mode 100644 index 0000000..4a6fb56 --- /dev/null +++ b/tools/ghidra/gsp-tu104-570.144/scripts/analyze_gsp_fwimage.py @@ -0,0 +1,207 @@ +# Ghidra script to analyze GSP .fwimage binary +# Usage: analyzeHeadless ... -postScript analyze_gsp_fwimage.py + +import sys +import os +import json + +# Get the current program +program = getCurrentProgram() +listing = program.getListing() +memory = program.getMemory() +function_manager = program.getFunctionManager() + +print("=== GSP Firmware .fwimage Analysis ===") +print(f"Program: {program.getName()}") +print(f"Architecture: {program.getLanguage().toString()}") + +# Create output directory +output_dir = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP" +os.makedirs(output_dir, exist_ok=True) + +# 1. Find all functions +print("\n--- Finding Functions ---") +functions = [] +func_iter = function_manager.getFunctions(True) +while func_iter.hasNext(): + func = func_iter.next() + functions.append({ + "name": func.getName(), + "entry": str(func.getEntryPoint()), + "size": func.getBody().getNumAddresses(), + "signature": func.getSignature().toString() if func.getSignature() else "" + }) + +print(f"Found {len(functions)} functions") + +# Save function list +with open(os.path.join(output_dir, "functions.json"), "w") as f: + json.dump(functions, f, indent=2) + +# 2. Find RPC-related functions (vRPC/pRPC handlers) +print("\n--- Finding RPC Handlers ---") +rpc_keywords = [ + "rpc", "Rpc", "RPC", + "vgpu", "Vgpu", "VGPU", + "vmiop", "Vmiop", "VMIOP", + "hal", "Hal", "HAL", + "bar", "Bar", "BAR", + "migrate", "Migrate", "MIGRATE", + "init", "Init", "INIT", + "boot", "Boot", "BOOT", + "kgsp", "Kgsp", "KGSP" +] + +rpc_functions = [] +for func in functions: + name_lower = func["name"].lower() + for kw in rpc_keywords: + if kw.lower() in name_lower: + rpc_functions.append(func) + break + +print(f"Found {len(rpc_functions)} potential RPC-related functions") +with open(os.path.join(output_dir, "rpc_functions.json"), "w") as f: + json.dump(rpc_functions, f, indent=2) + +# 3. Find strings related to vGPU initialization +print("\n--- Finding vGPU-related Strings ---") +vgpu_strings = [] +string_iter = listing.getDefinedData(True) +while string_iter.hasNext(): + data = string_iter.next() + if data.getDataType().getName() == "string": + value = data.getValue() + if value: + val_str = str(value).lower() + if any(kw in val_str for kw in ["vgpu", "vmiop", "rpc", "hal", "bar", "migrate", "init", "boot", "kgsp", "wpr", "firmware", "version"]): + vgpu_strings.append({ + "address": str(data.getAddress()), + "value": str(value) + }) + +print(f"Found {len(vgpu_strings)} vGPU-related strings") +with open(os.path.join(output_dir, "vgpu_strings.json"), "w") as f: + json.dump(vgpu_strings, f, indent=2) + +# 4. Find entry points and initialization functions +print("\n--- Finding Entry Points ---") +entry_points = [] +# Look for common entry point names +entry_names = [ + "kgspInitRm_IMPL", "kgspInitRm", "kgspBootGspRm", "kgspBootGspRm_IMPL", + "gspBoot", "gspInit", "gspMain", + "nv_vgpu_vfio_get_ops", "nv_vgpu_vfio_set_ops", + "vgpu_init", "vgpu_init_IMPL", + "vmiop_init", "vmiop_init_IMPL", + "hal_init", "hal_init_IMPL" +] + +for name in entry_names: + func = function_manager.getFunction(name) + if func: + entry_points.append({ + "name": name, + "entry": str(func.getEntryPoint()), + "size": func.getBody().getNumAddresses() + }) + print(f" Found: {name} at {func.getEntryPoint()}") + +# Also check for functions called from reset vector / entry +# Look at the first few instructions +entry_addr = program.getMinAddress() +print(f"\nProgram entry address: {entry_addr}") + +# 5. Find WPR (Write Protected Region) related code +print("\n--- Finding WPR-related Code ---") +wpr_functions = [] +for func in functions: + name_lower = func["name"].lower() + if "wpr" in name_lower or "write_protect" in name_lower or "protect" in name_lower: + wpr_functions.append(func) + +print(f"Found {len(wpr_functions)} WPR-related functions") +with open(os.path.join(output_dir, "wpr_functions.json"), "w") as f: + json.dump(wpr_functions, f, indent=2) + +# 6. Find memory management functions +print("\n--- Finding Memory Management Functions ---") +mem_keywords = ["alloc", "free", "map", "unmap", "pin", "unpin", "dma", "iommu", "mmio", "bar"] +mem_functions = [] +for func in functions: + name_lower = func["name"].lower() + for kw in mem_keywords: + if kw in name_lower: + mem_functions.append(func) + break + +print(f"Found {len(mem_functions)} memory management functions") +with open(os.path.join(output_dir, "mem_functions.json"), "w") as f: + json.dump(mem_functions, f, indent=2) + +# 7. Find interrupt/MSI-X related functions +print("\n--- Finding Interrupt/MSI-X Functions ---") +irq_keywords = ["irq", "interrupt", "msi", "msix", "eventfd", "doorbell"] +irq_functions = [] +for func in functions: + name_lower = func["name"].lower() + for kw in irq_keywords: + if kw in name_lower: + irq_functions.append(func) + break + +print(f"Found {len(irq_functions)} interrupt-related functions") +with open(os.path.join(output_dir, "irq_functions.json"), "w") as f: + json.dump(irq_functions, f, indent=2) + +# 8. Generate summary report +print("\n--- Generating Summary Report ---") +report = { + "firmware": "gsp-570.144", + "architecture": program.getLanguage().toString(), + "total_functions": len(functions), + "rpc_functions": len(rpc_functions), + "vgpu_strings": len(vgpu_strings), + "entry_points": entry_points, + "wpr_functions": len(wpr_functions), + "mem_functions": len(mem_functions), + "irq_functions": len(irq_functions) +} + +with open(os.path.join(output_dir, "analysis_summary.json"), "w") as f: + json.dump(report, f, indent=2) + +# Print summary +print(json.dumps(report, indent=2)) + +# 9. Decompile key functions and save +print("\n--- Decompiling Key Functions ---") +from ghidra.app.decompiler import DecompInterface +from ghidra.util.task import ConsoleTaskMonitor + +decompiler = DecompInterface() +decompiler.openProgram(program) +monitor = ConsoleTaskMonitor() + +key_functions = entry_points + rpc_functions[:20] + wpr_functions + mem_functions[:10] + irq_functions[:10] +decompiled = {} + +for func_info in key_functions: + func = function_manager.getFunctionAt(func_info["entry"]) + if func: + result = decompiler.decompileFunction(func, 60, monitor) + if result.decompileCompleted(): + decompiled[func_info["name"]] = result.getDecompiledFunction().getC() + print(f" Decompiled: {func_info['name']}") + else: + print(f" Failed to decompile: {func_info['name']}") + +# Save decompiled code +with open(os.path.join(output_dir, "decompiled_key_functions.c"), "w") as f: + for name, code in decompiled.items(): + f.write(f"\n// ===== {name} =====\n") + f.write(code) + f.write("\n") + +print(f"\nDecompiled {len(decompiled)} key functions") +print("Analysis complete!")
\ No newline at end of file diff --git a/tools/ghidra/gsp-tu104-570.144/scripts/basic_analysis.py b/tools/ghidra/gsp-tu104-570.144/scripts/basic_analysis.py new file mode 100644 index 0000000..4992106 --- /dev/null +++ b/tools/ghidra/gsp-tu104-570.144/scripts/basic_analysis.py @@ -0,0 +1,108 @@ +# Ghidra script to do basic disassembly analysis of GSP .fwimage binary +# Usage: analyzeHeadless ... -postScript basic_analysis.py + +import sys +import os +import json + +# Get the current program +program = getCurrentProgram() +listing = program.getListing() +memory = program.getMemory() +function_manager = program.getFunctionManager() + +print("=== GSP Firmware .fwimage Basic Analysis ===") +print(f"Program: {program.getName()}") +print(f"Architecture: {program.getLanguage().toString()}") + +# Create output directory +output_dir = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP" +os.makedirs(output_dir, exist_ok=True) + +# 1. Check memory blocks +print("\n--- Memory Blocks ---") +mem_blocks = [] +for block in memory.getBlocks(): + mem_blocks.append({ + "name": block.getName(), + "start": str(block.getStart()), + "end": str(block.getEnd()), + "size": block.getSize(), + "execute": block.isExecute(), + "read": block.isRead(), + "write": block.isWrite() + }) + print(f" {block.getName()}: {block.getStart()} - {block.getEnd()} ({block.getSize()} bytes) r={block.isRead()} w={block.isWrite()} x={block.isExecute()}") + +with open(os.path.join(output_dir, "memory_blocks.json"), "w") as f: + json.dump(mem_blocks, f, indent=2) + +# 2. Find all functions (but don't decompile yet) +print("\n--- Finding Functions ---") +functions = [] +func_iter = function_manager.getFunctions(True) +count = 0 +while func_iter.hasNext(): + func = func_iter.next() + functions.append({ + "name": func.getName(), + "entry": str(func.getEntryPoint()), + "size": func.getBody().getNumAddresses() + }) + count += 1 + if count % 1000 == 0: + print(f" Found {count} functions...") + +print(f"Total functions: {len(functions)}") + +with open(os.path.join(output_dir, "functions.json"), "w") as f: + json.dump(functions, f, indent=2) + +# 3. Find strings +print("\n--- Finding Strings ---") +strings = [] +string_iter = listing.getDefinedData(True) +while string_iter.hasNext(): + data = string_iter.next() + if data.getDataType().getName() == "string": + value = data.getValue() + if value and len(str(value)) > 4: + strings.append({ + "address": str(data.getAddress()), + "value": str(value) + }) + +print(f"Found {len(strings)} strings") + +# Filter for interesting strings +interesting_keywords = ["vgpu", "vmiop", "rpc", "hal", "bar", "migrate", "init", "boot", "kgsp", "wpr", "firmware", "version", "error", "fail", "assert", "panic", "nvidia", "rmapi", "gsp", "falcon", "mailbox", "interrupt", "dma", "iommu", "page", "mmu", "pde", "pte"] +interesting_strings = [s for s in strings if any(kw in s['value'].lower() for kw in interesting_keywords)] +print(f"Interesting strings: {len(interesting_strings)}") + +with open(os.path.join(output_dir, "interesting_strings.json"), "w") as f: + json.dump(interesting_strings, f, indent=2) + +# 4. Find entry point (offset 0) +print("\n--- Entry Point Analysis ---") +entry_addr = program.getMinAddress() +print(f"Program entry (min address): {entry_addr}") + +# Check what's at address 0 +func_at_zero = function_manager.getFunctionAt(entry_addr) +if func_at_zero: + print(f"Function at entry: {func_at_zero.getName()}") +else: + print("No function at entry point - creating one...") + # Create function at entry point + func = function_manager.createFunction("entry_point", entry_addr, None, None) + if func: + print(f"Created function: {func.getName()}") + +# 5. Look for RISC-V standard function prologues +print("\n--- Looking for Function Prologues ---") +# RISC-V typical prologue: addi sp, sp, -X; sd ra, X-8(sp); sd s0, X-16(sp); addi s0, sp, X +# Let's search for common patterns + +# 6. Find cross-references to interesting addresses +print("\n--- Analysis Complete ---") +print(f"Results saved to {output_dir}")
\ No newline at end of file diff --git a/tools/ghidra/gsp-tu104-570.144/scripts/extract_fwimage.py b/tools/ghidra/gsp-tu104-570.144/scripts/extract_fwimage.py new file mode 100644 index 0000000..d8b0bfd --- /dev/null +++ b/tools/ghidra/gsp-tu104-570.144/scripts/extract_fwimage.py @@ -0,0 +1,56 @@ +# Ghidra script to extract .fwimage section from GSP firmware ELF +# Usage: analyzeHeadless ... -postScript extract_fwimage.py + +import sys +import os + +# Get the current program +program = getCurrentProgram() + +# Get the .fwimage section +fwimage_section = program.getSection(".fwimage") +if not fwimage_section: + print("No .fwimage section found") + sys.exit(1) + +print(f"Found .fwimage section: {fwimage_section.getStart()} - {fwimage_section.getEnd()}") +print(f"Size: {fwimage_section.getLength()} bytes") + +# Get the data from the section +data = getBytes(fwimage_section.getStart(), fwimage_section.getLength()) + +# Save to file +output_dir = "/home/rmcore/Git/integdev_gpu_drv/open-iov/firmware/570-GSP" +os.makedirs(output_dir, exist_ok=True) +fwimage_path = os.path.join(output_dir, "gsp-570.144.fwimage.bin") + +with open(fwimage_path, "wb") as f: + f.write(data) + +print(f"Saved .fwimage to {fwimage_path}") + +# Also extract .fwversion +fwversion_section = program.getSection(".fwversion") +if fwversion_section: + version_data = getBytes(fwversion_section.getStart(), fwversion_section.getLength()) + version_str = version_data.decode('ascii').strip('\x00') + print(f"GSP Firmware Version: {version_str}") + + with open(os.path.join(output_dir, "gsp_version.txt"), "w") as f: + f.write(version_str) + +# Extract signatures +for i in range(3): + sig_section = program.getSection(f".fwsignature{i}") + if sig_section: + sig_data = getBytes(sig_section.getStart(), sig_section.getLength()) + sig_path = os.path.join(output_dir, f"gsp-570.144.fwsignature{i}.bin") + with open(sig_path, "wb") as f: + f.write(sig_data) + print(f"Saved signature {i} to {sig_path} ({len(sig_data)} bytes)") + +# Create a new Ghidra project for the .fwimage binary +print("Creating Ghidra project for .fwimage binary...") + +# We'll do this via a separate script call +print("Done extracting sections.")
\ No newline at end of file diff --git a/tools/ghidra/gsp-tu104-570.144/scripts/extract_gsp_version.py b/tools/ghidra/gsp-tu104-570.144/scripts/extract_gsp_version.py new file mode 100644 index 0000000..ef44d7d --- /dev/null +++ b/tools/ghidra/gsp-tu104-570.144/scripts/extract_gsp_version.py @@ -0,0 +1,30 @@ +# Ghidra script to extract version from .fwversion section of GSP binary +# Usage: analyzeHeadless ... -postScript extract_gsp_version.py + +import sys + +# Get the current program +program = getCurrentProgram() + +# Get the .fwversion section +fwversion_section = program.getSection(".fwversion") +if fwversion_section: + # Get the data from the section + data = getBytes(fwversion_section.getStart(), fwversion_section.getLength()) + version_str = data.decode('ascii').strip('\x00') + print(f"GSP Firmware Version: {version_str}") + + # Also save to a file + with open("/home/rmcore/Git/integdev_gpu_drv/open-iov/tools/ghidra/gsp-tu104-570.144/gsp_version.txt", "w") as f: + f.write(version_str) +else: + print("No .fwversion section found") + + # Try to find version in memory + listing = program.getListing() + for i in range(listing.getNumAddresses()): + addr = listing.getAddress(i) + data = getBytes(addr, 32) + if data and b"570.144" in data: + print(f"Found version at {addr}: {data}") + break diff --git a/tools/guest-diag/README.md b/tools/guest-diag/README.md new file mode 100644 index 0000000..618663b --- /dev/null +++ b/tools/guest-diag/README.md @@ -0,0 +1,103 @@ +# guest-diag - tracing tools for the emulated vGPU guest + +Tools used while driving the guest's stock NVIDIA driver through CUDA bring-up. +All are tiny and dependency-free (libc only). + +## ptrace_ioctl.c - ioctl/result tracer (the important one) + +libcuda issues **raw `syscall` instructions**, so LD_PRELOAD shims and ltrace +never see its ioctls. This ptrace tracer follows a child with +PTRACE_SYSCALL + PTRACE_GET_SYSCALL_INFO and dumps: + +- `req=0x25` (UVM_REGISTER_GPU) buffer on entry and exit + (`rmStatus` is the high dword of the 5th qword) - this is how the current + failure was found: `rmStatus = 0x56 NV_ERR_NOT_SUPPORTED` +- `req=0xc020462a` (NV_ESC_RM_CONTROL) cmd + status +- `req=0xc020462b` / `0x...4629` (RM alloc/free) status when nonzero +- UVM ioctls 0x1/0x2/0x17/0x19/0x25/0x27 + +Build/run **in the guest**: + + gcc -O1 -w -o /tmp/ptio ptrace_ioctl.c + /tmp/ptio /tmp/cdt 2>&1 | grep -E 'ptio|rmctl|rmio|\^' + +Note: `op` in `struct ptrace_syscall_info` is ENTRY=1 / EXIT=2, and the struct +has `pad[3], arch, instruction_pointer, stack_pointer` before the union. + +## sysc_shim.c - known dead end + +An `syscall()` LD_PRELOAD interceptor: libcuda bypasses it entirely (inline +syscall instruction). Kept only as documentation of that fact. + +## vmcycle.sh - reload cycle + +pkill QEMU -> remove mdevs -> rmmod/insmod open-iov-vfio.ko -> create a fresh +`nvidia-nvidia-vgpu-turing` mdev -> patch its UUID into +launch_qemu_rootport.sh -> relaunch. Guest boots in ~60-75 s. + +## launch_qemu_rootport.sh - QEMU command line with pcie-root-port + +Snapshot of the working launch command (derived from 's argv of a stopped +VM) plus the root-port fix. **Requires** the mdev UUID to be substituted and: + + mkdir -p /tmp/sockets/<domain-uuid> # else QMP bind fails + rm -f /tmp/qemu-guest-agent.sock /tmp/qemu-qmp.sock + +Why the root port matters: on the Q35 root bus (`pcie.0`) QEMU's +`vfio_setup_pcie_cap()` (hw/vfio/pci.c) relabels the mdev as "Root Complex +Integrated Endpoint" and clears LNK/LNK2 + DeviceCaps, so the guest reads +`LnkCap = 0` and `getPCIELinkRateMBps()` fails with "Unknown PCIe speed". +Behind a `pcie-root-port` QEMU leaves the capability alone. + +## rmq_probe.c - legacy-ioctl RM control probe (guest or host) + +Minimal RM client (client alloc + device alloc + NV_ESC_RM_CONTROL) to query +device/subdevice controls from userspace, e.g. +`0x800289 NV0080_CTRL_CMD_GPU_GET_VIRTUALIZATION_MODE`, +`0x800280 GET_NUM_SUBDEVICES`, `0x20800123 GPU_GET_ENGINES`. +NOTE: the device alloc currently fails (0x40 in guest, 0x1b on host) - the +device allocation sequence needs the proper NV0080_ALLOC_PARAMETERS handling +(or the NVOS64 path that tools/rmq/rmq.c uses); fix before relying on it. + +## kret.sh - kretprobe attempt (blocked) + +Tries to kretprobe the nvGpuOps*/nvUvmInterface* chain to read return values. +Blocked on this kernel: "trace_kprobe: Could not probe notrace function +<name> [nvidia]" - kprobes are not available for nvidia module text. +Kept for documentation. + +## push_guest_tools.sh - rebuild the guest-side binaries from these sources + +The guest's /tmp is wiped on boot; this pushes+compiles ptio/ge/rmqp/cdt into +the guest from the sources kept here. Run it after every VM start. + +## guest_engines.c - client RM introspection (used with the ptrace tracer) + +Queries the guest client RM for exactly the data UVM's register path uses: +virtualization mode (0x800289), subdevice count (0x800280), CLASS LIST V2 +(0x800292), ENGINE LIST V2 (0x20800170), CE caps (0x20802a03) and CE->PCE +masks (0x20802a02). Needs the NVOS64 alloc path; see rmq_probe.c note. + +## Guest-side UVM logging (critical for diagnosis!) + +UVM's messages are silent until enabled: + + echo 1 | sudo tee /sys/module/nvidia_uvm/parameters/uvm_debug_prints + +Then run /tmp/cdt and `dmesg | grep nvidia-uvm`. This produced the current +blocker's exact message: +`nvidia-uvm: uvm_gpu.c:1574 init_gpu[pid:..] Failed to initialize the +channel manager: Call not supported [NV_ERR_NOT_SUPPORTED]` +(also useful: uvm_enable_debug_procfs, uvm_debug_enable_push_desc, +uvm_release_asserts, uvm_release_asserts_dump_stack) + +## launch_qemu_rootport.sh / .run.sh - QEMU command line + +Template (root-port fix) + runtime copy with the live mdev UUID, patched by +vmcycle.sh. Sockets still live under /tmp because scripts/vgpu_vm.py talks to +/tmp/qemu-guest-agent.sock by design. + +## dbg_seq.sh / dbg_seq2.sh - control-sequence helpers (guest, needs ptio) + +Print the RM control sequence immediately before/after the failing +UVM_REGISTER_GPU ioctl. diff --git a/tools/guest-diag/dbg_seq.sh b/tools/guest-diag/dbg_seq.sh new file mode 100755 index 0000000..247bc91 --- /dev/null +++ b/tools/guest-diag/dbg_seq.sh @@ -0,0 +1,5 @@ +#!/bin/bash +/tmp/ptio /tmp/cdt > /dev/null 2>/tmp/tr.log +n=$(grep -n 'req=0x25' /tmp/tr.log | head -1 | cut -d: -f1) +echo "marker line: $n" +sed -n "$((n-28)),$((n+2))p" /tmp/tr.log diff --git a/tools/guest-diag/dbg_seq2.sh b/tools/guest-diag/dbg_seq2.sh new file mode 100755 index 0000000..8710fd9 --- /dev/null +++ b/tools/guest-diag/dbg_seq2.sh @@ -0,0 +1,5 @@ +#!/bin/bash +/tmp/ptio /tmp/cdt > /dev/null 2>/tmp/tr.log +n=$(grep -n 'req=0x25' /tmp/tr.log | head -1 | cut -d: -f1) +echo "=== controls AFTER register (line $n), first 40: ===" +sed -n "$((n+1)),$((n+45))p" /tmp/tr.log diff --git a/tools/guest-diag/guest_engines.c b/tools/guest-diag/guest_engines.c new file mode 100644 index 0000000..10209e2 --- /dev/null +++ b/tools/guest-diag/guest_engines.c @@ -0,0 +1,192 @@ +/* guest_engines.c - query the guest client RM for the data UVM's register + * path depends on: virtualization mode, subdevice count, ENGINE LIST, + * CLASS LIST, CE caps and CE->PCE masks. Uses the NVOS64 serialized alloc + * path (same as tools/rmq/rmq.c) so device allocs succeed. + * + * Build in guest: gcc -O1 -o /tmp/ge guest_engines.c ; run /tmp/ge + */ +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <sys/ioctl.h> +#include <fcntl.h> +#include <unistd.h> +#include <stdlib.h> + +typedef uint32_t NvU32; +typedef uint32_t NvHandle; +typedef uint64_t NvP64; + +#define NV_IOCTL_MAGIC 'F' +#define NV_ESC_RM_ALLOC 0x2B +#define NV_ESC_RM_CONTROL 0x2A +#define NV01_ROOT 0x0 +#define NV01_ROOT_CLIENT 0x41 +#define NV01_DEVICE_0 0x80 +#define NV20_SUBDEVICE_0 0x2080 + +typedef struct { + NvHandle hRoot, hObjectParent, hObjectNew; + NvU32 hClass; + NvP64 pAllocParms; + NvU32 paramsSize, status; +} NVOS21_PARAMETERS; + +typedef struct { + NvHandle hRoot, hObjectParent, hObjectNew; + NvU32 hClass; + NvP64 pAllocParms; + NvP64 pRightsRequested; + NvU32 paramsSize, flags, status; +} NVOS64_PARAMETERS; + +typedef struct { + NvHandle hClient, hObject; + NvU32 cmd, flags; + NvP64 params; + NvU32 paramsSize, status; +} NVOS54_PARAMETERS; + +static int fd; + +static NvU32 rm_alloc(NvHandle hRoot, NvHandle hParent, NvU32 hClass, + NvHandle *pOut) +{ + NVOS21_PARAMETERS p; + memset(&p, 0, sizeof(p)); + p.hRoot = hRoot; p.hObjectParent = hParent; p.hClass = hClass; + if (ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS21_PARAMETERS), &p) < 0) { + perror("alloc ioctl"); return 0xffffffff; + } + *pOut = p.hObjectNew; + return p.status; +} + +static NvU32 rm_alloc64(NvHandle hRoot, NvHandle hParent, NvU32 hClass, + NvHandle *pOut) +{ + NVOS64_PARAMETERS p; + memset(&p, 0, sizeof(p)); + p.hRoot = hRoot; p.hObjectParent = hParent; p.hClass = hClass; + p.flags = 1; /* FINN_SERIALIZED */ + if (ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS64_PARAMETERS), &p) < 0) { + perror("alloc64 ioctl"); return 0xffffffff; + } + *pOut = p.hObjectNew; + return p.status; +} + +static NvU32 rm_control(NvHandle hClient, NvHandle hObject, NvU32 cmd, + void *params, NvU32 size) +{ + NVOS54_PARAMETERS p; + memset(&p, 0, sizeof(p)); + p.hClient = hClient; p.hObject = hObject; p.cmd = cmd; + p.params = (NvP64)params; p.paramsSize = size; + if (ioctl(fd, _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS), &p) < 0) { + perror("control ioctl"); return 0xffffffff; + } + return p.status; +} + +#define MAX_CLASSES 256 + +int main(void) +{ + NvHandle hClient = 0, hDevice = 0, hSubdevice = 0; + NvU32 st; + + fd = open("/dev/nvidiactl", O_RDWR | O_CLOEXEC); + if (fd < 0) { perror("open nvidiactl"); return 1; } + + st = rm_alloc(0, 0, getenv("RC") ? NV01_ROOT_CLIENT : NV01_ROOT, &hClient); + printf("alloc client: st=0x%x hClient=0x%x\n", st, hClient); + + st = rm_alloc64(hClient, hClient, NV01_DEVICE_0, &hDevice); + printf("alloc device: st=0x%x hDevice=0x%x\n", st, hDevice); + if (st) return 2; + + st = rm_alloc64(hClient, hDevice, NV20_SUBDEVICE_0, &hSubdevice); + printf("alloc subdevice: st=0x%x hSub=0x%x\n", st, hSubdevice); + + { + struct { NvU32 mode; NvU32 isGridBuild; } v = {0xdead, 0xdead}; + st = rm_control(hClient, hDevice, 0x800289, &v, sizeof(v)); + printf("GET_VIRTUALIZATION_MODE: st=0x%x mode=%u (0=NONE,1=NMOS,2=VGX) gridBuild=%u\n", + st, v.mode, v.isGridBuild); + } + { + struct { NvU32 n; } s = {0xdead}; + st = rm_control(hClient, hDevice, 0x800280, &s, sizeof(s)); + printf("GET_NUM_SUBDEVICES: st=0x%x n=%u\n", st, s.n); + } + { + struct { NvU32 numClasses; NvU32 classList[MAX_CLASSES]; } c; + int i; + memset(&c, 0, sizeof(c)); + st = rm_control(hClient, hDevice, 0x800292, &c, sizeof(c)); + printf("GET_CLASSLIST_V2: st=0x%x numClasses=%u\n", st, c.numClasses); + if (!st && c.numClasses) { + printf(" classes:"); + for (i = 0; i < (int)c.numClasses && i < MAX_CLASSES; i++) + printf(" 0x%x", c.classList[i]); + printf("\n"); + } + } + if (hSubdevice) { + struct { NvU32 engineCount; NvU32 engineList[0x54]; } e; + int i; + memset(&e, 0, sizeof(e)); + st = rm_control(hClient, hSubdevice, 0x20800170, &e, sizeof(e)); + printf("GPU_GET_ENGINES_V2: st=0x%x engineCount=%u\n", st, e.engineCount); + if (!st && e.engineCount) { + printf(" engines:"); + for (i = 0; i < (int)e.engineCount && i < 0x54; i++) + printf(" 0x%x", e.engineList[i]); + printf("\n"); + } + for (i = 0; i < (int)e.engineCount && i < 0x54; i++) { + struct { NvU32 t, mask; } m; + struct { NvU32 t; unsigned char caps[2]; } cc; + NvU32 t = e.engineList[i]; + memset(&m, 0, sizeof(m)); + st = rm_control(hClient, hSubdevice, 0x20802a02, &m, sizeof(m)); + printf(" CE_GET_CE_PCE_MASK(type=0x%x): st=0x%x mask=0x%x\n", t, st, m.mask); + memset(&cc, 0, sizeof(cc)); + cc.t = t; + st = rm_control(hClient, hSubdevice, 0x20802a03, &cc, sizeof(cc)); + printf(" CE_GET_CAPS_V2(type=0x%x): st=0x%x caps=%02x %02x\n", t, st, cc.caps[0], cc.caps[1]); + } + } + if (hSubdevice) { + struct { + NvU32 engineData[16]; + NvU32 pbdmaIds[2]; + NvU32 pbdmaFaultIds[2]; + NvU32 numPbdmas; + char engineName[16]; + } e[32]; + struct { + NvU32 baseIndex, numEntries, bMore; + typeof(e) entries; + } t; + int i, j; + NvU32 base = 0; + memset(&t, 0, sizeof(t)); + do { + t.baseIndex = base; + st = rm_control(hClient, hSubdevice, 0x20801112, &t, sizeof(t)); + if (st) { printf("FIFO_GET_DEVICE_INFO_TABLE: st=0x%x\n", st); break; } + for (i = 0; i < (int)t.numEntries; i++) { + printf(" [%2u] %-8s numPbdmas=%u pbdmaIds=[%u,%u] engineData=[", + base + i, t.entries[i].engineName, t.entries[i].numPbdmas, + t.entries[i].pbdmaIds[0], t.entries[i].pbdmaIds[1]); + for (j = 0; j < 16; j++) + printf("%s%u", j ? "," : "", t.entries[i].engineData[j]); + printf("]\n"); + } + base += t.numEntries; + } while (t.bMore); + } + return 0; +} diff --git a/tools/guest-diag/kret.sh b/tools/guest-diag/kret.sh new file mode 100755 index 0000000..9ed1220 --- /dev/null +++ b/tools/guest-diag/kret.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# kret.sh - kretprobe the nvGpuOps* call chain in the guest's nvidia.ko and +# capture return values while /tmp/cdt runs. Run with sudo. +set -u + +TR=/sys/kernel/tracing +SYMS="nvGpuOpsGetGpuInfo nvGpuOpsDeviceCreate nvGpuOpsRmDeviceCreate \ +nvGpuOpsRmSubDeviceCreate queryFbInfo findDeviceClasses queryVirtMode \ +getSysmemLinkInfo getSystemMemoryWindow getNvswitchInfo getEgmInfo getAtsInfo \ +nvGpuOpsQueryGpuConfidentialComputeCaps allocNvlinkStatus" + +echo > $TR/kprobe_events +for f in $SYMS; do + if ! grep -q " [tT] $f\b" /proc/kallsyms; then + echo "skip $f (not in kallsyms)" + continue + fi + if ! echo "r:ret_$f $f \$retval" >> $TR/kprobe_events 2>/dev/null; then + echo "FAILED to register $f" + fi +done +echo "--- registered:" +grep ret_ $TR/kprobe_events 2>/dev/null | head -20 + +if [ -d $TR/events/kprobes ]; then + echo 1 > $TR/events/kprobes/enable + dmesg -C + /tmp/cdt >/dev/null 2>&1 + echo "--- results (retval per probe):" + grep -E "ret_" $TR/trace | awk '{n=$4; sub(/^ret_/,"",n); printf "%s %s\n", n, $NF}' | sort | uniq -c | sort -rn | head -30 + echo 0 > $TR/events/kprobes/enable + echo > $TR/trace +fi diff --git a/tools/guest-diag/launch_qemu_rootport.sh b/tools/guest-diag/launch_qemu_rootport.sh new file mode 100755 index 0000000..52728bd --- /dev/null +++ b/tools/guest-diag/launch_qemu_rootport.sh @@ -0,0 +1,2 @@ +#!/bin/bash +exec /bin/qemu-system-x86_64 -D /home/rmcore/.local/libvf.io/logs/qemu/e726ba0b-204f-4ec9-91da-983ed7f41d06-session.txt -no-hpet -mem-prealloc -qmp unix:/tmp/qemu-qmp.sock,server,nowait -chardev socket,path=/tmp/qemu-guest-agent.sock,server=on,wait=off,id=qga0 -device virtio-serial-pci -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 -serial file:/tmp/vgpu-guest-serial.log -gdb tcp::1234 -display none -device ivshmem-plain,id=shmem0,memdev=ivshmem_kvmfr -object memory-backend-file,id=ivshmem_kvmfr,mem-path=/dev/shm/kvmfr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=128M,share=yes -device ivshmem-plain,id=shmem1,memdev=ivshmem_kvmsr -object memory-backend-file,id=ivshmem_kvmsr,mem-path=/dev/shm/kvmsr-e726ba0b-204f-4ec9-91da-983ed7f41d06,size=2M,share=yes -uuid e726ba0b-204f-4ec9-91da-983ed7f41d06 -machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off -cpu host,ss=on,vmx=on,pcid=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,md-clear=on,stibp=on,arch-capabilities=on,ssbd=on,xsaveopt=on,pdpe1gb=on,ibpb=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on,hv-vapic,hv_time,hv-spinlocks=0x1fff,hv-vendor-id=null,kvm=on,topoext=on -rtc clock=host,base=localtime -m 4096 -smp cores=4,threads=1,sockets=1 -hda /home/rmcore/.local/libvf.io/kernel/linux.qcow2 --enable-kvm -device pcie-root-port,id=rp0,chassis=1,slot=30,bus=pcie.0 -device vfio-pci,id=hostdev0,bus=rp0,sysfsdev=/sys/bus/mdev/devices/<MDEV-UUID>,display=off -device rtl8139,netdev=net0 -netdev user,id=net0,hostfwd=tcp::2222-:22 -net user -net nic -device vhost-vsock-pci,guest-cid=50718 -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/main.sock,server,nowait -qmp unix:/tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06/master.sock,server,nowait diff --git a/tools/guest-diag/ptrace_ioctl.c b/tools/guest-diag/ptrace_ioctl.c new file mode 100644 index 0000000..5f96b31 --- /dev/null +++ b/tools/guest-diag/ptrace_ioctl.c @@ -0,0 +1,236 @@ +/* ptrace_ioctl.c - follow a child with PTRACE_SYSCALL and dump the buffer + * results of selected UVM ioctls. Needed because libcuda issues raw syscall + * instructions (LD_PRELOAD/ltrace never see these). + * + * Build (guest): gcc -O1 -w -o /tmp/ptio ptrace_ioctl.c + * Run: /tmp/ptio /tmp/cdt + */ +#define _GNU_SOURCE +#include <errno.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ptrace.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + +#define SYS_IOCTL_X86_64 16 + +struct syscall_info { + unsigned char op; + unsigned char pad[3]; + unsigned int arch; + unsigned long long instruction_pointer; + unsigned long long stack_pointer; + union { + struct { + unsigned long long nr; + unsigned long long args[6]; + } entry; + struct { + long long rval; + unsigned char is_error; + } exit; + } u; +}; + +static int interesting(unsigned long req) +{ + switch (req) { + case 0x1: /* UVM_RESERVE_VA */ + case 0x2: /* UVM_RELEASE_VA */ + case 0x25: /* UVM_REGISTER_GPU */ + case 0x27: /* UVM_PAGEABLE_MEM_ACCESS */ + case 0x17: /* UVM_EVENT_CTRL */ + case 0x19: /* UVM_REGISTER_MPS_CLIENT */ + return 1; + default: + return 0; + } +} + +static void dump_words(pid_t pid, unsigned long addr, int nwords) +{ + int i; + + fprintf(stderr, " buffer @%#lx:", addr); + for (i = 0; i < nwords; i++) { + errno = 0; + long v = ptrace(PTRACE_PEEKDATA, pid, addr + i * 8, NULL); + if (errno) + break; + fprintf(stderr, " %016lx", (unsigned long)v); + } + fprintf(stderr, "\n"); +} + +struct saved { + pid_t pid; + int used; + unsigned long req, arg; +} saved[64]; + +static struct saved *slot(pid_t pid) +{ + int i, free_i = -1; + + for (i = 0; i < 64; i++) { + if (saved[i].used && saved[i].pid == pid) + return &saved[i]; + if (!saved[i].used && free_i < 0) + free_i = i; + } + if (free_i >= 0) { + saved[free_i].used = 1; + saved[free_i].pid = pid; + saved[free_i].req = 0; + return &saved[free_i]; + } + return NULL; +} + +int main(int argc, char **argv) +{ + pid_t pid; + int nprint = 0; + + if (argc < 2) { + fprintf(stderr, "usage: %s prog [args...]\n", argv[0]); + return 2; + } + + pid = fork(); + if (pid == 0) { + ptrace(PTRACE_TRACEME, 0, 0, 0); + raise(SIGSTOP); + execvp(argv[1], &argv[1]); + perror("execvp"); + _exit(127); + } + + waitpid(pid, NULL, WUNTRACED); + ptrace(PTRACE_SETOPTIONS, pid, 0, + (void *)(PTRACE_O_TRACESYSGOOD | PTRACE_O_TRACECLONE | + PTRACE_O_TRACEFORK | PTRACE_O_TRACEVFORK | PTRACE_O_EXITKILL)); + ptrace(PTRACE_SYSCALL, pid, 0, 0); + + fprintf(stderr, "[ptio] attached to pid %d\n", pid); + for (;;) { + int status; + pid_t p = waitpid(-1, &status, __WALL); + + if (p < 0) { + if (errno == EINTR) + continue; + break; + } + if (WIFEXITED(status) || WIFSIGNALED(status)) + continue; + if (!WIFSTOPPED(status)) + continue; + + { + int sig = WSTOPSIG(status); + struct syscall_info si; + + memset(&si, 0, sizeof(si)); + if ((sig == (SIGTRAP | 0x80)) && + ptrace(PTRACE_GET_SYSCALL_INFO, p, sizeof(si), &si) > 0) { + struct saved *sv = slot(p); + + if (si.op == 1 /* entry */) { + if (sv && si.u.entry.nr == SYS_IOCTL_X86_64) { + sv->req = si.u.entry.args[1]; + sv->arg = si.u.entry.args[2]; + if (sv->req == 0x25) { + fprintf(stderr, "[ptio] ENTER UVM_REGISTER_GPU pid=%d\n", p); + dump_words(p, sv->arg, 6); + fprintf(stderr, " ^ IN: uuid(2q) | numaEnabled numaNodeId | rmCtrlFd hClient | hSmcPartRef rmStatus\n"); + } + } else if (sv) { + sv->req = 0; + } + } else if (si.op == 2 /* exit */ && sv && + ((sv->req & 0xff) == 0x4629ul || + ((sv->req & 0xffff00fful) == 0x0046002bul) || + ((sv->req & 0xffff00fful) == 0x0046002aul))) { + unsigned int own = (unsigned int)(sv->req & 0xff); + long q0 = 0, q1 = 0, q2 = 0, q3 = 0; + unsigned long st = 0; + + errno = 0; q0 = ptrace(PTRACE_PEEKDATA, p, sv->arg, NULL); + errno = 0; q1 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 8, NULL); + errno = 0; q2 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 16, NULL); + errno = 0; q3 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 24, NULL); + if (own == 0x29) + st = (unsigned long)((q1 >> 32) & 0xfffffffful); + else + st = (unsigned long)((q3 >> 32) & 0xfffffffful); + if (st != 0) + fprintf(stderr, + "[rmio] pid=%d nr=0x%x status=0x%lx hClient=0x%lx hObj=0x%lx cmd/class=0x%lx size=%lu\n", + p, own, + st, + (unsigned long)(q0 & 0xfffffffful), + (unsigned long)((q0 >> 32) & 0xfffffffful), + (unsigned long)(q1 & 0xfffffffful), + (unsigned long)((q3 >> 32) & 0xfffffffful)); + } else if (si.op == 2 /* exit */ && sv && + sv->req == 0xc020462aul) { + long q0, q1, q2, q3; + errno = 0; + q1 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 8, NULL); + errno = 0; + q2 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 16, NULL); + errno = 0; + q3 = ptrace(PTRACE_PEEKDATA, p, sv->arg + 24, NULL); + q0 = ptrace(PTRACE_PEEKDATA, p, sv->arg, NULL); + (void)q0; + fprintf(stderr, + "[rmctl] pid=%d cmd=0x%lx hClient=0x%lx params=%#lx size=%lu status=0x%lx\n", + p, (unsigned long)(q1 & 0xfffffffful), + (unsigned long)(q0 & 0xfffffffful), + (unsigned long)(q2 & 0xfffffffffffful), + (unsigned long)((q2 >> 32) & 0xfffffffful), + (unsigned long)((q3 >> 32) & 0xfffffffful)); + if (((q3 >> 32) & 0xfffffffful) != 0 && + (q2 & 0xfffffffffffful) > 0x10000) { + fprintf(stderr, " params:"); + { + int i2; + for (i2 = 0; i2 < 6; i2++) { + errno = 0; + long v = ptrace(PTRACE_PEEKDATA, p, + (unsigned long)(q2 & 0xfffffffffffful) + i2 * 8, NULL); + if (errno) + break; + fprintf(stderr, " %016lx", + (unsigned long)v); + } + } + fprintf(stderr, "\n"); + } + } else if (si.op == 2 /* exit */ && sv && + interesting(sv->req)) { + fprintf(stderr, + "[ptio] pid=%d ioctl req=0x%lx arg=%#lx ret=%ld%s\n", + p, sv->req, sv->arg, + si.u.exit.rval, + interesting(sv->req) ? " *" : ""); + if (sv->req == 0x25) { + dump_words(p, sv->arg, 6); + fprintf(stderr, + " ^ UVM_REGISTER_GPU: uuid(2q) | numaEnabled numaNodeId rmCtrlFd hClient | hSmcPartRef rmStatus\n"); + } else { + dump_words(p, sv->arg, 4); + } + } + } + ptrace(PTRACE_SYSCALL, p, 0, sig == (SIGTRAP | 0x80) ? 0 : + (sig == SIGTRAP ? 0 : sig)); + } + } + return 0; +} diff --git a/tools/guest-diag/push_guest_tools.sh b/tools/guest-diag/push_guest_tools.sh new file mode 100755 index 0000000..a2a3dbb --- /dev/null +++ b/tools/guest-diag/push_guest_tools.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# push_guest_tools.sh - (re)build the guest-side diagnostic binaries from the +# sources kept in this directory. The guest's /tmp is wiped on boot, so run +# this after every VM start; the sources here are the persistent copies. +# +# Usage: tools/guest-diag/push_guest_tools.sh +# (env: VM="vgpu-linux" is not needed; scripts/vgpu_vm.py talks to the +# running VM's guest agent socket) +set -e + +HERE="$(cd "$(dirname "$0")" && pwd)" +VMEXEC="$HERE/../../scripts/vgpu_vm.py" +REPO="$HERE/../.." + +push_b64() { # push_b64 <local file> <remote file> + local b64 + b64=$(base64 -w0 "$1") + "$VMEXEC" exec "echo '$b64' | base64 -d > '$2'" >/dev/null 2>&1 +} + +echo "[*] pushing sources..." +push_b64 "$HERE/ptrace_ioctl.c" /tmp/ptio.c +push_b64 "$HERE/guest_engines.c" /tmp/ge.c +push_b64 "$HERE/rmq_probe.c" /tmp/rmqp.c +push_b64 "$REPO/tools/test/cuda_drv_test.c" /tmp/cdt.c + +echo "[*] compiling in guest..." +"$VMEXEC" exec "gcc -O1 -w -o /tmp/ptio /tmp/ptio.c && \ +gcc -O1 -w -o /tmp/ge /tmp/ge.c && \ +gcc -O1 -w -o /tmp/rmqp /tmp/rmqp.c && \ +gcc -O1 -o /tmp/cdt /tmp/cdt.c -ldl && echo BUILD_OK" diff --git a/tools/guest-diag/rmq_probe.c b/tools/guest-diag/rmq_probe.c new file mode 100644 index 0000000..7554309 --- /dev/null +++ b/tools/guest-diag/rmq_probe.c @@ -0,0 +1,132 @@ +/* rmq_probe.c - minimal legacy-ioctl RM client to read device controls: + * 0x800289 NV0080_CTRL_CMD_GPU_GET_VIRTUALIZATION_MODE + * 0x800280 NV0080_CTRL_CMD_GPU_GET_NUM_SUBDEVICES + * 0x800292 NV0080_CTRL_CMD_GPU_GET_CLASSLIST_V2 (count only) + * plus (optional) the A080 VGPU config query used by UVM's queryVirtMode. + * + * Build in guest: gcc -O1 -o /tmp/rmqp rmq_probe.c ; run /tmp/rmqp + */ +#include <errno.h> +#include <fcntl.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <sys/ioctl.h> +#include <unistd.h> + +#define NV_IOCTL_MAGIC 'F' +#define IOCTL_DIR(nr, sz) _IOC(_IOC_READ | _IOC_WRITE, NV_IOCTL_MAGIC, nr, sz) + +#define NV_ESC_RM_ALLOC 0x2b +#define NV_ESC_RM_CONTROL 0x2a +#define NV_ESC_RM_FREE 0x29 + +struct nvos21 { + uint32_t hRoot, hObjectParent, hObjectNew, hClass; + uint64_t pAllocParms; + uint32_t paramsSize, status; +}; + +struct nvos54 { + uint32_t hClient, hObject, cmd, flags; + uint64_t params; + uint32_t paramsSize, status; +}; + +#define NV01_ROOT_CLIENT 0x00000041 +#define NV01_DEVICE_0 0x00000080 + +static int fd; + +static int rm_alloc(uint32_t hRoot, uint32_t hParent, uint32_t *hNew, + uint32_t hClass, void *parms, uint32_t parmsSize) +{ + struct nvos21 a; + + memset(&a, 0, sizeof(a)); + a.hRoot = hRoot; + a.hObjectParent = hParent; + a.hObjectNew = *hNew; + a.hClass = hClass; + a.pAllocParms = (uint64_t)(uintptr_t)parms; + a.paramsSize = parmsSize; + if (ioctl(fd, IOCTL_DIR(NV_ESC_RM_ALLOC, sizeof(a)), &a) < 0) { + perror("ioctl alloc"); + return -1; + } + *hNew = a.hObjectNew; + return (int)a.status; +} + +static int rm_control(uint32_t hClient, uint32_t hObject, uint32_t cmd, + void *params, uint32_t paramsSize) +{ + struct nvos54 c; + + memset(&c, 0, sizeof(c)); + c.hClient = hClient; + c.hObject = hObject; + c.cmd = cmd; + c.params = (uint64_t)(uintptr_t)params; + c.paramsSize = paramsSize; + if (ioctl(fd, IOCTL_DIR(NV_ESC_RM_CONTROL, sizeof(c)), &c) < 0) { + perror("ioctl control"); + return -1; + } + return (int)c.status; +} + +int main(void) +{ + uint32_t hClient = 0, hDevice = 0; + struct { + uint32_t deviceId; + uint32_t hClientShare; + uint32_t hTargetClient; + uint32_t hTargetDevice; + uint32_t flags; + uint32_t pad0; + uint64_t vaSpaceSize; + uint64_t vaStartInternal; + uint64_t vaLimitInternal; + uint32_t vaMode; + uint32_t pad1; + } devAlloc; + int st; + struct { + uint32_t virtualizationMode; + uint32_t isGridBuild; + } virt = { 0xdeadbeef, 0xdeadbeef }; + struct { + uint32_t numSubDevices; + } subdev = { 0xdeadbeef }; + + fd = open("/dev/nvidiactl", O_RDWR | O_CLOEXEC); + if (fd < 0) { + perror("open nvidiactl"); + return 1; + } + + st = rm_alloc(0, 0, &hClient, NV01_ROOT_CLIENT, NULL, 0); + printf("alloc client: st=0x%x hClient=0x%x\n", st, hClient); + if (st != 0) + return 2; + + memset(&devAlloc, 0, sizeof(devAlloc)); + devAlloc.deviceId = 0; + st = rm_alloc(hClient, hClient, &hDevice, NV01_DEVICE_0, &devAlloc, + sizeof(devAlloc)); + printf("alloc device: st=0x%x hDevice=0x%x\n", st, hDevice); + if (st != 0) + return 3; + + st = rm_control(hClient, hDevice, 0x800289, &virt, sizeof(virt)); + printf("GET_VIRTUALIZATION_MODE: st=0x%x mode=0x%x isGridBuild=%u\n", + st, virt.virtualizationMode, virt.isGridBuild); + + st = rm_control(hClient, hDevice, 0x800280, &subdev, sizeof(subdev)); + printf("GET_NUM_SUBDEVICES: st=0x%x numSubDevices=%u\n", + st, subdev.numSubDevices); + + return 0; +} diff --git a/tools/guest-diag/sysc_shim.c b/tools/guest-diag/sysc_shim.c new file mode 100644 index 0000000..9b07e4c --- /dev/null +++ b/tools/guest-diag/sysc_shim.c @@ -0,0 +1,73 @@ +/* sysc_shim.c - intercept syscall() so we can dump ioctl buffer *results* + * for UVM ioctls (libcuda calls syscall(SYS_ioctl, ...) directly, so an + * ioctl() wrapper is bypassed but syscall() is not). + * Build in guest: gcc -shared -fPIC -O1 -o /tmp/sysc.so sysc_shim.c -ldl + * Run: LD_PRELOAD=/tmp/sysc.so /tmp/cdt + */ +#define _GNU_SOURCE +#include <dlfcn.h> +#include <stdarg.h> +#include <stddef.h> +#include <sys/syscall.h> +#include <unistd.h> + +static long (*real_syscall)(long, ...); + +static void dump(const char *tag, unsigned long req, void *arg, long ret) +{ + unsigned int *p = (unsigned int *)arg; + char buf[512]; + int n, i; + long *vp = (long *)arg; + + if (!p) + return; + n = 0; + n += __builtin_snprintf(buf + n, sizeof(buf) - n, + "SYSC[%s] req=0x%lx ret=%ld dwords: ", tag, req, + ret); + for (i = 0; i < 16 && n < (int)sizeof(buf) - 16; i++) + n += __builtin_snprintf(buf + n, sizeof(buf) - n, "%08x ", + p[i]); + (void)vp; + write(2, buf, n); + write(2, "\n", 1); +} + +long syscall(long nr, ...) +{ + va_list ap; + long a1, a2, a3, a4, a5, a6; + long ret; + + if (!real_syscall) + real_syscall = dlsym(RTLD_NEXT, "syscall"); + + va_start(ap, nr); + a1 = va_arg(ap, long); + a2 = va_arg(ap, long); + a3 = va_arg(ap, long); + a4 = va_arg(ap, long); + a5 = va_arg(ap, long); + a6 = va_arg(ap, long); + va_end(ap); + + ret = real_syscall(nr, a1, a2, a3, a4, a5, a6); + + if (nr == SYS_ioctl) { + unsigned long req = (unsigned long)a2; + switch (req) { + case 0x1: /* UVM_RESERVE_VA */ + case 0x2: /* UVM_RELEASE_VA */ + case 0x25: /* UVM_REGISTER_GPU */ + case 0x27: /* UVM_PAGEABLE_MEM_ACCESS */ + case 0x17: /* UVM_EVENT_CTRL */ + case 0x19: /* UVM_REGISTER_MPS_CLIENT */ + dump("uvmioctl", req, (void *)a3, ret); + break; + default: + break; + } + } + return ret; +} diff --git a/tools/guest-diag/vmcycle.sh b/tools/guest-diag/vmcycle.sh new file mode 100755 index 0000000..6faaa75 --- /dev/null +++ b/tools/guest-diag/vmcycle.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# vmcycle.sh - reload cycle for the emulated vGPU stack: +# stop QEMU -> remove mdevs -> rmmod/insmod open-iov-vfio.ko -> create a +# fresh mdev -> patch its UUID into the runtime launch script -> relaunch. +# +# ALL artifacts stay inside this directory (tools/guest-diag/) so nothing is +# lost on reboot: +# launch_qemu_rootport.sh - template QEMU command line (root-port fix) +# launch_qemu_rootport.run.sh - runtime copy with the current mdev UUID +# run/qemu.log - QEMU stderr/stdout of the last launch +# +# Usage: tools/guest-diag/vmcycle.sh +# Env: MDEV_PARENT (default: 0000:01:00.0) +# MDEV_TYPE (default: nvidia-nvidia-vgpu-turing) +set -e + +HERE="$(cd "$(dirname "$0")" && pwd)" +TEMPLATE="$HERE/launch_qemu_rootport.sh" +QEMU_SCRIPT="${QEMU_SCRIPT:-$HERE/launch_qemu_rootport.run.sh}" +MODULE="${MODULE:-$HERE/../../kernel/open-iov-vfio.ko}" +MDEV_PARENT="${MDEV_PARENT:-0000:01:00.0}" +MDEV_TYPE="${MDEV_TYPE:-nvidia-nvidia-vgpu-turing}" +RUNDIR="$HERE/run" + +mkdir -p "$RUNDIR" +[ -f "$QEMU_SCRIPT" ] || cp "$TEMPLATE" "$QEMU_SCRIPT" + +pkill -f qemu-system-x86_64 2>/dev/null || true +sleep 4 +for m in /sys/bus/mdev/devices/*; do + [ -d "$m" ] && echo 1 | sudo tee "$m/remove" >/dev/null +done +sudo rmmod open_iov_vfio 2>/dev/null || true +sudo insmod "$MODULE" +U=$(uuidgen) +echo "$U" | sudo tee "/sys/class/mdev_bus/$MDEV_PARENT/mdev_supported_types/$MDEV_TYPE/create" >/dev/null +sed -i "s|/sys/bus/mdev/devices/[0-9a-f-]*|/sys/bus/mdev/devices/$U|" "$QEMU_SCRIPT" +mkdir -p /tmp/sockets/e726ba0b-204f-4ec9-91da-983ed7f41d06 +rm -f /tmp/qemu-guest-agent.sock /tmp/qemu-qmp.sock +(setsid nohup "$QEMU_SCRIPT" > "$RUNDIR/qemu.log" 2>&1 &) +sleep 6 +if pgrep -f qemu-system-x86_64 >/dev/null; then + echo "VM up (mdev $U) [qemu log: $RUNDIR/qemu.log]" +else + echo "VM FAILED"; tail -3 "$RUNDIR/qemu.log"; exit 1 +fi diff --git a/tools/guest_agent/guest_test_agent.py b/tools/guest_agent/guest_test_agent.py new file mode 100644 index 0000000..15a578d --- /dev/null +++ b/tools/guest_agent/guest_test_agent.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +""" +guest_test_agent.py - Runs inside guest VM, executes tests, reports results +Communication via vsock, serial, or HTTP +""" + +import sys +import os +import json +import time +import subprocess +import threading +import socket +import argparse +from datetime import datetime +from pathlib import Path + +class GuestTestAgent: + def __init__(self, listen_port=9999, report_url=None, vsock_cid=3, vsock_port=9999): + self.listen_port = listen_port + self.report_url = report_url + self.vsock_cid = vsock_cid + self.vsock_port = vsock_port + self.running = False + self.results = [] + + def start_server(self): + """Start command listener (TCP or vsock)""" + self.running = True + + # Try vsock first (preferred for VM-host communication) + try: + import socket + if hasattr(socket, 'AF_VSOCK'): + self.server = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) + self.server.bind((self.vsock_cid, self.vsock_port)) + self.server.listen(5) + print(f"[{datetime.now()}] Guest agent listening on vsock CID:{self.vsock_cid} PORT:{self.vsock_port}") + else: + raise ImportError("vsock not available") + except Exception: + # Fallback to TCP + self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.server.bind(('0.0.0.0', self.listen_port)) + self.server.listen(5) + print(f"[{datetime.now()}] Guest agent listening on TCP port {self.listen_port}") + + while self.running: + try: + conn, addr = self.server.accept() + threading.Thread(target=self._handle_connection, args=(conn,), daemon=True).start() + except Exception as e: + if self.running: + print(f"[{datetime.now()}] Server error: {e}") + + def _handle_connection(self, conn): + """Handle incoming test commands""" + try: + data = conn.recv(4096).decode('utf-8') + if not data: + return + + cmd = json.loads(data) + action = cmd.get('action') + test_id = cmd.get('test_id', 'unknown') + + print(f"[{datetime.now()}] Received command: {action} (test_id: {test_id})") + + if action == 'ping': + result = {'status': 'ok', 'agent': 'guest_test_agent', 'version': '1.0'} + elif action == 'run_test': + result = self._run_test(cmd.get('test'), cmd.get('args', {})) + elif action == 'run_suite': + result = self._run_suite(cmd.get('suite')) + elif action == 'get_results': + result = {'results': self.results} + elif action == 'shutdown': + self.running = False + result = {'status': 'shutting_down'} + else: + result = {'error': f'Unknown action: {action}'} + + result['test_id'] = test_id + conn.send(json.dumps(result).encode('utf-8')) + + except Exception as e: + conn.send(json.dumps({'error': str(e)}).encode('utf-8')) + finally: + conn.close() + + def _run_test(self, test_name, args): + """Run a single test""" + test_method = getattr(self, f'test_{test_name}', None) + if not test_method: + return {'status': 'error', 'message': f'Unknown test: {test_name}'} + + start = time.time() + try: + result = test_method(**args) + result['duration'] = time.time() - start + result['timestamp'] = datetime.now().isoformat() + self.results.append(result) + return result + except Exception as e: + return {'status': 'error', 'message': str(e), 'duration': time.time() - start} + + def _run_suite(self, suite_name): + """Run a test suite""" + suite_method = getattr(self, f'suite_{suite_name}', None) + if not suite_method: + return {'status': 'error', 'message': f'Unknown suite: {suite_name}'} + + return suite_method() + + # ============================================================ + # INDIVIDUAL TESTS + # ============================================================ + + def test_nvidia_smi(self): + """Run nvidia-smi and verify GPU detection""" + try: + out = subprocess.check_output(['nvidia-smi'], text=True, timeout=30) + return { + 'test': 'nvidia_smi', + 'status': 'pass' if 'NVIDIA' in out else 'fail', + 'output': out[:2000] + } + except subprocess.CalledProcessError as e: + return {'test': 'nvidia_smi', 'status': 'fail', 'error': str(e), 'output': e.output} + except Exception as e: + return {'test': 'nvidia_smi', 'status': 'error', 'message': str(e)} + + def test_cuda_vectoradd(self): + """Compile and run CUDA vectorAdd sample""" + try: + # Find CUDA samples + cuda_samples = Path('/usr/local/cuda/samples') + if not cuda_samples.exists(): + cuda_samples = Path('/opt/cuda/samples') + + vecadd_dir = cuda_samples / '1_Utilities' / 'vectorAdd' + if not vecadd_dir.exists(): + return {'test': 'cuda_vectoradd', 'status': 'skip', 'message': 'vectorAdd sample not found'} + + # Build + subprocess.check_call(['make'], cwd=vecadd_dir, timeout=60) + + # Run + out = subprocess.check_output(['./vectorAdd'], cwd=vecadd_dir, text=True, timeout=30) + + return { + 'test': 'cuda_vectoradd', + 'status': 'pass' if 'Test passed' in out else 'fail', + 'output': out[:2000] + } + except subprocess.CalledProcessError as e: + return {'test': 'cuda_vectoradd', 'status': 'fail', 'error': str(e), 'output': e.output} + except Exception as e: + return {'test': 'cuda_vectoradd', 'status': 'error', 'message': str(e)} + + def test_cuda_devquery(self): + """Run CUDA deviceQuery""" + try: + cuda_samples = Path('/usr/local/cuda/samples') + if not cuda_samples.exists(): + cuda_samples = Path('/opt/cuda/samples') + + devq_dir = cuda_samples / '1_Utilities' / 'deviceQuery' + if not devq_dir.exists(): + return {'test': 'cuda_devquery', 'status': 'skip', 'message': 'deviceQuery sample not found'} + + subprocess.check_call(['make'], cwd=devq_dir, timeout=60) + out = subprocess.check_output(['./deviceQuery'], cwd=devq_dir, text=True, timeout=30) + + return { + 'test': 'cuda_devquery', + 'status': 'pass' if 'Result = PASS' in out else 'fail', + 'output': out[:3000] + } + except Exception as e: + return {'test': 'cuda_devquery', 'status': 'error', 'message': str(e)} + + def test_vulkan_info(self): + """Run vulkaninfo""" + try: + out = subprocess.check_output(['vulkaninfo'], text=True, timeout=30) + return { + 'test': 'vulkan_info', + 'status': 'pass' if 'GPU id' in out else 'fail', + 'output': out[:3000] + } + except Exception as e: + return {'test': 'vulkan_info', 'status': 'error', 'message': str(e)} + + def test_vkcube(self): + """Run vkcube (headless)""" + try: + # vkcube may need display; try with xvfb or skip + out = subprocess.check_output(['vkcube', '--duration', '2'], text=True, timeout=30) + return {'test': 'vkcube', 'status': 'pass', 'output': out[:2000]} + except FileNotFoundError: + return {'test': 'vkcube', 'status': 'skip', 'message': 'vkcube not installed'} + except Exception as e: + return {'test': 'vkcube', 'status': 'error', 'message': str(e)} + + def test_glxgears(self): + """Run glxgears (brief)""" + try: + out = subprocess.check_output(['timeout', '5', 'glxgears'], text=True, timeout=15) + return {'test': 'glxgears', 'status': 'pass', 'output': out[:1000]} + except Exception as e: + return {'test': 'glxgears', 'status': 'error', 'message': str(e)} + + def test_ffmpeg_nvenc(self): + """Test FFmpeg NVENC""" + try: + out = subprocess.check_output(['ffmpeg', '-encoders'], text=True, timeout=10) + has_nvenc = 'h264_nvenc' in out or 'hevc_nvenc' in out + return { + 'test': 'ffmpeg_nvenc', + 'status': 'pass' if has_nvenc else 'fail', + 'output': out[:2000] + } + except Exception as e: + return {'test': 'ffmpeg_nvenc', 'status': 'error', 'message': str(e)} + + def test_pytorch(self): + """Test PyTorch CUDA""" + try: + code = """ +import torch +print(f'CUDA available: {torch.cuda.is_available()}') +if torch.cuda.is_available(): + print(f'Device count: {torch.cuda.device_count()}') + print(f'Device name: {torch.cuda.get_device_name(0)}') + x = torch.randn(100, 100).cuda() + y = torch.randn(100, 100).cuda() + z = torch.mm(x, y) + print(f'Matmul result shape: {z.shape}') + print('PyTorch CUDA test: PASS') +else: + print('PyTorch CUDA test: FAIL - CUDA not available') +""" + out = subprocess.check_output([sys.executable, '-c', code], text=True, timeout=60) + return { + 'test': 'pytorch', + 'status': 'pass' if 'PASS' in out else 'fail', + 'output': out + } + except Exception as e: + return {'test': 'pytorch', 'status': 'error', 'message': str(e)} + + # ============================================================ + # TEST SUITES + # ============================================================ + + def suite_smoke(self): + """Quick smoke test suite""" + tests = ['nvidia_smi', 'cuda_devquery', 'cuda_vectoradd'] + return self._run_test_list(tests) + + def suite_compute(self): + """Compute-focused test suite""" + tests = ['nvidia_smi', 'cuda_devquery', 'cuda_vectoradd', 'pytorch'] + return self._run_test_list(tests) + + def suite_graphics(self): + """Graphics-focused test suite""" + tests = ['nvidia_smi', 'vulkan_info', 'vkcube', 'glxgears'] + return self._run_test_list(tests) + + def suite_video(self): + """Video encode/decode test suite""" + tests = ['nvidia_smi', 'ffmpeg_nvenc'] + return self._run_test_list(tests) + + def suite_full(self): + """Full test suite""" + tests = ['nvidia_smi', 'cuda_devquery', 'cuda_vectoradd', 'vulkan_info', 'vkcube', 'glxgears', 'ffmpeg_nvenc', 'pytorch'] + return self._run_test_list(tests) + + def _run_test_list(self, tests): + """Run a list of tests""" + results = [] + for test in tests: + print(f"[{datetime.now()}] Running suite test: {test}") + result = self._run_test(test, {}) + results.append(result) + print(f" -> {result.get('status', 'unknown')}") + + passed = sum(1 for r in results if r.get('status') == 'pass') + failed = sum(1 for r in results if r.get('status') in ('fail', 'error')) + skipped = sum(1 for r in results if r.get('status') == 'skip') + + return { + 'suite': 'custom', + 'total': len(tests), + 'passed': passed, + 'failed': failed, + 'skipped': skipped, + 'results': results + } + +def main(): + parser = argparse.ArgumentParser(description="Guest Test Agent") + parser.add_argument('--port', type=int, default=9999, help='TCP listen port') + parser.add_argument('--vsock-cid', type=int, default=3, help='VSOCK CID') + parser.add_argument('--vsock-port', type=int, default=9999, help='VSOCK port') + parser.add_argument('--run', help='Run single test and exit (for standalone mode)') + parser.add_argument('--suite', help='Run test suite and exit') + args = parser.parse_args() + + agent = GuestTestAgent(listen_port=args.port, vsock_cid=args.vsock_cid, vsock_port=args.vsock_port) + + if args.run: + # Standalone mode + result = agent._run_test(args.run, {}) + print(json.dumps(result, indent=2)) + elif args.suite: + result = getattr(agent, f'suite_{args.suite}')() + print(json.dumps(result, indent=2)) + else: + # Server mode + try: + agent.start_server() + except KeyboardInterrupt: + print("\nShutting down...") + agent.running = False + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/tools/guest_init_monitor.py b/tools/guest_init_monitor.py new file mode 100644 index 0000000..bc34a08 --- /dev/null +++ b/tools/guest_init_monitor.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +""" +guest_init_monitor.py - Monitor guest kernel initialization for NVIDIA driver +Correlates host tracepoints with guest dmesg/serial output +""" + +import sys +import os +import re +import time +import subprocess +import threading +import json +from datetime import datetime +from collections import deque +from pathlib import Path + +class GuestInitMonitor: + def __init__(self, guest_serial="/dev/ttyS0", host_trace_pipe="/sys/kernel/debug/tracing/trace_pipe"): + self.guest_serial = guest_serial + self.host_trace_pipe = host_trace_pipe + self.guest_log = deque(maxlen=1000) + self.host_events = deque(maxlen=1000) + self.failure_patterns = self._load_failure_patterns() + self.running = False + + def _load_failure_patterns(self): + return { + "pci_probe_fail": [ + r"nvidia: probe of .* failed with error -?\d+", + r"pci .*: can't claim BAR \d+", + r"pci .*: BAR \d+: no space for", + ], + "gsp_firmware_fail": [ + r"GSP.*firmware.*load.*failed", + r"GSP.*bootstrap.*failed", + r"GSP.*RPC.*timeout", + r"nv_gsp_firmware_load.*error", + ], + "msix_fail": [ + r"MSI-X.*failed", + r"pci_enable_msix.*failed", + r"request_threaded_irq.*failed", + ], + "bar_map_fail": [ + r"ioremap.*failed", + r"pci_iomap.*failed", + r"BAR.*mapping.*failed", + ], + "dma_fail": [ + r"dma_map.*failed", + r"IOMMU.*fault", + r"DMAR.*error", + ], + "vgpu_vfio_fail": [ + r"vGPU VFIO.*failed", + r"nvidia_vgpu_vfio_get_ops.*failed", + r"mdev.*register.*failed", + ], + } + + def start(self): + self.running = True + self.guest_thread = threading.Thread(target=self._monitor_guest_serial, daemon=True) + self.host_thread = threading.Thread(target=self._monitor_host_traces, daemon=True) + self.guest_thread.start() + self.host_thread.start() + print(f"[{datetime.now()}] Guest init monitor started") + print(f" Guest serial: {self.guest_serial}") + print(f" Host trace pipe: {self.host_trace_pipe}") + + def stop(self): + self.running = False + if hasattr(self, 'guest_thread'): + self.guest_thread.join(timeout=2) + if hasattr(self, 'host_thread'): + self.host_thread.join(timeout=2) + print(f"[{datetime.now()}] Guest init monitor stopped") + + def _monitor_guest_serial(self): + """Monitor guest kernel log via serial console""" + try: + # Try to open serial device or use virsh console + proc = subprocess.Popen( + ["timeout", "300", "cat", self.guest_serial], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + for line in proc.stdout: + if not self.running: + break + self._process_guest_line(line.strip()) + except Exception as e: + print(f"[{datetime.now()}] Guest serial monitor error: {e}") + + def _monitor_host_traces(self): + """Monitor host kernel tracepoints""" + try: + proc = subprocess.Popen( + ["cat", self.host_trace_pipe], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + for line in proc.stdout: + if not self.running: + break + self._process_host_trace(line.strip()) + except Exception as e: + print(f"[{datetime.now()}] Host trace monitor error: {e}") + + def _process_guest_line(self, line): + """Process a line from guest serial console""" + timestamp = datetime.now().isoformat() + entry = {"timestamp": timestamp, "source": "guest", "line": line} + self.guest_log.append(entry) + + # Check for failure patterns + failure = self._classify_failure(line) + if failure: + self._report_failure(failure, line, "guest") + + # Print important lines + if any(kw in line.lower() for kw in ["nvidia", "gsp", "vfio", "mdev", "error", "fail", "timeout"]): + print(f"[GUEST] {line}") + + def _process_host_trace(self, line): + """Process a line from host trace_pipe""" + timestamp = datetime.now().isoformat() + entry = {"timestamp": timestamp, "source": "host_trace", "line": line} + self.host_events.append(entry) + + # Look for GSP emulation tracepoints + if "gsp_emu" in line: + print(f"[HOST TRACE] {line}") + # Correlate with recent guest logs + self._correlate_events(line) + + def _classify_failure(self, line): + """Classify failure type based on patterns""" + for failure_type, patterns in self.failure_patterns.items(): + for pattern in patterns: + if re.search(pattern, line, re.IGNORECASE): + return failure_type + return None + + def _report_failure(self, failure_type, line, source): + """Report a detected failure""" + report = { + "timestamp": datetime.now().isoformat(), + "failure_type": failure_type, + "source": source, + "line": line, + "recent_guest_logs": list(self.guest_log)[-20:], + "recent_host_events": list(self.host_events)[-20:], + } + print(f"\n{'='*60}") + print(f"FAILURE DETECTED: {failure_type}") + print(f"Source: {source}") + print(f"Line: {line}") + print(f"{'='*60}\n") + + # Save failure report + report_dir = Path("/tmp/gsp_emu_failures") + report_dir.mkdir(exist_ok=True) + report_file = report_dir / f"failure_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + with open(report_file, 'w') as f: + json.dump(report, f, indent=2) + print(f"Failure report saved: {report_file}") + + def _correlate_events(self, host_event): + """Correlate host tracepoint with recent guest activity""" + # Look for matching sequence IDs, RPC types, etc. + pass + + def get_summary(self): + """Get monitoring summary""" + return { + "guest_log_entries": len(self.guest_log), + "host_events": len(self.host_events), + "recent_guest": list(self.guest_log)[-10:], + "recent_host": list(self.host_events)[-10:], + } + +def main(): + import argparse + parser = argparse.ArgumentParser(description="Monitor guest NVIDIA driver initialization") + parser.add_argument("--guest-serial", default="/dev/ttyS0", help="Guest serial device") + parser.add_argument("--host-trace", default="/sys/kernel/debug/tracing/trace_pipe", help="Host trace pipe") + parser.add_argument("--duration", type=int, default=300, help="Monitor duration (seconds)") + args = parser.parse_args() + + monitor = GuestInitMonitor(args.guest_serial, args.host_trace) + monitor.start() + + try: + time.sleep(args.duration) + except KeyboardInterrupt: + pass + finally: + monitor.stop() + summary = monitor.get_summary() + print(json.dumps(summary, indent=2)) + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/tools/module_ops/dry_run.sh b/tools/module_ops/dry_run.sh new file mode 100755 index 0000000..e6bb6f2 --- /dev/null +++ b/tools/module_ops/dry_run.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Run vGPU dry-run validation +# Usage: sudo ./dry_run.sh + +set -euo pipefail + +VGPU_KERNEL="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}/kernel" + +echo "=== Open-IOV vGPU Dry-Run Validation ===" +echo "" + +# Check if host nvidia.ko is loaded +if ! lsmod | grep -q "^nvidia "; then + echo "ERROR: Host nvidia.ko not loaded. Load it first with:" + echo " rzsh nvidia vgpu load" + exit 1 +fi + +echo "Host nvidia.ko: LOADED ✓" +echo "" + +# Load vGPU module in dry-run mode +echo "Loading nvidia-vgpu-vfio.ko in dry-run mode..." +sudo insmod "$VGPU_KERNEL/nvidia-vgpu-vfio.ko" dyndbg='+p' dry_run=1 + +# Check result +sleep 2 +if dmesg -T | grep -q "DRY-RUN MODE: validating init without hardware changes"; then + echo "" + echo "✓ Dry-run validation PASSED" + echo " Firmware, BAR layout, IRQ caps, IOMMU validated" +else + echo "" + echo "✗ Dry-run validation FAILED" + echo "Check dmesg for errors:" + dmesg -T | grep -i -e nvidia-vgpu -e vgpu | tail -20 +fi + +# Unload +echo "" +echo "Unloading dry-run module..." +sudo rmmod nvidia-vgpu-vfio + +echo "" +echo "=== Dry-Run Complete ===" +echo "If validation passed, load for real with:" +echo " sudo insmod $VGPU_KERNEL/nvidia-vgpu-vfio.ko dyndbg='+p'"
\ No newline at end of file diff --git a/tools/module_ops/load_converged.sh b/tools/module_ops/load_converged.sh new file mode 100755 index 0000000..dbd090e --- /dev/null +++ b/tools/module_ops/load_converged.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Load vGPU stack in converged mode +# Usage: sudo ./load_converged.sh [dry_run] + +set -euo pipefail + +OPEN_IOV_ROOT="${OPEN_IOV_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}" +NVIDIA_OPEN_KERNEL="$OPEN_IOV_ROOT/nvidia-open-gpu-kernel-modules/kernel-open" +VGPU_KERNEL="$OPEN_IOV_ROOT/kernel" + +DRY_RUN="${1:-false}" + +echo "=== Open-IOV Converged Mode Module Load ===" +echo "Open-IOV Root: $OPEN_IOV_ROOT" +echo "Dry-run mode: $DRY_RUN" +echo "" + +# Function to check if module is loaded +is_loaded() { + lsmod | grep -q "^$1 " +} + +# Function to load module with error checking +load_module() { + local module_path=$1 + local module_name=$(basename $module_path .ko) + local params="${2:-}" + + if is_loaded "$module_name"; then + echo "Module $module_name already loaded" + return 0 + fi + + echo "Loading $module_name..." + if sudo insmod "$module_path" $params; then + echo " ✓ $module_name loaded successfully" + return 0 + else + echo " ✗ Failed to load $module_name" + return 1 + fi +} + +# 1. Load mdev core +echo "Step 1: Loading mdev core module..." +sudo modprobe mdev +echo " ✓ mdev loaded" +echo "" + +# 2. Load host nvidia.ko with vGPU support +echo "Step 2: Loading host nvidia.ko (patched open-gpu-kernel-modules)..." +load_module "$NVIDIA_OPEN_KERNEL/nvidia.ko" "NVreg_RegisterPCIDriver=1 NVreg_RmMsg=:" +echo "" + +# 3. Load dependent modules +echo "Step 3: Loading dependent modules..." +load_module "$NVIDIA_OPEN_KERNEL/nvidia-modeset.ko" +load_module "$NVIDIA_OPEN_KERNEL/nvidia-uvm.ko" +load_module "$NVIDIA_OPEN_KERNEL/nvidia-drm.ko" +load_module "$NVIDIA_OPEN_KERNEL/nvidia-peermem.ko" +echo "" + +# 4. Load nvidia-vgpu-vfio.ko +echo "Step 4: Loading nvidia-vgpu-vfio.ko..." +if [ "$DRY_RUN" = "dry_run" ] || [ "$DRY_RUN" = "true" ]; then + echo " Running in DRY-RUN mode first..." + load_module "$VGPU_KERNEL/nvidia-vgpu-vfio.ko" "dyndbg='+p' dry_run=1" + echo " Dry-run successful, unloading..." + sudo rmmod nvidia-vgpu-vfio + echo " Loading for real..." + load_module "$VGPU_KERNEL/nvidia-vgpu-vfio.ko" "dyndbg='+p'" +else + load_module "$VGPU_KERNEL/nvidia-vgpu-vfio.ko" "dyndbg='+p'" +fi +echo "" + +# 5. Verify +echo "Step 5: Verifying vGPU registration..." +sleep 2 +if [ -d "/sys/bus/pci/devices/0000:01:00.0/mdev_supported_types" ]; then + echo " ✓ mdev_supported_types directory exists" + ls -la /sys/bus/pci/devices/0000:01:00.0/mdev_supported_types/ +else + echo " ✗ mdev_supported_types NOT found" + echo " Check dmesg for errors: dmesg -T | grep -i vgpu" + exit 1 +fi + +echo "" +echo "=== Converged Mode Load Complete ===" +echo "" +echo "Next steps:" +echo " - Create mdev device: echo <uuid> | sudo tee /sys/bus/pci/devices/0000:01:00.0/mdev_supported_types/nvidia-vgpu-turing/create" +echo " - Run QEMU with vGPU passthrough" +echo " - Run tests: $OPEN_IOV_ROOT/tools/test/run_integration_tests.sh"
\ No newline at end of file diff --git a/tools/module_ops/unload_all.sh b/tools/module_ops/unload_all.sh new file mode 100755 index 0000000..4c73a0b --- /dev/null +++ b/tools/module_ops/unload_all.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Unload all vGPU modules cleanly +# Usage: sudo ./unload_all.sh + +set -euo pipefail + +echo "=== Unloading Open-IOV vGPU Stack ===" + +# Unload in reverse order of loading +modules=( + "nvidia-vgpu-vfio" + "nvidia-drm" + "nvidia-modeset" + "nvidia-uvm" + "nvidia-peermem" + "nvidia" + "mdev" +) + +for module in "${modules[@]}"; do + if lsmod | grep -q "^$module "; then + echo "Unloading $module..." + if sudo rmmod "$module" 2>/dev/null; then + echo " ✓ $module unloaded" + else + echo " ⚠ $module could not be unloaded (may be in use)" + # Check refcnt + if [[ -f /sys/module/$module/refcnt ]]; then + echo " refcnt: $(cat /sys/module/$module/refcnt)" + fi + if [[ -f /sys/module/$module/initstate ]]; then + echo " initstate: $(cat /sys/module/$module/initstate)" + fi + fi + else + echo " - $module not loaded" + fi +done + +echo "" +echo "=== Unload Complete ===" +lsmod | grep -E 'nvidia|mdev' || echo "No NVIDIA/mdev modules loaded"
\ No newline at end of file diff --git a/tools/rmq/Makefile b/tools/rmq/Makefile new file mode 100644 index 0000000..f58e4d9 --- /dev/null +++ b/tools/rmq/Makefile @@ -0,0 +1,19 @@ +# rmq - RM query tool (userspace RM API client) +# +# Queries the host NVIDIA Resource Manager via /dev/nvidia0 ioctls. +# Used as an "oracle" to extract ground-truth static GPU data (device info +# tables, static config) from the host's real GSP-RM for the GSP-RM emulation +# layer (kernel/gsp_rm_emulation.c). + +CC ?= gcc +CFLAGS ?= -O2 -Wall + +all: rmq + +rmq: rmq.c + $(CC) $(CFLAGS) -o $@ $< + +clean: + rm -f rmq + +.PHONY: all clean diff --git a/tools/rmq/grctx_oracle.c b/tools/rmq/grctx_oracle.c new file mode 100644 index 0000000..a2fc5fa --- /dev/null +++ b/tools/rmq/grctx_oracle.c @@ -0,0 +1,189 @@ +/* grctx_oracle - query the host NVIDIA RM for the engine context buffer sizes. + * + * Goal: recover the table that GSP-RM serves for + * NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO (0x20800a32), + * so the emulation layer does not have to guess the per-engine ctx buffer + * sizes (kernel/gsp_rm_tu104.h: TU104_CTX_SIZE_*). + * + * Outcome (2026-09-22): NOT REACHABLE from userspace on a bare-metal PF: + * - 0x801707 NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES + * hObject=hDevice -> 0x57 NV_ERR_OBJECT_NOT_FOUND (not exported) + * hObject=hClient -> 0x56 NV_ERR_NOT_SUPPORTED (bound to the + * _92bfc3 hal: the client RM does not manage ctx + * buffers on a PF, only a VF's VGPU_STATIC_INFO) + * - 0x2080122d NV2080_CTRL_CMD_GR_GET_ENGINE_CONTEXT_PROPERTIES + * (subdevice) -> 0x57 / 0x56, never returns sizes + * - The producing control (0x20800a32) is internal-only. + * The real values are probed from the ctxsw ucode by the physical RM + * (grSetupInitialCtxState -> grDiscoverCtxBufferSize) and are internal to + * GSP-RM, so the emulation currently serves synthesized page-aligned sizes + * (see tu104_ctx_buffer_info in kernel/gsp_rm_tu104.h). + * + * Kept as a reference for the RM client bring-up sequence and for a future + * attempt via a kernel-side RM client. + */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <stdint.h> + +typedef uint32_t NvU32; +typedef uint64_t NvU64; +typedef uint32_t NvHandle; +typedef uint32_t NvV32; +typedef uint64_t NvP64; + +#define NV_IOCTL_MAGIC 'F' +#define NV_ESC_RM_ALLOC 0x2B +#define NV_ESC_RM_CONTROL 0x2A + +#define NV01_ROOT 0x0 +#define NV01_DEVICE_0 0x80 +#define NV20_SUBDEVICE_0 0x2080 + +typedef struct { + NvHandle hRoot; + NvHandle hObjectParent; + NvHandle hObjectNew; + NvV32 hClass; + NvP64 pAllocParms __attribute__((aligned(8))); + NvU32 paramsSize; + NvV32 status; +} NVOS21_PARAMETERS; + +typedef struct { + NvHandle hRoot; + NvHandle hObjectParent; + NvHandle hObjectNew; + NvV32 hClass; + NvP64 pAllocParms __attribute__((aligned(8))); + NvP64 pRightsRequested __attribute__((aligned(8))); + NvU32 paramsSize; + NvU32 flags; + NvV32 status; +} NVOS64_PARAMETERS; + +typedef struct { + NvHandle hClient; + NvHandle hObject; + NvV32 cmd; + NvU32 flags; + NvP64 params __attribute__((aligned(8))); + NvU32 paramsSize; + NvV32 status; +} NVOS54_PARAMETERS; + +#define NV_IOCTL_RM_ALLOC _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS21_PARAMETERS) +#define NV_IOCTL_RM_CONTROL _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS) + +#define NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES 0x801707U +#define NV2080_CTRL_CMD_GR_GET_ENGINE_CONTEXT_PROPERTIES 0x2080122dU +#define ENGINE_ID_COUNT 0x1a + +typedef struct { + NvU32 engineId; + NvU32 alignment; + NvU32 size; +} FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS; + +typedef struct { + NvU32 flags; + NvU32 pad; + NvU64 route; +} GR_ROUTE_INFO; + +typedef struct { + GR_ROUTE_INFO grRouteInfo; + NvU32 engineId; + NvU32 alignment; + NvU32 size; + NvU32 bInfoPopulated; +} GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS; + +static int fd; + +static NvU32 rm_alloc(NvHandle parent, NvHandle client, NvU32 hClass, NvHandle *pOut) +{ + NVOS21_PARAMETERS p = {0}; + p.hRoot = client; + p.hObjectParent = parent; + p.hClass = hClass; + if (ioctl(fd, NV_IOCTL_RM_ALLOC, &p) < 0) { perror("alloc ioctl"); return 1; } + if (p.status) { fprintf(stderr, "alloc class 0x%x failed: 0x%x\n", hClass, p.status); return p.status; } + *pOut = p.hObjectNew; + return 0; +} + +static NvU32 rm_alloc64(NvHandle parent, NvHandle client, NvU32 hClass, + NvHandle newHandle, void *parms, NvU32 parmsSize, NvHandle *pOut) +{ + NVOS64_PARAMETERS p = {0}; + p.hRoot = client; + p.hObjectParent = parent; + p.hObjectNew = newHandle; /* 0 = let RM generate (device); subdevices take an explicit handle */ + p.hClass = hClass; + p.pAllocParms = (NvP64)parms; + p.paramsSize = parmsSize; + p.flags = 0; + if (ioctl(fd, NV_IOCTL_RM_ALLOC, &p) < 0) { perror("alloc64 ioctl"); return 1; } + if (p.status) { + fprintf(stderr, "alloc64 class 0x%x failed: 0x%x\n", hClass, p.status); + return p.status; + } + *pOut = p.hObjectNew; + return 0; +} + +static NvU32 rm_control(NvHandle hClient, NvHandle hObject, NvU32 cmd, void *params, NvU32 size) +{ + NVOS54_PARAMETERS p = {0}; + p.hClient = hClient; p.hObject = hObject; p.cmd = cmd; + p.params = (NvP64)params; p.paramsSize = size; + if (ioctl(fd, NV_IOCTL_RM_CONTROL, &p) < 0) { perror("control ioctl"); return 1; } + return p.status; +} + +int main(void) +{ + NvHandle hClient = 0, hDevice = 0, hSubdevice = 0; + NvU32 deviceParams = 0; /* deviceId 0 */ + NvU32 subdeviceParams = 0; /* subDeviceId 0 */ + NvU32 st, i; + + fd = open("/dev/nvidiactl", O_RDWR); + if (fd < 0) { perror("open /dev/nvidiactl"); return 1; } + + if ((st = rm_alloc(0, 0, NV01_ROOT, &hClient))) return 1; + if ((st = rm_alloc64(hClient, hClient, NV01_DEVICE_0, 0, + &deviceParams, sizeof(deviceParams), &hDevice))) return 1; + if ((st = rm_alloc64(hDevice, hClient, NV20_SUBDEVICE_0, 0xcaf00001, + &subdeviceParams, sizeof(subdeviceParams), &hSubdevice))) return 1; + printf("hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n", hClient, hDevice, hSubdevice); + + printf("=== device-level FIFO_GET_ENGINE_CONTEXT_PROPERTIES (0x801707) ===\n"); + for (i = 0; i < ENGINE_ID_COUNT; i++) { + FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS p; + memset(&p, 0, sizeof(p)); + p.engineId = i; + st = rm_control(hClient, hDevice, NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES, + &p, sizeof(p)); + printf("engine[%2u] status=0x%x size=0x%08x alignment=0x%08x\n", + i, st, p.size, p.alignment); + } + + printf("=== subdevice-level GR_GET_ENGINE_CONTEXT_PROPERTIES (0x2080122d) ===\n"); + for (i = 0; i < ENGINE_ID_COUNT; i++) { + GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS p; + memset(&p, 0, sizeof(p)); + p.engineId = i; + st = rm_control(hClient, hSubdevice, NV2080_CTRL_CMD_GR_GET_ENGINE_CONTEXT_PROPERTIES, + &p, sizeof(p)); + printf("engine[%2u] status=0x%x size=0x%08x alignment=0x%08x\n", + i, st, p.size, p.alignment); + } + + return 0; +} diff --git a/tools/rmq/kmod/Makefile b/tools/rmq/kmod/Makefile new file mode 100644 index 0000000..2fde4ef --- /dev/null +++ b/tools/rmq/kmod/Makefile @@ -0,0 +1,7 @@ +obj-m += wpr2_reset_kmod.o + +all: + make -C /lib/modules/$(shell uname -r)/build M=$(CURDIR) modules + +clean: + make -C /lib/modules/$(shell uname -r)/build M=$(CURDIR) clean diff --git a/tools/rmq/kmod/wpr2_reset_kmod.c b/tools/rmq/kmod/wpr2_reset_kmod.c new file mode 100644 index 0000000..523fac7 --- /dev/null +++ b/tools/rmq/kmod/wpr2_reset_kmod.c @@ -0,0 +1,90 @@ +/* wpr2_reset_kmod - kernel module that clears GPU WPR2 registers via BAR0 ioremap. + * + * When nvidia.ko is unloaded without a clean GSP teardown, GSP firmware stays + * resident in WPR2 (a write-protected FB region described by + * PFB_PRI_MMU_WPR2_ADDR_LO/HI). Every subsequent RmInitAdapter then fails with + * "unexpected WPR2 already up" and FLR does not help (GSP survives warm resets). + * + * This module ioremaps GPU BAR0 and pokes WPR2_ADDR_LO/HI to zero so the next + * driver load takes the normal (clean) GSP boot path. Equivalent to the effect + * of FWSEC-SB + Booter Unload teardown, minus running the ucodes. + * + * Usage: sudo insmod wpr2_reset_kmod.ko [pci_bdf=0000:01:00.0] + * result printed via pr_info (check dmesg) + */ +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/io.h> + +#define NV_PFB_PRI_MMU_WPR2_ADDR_LO 0x1fa824 +#define NV_PFB_PRI_MMU_WPR2_ADDR_HI 0x1fa828 + +static char *pci_bdf = "0000:01:00.0"; +module_param(pci_bdf, charp, 0444); +MODULE_PARM_DESC(pci_bdf, "PCI BDF of the GPU (default 0000:01:00.0)"); + +static int __init wpr2_reset_init(void) +{ + struct pci_dev *pdev = NULL; + unsigned int bus, dev, fn; + void __iomem *bar0; + u32 lo_before, hi_before, lo_after, hi_after; + + if (sscanf(pci_bdf, "0000:%x:%x.%u", &bus, &dev, &fn) != 3) { + pr_err("wpr2_reset: bad pci_bdf %s\n", pci_bdf); + return -EINVAL; + } + pdev = pci_get_domain_bus_and_slot(0, bus, PCI_DEVFN(dev, fn)); + if (!pdev) { + pr_err("wpr2_reset: device %s not found\n", pci_bdf); + return -ENODEV; + } + if (!pci_resource_start(pdev, 0)) { + pr_err("wpr2_reset: BAR0 not assigned\n"); + pci_dev_put(pdev); + return -ENODEV; + } + + if (pci_enable_device(pdev)) { + pr_err("wpr2_reset: pci_enable_device failed\n"); + pci_dev_put(pdev); + return -EIO; + } + pci_set_master(pdev); + bar0 = ioremap_wc(pci_resource_start(pdev, 0), pci_resource_len(pdev, 0)); + if (!bar0) { + pr_err("wpr2_reset: ioremap failed\n"); + pci_dev_put(pdev); + return -ENOMEM; + } + + lo_before = readl(bar0 + NV_PFB_PRI_MMU_WPR2_ADDR_LO); + hi_before = readl(bar0 + NV_PFB_PRI_MMU_WPR2_ADDR_HI); + pr_info("wpr2_reset: before LO=0x%08x HI=0x%08x\n", lo_before, hi_before); + + writel(0, bar0 + NV_PFB_PRI_MMU_WPR2_ADDR_LO); + writel(0, bar0 + NV_PFB_PRI_MMU_WPR2_ADDR_HI); + + lo_after = readl(bar0 + NV_PFB_PRI_MMU_WPR2_ADDR_LO); + hi_after = readl(bar0 + NV_PFB_PRI_MMU_WPR2_ADDR_HI); + pr_info("wpr2_reset: after LO=0x%08x HI=0x%08x\n", lo_after, hi_after); + + if (hi_after == 0) + pr_info("wpr2_reset: WPR2 cleared, clean GSP boot now possible\n"); + else + pr_info("wpr2_reset: write dropped (PLM?) - WPR2 still up\n"); + + iounmap(bar0); + pci_dev_put(pdev); + return hi_after ? -EINVAL : 0; +} + +static void __exit wpr2_reset_exit(void) +{ + pr_info("wpr2_reset: unloaded\n"); +} + +module_init(wpr2_reset_init); +module_exit(wpr2_reset_exit); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Clear GPU WPR2 registers to unstick GSP boot"); diff --git a/tools/rmq/rmq.c b/tools/rmq/rmq.c new file mode 100644 index 0000000..189d647 --- /dev/null +++ b/tools/rmq/rmq.c @@ -0,0 +1,189 @@ +/* rmq - query host NVIDIA RM via /dev/nvidia0 ioctls (oracle for TU104 static data) */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <stdint.h> + +typedef uint32_t NvU32; +typedef uint16_t NvU16; +typedef uint8_t NvU8; +typedef uint64_t NvU64; +typedef uint32_t NvHandle; +typedef uint32_t NvV32; +typedef uint64_t NvP64; + +/* NVIDIA ioctl encoding: _IOWR('F', nr, type) */ +#define NV_IOCTL_MAGIC 'F' +#define NV_IOCTL_BASE 200 +#define NV_ESC_RM_ALLOC 0x2B +#define NV_ESC_RM_CONTROL 0x2A + +#define NV01_ROOT 0x0 +#define NV01_DEVICE_0 0x80 +#define NV20_SUBDEVICE_0 0x2080 + +typedef struct { + NvHandle hRoot; + NvHandle hObjectParent; + NvHandle hObjectNew; + NvV32 hClass; + NvP64 pAllocParms __attribute__((aligned(8))); + NvU32 paramsSize; + NvV32 status; +} NVOS21_PARAMETERS; + +typedef struct { + NvHandle hClient; + NvHandle hObject; + NvV32 cmd; + NvU32 flags; + NvP64 params __attribute__((aligned(8))); + NvU32 paramsSize; + NvV32 status; +} NVOS54_PARAMETERS; + +#define NV_IOCTL_RM_ALLOC _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS21_PARAMETERS) +#define NV_IOCTL_RM_CONTROL _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS) + +static int fd; + +/* NVOS64: 48 bytes (0x30), used for device/subdevice allocs (with rights) */ +typedef struct { + NvHandle hRoot; + NvHandle hObjectParent; + NvHandle hObjectNew; + NvV32 hClass; + NvP64 pAllocParms __attribute__((aligned(8))); + NvP64 pRightsRequested __attribute__((aligned(8))); + NvU32 paramsSize; + NvU32 flags; + NvV32 status; +} NVOS64_PARAMETERS; + +#define NV_IOCTL_RM_ALLOC64 _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS64_PARAMETERS) + +static NvU32 rm_alloc(NvHandle hRoot, NvHandle hParent, NvU32 hClass, NvHandle *pOut) +{ + NVOS21_PARAMETERS p = {0}; + p.hRoot = hRoot; p.hObjectParent = hParent; p.hClass = hClass; + if (ioctl(fd, NV_IOCTL_RM_ALLOC, &p) < 0) { perror("alloc ioctl"); return 1; } + if (p.status) { fprintf(stderr, "alloc class 0x%x failed: 0x%x\n", hClass, p.status); return p.status; } + *pOut = p.hObjectNew; + return 0; +} + +/* Device/subdevice allocs use NVOS64 (48 bytes) with rights */ +static NvU32 rm_alloc64(NvHandle hRoot, NvHandle hParent, NvU32 hClass, NvHandle *pOut) +{ + NVOS64_PARAMETERS p = {0}; + p.hRoot = hRoot; p.hObjectParent = hParent; p.hClass = hClass; + p.flags = 1; /* FINN_SERIALIZED */ + if (ioctl(fd, NV_IOCTL_RM_ALLOC64, &p) < 0) { perror("alloc64 ioctl"); return 1; } + if (p.status) { fprintf(stderr, "alloc64 class 0x%x failed: 0x%x\n", hClass, p.status); return p.status; } + *pOut = p.hObjectNew; + return 0; +} + +static NvU32 rm_control(NvHandle hClient, NvHandle hObject, NvU32 cmd, void *params, NvU32 size) +{ + NVOS54_PARAMETERS p = {0}; + p.hClient = hClient; p.hObject = hObject; p.cmd = cmd; + p.params = (NvP64)params; p.paramsSize = size; + if (ioctl(fd, NV_IOCTL_RM_CONTROL, &p) < 0) { perror("control ioctl"); return 1; } + return p.status; +} + +/* NV2080_CTRL_FIFO_DEVICE_ENTRY (16 u32 engineData + 2 pbdmaIds + 2 pbdmaFaultIds + numPbdmas + name[16]) */ +typedef struct { + NvU32 engineData[16]; + NvU32 pbdmaIds[2]; + NvU32 pbdmaFaultIds[2]; + NvU32 numPbdmas; + char engineName[16]; +} FIFO_DEVICE_ENTRY; + +typedef struct { + NvU32 baseIndex; + NvU32 numEntries; + NvU32 bMore; + FIFO_DEVICE_ENTRY entries[32]; +} FIFO_GET_DEVICE_INFO_TABLE_PARAMS; + +typedef struct { + NvU32 faultId, instanceId, typeEnum, resetId, devicePriBase, isEngine; + NvU32 rlEngId, runlistPriBase, groupId, ginTargetId, deviceBroadcastPriBase, groupLocalInstanceId; +} INTERNAL_DEVICE_INFO; + +typedef struct { + NvU32 numEntries; + INTERNAL_DEVICE_INFO deviceInfoTable[512]; +} INTERNAL_GET_DEVICE_INFO_TABLE_PARAMS; + +int main(int argc, char **argv) +{ + NvHandle hClient = 0, hDevice = 0, hSubdevice = 0; + NvU32 st; + + /* RM_ALLOC creates clients on /dev/nvidiactl, not /dev/nvidia0 */ + fd = open("/dev/nvidiactl", O_RDWR); + if (fd < 0) { perror("open /dev/nvidiactl"); return 1; } + + if ((st = rm_alloc(0, 0, NV01_ROOT, &hClient))) return 1; + if ((st = rm_alloc64(hClient, hClient, NV01_DEVICE_0, &hDevice))) return 1; + if ((st = rm_alloc64(hClient, hDevice, NV20_SUBDEVICE_0, &hSubdevice))) return 1; + printf("hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n", hClient, hDevice, hSubdevice); + + /* FIFO_GET_DEVICE_INFO_TABLE (0x20801112) */ + { + FIFO_GET_DEVICE_INFO_TABLE_PARAMS *t = calloc(1, sizeof(*t)); + NvU32 base = 0; + printf("\n=== FIFO_GET_DEVICE_INFO_TABLE (0x20801112) ===\n"); + do { + memset(t, 0, sizeof(*t)); + t->baseIndex = base; + st = rm_control(hClient, hSubdevice, 0x20801112, t, sizeof(*t)); + if (st) { printf("control failed: 0x%x\n", st); break; } + NvU32 i; + for (i = 0; i < t->numEntries; i++) { + FIFO_DEVICE_ENTRY *e = &t->entries[i]; + printf("[%3u] name=%-16s numPbdmas=%u pbdmaIds=[%u,%u] faultIds=[%u,%u] engineData=[", + base + i, e->engineName, e->numPbdmas, + e->pbdmaIds[0], e->pbdmaIds[1], + e->pbdmaFaultIds[0], e->pbdmaFaultIds[1]); + int j; + for (j = 0; j < 16; j++) printf("%s%u", j ? "," : "", e->engineData[j]); + printf("]\n"); + } + base += t->numEntries; + if (!t->bMore) break; + } while (1); + free(t); + } + + /* INTERNAL_GET_DEVICE_INFO_TABLE (0x20800a40) */ + { + INTERNAL_GET_DEVICE_INFO_TABLE_PARAMS *t = calloc(1, sizeof(*t)); + printf("\n=== INTERNAL_GET_DEVICE_INFO_TABLE (0x20800a40) ===\n"); + st = rm_control(hClient, hSubdevice, 0x20800a40, t, sizeof(*t)); + if (st) { + printf("control failed: 0x%x (expected from userspace if internal-only)\n", st); + } else { + NvU32 i; + printf("numEntries=%u\n", t->numEntries); + for (i = 0; i < t->numEntries; i++) { + INTERNAL_DEVICE_INFO *e = &t->deviceInfoTable[i]; + printf("[%3u] faultId=%u instanceId=%u typeEnum=%u resetId=%u devicePriBase=0x%x isEngine=%u rlEngId=%u runlistPriBase=0x%x groupId=%u ginTargetId=%u deviceBroadcastPriBase=0x%x groupLocalInstanceId=%u\n", + i, e->faultId, e->instanceId, e->typeEnum, e->resetId, + e->devicePriBase, e->isEngine, e->rlEngId, e->runlistPriBase, + e->groupId, e->ginTargetId, e->deviceBroadcastPriBase, + e->groupLocalInstanceId); + } + } + free(t); + } + + return 0; +} diff --git a/tools/rmq/wpr2_reset b/tools/rmq/wpr2_reset Binary files differnew file mode 100755 index 0000000..08d4f55 --- /dev/null +++ b/tools/rmq/wpr2_reset diff --git a/tools/rmq/wpr2_reset.c b/tools/rmq/wpr2_reset.c new file mode 100644 index 0000000..4435843 --- /dev/null +++ b/tools/rmq/wpr2_reset.c @@ -0,0 +1,50 @@ +/* wpr2_reset - clear GPU WPR2 registers via BAR0 MMIO to unstick GSP boot. + * + * When nvidia.ko is unloaded, GSP firmware can remain resident in WPR2 + * (write-protected region 2 in FB). The next RmInitAdapter then fails with + * "unexpected WPR2 already up". FLR does not clear WPR2; only a cold (power) + * reset does. This tool pokes WPR2_ADDR_LO/HI to zero directly through + * /sys/bus/pci/devices/<dev>/resource0 while no driver is bound. + * + * Usage: sudo ./wpr2_reset [pci.bdf] (default 0000:01:00.0) + */ +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/mman.h> + +#define NV_PFB_PRI_MMU_WPR2_ADDR_LO 0x1fa824 +#define NV_PFB_PRI_MMU_WPR2_ADDR_HI 0x1fa828 + +int main(int argc, char **argv) +{ + const char *bdf = argc > 1 ? argv[1] : "0000:01:00.0"; + char path[256]; + snprintf(path, sizeof(path), "/sys/bus/pci/devices/%s/resource0", bdf); + + int fd = open(path, O_RDWR | O_SYNC); + if (fd < 0) { perror(path); return 1; } + void *bar = mmap(NULL, 0x200000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (bar == MAP_FAILED) { perror("mmap"); return 1; } + + volatile uint32_t *lo = (volatile uint32_t *)((char *)bar + NV_PFB_PRI_MMU_WPR2_ADDR_LO); + volatile uint32_t *hi = (volatile uint32_t *)((char *)bar + NV_PFB_PRI_MMU_WPR2_ADDR_HI); + + printf("before: WPR2_ADDR_LO=0x%08x WPR2_ADDR_HI=0x%08x\n", *lo, *hi); + *lo = 0; + *hi = 0; + /* read back */ + printf("after : WPR2_ADDR_LO=0x%08x WPR2_ADDR_HI=0x%08x\n", *lo, *hi); + + if (*hi == 0) + printf("WPR2 cleared - driver load should now boot GSP cleanly\n"); + else + printf("write dropped (PLM?) - WPR2 still up\n"); + + munmap(bar, 0x200000); + close(fd); + return *hi ? 1 : 0; +} diff --git a/tools/test/cuda_drv_test.c b/tools/test/cuda_drv_test.c new file mode 100644 index 0000000..6868a2e --- /dev/null +++ b/tools/test/cuda_drv_test.c @@ -0,0 +1,87 @@ +/* cuda_drv_test.c - minimal CUDA driver-API smoke test (dlopen, no toolkit). + * + * Run inside the guest. Exercises the paths CUDA needs beyond NVML: + * cuInit -> cuDeviceGetCount/Get -> cuCtxCreate -> cuMemAlloc -> + * cuMemcpyHtoD/DtoH -> cuMemFree -> cuCtxDestroy + * + * Expected output on success: "ALL OK". + * + * Status (2026-09-23): cuInit returns 3 (CUDA_ERROR_NOT_INITIALIZED) on the + * emulated GPU. strace shows every nvidia.ko ioctl succeeding (RM status 0), + * the driver completing RmInitAdapter and then closing the device, and + * libcuda opening /dev/nvidia-uvm before /dev/nvidia0 - i.e. the next blocker + * is the UVM driver (nvidia-uvm.ko), which needs the GPU MMU/VAS model that + * the GSP-RM emulation does not provide yet. Build in the guest with: + * gcc -O1 -o /tmp/cdt cuda_drv_test.c -ldl + */ +#include <stdio.h> +#include <string.h> +#include <dlfcn.h> +#include <stdint.h> + +typedef int CUresult; +typedef int CUdevice; +typedef void *CUcontext; +typedef uint64_t CUdeviceptr; + +#define CHECK(expr) do { \ + CUresult r = (expr); \ + if (r) { printf("%-24s FAILED rc=%d\n", #expr, r); return 1; } \ + printf("%-24s OK\n", #expr); \ +} while (0) + +int main(void) +{ + void *h = dlopen("libcuda.so.1", RTLD_NOW); + if (!h) { printf("dlopen libcuda.so.1 failed: %s\n", dlerror()); return 1; } + + CUresult (*cuInit)(unsigned int) = dlsym(h, "cuInit"); + CUresult (*cuDeviceGetCount)(int *) = dlsym(h, "cuDeviceGetCount"); + CUresult (*cuDeviceGet)(CUdevice *, int) = dlsym(h, "cuDeviceGet"); + CUresult (*cuCtxCreate)(CUcontext *, unsigned int, CUdevice) = dlsym(h, "cuCtxCreate_v2"); + CUresult (*cuMemAlloc)(CUdeviceptr *, size_t) = dlsym(h, "cuMemAlloc_v2"); + CUresult (*cuMemcpyHtoD)(CUdeviceptr, const void *, size_t) = dlsym(h, "cuMemcpyHtoD_v2"); + CUresult (*cuMemcpyDtoH)(void *, CUdeviceptr, size_t) = dlsym(h, "cuMemcpyDtoH_v2"); + CUresult (*cuMemFree)(CUdeviceptr) = dlsym(h, "cuMemFree_v2"); + CUresult (*cuCtxDestroy)(CUcontext) = dlsym(h, "cuCtxDestroy_v2"); + + if (!cuInit || !cuDeviceGetCount || !cuDeviceGet || !cuCtxCreate || + !cuMemAlloc || !cuMemcpyHtoD || !cuMemcpyDtoH || !cuMemFree || !cuCtxDestroy) { + printf("dlsym failed\n"); + return 1; + } + + int count = 0; + CUdevice dev; + CUcontext ctx = NULL; + CUdeviceptr dptr = 0; + unsigned int buf[64]; + unsigned int back[64]; + size_t i; + + CHECK(cuInit(0)); + CHECK(cuDeviceGetCount(&count)); + printf("device count = %d\n", count); + if (count < 1) return 1; + + CHECK(cuDeviceGet(&dev, 0)); + printf("device 0 = %d\n", dev); + + CHECK(cuCtxCreate(&ctx, 0, dev)); + printf("context = %p\n", ctx); + + CHECK(cuMemAlloc(&dptr, sizeof(buf))); + printf("device ptr = 0x%llx (%zu bytes)\n", (unsigned long long)dptr, sizeof(buf)); + + for (i = 0; i < 64; i++) buf[i] = (unsigned int)(i * 3 + 1); + CHECK(cuMemcpyHtoD(dptr, buf, sizeof(buf))); + memset(back, 0, sizeof(back)); + CHECK(cuMemcpyDtoH(back, dptr, sizeof(buf))); + printf("roundtrip %s (back[7]=%u expected 22)\n", + memcmp(buf, back, sizeof(buf)) ? "MISMATCH" : "match", back[7]); + + CHECK(cuMemFree(dptr)); + CHECK(cuCtxDestroy(ctx)); + printf("ALL OK\n"); + return 0; +} diff --git a/tools/test/run_integration_tests.sh b/tools/test/run_integration_tests.sh new file mode 100755 index 0000000..06e309c --- /dev/null +++ b/tools/test/run_integration_tests.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# GSP vGPU-IO Integration Test Runner + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +KERNEL_TEST_DIR="$PROJECT_ROOT/kernel/tests" +DRIVER_DIR="$PROJECT_ROOT/driver" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# Check if running as root +check_root() { + if [[ $EUID -ne 0 ]]; then + log_error "This script must be run as root" + exit 1 + fi +} + +# Build kernel test module +build_test_module() { + log_info "Building kernel test module..." + cd "$KERNEL_TEST_DIR" + make clean + make + if [[ $? -ne 0 ]]; then + log_error "Failed to build test module" + exit 1 + fi + log_info "Test module built successfully" +} + +# Load test module and capture output +run_test_module() { + log_info "Loading test module..." + cd "$KERNEL_TEST_DIR" + + # Clear dmesg + dmesg -C + + # Load module + insmod gsp_vgpu_comprehensive_test.ko 2>&1 || { + log_error "Failed to load test module" + dmesg -T | tail -30 + return 1 + } + + # Wait for tests to complete + sleep 2 + + # Get test results from dmesg + dmesg -T | grep -E "(PASS|FAIL|Test Results|TEST SUITE)" | tail -100 + + # Check if tests passed + if dmesg | grep -q "TEST SUITE PASSED"; then + log_info "All tests PASSED!" + return 0 + elif dmesg | grep -q "TEST SUITE FAILED"; then + log_error "Tests FAILED!" + return 1 + else + log_warn "Could not determine test result" + return 1 + fi +} + +# Unload test module +unload_test_module() { + log_info "Unloading test module..." + rmmod gsp_vgpu_comprehensive_test 2>/dev/null || true + sleep 1 +} + +# Build driver +build_driver() { + log_info "Building vGPU-IO driver..." + cd "$DRIVER_DIR" + make clean + make + if [[ $? -ne 0 ]]; then + log_error "Failed to build driver" + exit 1 + fi + log_info "Driver built successfully" +} + +# Load driver +load_driver() { + log_info "Loading vGPU-IO driver..." + cd "$DRIVER_DIR" + insmod gsp-vgpu.ko firmware_name=gsp-570.144.fwimage.bin 2>&1 || { + log_error "Failed to load driver" + dmesg -T | tail -30 + return 1 + } + + # Check if device created + sleep 2 + if ls /dev/gsp-vgpu* 2>/dev/null; then + log_info "Driver loaded, device created" + return 0 + else + log_warn "Driver loaded but no device node found" + dmesg -T | tail -20 + fi +} + +# Unload driver +unload_driver() { + log_info "Unloading vGPU-IO driver..." + rmmod gsp-vgpu 2>/dev/null || true + sleep 1 +} + +# Run VF creation test +test_vf_creation() { + log_info "Testing VF creation via debugfs..." + + # Check debugfs + if [[ -d /sys/kernel/debug/gsp-vgpu ]]; then + log_info "Debugfs interface available" + cat /sys/kernel/debug/gsp-vgpu/vfs + else + log_warn "Debugfs not mounted or driver not loaded" + return 1 + fi +} + +# Run RPC test via debugfs +test_rpc_stats() { + log_info "Checking RPC statistics..." + if [[ -f /sys/kernel/debug/gsp-vgpu/rpc ]]; then + cat /sys/kernel/debug/gsp-vgpu/rpc + fi +} + +# Main test runner +main() { + local test_type="${1:-all}" + + check_root + + case "$test_type" in + "unit") + build_test_module + run_test_module + unload_test_module + ;; + "driver") + build_driver + load_driver + test_vf_creation + test_rpc_stats + unload_driver + ;; + "all"|*) + log_info "Running ALL tests..." + build_test_module + run_test_module + unload_test_module + + build_driver + load_driver + test_vf_creation + test_rpc_stats + unload_driver + ;; + esac + + log_info "Test run completed" +} + +main "$@"
\ No newline at end of file diff --git a/tools/vm/cloud-init.yaml b/tools/vm/cloud-init.yaml new file mode 100644 index 0000000..05f11b5 --- /dev/null +++ b/tools/vm/cloud-init.yaml @@ -0,0 +1,26 @@ +#cloud-config +users: + - name: ubuntu + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + lock_passwd: false + passwd: $6$rounds=4096$ubuntu$ubuntu + ssh_authorized_keys: [] + +package_update: true +package_upgrade: true +packages: + - linux-headers-generic + - build-essential + - dkms + - git + - wget + - curl + - vim + - pciutils + - lshw + +runcmd: + - echo "ubuntu:ubuntu" | chpasswd + - systemctl enable ssh + - systemctl start ssh diff --git a/tools/vm/launch_vgpu_qemu.sh b/tools/vm/launch_vgpu_qemu.sh new file mode 100755 index 0000000..0355ef1 --- /dev/null +++ b/tools/vm/launch_vgpu_qemu.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# QEMU launch script for NVIDIA vGPU testing +# This script assumes: +# 1. nvidia-vgpu-vfio.ko is loaded on host (with dry_run=1 for testing) +# 2. An mdev device has been created via sysfs +# 3. Guest OS has NVIDIA vGPU guest driver installed + +set -euo pipefail + +# Configuration +MDEV_UUID="${MDEV_UUID:-$(cat /proc/sys/kernel/random/uuid)}" +VGPU_TYPE="${VGPU_TYPE:-nvidia-vgpu-turing}" # or ampere, hopper, pascal +QEMU_BIN="${QEMU_BIN:-qemu-system-x86_64}" +GUEST_IMG="${GUEST_IMG:-/var/lib/libvirt/images/ubuntu-vgpu.qcow2}" +GUEST_KERNEL="${GUEST_KERNEL:-/boot/vmlinuz-$(uname -r)}" +GUEST_INITRD="${GUEST_INITRD:-/boot/initrd.img-$(uname -r)}" + +# PCI address of the NVIDIA GPU (TU104: 10de:1ed3) +GPU_PCI_ADDR="${GPU_PCI_ADDR:-0000:01:00.0}" + +echo "=== NVIDIA vGPU QEMU Launch Script ===" +echo "MDEV UUID: $MDEV_UUID" +echo "VGPU Type: $VGPU_TYPE" +echo "GPU PCI: $GPU_PCI_ADDR" +echo "" + +# Check if mdev device exists +MDEV_PATH="/sys/bus/mdev/devices/$MDEV_UUID" +if [[ ! -d "$MDEV_PATH" ]]; then + echo "ERROR: mdev device $MDEV_UUID not found" + echo "" + echo "To create an mdev device (requires NVIDIA vGPU host driver):" + echo " echo $MDEV_UUID > /sys/bus/pci/devices/$GPU_PCI_ADDR/mdev_supported_types/$VGPU_TYPE/create" + echo "" + echo "For dry-run testing without host driver, this script will not work for actual QEMU launch." + echo "The kernel module loads in dry-run mode but cannot create real mdev devices." + exit 1 +fi + +# Get the mdev device's VFIO group +VFIO_GROUP=$(basename $(readlink -f "$MDEV_PATH/iommu_group")) +VFIO_DEV="/dev/vfio/$VFIO_GROUP" + +if [[ ! -e "$VFIO_DEV" ]]; then + echo "ERROR: VFIO device $VFIO_DEV not found" + exit 1 +fi + +echo "Using VFIO device: $VFIO_DEV" +echo "" + +# QEMU command +exec $QEMU_BIN \ + -name "vgpu-guest" \ + -enable-kvm \ + -cpu host,host-passthrough=on \ + -smp 4,sockets=1,cores=4,threads=1 \ + -m 8G \ + -device vfio-pci,sysfsdev=/sys/bus/mdev/devices/$MDEV_UUID,display=on,x-igd-opregion=on \ + -device virtio-net-pci,netdev=net0 \ + -netdev user,id=net0,hostfwd=tcp::2222-:22 \ + -drive file=$GUEST_IMG,if=virtio,format=qcow2 \ + -display gtk,gl=on \ + -vga none \ + "$@" diff --git a/tools/vm/setup_mdev.sh b/tools/vm/setup_mdev.sh new file mode 100755 index 0000000..ef31f46 --- /dev/null +++ b/tools/vm/setup_mdev.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Setup script for NVIDIA vGPU mdev device +# This script creates an mdev device for QEMU testing + +set -euo pipefail + +GPU_PCI_ADDR="${GPU_PCI_ADDR:-0000:01:00.0}" +VGPU_TYPE="${VGPU_TYPE:-nvidia-vgpu-turing}" +MDEV_UUID="${MDEV_UUID:-$(cat /proc/sys/kernel/random/uuid)}" + +MDEV_CREATE_PATH="/sys/bus/pci/devices/$GPU_PCI_ADDR/mdev_supported_types/$VGPU_TYPE/create" + +echo "=== NVIDIA vGPU mdev Device Setup ===" +echo "GPU PCI: $GPU_PCI_ADDR" +echo "VGPU Type: $VGPU_TYPE" +echo "MDEV UUID: $MDEV_UUID" +echo "" + +# Check if GPU device exists +if [[ ! -d "/sys/bus/pci/devices/$GPU_PCI_ADDR" ]]; then + echo "ERROR: GPU device $GPU_PCI_ADDR not found" + echo "Available NVIDIA GPUs:" + lspci -nn -d 10de: | grep -E "VGA|3D" + exit 1 +fi + +# Check if mdev_supported_types exists (requires NVIDIA vGPU host driver) +if [[ ! -d "/sys/bus/pci/devices/$GPU_PCI_ADDR/mdev_supported_types" ]]; then + echo "ERROR: mdev_supported_types not found for $GPU_PCI_ADDR" + echo "" + echo "This requires the NVIDIA vGPU host driver (licensed product for data center GPUs)." + echo "The regular NVIDIA driver does not provide vGPU mdev support." + echo "" + echo "For testing on consumer GPUs (TU104, etc.), use dry-run mode:" + echo " sudo insmod /path/to/nvidia-vgpu-vfio.ko dry_run=1" + echo "" + echo "In dry-run mode, the module loads but cannot create real mdev devices." + exit 1 +fi + +# Check if requested vGPU type exists +if [[ ! -d "/sys/bus/pci/devices/$GPU_PCI_ADDR/mdev_supported_types/$VGPU_TYPE" ]]; then + echo "ERROR: vGPU type $VGPU_TYPE not supported" + echo "Available types:" + ls "/sys/bus/pci/devices/$GPU_PCI_ADDR/mdev_supported_types/" + exit 1 +fi + +# Create mdev device +echo "Creating mdev device..." +echo "$MDEV_UUID" | sudo tee "$MDEV_CREATE_PATH" > /dev/null + +if [[ -d "/sys/bus/mdev/devices/$MDEV_UUID" ]]; then + echo "SUCCESS: mdev device created at /sys/bus/mdev/devices/$MDEV_UUID" + echo "" + echo "Device info:" + cat "/sys/bus/mdev/devices/$MDEV_UUID/description" 2>/dev/null || echo " (no description)" + echo "" + echo "To use with QEMU:" + echo " export MDEV_UUID=$MDEV_UUID" + echo " /path/to/launch_vgpu_qemu.sh" +else + echo "ERROR: Failed to create mdev device" + exit 1 +fi |