From 2302fc09080bad51c73f293294e19fd4db464eb0 Mon Sep 17 00:00:00 2001 From: Open-IOV Date: Wed, 23 Sep 2026 05:44:08 -0400 Subject: Open-IOV: initial public release Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with real GPU functionality - CUDA first, 3D and video to follow - on unmodified hosts, without a proprietary vGPU license and without patching any driver, firmware or hypervisor. Core principle: emulate the interface, pass through the work. kernel/ open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device (PCI config space, BAR shadows, MSI-X), GSP boot emulation, GSP-RM message-queue/RPC emulation, memory model (GPU-VA to guest-PA discovery) and the semaphore-completion service that advances guest UVM progress. userspace/ libiov.so management API and examples. tools/ guest diagnostics, firmware analysis wrappers, RM probes. scripts/ stack manager, VM harness, build/load/test automation. config/ hardware/version profiles (the portability surface). docs/ documentation tree - start at docs/README.md. wiki/ long-form firmware and protocol deep dives. Status: initialization works end to end for a stock 570.144 Linux guest on Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery, object creation and the early CUDA/UVM start-up path. GPU work forwarding - the step that makes CUDA kernels execute on real hardware - is the active work item; see docs/04-implementation/01-known-knowns-and-unknowns.md. License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md. --- tools/rmq/rmq.c | 189 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 tools/rmq/rmq.c (limited to 'tools/rmq/rmq.c') diff --git a/tools/rmq/rmq.c b/tools/rmq/rmq.c new file mode 100644 index 0000000..189d647 --- /dev/null +++ b/tools/rmq/rmq.c @@ -0,0 +1,189 @@ +/* rmq - query host NVIDIA RM via /dev/nvidia0 ioctls (oracle for TU104 static data) */ +#include +#include +#include +#include +#include +#include +#include + +typedef uint32_t NvU32; +typedef uint16_t NvU16; +typedef uint8_t NvU8; +typedef uint64_t NvU64; +typedef uint32_t NvHandle; +typedef uint32_t NvV32; +typedef uint64_t NvP64; + +/* NVIDIA ioctl encoding: _IOWR('F', nr, type) */ +#define NV_IOCTL_MAGIC 'F' +#define NV_IOCTL_BASE 200 +#define NV_ESC_RM_ALLOC 0x2B +#define NV_ESC_RM_CONTROL 0x2A + +#define NV01_ROOT 0x0 +#define NV01_DEVICE_0 0x80 +#define NV20_SUBDEVICE_0 0x2080 + +typedef struct { + NvHandle hRoot; + NvHandle hObjectParent; + NvHandle hObjectNew; + NvV32 hClass; + NvP64 pAllocParms __attribute__((aligned(8))); + NvU32 paramsSize; + NvV32 status; +} NVOS21_PARAMETERS; + +typedef struct { + NvHandle hClient; + NvHandle hObject; + NvV32 cmd; + NvU32 flags; + NvP64 params __attribute__((aligned(8))); + NvU32 paramsSize; + NvV32 status; +} NVOS54_PARAMETERS; + +#define NV_IOCTL_RM_ALLOC _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS21_PARAMETERS) +#define NV_IOCTL_RM_CONTROL _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS) + +static int fd; + +/* NVOS64: 48 bytes (0x30), used for device/subdevice allocs (with rights) */ +typedef struct { + NvHandle hRoot; + NvHandle hObjectParent; + NvHandle hObjectNew; + NvV32 hClass; + NvP64 pAllocParms __attribute__((aligned(8))); + NvP64 pRightsRequested __attribute__((aligned(8))); + NvU32 paramsSize; + NvU32 flags; + NvV32 status; +} NVOS64_PARAMETERS; + +#define NV_IOCTL_RM_ALLOC64 _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS64_PARAMETERS) + +static NvU32 rm_alloc(NvHandle hRoot, NvHandle hParent, NvU32 hClass, NvHandle *pOut) +{ + NVOS21_PARAMETERS p = {0}; + p.hRoot = hRoot; p.hObjectParent = hParent; p.hClass = hClass; + if (ioctl(fd, NV_IOCTL_RM_ALLOC, &p) < 0) { perror("alloc ioctl"); return 1; } + if (p.status) { fprintf(stderr, "alloc class 0x%x failed: 0x%x\n", hClass, p.status); return p.status; } + *pOut = p.hObjectNew; + return 0; +} + +/* Device/subdevice allocs use NVOS64 (48 bytes) with rights */ +static NvU32 rm_alloc64(NvHandle hRoot, NvHandle hParent, NvU32 hClass, NvHandle *pOut) +{ + NVOS64_PARAMETERS p = {0}; + p.hRoot = hRoot; p.hObjectParent = hParent; p.hClass = hClass; + p.flags = 1; /* FINN_SERIALIZED */ + if (ioctl(fd, NV_IOCTL_RM_ALLOC64, &p) < 0) { perror("alloc64 ioctl"); return 1; } + if (p.status) { fprintf(stderr, "alloc64 class 0x%x failed: 0x%x\n", hClass, p.status); return p.status; } + *pOut = p.hObjectNew; + return 0; +} + +static NvU32 rm_control(NvHandle hClient, NvHandle hObject, NvU32 cmd, void *params, NvU32 size) +{ + NVOS54_PARAMETERS p = {0}; + p.hClient = hClient; p.hObject = hObject; p.cmd = cmd; + p.params = (NvP64)params; p.paramsSize = size; + if (ioctl(fd, NV_IOCTL_RM_CONTROL, &p) < 0) { perror("control ioctl"); return 1; } + return p.status; +} + +/* NV2080_CTRL_FIFO_DEVICE_ENTRY (16 u32 engineData + 2 pbdmaIds + 2 pbdmaFaultIds + numPbdmas + name[16]) */ +typedef struct { + NvU32 engineData[16]; + NvU32 pbdmaIds[2]; + NvU32 pbdmaFaultIds[2]; + NvU32 numPbdmas; + char engineName[16]; +} FIFO_DEVICE_ENTRY; + +typedef struct { + NvU32 baseIndex; + NvU32 numEntries; + NvU32 bMore; + FIFO_DEVICE_ENTRY entries[32]; +} FIFO_GET_DEVICE_INFO_TABLE_PARAMS; + +typedef struct { + NvU32 faultId, instanceId, typeEnum, resetId, devicePriBase, isEngine; + NvU32 rlEngId, runlistPriBase, groupId, ginTargetId, deviceBroadcastPriBase, groupLocalInstanceId; +} INTERNAL_DEVICE_INFO; + +typedef struct { + NvU32 numEntries; + INTERNAL_DEVICE_INFO deviceInfoTable[512]; +} INTERNAL_GET_DEVICE_INFO_TABLE_PARAMS; + +int main(int argc, char **argv) +{ + NvHandle hClient = 0, hDevice = 0, hSubdevice = 0; + NvU32 st; + + /* RM_ALLOC creates clients on /dev/nvidiactl, not /dev/nvidia0 */ + fd = open("/dev/nvidiactl", O_RDWR); + if (fd < 0) { perror("open /dev/nvidiactl"); return 1; } + + if ((st = rm_alloc(0, 0, NV01_ROOT, &hClient))) return 1; + if ((st = rm_alloc64(hClient, hClient, NV01_DEVICE_0, &hDevice))) return 1; + if ((st = rm_alloc64(hClient, hDevice, NV20_SUBDEVICE_0, &hSubdevice))) return 1; + printf("hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n", hClient, hDevice, hSubdevice); + + /* FIFO_GET_DEVICE_INFO_TABLE (0x20801112) */ + { + FIFO_GET_DEVICE_INFO_TABLE_PARAMS *t = calloc(1, sizeof(*t)); + NvU32 base = 0; + printf("\n=== FIFO_GET_DEVICE_INFO_TABLE (0x20801112) ===\n"); + do { + memset(t, 0, sizeof(*t)); + t->baseIndex = base; + st = rm_control(hClient, hSubdevice, 0x20801112, t, sizeof(*t)); + if (st) { printf("control failed: 0x%x\n", st); break; } + NvU32 i; + for (i = 0; i < t->numEntries; i++) { + FIFO_DEVICE_ENTRY *e = &t->entries[i]; + printf("[%3u] name=%-16s numPbdmas=%u pbdmaIds=[%u,%u] faultIds=[%u,%u] engineData=[", + base + i, e->engineName, e->numPbdmas, + e->pbdmaIds[0], e->pbdmaIds[1], + e->pbdmaFaultIds[0], e->pbdmaFaultIds[1]); + int j; + for (j = 0; j < 16; j++) printf("%s%u", j ? "," : "", e->engineData[j]); + printf("]\n"); + } + base += t->numEntries; + if (!t->bMore) break; + } while (1); + free(t); + } + + /* INTERNAL_GET_DEVICE_INFO_TABLE (0x20800a40) */ + { + INTERNAL_GET_DEVICE_INFO_TABLE_PARAMS *t = calloc(1, sizeof(*t)); + printf("\n=== INTERNAL_GET_DEVICE_INFO_TABLE (0x20800a40) ===\n"); + st = rm_control(hClient, hSubdevice, 0x20800a40, t, sizeof(*t)); + if (st) { + printf("control failed: 0x%x (expected from userspace if internal-only)\n", st); + } else { + NvU32 i; + printf("numEntries=%u\n", t->numEntries); + for (i = 0; i < t->numEntries; i++) { + INTERNAL_DEVICE_INFO *e = &t->deviceInfoTable[i]; + printf("[%3u] faultId=%u instanceId=%u typeEnum=%u resetId=%u devicePriBase=0x%x isEngine=%u rlEngId=%u runlistPriBase=0x%x groupId=%u ginTargetId=%u deviceBroadcastPriBase=0x%x groupLocalInstanceId=%u\n", + i, e->faultId, e->instanceId, e->typeEnum, e->resetId, + e->devicePriBase, e->isEngine, e->rlEngId, e->runlistPriBase, + e->groupId, e->ginTargetId, e->deviceBroadcastPriBase, + e->groupLocalInstanceId); + } + } + free(t); + } + + return 0; +} -- cgit v1.2.3