Open-IOV: initial public release

Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with
real GPU functionality - CUDA first, 3D and video to follow - on unmodified
hosts, without a proprietary vGPU license and without patching any driver,
firmware or hypervisor.

Core principle: emulate the interface, pass through the work.

  kernel/     open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device
              (PCI config space, BAR shadows, MSI-X), GSP boot emulation,
              GSP-RM message-queue/RPC emulation, memory model (GPU-VA to
              guest-PA discovery) and the semaphore-completion service that
              advances guest UVM progress.
  userspace/  libiov.so management API and examples.
  tools/      guest diagnostics, firmware analysis wrappers, RM probes.
  scripts/    stack manager, VM harness, build/load/test automation.
  config/     hardware/version profiles (the portability surface).
  docs/       documentation tree - start at docs/README.md.
  wiki/       long-form firmware and protocol deep dives.

Status: initialization works end to end for a stock 570.144 Linux guest on
Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery,
object creation and the early CUDA/UVM start-up path. GPU work forwarding -
the step that makes CUDA kernels execute on real hardware - is the active
work item; see docs/04-implementation/01-known-knowns-and-unknowns.md.

License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link
GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.

1 files changed, 189 insertions, 0 deletions

diff --git a/tools/rmq/grctx_oracle.c b/tools/rmq/grctx_oracle.c
new file mode 100644
index 0000000..a2fc5fa
--- /dev/null
+++ b/tools/rmq/grctx_oracle.c
@@ -0,0 +1,189 @@
+/* grctx_oracle - query the host NVIDIA RM for the engine context buffer sizes.
+ *
+ * Goal: recover the table that GSP-RM serves for
+ * NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO (0x20800a32),
+ * so the emulation layer does not have to guess the per-engine ctx buffer
+ * sizes (kernel/gsp_rm_tu104.h: TU104_CTX_SIZE_*).
+ *
+ * Outcome (2026-09-22): NOT REACHABLE from userspace on a bare-metal PF:
+ *   - 0x801707 NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES
+ *       hObject=hDevice  -> 0x57 NV_ERR_OBJECT_NOT_FOUND (not exported)
+ *       hObject=hClient  -> 0x56 NV_ERR_NOT_SUPPORTED (bound to the
+ *                           _92bfc3 hal: the client RM does not manage ctx
+ *                           buffers on a PF, only a VF's VGPU_STATIC_INFO)
+ *   - 0x2080122d NV2080_CTRL_CMD_GR_GET_ENGINE_CONTEXT_PROPERTIES
+ *       (subdevice)      -> 0x57 / 0x56, never returns sizes
+ *   - The producing control (0x20800a32) is internal-only.
+ * The real values are probed from the ctxsw ucode by the physical RM
+ * (grSetupInitialCtxState -> grDiscoverCtxBufferSize) and are internal to
+ * GSP-RM, so the emulation currently serves synthesized page-aligned sizes
+ * (see tu104_ctx_buffer_info in kernel/gsp_rm_tu104.h).
+ *
+ * Kept as a reference for the RM client bring-up sequence and for a future
+ * attempt via a kernel-side RM client.
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <stdint.h>
+
+typedef uint32_t NvU32;
+typedef uint64_t NvU64;
+typedef uint32_t NvHandle;
+typedef uint32_t NvV32;
+typedef uint64_t NvP64;
+
+#define NV_IOCTL_MAGIC      'F'
+#define NV_ESC_RM_ALLOC     0x2B
+#define NV_ESC_RM_CONTROL   0x2A
+
+#define NV01_ROOT           0x0
+#define NV01_DEVICE_0       0x80
+#define NV20_SUBDEVICE_0    0x2080
+
+typedef struct {
+    NvHandle hRoot;
+    NvHandle hObjectParent;
+    NvHandle hObjectNew;
+    NvV32    hClass;
+    NvP64    pAllocParms __attribute__((aligned(8)));
+    NvU32    paramsSize;
+    NvV32    status;
+} NVOS21_PARAMETERS;
+
+typedef struct {
+    NvHandle hRoot;
+    NvHandle hObjectParent;
+    NvHandle hObjectNew;
+    NvV32    hClass;
+    NvP64    pAllocParms __attribute__((aligned(8)));
+    NvP64    pRightsRequested __attribute__((aligned(8)));
+    NvU32    paramsSize;
+    NvU32    flags;
+    NvV32    status;
+} NVOS64_PARAMETERS;
+
+typedef struct {
+    NvHandle hClient;
+    NvHandle hObject;
+    NvV32    cmd;
+    NvU32    flags;
+    NvP64    params __attribute__((aligned(8)));
+    NvU32    paramsSize;
+    NvV32    status;
+} NVOS54_PARAMETERS;
+
+#define NV_IOCTL_RM_ALLOC   _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_ALLOC, NVOS21_PARAMETERS)
+#define NV_IOCTL_RM_CONTROL _IOWR(NV_IOCTL_MAGIC, NV_ESC_RM_CONTROL, NVOS54_PARAMETERS)
+
+#define NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES 0x801707U
+#define NV2080_CTRL_CMD_GR_GET_ENGINE_CONTEXT_PROPERTIES   0x2080122dU
+#define ENGINE_ID_COUNT 0x1a
+
+typedef struct {
+    NvU32 engineId;
+    NvU32 alignment;
+    NvU32 size;
+} FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS;
+
+typedef struct {
+    NvU32 flags;
+    NvU32 pad;
+    NvU64 route;
+} GR_ROUTE_INFO;
+
+typedef struct {
+    GR_ROUTE_INFO grRouteInfo;
+    NvU32 engineId;
+    NvU32 alignment;
+    NvU32 size;
+    NvU32 bInfoPopulated;
+} GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS;
+
+static int fd;
+
+static NvU32 rm_alloc(NvHandle parent, NvHandle client, NvU32 hClass, NvHandle *pOut)
+{
+    NVOS21_PARAMETERS p = {0};
+    p.hRoot = client;
+    p.hObjectParent = parent;
+    p.hClass = hClass;
+    if (ioctl(fd, NV_IOCTL_RM_ALLOC, &p) < 0) { perror("alloc ioctl"); return 1; }
+    if (p.status) { fprintf(stderr, "alloc class 0x%x failed: 0x%x\n", hClass, p.status); return p.status; }
+    *pOut = p.hObjectNew;
+    return 0;
+}
+
+static NvU32 rm_alloc64(NvHandle parent, NvHandle client, NvU32 hClass,
+                        NvHandle newHandle, void *parms, NvU32 parmsSize, NvHandle *pOut)
+{
+    NVOS64_PARAMETERS p = {0};
+    p.hRoot = client;
+    p.hObjectParent = parent;
+    p.hObjectNew = newHandle; /* 0 = let RM generate (device); subdevices take an explicit handle */
+    p.hClass = hClass;
+    p.pAllocParms = (NvP64)parms;
+    p.paramsSize = parmsSize;
+    p.flags = 0;
+    if (ioctl(fd, NV_IOCTL_RM_ALLOC, &p) < 0) { perror("alloc64 ioctl"); return 1; }
+    if (p.status) {
+        fprintf(stderr, "alloc64 class 0x%x failed: 0x%x\n", hClass, p.status);
+        return p.status;
+    }
+    *pOut = p.hObjectNew;
+    return 0;
+}
+
+static NvU32 rm_control(NvHandle hClient, NvHandle hObject, NvU32 cmd, void *params, NvU32 size)
+{
+    NVOS54_PARAMETERS p = {0};
+    p.hClient = hClient; p.hObject = hObject; p.cmd = cmd;
+    p.params = (NvP64)params; p.paramsSize = size;
+    if (ioctl(fd, NV_IOCTL_RM_CONTROL, &p) < 0) { perror("control ioctl"); return 1; }
+    return p.status;
+}
+
+int main(void)
+{
+    NvHandle hClient = 0, hDevice = 0, hSubdevice = 0;
+    NvU32 deviceParams = 0;    /* deviceId 0 */
+    NvU32 subdeviceParams = 0; /* subDeviceId 0 */
+    NvU32 st, i;
+
+    fd = open("/dev/nvidiactl", O_RDWR);
+    if (fd < 0) { perror("open /dev/nvidiactl"); return 1; }
+
+    if ((st = rm_alloc(0, 0, NV01_ROOT, &hClient))) return 1;
+    if ((st = rm_alloc64(hClient, hClient, NV01_DEVICE_0, 0,
+                         &deviceParams, sizeof(deviceParams), &hDevice))) return 1;
+    if ((st = rm_alloc64(hDevice, hClient, NV20_SUBDEVICE_0, 0xcaf00001,
+                         &subdeviceParams, sizeof(subdeviceParams), &hSubdevice))) return 1;
+    printf("hClient=0x%x hDevice=0x%x hSubdevice=0x%x\n", hClient, hDevice, hSubdevice);
+
+    printf("=== device-level FIFO_GET_ENGINE_CONTEXT_PROPERTIES (0x801707) ===\n");
+    for (i = 0; i < ENGINE_ID_COUNT; i++) {
+        FIFO_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS p;
+        memset(&p, 0, sizeof(p));
+        p.engineId = i;
+        st = rm_control(hClient, hDevice, NV0080_CTRL_CMD_FIFO_GET_ENGINE_CONTEXT_PROPERTIES,
+                        &p, sizeof(p));
+        printf("engine[%2u] status=0x%x size=0x%08x alignment=0x%08x\n",
+               i, st, p.size, p.alignment);
+    }
+
+    printf("=== subdevice-level GR_GET_ENGINE_CONTEXT_PROPERTIES (0x2080122d) ===\n");
+    for (i = 0; i < ENGINE_ID_COUNT; i++) {
+        GR_GET_ENGINE_CONTEXT_PROPERTIES_PARAMS p;
+        memset(&p, 0, sizeof(p));
+        p.engineId = i;
+        st = rm_control(hClient, hSubdevice, NV2080_CTRL_CMD_GR_GET_ENGINE_CONTEXT_PROPERTIES,
+                        &p, sizeof(p));
+        printf("engine[%2u] status=0x%x size=0x%08x alignment=0x%08x\n",
+               i, st, p.size, p.alignment);
+    }
+
+    return 0;
+}