Open-IOV: initial public release

Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with
real GPU functionality - CUDA first, 3D and video to follow - on unmodified
hosts, without a proprietary vGPU license and without patching any driver,
firmware or hypervisor.

Core principle: emulate the interface, pass through the work.

  kernel/     open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device
              (PCI config space, BAR shadows, MSI-X), GSP boot emulation,
              GSP-RM message-queue/RPC emulation, memory model (GPU-VA to
              guest-PA discovery) and the semaphore-completion service that
              advances guest UVM progress.
  userspace/  libiov.so management API and examples.
  tools/      guest diagnostics, firmware analysis wrappers, RM probes.
  scripts/    stack manager, VM harness, build/load/test automation.
  config/     hardware/version profiles (the portability surface).
  docs/       documentation tree - start at docs/README.md.
  wiki/       long-form firmware and protocol deep dives.

Status: initialization works end to end for a stock 570.144 Linux guest on
Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery,
object creation and the early CUDA/UVM start-up path. GPU work forwarding -
the step that makes CUDA kernels execute on real hardware - is the active
work item; see docs/04-implementation/01-known-knowns-and-unknowns.md.

License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link
GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.

1 files changed, 528 insertions, 0 deletions

diff --git a/userspace/lib/rpc_emulator.c b/userspace/lib/rpc_emulator.c
new file mode 100644
index 0000000..9f7f2c7
--- /dev/null
+++ b/userspace/lib/rpc_emulator.c
@@ -0,0 +1,528 @@
+/*
+ * GSP vGPU-IO Userspace RPC Emulator
+ * 
+ * Emulates missing vGPU RPC handlers (BAR_TRAP, WPR, etc.) that exist only
+ * in GRID host driver GSP firmware. Allows operation with consumer/OpenRM GSP.
+ */
+
+#define _GNU_SOURCE
+#include "rpc_emulator.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <errno.h>
+
+/* ============================================================
+ * BAR Emulation State
+ * ============================================================ */
+
+struct emu_bar_state {
+    uint64_t aperture_offset;
+    uint64_t aperture_size;
+    bool force_access;
+    uint32_t force_vf;
+    void *backing_store;
+    size_t backing_size;
+    pthread_mutex_t lock;
+};
+
+static struct emu_bar_state g_bar_state = {
+    .aperture_offset = 0,
+    .aperture_size = 0,
+    .force_access = false,
+    .force_vf = 0,
+    .backing_store = NULL,
+    .backing_size = 0,
+};
+
+/* ============================================================
+ * WPR Emulation State
+ * ============================================================ */
+
+struct emu_wpr_region {
+    uint64_t base;
+    uint64_t size;
+    uint32_t flags;
+    uint64_t timeout_ns;
+    struct emu_wpr_region *next;
+};
+
+struct emu_wpr_state {
+    struct emu_wpr_region *regions;
+    pthread_mutex_t lock;
+};
+
+static struct emu_wpr_state g_wpr_state = {
+    .regions = NULL,
+};
+
+/* ============================================================
+ * Emulation Mode Control
+ * ============================================================ */
+
+static bool g_emulation_enabled = false;
+static pthread_mutex_t g_emu_lock = PTHREAD_MUTEX_INITIALIZER;
+
+void rpc_emu_enable(bool enable)
+{
+    pthread_mutex_lock(&g_emu_lock);
+    g_emulation_enabled = enable;
+    pthread_mutex_unlock(&g_emu_lock);
+}
+
+bool rpc_emu_is_enabled(void)
+{
+    bool enabled;
+    pthread_mutex_lock(&g_emu_lock);
+    enabled = g_emulation_enabled;
+    pthread_mutex_unlock(&g_emu_lock);
+    return enabled;
+}
+
+/* ============================================================
+ * BAR Emulation Implementation
+ * ============================================================ */
+
+int rpc_emu_bar_config(uint32_t vf_id, uint64_t offset, uint64_t size)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    if (offset & 0xFFF || size & 0xFFF)
+        return -EINVAL;
+
+    if (size > 0x40000)  /* 256KB max per firmware spec */
+        return -EINVAL;
+
+    pthread_mutex_lock(&g_bar_state.lock);
+
+    g_bar_state.aperture_offset = offset;
+    g_bar_state.aperture_size = size;
+
+    /* Allocate/resize backing store */
+    if (g_bar_state.backing_store) {
+        free(g_bar_state.backing_store);
+        g_bar_state.backing_store = NULL;
+        g_bar_state.backing_size = 0;
+    }
+
+    if (size > 0) {
+        g_bar_state.backing_size = size;
+        g_bar_state.backing_store = calloc(1, size);
+        if (!g_bar_state.backing_store) {
+            g_bar_state.backing_size = 0;
+            pthread_mutex_unlock(&g_bar_state.lock);
+            return -ENOMEM;
+        }
+    }
+
+    pthread_mutex_unlock(&g_bar_state.lock);
+    return 0;
+}
+
+int rpc_emu_bar_force_access(uint32_t vf_id, bool enable)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    pthread_mutex_lock(&g_bar_state.lock);
+    g_bar_state.force_access = enable;
+    g_bar_state.force_vf = enable ? vf_id : 0;
+    pthread_mutex_unlock(&g_bar_state.lock);
+    return 0;
+}
+
+int rpc_emu_bar_save(uint32_t vf_id, void *buf, size_t *size)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    if (!buf || !size)
+        return -EINVAL;
+
+    pthread_mutex_lock(&g_bar_state.lock);
+
+    size_t needed = sizeof(struct emu_bar_state) + g_bar_state.backing_size;
+    if (*size < needed) {
+        *size = needed;
+        pthread_mutex_unlock(&g_bar_state.lock);
+        return -ENOSPC;
+    }
+
+    memcpy(buf, &g_bar_state, sizeof(struct emu_bar_state));
+    if (g_bar_state.backing_store && g_bar_state.backing_size > 0)
+        memcpy(buf + sizeof(struct emu_bar_state), g_bar_state.backing_store, g_bar_state.backing_size);
+
+    *size = needed;
+    pthread_mutex_unlock(&g_bar_state.lock);
+    return 0;
+}
+
+int rpc_emu_bar_restore(uint32_t vf_id, const void *buf, size_t size)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    if (!buf || size < sizeof(struct emu_bar_state))
+        return -EINVAL;
+
+    pthread_mutex_lock(&g_bar_state.lock);
+
+    memcpy(&g_bar_state, buf, sizeof(struct emu_bar_state));
+
+    if (size > sizeof(struct emu_bar_state) && g_bar_state.backing_size > 0) {
+        if (!g_bar_state.backing_store) {
+            g_bar_state.backing_store = calloc(1, g_bar_state.backing_size);
+            if (!g_bar_state.backing_store) {
+                pthread_mutex_unlock(&g_bar_state.lock);
+                return -ENOMEM;
+            }
+        }
+        memcpy(g_bar_state.backing_store, buf + sizeof(struct emu_bar_state),
+               size - sizeof(struct emu_bar_state));
+    }
+
+    pthread_mutex_unlock(&g_bar_state.lock);
+    return 0;
+}
+
+int rpc_emu_bar_get_state(uint32_t vf_id, uint64_t *offset, uint64_t *size)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    pthread_mutex_lock(&g_bar_state.lock);
+    if (offset) *offset = g_bar_state.aperture_offset;
+    if (size) *size = g_bar_state.aperture_size;
+    pthread_mutex_unlock(&g_bar_state.lock);
+    return 0;
+}
+
+/* ============================================================
+ * WPR Emulation Implementation
+ * ============================================================ */
+
+static struct emu_wpr_region *emu_wpr_find(uint64_t base)
+{
+    struct emu_wpr_region *rgn = g_wpr_state.regions;
+    while (rgn) {
+        if (rgn->base == base)
+            return rgn;
+        rgn = rgn->next;
+    }
+    return NULL;
+}
+
+static struct emu_wpr_region *emu_wpr_find_containing(uint64_t addr)
+{
+    struct emu_wpr_region *rgn = g_wpr_state.regions;
+    while (rgn) {
+        if (addr >= rgn->base && addr < rgn->base + rgn->size)
+            return rgn;
+        rgn = rgn->next;
+    }
+    return NULL;
+}
+
+int rpc_emu_wpr_protect_write(uint32_t vf_id, uint64_t base, uint64_t size)
+{
+    return rpc_emu_wpr_protect(vf_id, base, size, VGPU_WPR_WRITE);
+}
+
+int rpc_emu_wpr_protect_read(uint32_t vf_id, uint64_t base, uint64_t size)
+{
+    return rpc_emu_wpr_protect(vf_id, base, size, VGPU_WPR_READ);
+}
+
+int rpc_emu_wpr_protect(uint32_t vf_id, uint64_t base, uint64_t size, uint32_t flags)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    if (base & 0xFFF || size & 0xFFF)
+        return -EINVAL;
+
+    pthread_mutex_lock(&g_wpr_state.lock);
+
+    /* Check for overlap */
+    struct emu_wpr_region *rgn = g_wpr_state.regions;
+    while (rgn) {
+        if (base < rgn->base + rgn->size && base + size > rgn->base) {
+            pthread_mutex_unlock(&g_wpr_state.lock);
+            return -EBUSY;  /* Overlaps existing region */
+        }
+        rgn = rgn->next;
+    }
+
+    struct emu_wpr_region *new_rgn = calloc(1, sizeof(*new_rgn));
+    if (!new_rgn) {
+        pthread_mutex_unlock(&g_wpr_state.lock);
+        return -ENOMEM;
+    }
+
+    new_rgn->base = base;
+    new_rgn->size = size;
+    new_rgn->flags = flags;
+    new_rgn->timeout_ns = 10000000;  /* 10ms default */
+
+    new_rgn->next = g_wpr_state.regions;
+    g_wpr_state.regions = new_rgn;
+
+    pthread_mutex_unlock(&g_wpr_state.lock);
+    return 0;
+}
+
+int rpc_emu_wpr_unprotect(uint32_t vf_id, uint64_t base)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    pthread_mutex_lock(&g_wpr_state.lock);
+
+    struct emu_wpr_region **prev = &g_wpr_state.regions;
+    struct emu_wpr_region *rgn = g_wpr_state.regions;
+    bool found = false;
+
+    while (rgn) {
+        if (rgn->base == base) {
+            *prev = rgn->next;
+            free(rgn);
+            found = true;
+            break;
+        }
+        prev = &rgn->next;
+        rgn = rgn->next;
+    }
+
+    pthread_mutex_unlock(&g_wpr_state.lock);
+    return found ? 0 : -ENOENT;
+}
+
+int rpc_emu_wpr_set_timeout(uint32_t vf_id, uint64_t timeout_ns)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    pthread_mutex_lock(&g_wpr_state.lock);
+    struct emu_wpr_region *rgn = g_wpr_state.regions;
+    while (rgn) {
+        rgn->timeout_ns = timeout_ns;
+        rgn = rgn->next;
+    }
+    pthread_mutex_unlock(&g_wpr_state.lock);
+    return 0;
+}
+
+int rpc_emu_wpr_check_access(uint32_t vf_id, uint64_t addr, uint32_t access_type)
+{
+    if (!rpc_emu_is_enabled())
+        return 0;  /* Allow if emulation disabled */
+
+    pthread_mutex_lock(&g_wpr_state.lock);
+    struct emu_wpr_region *rgn = emu_wpr_find_containing(addr);
+    int ret = 0;
+
+    if (rgn) {
+        if ((access_type == VGPU_WPR_WRITE && (rgn->flags & VGPU_WPR_WRITE)) ||
+            (access_type == VGPU_WPR_READ && (rgn->flags & VGPU_WPR_READ))) {
+            ret = -EPERM;  /* Access violation */
+        }
+    }
+
+    pthread_mutex_unlock(&g_wpr_state.lock);
+    return 0;
+}
+
+/* ============================================================
+ * Scheduling Emulation Implementation
+ * ============================================================ */
+
+struct emu_sched_work {
+    uint64_t work_handle;
+    uint64_t callback;
+    uint64_t arg;
+    uint32_t priority;
+    uint32_t flags;
+    struct emu_sched_work *next;
+};
+
+static struct emu_sched_work *g_sched_queue = NULL;
+static pthread_mutex_t g_sched_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int rpc_emu_schedule(uint32_t vf_id, uint64_t work_handle, uint64_t callback, uint64_t arg, uint32_t priority, uint32_t flags)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    struct emu_sched_work *work = calloc(1, sizeof(*work));
+    if (!work)
+        return -ENOMEM;
+
+    work->work_handle = work_handle;
+    work->callback = callback;
+    work->arg = arg;
+    work->priority = priority;
+    work->flags = flags;
+
+    pthread_mutex_lock(&g_sched_lock);
+    work->next = g_sched_queue;
+    g_sched_queue = work;
+    pthread_mutex_unlock(&g_sched_lock);
+
+    /* In emulation, we just return success - work is queued locally */
+    (void)vf_id;
+    return 0;
+}
+
+int rpc_emu_yield(uint32_t vf_id)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    /* In emulation, yield is a no-op */
+    (void)vf_id;
+    return 0;
+}
+
+/* ============================================================
+ * Notification Emulation Implementation
+ * ============================================================ */
+
+struct emu_notify_cb {
+    uint32_t type;
+    rpc_emu_notify_cb_t callback;
+    void *ctx;
+    struct emu_notify_cb *next;
+};
+
+static struct emu_notify_cb *g_notify_callbacks = NULL;
+static pthread_mutex_t g_notify_lock = PTHREAD_MUTEX_INITIALIZER;
+
+int rpc_emu_notify_register(uint32_t vf_id, uint32_t type, rpc_emu_notify_cb_t cb, void *ctx)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    struct emu_notify_cb *nc = calloc(1, sizeof(*nc));
+    if (!nc)
+        return -ENOMEM;
+
+    nc->type = type;
+    nc->callback = cb;
+    nc->ctx = ctx;
+
+    pthread_mutex_lock(&g_notify_lock);
+    nc->next = g_notify_callbacks;
+    g_notify_callbacks = nc;
+    pthread_mutex_unlock(&g_notify_lock);
+
+    (void)vf_id;
+    return 0;
+}
+
+int rpc_emu_notify_unregister(uint32_t vf_id, uint32_t type)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    pthread_mutex_lock(&g_notify_lock);
+    struct emu_notify_cb **prev = &g_notify_callbacks;
+    struct emu_notify_cb *nc = g_notify_callbacks;
+    bool found = false;
+
+    while (nc) {
+        if (nc->type == type) {
+            *prev = nc->next;
+            free(nc);
+            found = true;
+            break;
+        }
+        prev = &nc->next;
+        nc = nc->next;
+    }
+    pthread_mutex_unlock(&g_notify_lock);
+
+    (void)vf_id;
+    return found ? 0 : -ENOENT;
+}
+
+int rpc_emu_notify_send(uint32_t vf_id, uint32_t type, void *data, size_t size)
+{
+    if (!rpc_emu_is_enabled())
+        return -ENOTSUP;
+
+    pthread_mutex_lock(&g_notify_lock);
+    struct emu_notify_cb *nc = g_notify_callbacks;
+    while (nc) {
+        if (nc->type == type || nc->type == 0) {
+            nc->callback(vf_id, type, data, size, nc->ctx);
+        }
+        nc = nc->next;
+    }
+    pthread_mutex_unlock(&g_notify_lock);
+
+    return 0;
+}
+
+/* ============================================================
+ * Initialization / Cleanup
+ * ============================================================ */
+
+int rpc_emu_init(void)
+{
+    pthread_mutex_init(&g_bar_state.lock, NULL);
+    pthread_mutex_init(&g_wpr_state.lock, NULL);
+    pthread_mutex_init(&g_sched_lock, NULL);
+    pthread_mutex_init(&g_notify_lock, NULL);
+    return 0;
+}
+
+void rpc_emu_fini(void)
+{
+    pthread_mutex_lock(&g_bar_state.lock);
+    if (g_bar_state.backing_store) {
+        free(g_bar_state.backing_store);
+        g_bar_state.backing_store = NULL;
+        g_bar_state.backing_size = 0;
+    }
+    pthread_mutex_unlock(&g_bar_state.lock);
+    pthread_mutex_destroy(&g_bar_state.lock);
+
+    pthread_mutex_lock(&g_wpr_state.lock);
+    struct emu_wpr_region *rgn = g_wpr_state.regions;
+    while (rgn) {
+        struct emu_wpr_region *next = rgn->next;
+        free(rgn);
+        rgn = next;
+    }
+    g_wpr_state.regions = NULL;
+    pthread_mutex_unlock(&g_wpr_state.lock);
+    pthread_mutex_destroy(&g_wpr_state.lock);
+
+    pthread_mutex_lock(&g_sched_lock);
+    struct emu_sched_work *work = g_sched_queue;
+    while (work) {
+        struct emu_sched_work *next = work->next;
+        free(work);
+        work = next;
+    }
+    g_sched_queue = NULL;
+    pthread_mutex_unlock(&g_sched_lock);
+    pthread_mutex_destroy(&g_sched_lock);
+
+    pthread_mutex_lock(&g_notify_lock);
+    struct emu_notify_cb *nc = g_notify_callbacks;
+    while (nc) {
+        struct emu_notify_cb *next = nc->next;
+        free(nc);
+        nc = next;
+    }
+    g_notify_callbacks = NULL;
+    pthread_mutex_unlock(&g_notify_lock);
+    pthread_mutex_destroy(&g_notify_lock);
+
+    pthread_mutex_destroy(&g_emu_lock);
+}