Open-IOV: initial public release

Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with
real GPU functionality - CUDA first, 3D and video to follow - on unmodified
hosts, without a proprietary vGPU license and without patching any driver,
firmware or hypervisor.

Core principle: emulate the interface, pass through the work.

  kernel/     open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device
              (PCI config space, BAR shadows, MSI-X), GSP boot emulation,
              GSP-RM message-queue/RPC emulation, memory model (GPU-VA to
              guest-PA discovery) and the semaphore-completion service that
              advances guest UVM progress.
  userspace/  libiov.so management API and examples.
  tools/      guest diagnostics, firmware analysis wrappers, RM probes.
  scripts/    stack manager, VM harness, build/load/test automation.
  config/     hardware/version profiles (the portability surface).
  docs/       documentation tree - start at docs/README.md.
  wiki/       long-form firmware and protocol deep dives.

Status: initialization works end to end for a stock 570.144 Linux guest on
Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery,
object creation and the early CUDA/UVM start-up path. GPU work forwarding -
the step that makes CUDA kernels execute on real hardware - is the active
work item; see docs/04-implementation/01-known-knowns-and-unknowns.md.

License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link
GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.

1 files changed, 355 insertions, 0 deletions

diff --git a/kernel/gsp_rpc_client.h b/kernel/gsp_rpc_client.h
new file mode 100644
index 0000000..4712754
--- /dev/null
+++ b/kernel/gsp_rpc_client.h
@@ -0,0 +1,355 @@
+/*
+ * GSP 570.144W RPC Client Header
+ * Auto-generated from firmware decompilation
+ */
+
+#ifndef _GSP_RPC_CLIENT_H_
+#define _GSP_RPC_CLIENT_H_
+
+#include <linux/types.h>
+#include <linux/device.h>
+
+/* ============================================================
+ * FIRMWARE BASE ADDRESS
+ * ============================================================ */
+#define GSP_FW_BASE                 0x04000000UL
+
+/* ============================================================
+ * RPC DISPATCHER ADDRESSES
+ * ============================================================ */
+
+/* Primary Dispatcher - Jump table switch for 6 RPC types */
+#define GSP_RPC_DISPATCH_PRIMARY    (GSP_FW_BASE + 0x17A1AC0)  /* 0x057a1ac0 */
+#define GSP_RPC_JUMP_TABLE          (GSP_FW_BASE + 0x17A3AD8)  /* 0x057a3ad8 - 32 sub-commands */
+
+/* Secondary Dispatcher - Full implementation for all 11 RPC types */
+#define GSP_RPC_DISPATCH_SECONDARY  (GSP_FW_BASE + 0x0ED39DC)  /* 0x04ed39dc */
+
+/* Tertiary Dispatcher - Stack=160 bytes */
+#define GSP_RPC_DISPATCH_TERTIARY   (GSP_FW_BASE + 0x0CC58F8)  /* 0x04cc58f8 */
+
+/* Quaternary Dispatcher - Stack=96 bytes */
+#define GSP_RPC_DISPATCH_QUATERNARY (GSP_FW_BASE + 0x1723EF4)  /* 0x05723ef4 */
+
+/* ============================================================
+ * SPECIALIZED HANDLER ADDRESSES
+ * ============================================================ */
+
+#define GSP_ALLOC_MAIN              (GSP_FW_BASE + 0x17E94EC)  /* 0x057e94ec */
+#define GSP_SCHEDULE_MAIN           (GSP_FW_BASE + 0x11EFBF4)  /* 0x051efbf4 */
+#define GSP_BAR_TRAP_HANDLER_JT     (GSP_FW_BASE + 0x17675B7)  /* 0x057675b7 */
+#define GSP_BAR_TRAP_HANDLER_PRIM   (GSP_FW_BASE + 0x0CB8189)  /* 0x04cb8189 */
+#define GSP_PIN_HANDLER_SEC         (GSP_FW_BASE + 0x0ED34B8)  /* 0x04ed34b8 */
+
+/* Jump Table Handlers (from decoded table at 0x059bfeb0) */
+#define GSP_MAP_HANDLER_A           (GSP_FW_BASE + 0x1766AB7)  /* 0x05766ab7 */
+#define GSP_MAP_HANDLER_B           (GSP_FW_BASE + 0x1778A37)  /* 0x05778a37 */
+#define GSP_UNMAP_HANDLER_A         (GSP_FW_BASE + 0x17669B7)  /* 0x057669b7 */
+#define GSP_UNMAP_HANDLER_B         (GSP_FW_BASE + 0x1779937)  /* 0x05779937 */
+#define GSP_PIN_HANDLER_A           (GSP_FW_BASE + 0x17664B7)  /* 0x057664b7 */
+#define GSP_PIN_HANDLER_B           (GSP_FW_BASE + 0x177A3B7)  /* 0x0577a3b7 */
+#define GSP_UNPIN_HANDLER_A         (GSP_FW_BASE + 0x17682B7)  /* 0x057682b7 */
+#define GSP_UNPIN_HANDLER_B         (GSP_FW_BASE + 0x1779FB7)  /* 0x05779fb7 */
+#define GSP_SCHEDULE_HANDLER_A      (GSP_FW_BASE + 0x1767F37)  /* 0x05767f37 */
+#define GSP_SCHEDULE_HANDLER_B      (GSP_FW_BASE + 0x175AEB7)  /* 0x0575aeb7 */
+#define GSP_YIELD_HANDLER_A         (GSP_FW_BASE + 0x175AE37)  /* 0x0575ae37 */
+#define GSP_YIELD_HANDLER_B         (GSP_FW_BASE + 0x1767337)  /* 0x05767337 */
+#define GSP_NOTIFY_HANDLER_A        (GSP_FW_BASE + 0x177C8B7)  /* 0x0577c8b7 */
+#define GSP_NOTIFY_HANDLER_B        (GSP_FW_BASE + 0x1767837)  /* 0x05767837 */
+
+/* Entry Point */
+#define GSP_ENTRY_POINT             (GSP_FW_BASE + 0x0009000)  /* 0x04009000 */
+
+/* ============================================================
+ * RPC TYPE DEFINITIONS
+ * ============================================================ */
+enum gsp_rpc_type {
+    GSP_RPC_ALLOC       = 0x01,
+    GSP_RPC_FREE        = 0x02,
+    GSP_RPC_MAP         = 0x03,
+    GSP_RPC_UNMAP       = 0x04,
+    GSP_RPC_PIN         = 0x05,
+    GSP_RPC_UNPIN       = 0x06,
+    GSP_RPC_SCHEDULE    = 0x07,
+    GSP_RPC_YIELD       = 0x08,
+    GSP_RPC_NOTIFY      = 0x09,
+    GSP_RPC_BAR_TRAP    = 0x0A,
+    GSP_RPC_MIGRATE     = 0x0B,
+    GSP_RPC_DEBUG       = 0xFF,
+    GSP_RPC_MAX_TYPE    = 0x10,  /* Must be > highest RPC type value */
+};
+
+/* ============================================================
+ * ALLOC SUB-COMMANDS (Primary Dispatcher Jump Table Indices)
+ * ============================================================ */
+enum gsp_alloc_subcmd {
+    GSP_ALLOC_SUBCMD_DEFAULT    = 0,
+    GSP_ALLOC_SUBCMD_TYPE1      = 1,
+    GSP_ALLOC_SUBCMD_TYPE2      = 3,
+    GSP_ALLOC_SUBCMD_TYPE3      = 4,
+    GSP_ALLOC_SUBCMD_TYPE4      = 5,
+    GSP_ALLOC_SUBCMD_TYPE5      = 8,
+    GSP_ALLOC_SUBCMD_TYPE6      = 10,
+    GSP_ALLOC_SUBCMD_TYPE7      = 12,
+    GSP_ALLOC_SUBCMD_TYPE8      = 17,
+    GSP_ALLOC_SUBCMD_TYPE9      = 19,
+    GSP_ALLOC_SUBCMD_TYPE10     = 20,
+    GSP_ALLOC_SUBCMD_TYPE11     = 21,
+    GSP_ALLOC_SUBCMD_TYPE12     = 22,
+    GSP_ALLOC_SUBCMD_TYPE13     = 23,
+    GSP_ALLOC_SUBCMD_TYPE14     = 24,
+    GSP_ALLOC_SUBCMD_TYPE15     = 25,
+    GSP_ALLOC_SUBCMD_TYPE16     = 26,
+    GSP_ALLOC_SUBCMD_TYPE17     = 27,
+    GSP_ALLOC_SUBCMD_TYPE18     = 28,
+    GSP_ALLOC_SUBCMD_TYPE19     = 29,
+    GSP_ALLOC_SUBCMD_TYPE20     = 30,
+    GSP_ALLOC_SUBCMD_TYPE21     = 31,
+};
+
+/* ============================================================
+ * MIGRATE SUB-COMMANDS
+ * ============================================================ */
+enum gsp_migrate_subcmd {
+    GSP_MIGRATE_SUBCMD_SAVE     = 0x04,
+    GSP_MIGRATE_SUBCMD_RESTORE  = 0x05,
+};
+
+/* ============================================================
+ * PIN SUB-COMMANDS
+ * ============================================================ */
+enum gsp_pin_subcmd {
+    GSP_PIN_SUBCMD_DEFAULT      = 0x05,
+};
+
+/* ============================================================
+ * SCHEDULE SUB-COMMANDS
+ * ============================================================ */
+enum gsp_sched_subcmd {
+    GSP_SCHED_SUBCMD_DEFAULT    = 0x07,
+};
+
+/* ============================================================
+ * BAR TRAP SUB-COMMANDS
+ * ============================================================ */
+enum gsp_bar_trap_subcmd {
+    GSP_BAR_TRAP_SUBCMD_DEFAULT = 0x0A,
+};
+
+/* ============================================================
+ * RPC MESSAGE STRUCTURES
+ * ============================================================ */
+
+struct gsp_rpc_msg {
+    __le32 type;
+    __le32 subcmd;
+    __le64 payload[6];
+    __le32 seq_id;
+    __le32 flags;
+    __le32 reserved[2];
+} __packed;
+
+struct gsp_rpc_response {
+    __le32 status;
+    __le32 type;
+    __le64 result[6];
+    __le32 seq_id;
+} __packed;
+
+/* ALLOC request/response */
+struct gsp_alloc_req {
+    __le64 size;
+    __le64 alignment;
+    __le32 mem_type;
+    __le32 flags;
+    __le64 addr_hint;
+    __le32 pid;
+    __le32 reserved;
+} __packed;
+
+struct gsp_alloc_rsp {
+    __le64 gpu_va;
+    __le64 handle;
+    __le64 size;
+} __packed;
+
+/* MAP request/response */
+struct gsp_map_req {
+    __le64 handle;
+    __le64 gpu_va;
+    __le64 size;
+    __le32 prot;
+    __le32 flags;
+    __le64 offset;
+    __le32 reserved[2];
+} __packed;
+
+struct gsp_map_rsp {
+    __le64 gpu_va;
+} __packed;
+
+/* PIN/UNPIN request/response */
+struct gsp_pin_req {
+    __le64 handle;
+    __le64 offset;
+    __le64 size;
+    __le32 flags;
+    __le32 reserved[3];
+} __packed;
+
+struct gsp_pin_rsp {
+    __le64 dma_addr;
+    __le32 num_pages;
+    __le32 reserved;
+} __packed;
+
+/* SCHEDULE request */
+struct gsp_sched_req {
+    __le64 work_handle;
+    __le64 callback;
+    __le64 arg;
+    __le32 priority;
+    __le32 flags;
+    __le32 reserved[2];
+} __packed;
+
+/* BAR TRAP request/response */
+struct gsp_bar_trap_req {
+    __le64 bar_addr;
+    __le32 access_type;
+    __le32 size;
+    __le64 value;
+    __le32 bar_num;
+    __le32 reserved;
+} __packed;
+
+struct gsp_bar_trap_rsp {
+    __le64 value;
+    __le32 handled;
+    __le32 reserved;
+} __packed;
+
+/* MIGRATE request/response */
+struct gsp_migrate_req {
+    __le64 src_gpu_id;
+    __le64 dst_gpu_id;
+    __le64 memory_bitmap;
+    __le64 page_table_ptr;
+    __le32 state;
+    __le32 flags;
+    __le64 context;
+    __le32 reserved;
+} __packed;
+
+struct gsp_migrate_rsp {
+    __le32 status;
+    __le32 progress;
+    __le64 checkpoint;
+} __packed;
+
+/* ============================================================
+ * CLIENT OPAQUE STRUCT
+ * ============================================================ */
+struct gsp_client;
+
+/* ============================================================
+ * PUBLIC API
+ * ============================================================ */
+
+ /* Client lifecycle */
+ struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell);
+ struct gsp_client *gsp_client_init_emulation(struct device *dev, void __iomem *fw_base_virt, void __iomem *doorbell_virt);
+ void gsp_client_fini(struct gsp_client *client);
+
+/* Memory management */
+int gsp_rpc_alloc(struct gsp_client *client,
+                   u64 size, u64 alignment, u32 mem_type,
+                   u32 flags, u64 addr_hint, u32 pid,
+                   u64 *gpu_va, u64 *handle, u64 *actual_size);
+
+int gsp_rpc_free(struct gsp_client *client, u64 handle);
+
+int gsp_rpc_map(struct gsp_client *client,
+                 u64 handle, u64 gpu_va, u64 size,
+                 u32 prot, u32 flags, u64 offset,
+                 u64 *mapped_va);
+
+int gsp_rpc_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size);
+
+/* Page pinning */
+int gsp_rpc_pin(struct gsp_client *client,
+                 u64 handle, u64 offset, u64 size,
+                 u32 flags, u64 *dma_addr, u32 *num_pages);
+
+int gsp_rpc_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size);
+
+/* Scheduling */
+int gsp_rpc_schedule(struct gsp_client *client,
+                      u64 work_handle, u64 callback, u64 arg,
+                      u32 priority, u32 flags);
+
+int gsp_rpc_yield(struct gsp_client *client);
+
+/* BAR trap emulation */
+int gsp_rpc_bar_trap(struct gsp_client *client,
+                      u64 bar_addr, u32 access_type, u32 size,
+                      u64 value, u32 bar_num, u64 *ret_value);
+
+/* Migration */
+int gsp_rpc_migrate_save(struct gsp_client *client,
+                          u64 src_gpu_id, u64 dst_gpu_id,
+                          u64 memory_bitmap, u64 page_table_ptr,
+                          u64 *checkpoint);
+
+int gsp_rpc_migrate_restore(struct gsp_client *client,
+                             u64 src_gpu_id, u64 dst_gpu_id,
+                             u64 memory_bitmap, u64 page_table_ptr,
+                             u64 checkpoint);
+
+/* WPR Management */
+int gsp_wpr_write_mutex_acquire(struct gsp_client *client, u32 wpr_id);
+int gsp_wpr_write_mutex_release(struct gsp_client *client, u32 wpr_id);
+int gsp_wpr_scratch_mutex_acquire(struct gsp_client *client, u32 wpr_id);
+int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id);
+
+/* Emulation control */
+void gsp_rpc_emulation_set(bool enable);
+bool gsp_rpc_emulation_is_enabled(void);
+
+/* Emulation implementations */
+int gsp_rpc_alloc_emul(struct gsp_client *client,
+                        u64 size, u64 alignment, u32 mem_type,
+                        u32 flags, u64 addr_hint, u32 pid,
+                        u64 *gpu_va, u64 *handle, u64 *actual_size);
+int gsp_rpc_free_emul(struct gsp_client *client, u64 handle);
+int gsp_rpc_map_emul(struct gsp_client *client,
+                      u64 handle, u64 gpu_va, u64 size,
+                      u32 prot, u32 flags, u64 offset,
+                      u64 *mapped_va);
+int gsp_rpc_unmap_emul(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size);
+int gsp_rpc_pin_emul(struct gsp_client *client,
+                      u64 handle, u64 offset, u64 size,
+                      u32 flags, u64 *dma_addr, u32 *num_pages);
+int gsp_rpc_unpin_emul(struct gsp_client *client, u64 handle, u64 offset, u64 size);
+
+/* Notification */
+int gsp_rpc_notify_register(struct gsp_client *client,
+                             u32 type,
+                             void (*callback)(struct gsp_client *,
+                                              struct gsp_rpc_response *,
+                                              void *),
+                             void *ctx);
+void gsp_rpc_notify_unregister(struct gsp_client *client, u32 type);
+void gsp_handle_notification(struct gsp_client *client, struct gsp_rpc_msg *msg);
+
+/* Client lifecycle */
+struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell);
+void gsp_client_fini(struct gsp_client *client);
+
+/* WPR Management */
+int gsp_wpr_write_mutex_acquire(struct gsp_client *client, u32 wpr_id);
+int gsp_wpr_write_mutex_release(struct gsp_client *client, u32 wpr_id);
+int gsp_wpr_scratch_mutex_acquire(struct gsp_client *client, u32 wpr_id);
+int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id);
+
+#endif /* _GSP_RPC_CLIENT_H_ */