Open-IOV: initial public release

Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with
real GPU functionality - CUDA first, 3D and video to follow - on unmodified
hosts, without a proprietary vGPU license and without patching any driver,
firmware or hypervisor.

Core principle: emulate the interface, pass through the work.

  kernel/     open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device
              (PCI config space, BAR shadows, MSI-X), GSP boot emulation,
              GSP-RM message-queue/RPC emulation, memory model (GPU-VA to
              guest-PA discovery) and the semaphore-completion service that
              advances guest UVM progress.
  userspace/  libiov.so management API and examples.
  tools/      guest diagnostics, firmware analysis wrappers, RM probes.
  scripts/    stack manager, VM harness, build/load/test automation.
  config/     hardware/version profiles (the portability surface).
  docs/       documentation tree - start at docs/README.md.
  wiki/       long-form firmware and protocol deep dives.

Status: initialization works end to end for a stock 570.144 Linux guest on
Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery,
object creation and the early CUDA/UVM start-up path. GPU work forwarding -
the step that makes CUDA kernels execute on real hardware - is the active
work item; see docs/04-implementation/01-known-knowns-and-unknowns.md.

License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link
GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md.

1 files changed, 316 insertions, 0 deletions

diff --git a/wiki/RPC-Message-Format.md b/wiki/RPC-Message-Format.md
new file mode 100644
index 0000000..e627871
--- /dev/null
+++ b/wiki/RPC-Message-Format.md
@@ -0,0 +1,316 @@
+# GSP RPC Message Format Specification
+
+## Overview
+The GSP firmware uses a dispatcher-based RPC system with two dispatchers:
+- **Primary** (0x057A1AC0): ALLOC, FREE, MAP, UNMAP, PIN, UNPIN, SCHEDULE, YIELD, NOTIFY, BAR_TRAP, MIGRATE, DEBUG
+- **Secondary** (0x04ED39DC): Branch-based dispatch for all 11 types
+
+## Primary Dispatcher Call Convention
+
+### Entry Point
+```
+FUN_ram_057a1ac0(
+    uint32_t rpc_type,      // param_1: RPC type (0x01-0x0C)
+    uint8_t  flags_a,       // param_2
+    uint8_t  flags_b,       // param_3
+    uint8_t  flags_c,       // param_4
+    uint32_t *out_subcmd_id,// param_5: Returns sub-command ID
+    uint8_t  *out_a,        // param_6: Echoes flags_b
+    uint8_t  *out_b,        // param_7: Echoes flags_a
+    uint8_t  *out_c         // param_8: Echoes flags_c
+) -> int64_t
+```
+
+### RPC Type to Sub-Command Mapping
+The primary dispatcher validates RPC type, then indexes into the 32-entry jump table at **0x057A3AD8**.
+
+| RPC Type | Name | Subcmd Range | Notes |
+|----------|------|--------------|-------|
+| 0x01 | ALLOC | 0-31 | Uses 0x057E94EC main |
+| 0x02 | FREE | 0-31 | |
+| 0x03 | MAP | — | Uses secondary dispatcher |
+| 0x04 | UNMAP | 0-31 | Primary + secondary |
+| 0x05 | PIN | — | Uses secondary dispatcher |
+| 0x06 | UNPIN | — | Uses secondary dispatcher |
+| 0x07 | SCHEDULE | 0-31 | Uses 0x051EFBF4 main |
+| 0x08 | YIELD | — | Uses secondary dispatcher |
+| 0x09 | NOTIFY | — | Uses secondary dispatcher |
+| 0x0A | BAR_TRAP | — | Uses 0x04CB8188 |
+| 0x0B | MIGRATE | 0-15 x2 | Dual jump tables |
+| 0x0C | DEBUG | — | Unimplemented |
+
+### Sub-Command Descriptor (Returned by Jump Table Handlers)
+Each sub-command handler at the jump table returns:
+```c
+typedef struct {
+    uint32_t subcmd_id;     // Constant ID (e.g., 0x10, 0x14, 0x40)
+    uint8_t  echo_flags_b;  // Copy of input flags_b
+    uint8_t  echo_flags_a;  // Copy of input flags_a
+    uint8_t  echo_flags_c;  // Copy of input flags_c
+} __attribute__((packed)) subcmd_descriptor_t;
+```
+
+**Example: Subcmd 3 (0x057A1BCC)**
+```c
+// Returns ID = 0x10
+*out_subcmd_id = 0x10;
+*out_a = flags_b;
+*out_b = flags_a;
+*out_c = flags_c;
+return 0;
+```
+
+**Example: Subcmd 0 (0x057A1DB4)**
+```c
+// Returns ID = 0x40
+*out_subcmd_id = 0x40;
+*out_a = flags_b;
+*out_b = flags_a;
+*out_c = flags_c;
+return 0;
+```
+
+## Secondary Dispatcher Call Convention
+
+### Entry Point
+```
+FUN_ram_04ed39dc(rpc_message_t *msg) -> int64_t
+```
+
+### Message Structure (Inferred)
+```c
+typedef struct {
+    uint32_t rpc_type;      // 0x01-0x0B
+    uint32_t sub_type;      // Sub-operation code
+    uint64_t payload_ptr;   // Pointer to RPC-specific payload
+    uint32_t payload_size;  // Size of payload
+    uint32_t flags;         // Control flags
+    uint64_t context;       // Opaque context
+    uint32_t sequence;      // Sequence number
+    uint32_t reserved;
+} __attribute__((packed)) rpc_message_t;
+```
+
+### Branch-Based Dispatch
+Secondary dispatcher uses `bgeu`/`bltu` branches on `rpc_type` register:
+```assembly
+# Pseudocode
+if (rpc_type >= 0x0C) goto error;
+switch(rpc_type):
+  case 0x03: call MAP_handler_A;
+  case 0x04: call UNMAP_handler_A;
+  case 0x05: call PIN_handler_A (0x04ED34B8);
+  case 0x06: call UNPIN_handler_A;
+  case 0x07: call SCHEDULE_handler_A;
+  case 0x08: call YIELD_handler_A;
+  case 0x09: call NOTIFY_handler_A;
+  case 0x0A: call BAR_TRAP_handler (0x057675B7);
+```
+
+## RPC-Specific Payloads
+
+### ALLOC (0x01) - Via 0x057E94EC
+```c
+typedef struct {
+    uint64_t size;          // Allocation size in bytes
+    uint32_t type;          // Memory type (VRAM, sysmem, etc.)
+    uint32_t alignment;     // Alignment requirement
+    uint64_t va_hint;       // Virtual address hint
+    uint32_t flags;         // CACHE_COHERENT, UNCACHED, etc.
+    uint32_t pid;           // Process ID
+    uint32_t vf_id;         // VF ID (for vGPU)
+    uint32_t reserved;
+    uint64_t *out_handle;   // Output: allocation handle
+    uint64_t *out_va;       // Output: mapped virtual address
+    uint64_t *out_pa;       // Output: physical address
+} alloc_payload_t;
+```
+
+### MAP (0x03) - Handlers 0x05766AB7 / 0x05778A37
+```c
+typedef struct {
+    uint64_t handle;        // Allocation handle from ALLOC
+    uint64_t va;            // Target virtual address
+    uint64_t size;          // Map size
+    uint32_t prot;          // PROT_READ|PROT_WRITE|PROT_EXEC
+    uint32_t flags;         // MAP_FIXED, MAP_SHARED, etc.
+    uint32_t vf_id;         // Target VF
+    uint32_t reserved;
+} map_payload_t;
+```
+
+### PIN (0x05) - Handler 0x04ED34B8
+```c
+typedef struct {
+    uint64_t handle;        // Allocation handle
+    uint64_t offset;        // Offset within allocation
+    uint64_t size;          // Size to pin
+    uint32_t vf_id;         // Target VF
+    uint32_t flags;         // PIN_WRITE, PIN_READ, PIN_LONGTERM
+    uint64_t *out_dma_addr; // Output: DMA address for device
+} pin_payload_t;
+```
+
+### SCHEDULE (0x07) - Via 0x051EFBF4 / 0x05767F37
+```c
+typedef struct {
+    uint32_t vf_id;         // VF to schedule
+    uint32_t priority;      // Scheduling priority
+    uint64_t time_slice_ns; // Time slice in nanoseconds
+    uint32_t flags;         // PREEMPTIBLE, REALTIME, etc.
+    uint32_t reserved;
+} schedule_payload_t;
+```
+
+### BAR_TRAP (0x0A) - Handler 0x04CB8188
+```c
+typedef struct {
+    uint32_t bar_index;     // BAR number (0-5)
+    uint32_t trap_type;     // READ, WRITE, ATOMIC
+    uint64_t offset;        // Offset within BAR
+    uint64_t value;         // Value for writes
+    uint32_t size;          // Access size (1,2,4,8)
+    uint32_t vf_id;         // Trapping VF
+    uint64_t *out_value;    // Output: read value
+} bar_trap_payload_t;
+```
+
+### MIGRATE (0x0B) - Dual Jump Tables
+```c
+typedef struct {
+    uint32_t phase;         // 0=prepare, 1=transfer, 2=commit, 3=cleanup
+    uint32_t subcmd;        // Sub-command index (0-15 for each table)
+    uint64_t src_vf;        // Source VF
+    uint64_t dst_vf;        // Destination VF
+    uint64_t data_ptr;      // Migration data buffer
+    uint64_t data_size;     // Buffer size
+    uint32_t flags;         // LIVE, POSTCOPY, etc.
+    uint32_t reserved;
+} migrate_payload_t;
+```
+
+## Wire Format (Host → GSP)
+
+### Message Header (64 bytes)
+```c
+typedef struct {
+    uint32_t magic;         // 0x47535052 ("GSPR")
+    uint32_t version;       // Protocol version (1)
+    uint32_t rpc_type;      // 0x01-0x0C
+    uint32_t sequence;      // Monotonic sequence
+    uint64_t timestamp_ns;  // Submission timestamp
+    uint32_t payload_size;  // Size of payload
+    uint32_t flags;         // MSG_SYNC, MSG_ASYNC, MSG_NOREPLY
+    uint64_t correlation_id;// For async matching
+    uint64_t reserved[3];   // Padding to 64 bytes
+} __attribute__((packed)) rpc_header_t;
+```
+
+### Complete Message
+```
++------------------+
+| rpc_header_t     |  64 bytes
++------------------+
+| payload (varies) |  payload_size bytes
++------------------+
+| padding to 256B  |  (alignment)
++------------------+
+```
+
+## Response Format
+
+### Synchronous Response
+```c
+typedef struct {
+    int64_t status;         // 0 = success, negative = error
+    uint32_t rpc_type;      // Echo of request
+    uint32_t sequence;      // Echo of request
+    uint64_t timestamp_ns;  // Completion timestamp
+    uint32_t payload_size;  // Response payload size
+    uint32_t reserved;
+} __attribute__((packed)) rpc_response_header_t;
+```
+
+### Error Codes
+| Code | Name | Description |
+|------|------|-------------|
+| 0 | OK | Success |
+| -1 (0xFFFF) | EPERM | Permission denied |
+| -2 (0xFFFE) | ENOENT | Not found |
+| -3 (0xFFFD) | EINVAL | Invalid argument |
+| -6 (0xFFFA) | ENODEV | No such device |
+| -12 (0xFFF4) | ENOMEM | Out of memory |
+| -16 (0xFFEE) | EBUSY | Resource busy |
+| -22 (0xFFEA) | EINVAL | Invalid argument |
+| -28 (0xFFE4) | ENOSPC | No space left |
+| -37 (0xFFDB) | ENOLCK | No locks available |
+| -56 (0xFFC8) | EBADMSG | Bad message |
+| -84 (0xFFAC) | EOVERFLOW | Value too large |
+| -90 (0xFFA6) | ENOSTR | Not a stream |
+| -91 (0xFFA5) | ENODATA | No data available |
+| -94 (0xFFA2) | ENOSR | Out of streams resources |
+| -97 (0xFF9F) | EPROTO | Protocol error |
+| -111 (0xFF91) | ETIMEDOUT | Connection timed out |
+
+## Kernel Client Implementation
+
+### Message Builder
+```c
+static int gsp_rpc_send(gsp_client_t *client, uint32_t rpc_type, 
+                        void *payload, size_t payload_size,
+                        void *response, size_t *response_size) {
+    rpc_header_t hdr = {
+        .magic = 0x47535052,
+        .version = 1,
+        .rpc_type = rpc_type,
+        .sequence = atomic_inc(&client->seq),
+        .timestamp_ns = ktime_get_ns(),
+        .payload_size = payload_size,
+        .flags = MSG_SYNC,
+    };
+    
+    // Build complete message
+    uint8_t msg[256];
+    memcpy(msg, &hdr, sizeof(hdr));
+    memcpy(msg + 64, payload, payload_size);
+    
+    // Submit to GSP doorbell
+    return gsp_doorbell_ring(client->gsp, msg, sizeof(msg));
+}
+```
+
+### Doorbell Mechanism
+```c
+// GSP doorbell register (PCIe BAR0 + offset)
+#define GSP_DOORBELL_OFFSET  0x1000
+
+static void gsp_doorbell_ring(gsp_device_t *gsp, void *msg, size_t len) {
+    // Copy message to shared memory ring
+    // Ring GSP doorbell
+    writel(1, gsp->bar0 + GSP_DOORBELL_OFFSET);
+}
+```
+
+## Async Completion
+
+### Notification Mechanism
+```c
+typedef struct {
+    uint32_t rpc_type;
+    uint32_t sequence;
+    int64_t status;
+    uint64_t timestamp_ns;
+    uint32_t payload_size;
+    uint32_t reserved;
+} async_completion_t;
+
+// Delivered via MSI-X or GSP notification queue
+```
+
+---
+
+## Reference
+- Primary Dispatcher: 0x057A1AC0
+- Secondary Dispatcher: 0x04ED39DC
+- Jump Table: 0x057A3AD8 (32 entries)
+- RPC Type Jump Table: 0x059BFEB0 (12 types)
\ No newline at end of file