path: root/wiki/vGPU-IO-Integration-Guide.md

# vGPU-IO Integration Guide

## Overview
This guide covers implementing a Linux kernel driver for vGPU-IO using the GSP 570.144W firmware RPC interface.

## Architecture

```
┌─────────────────────────────────────────────────────────────┐
│                    vGPU-IO Kernel Driver                     │
├─────────────────────────────────────────────────────────────┤
│  gsp_core.c      │  gsp_rpc.c    │  gsp_vf.c               │
│  - Device probe  │  - RPC client │  - VF lifecycle         │
│  - BAR mapping   │  - Doorbell   │  - Migration            │
│  - IRQ/MSI-X     │  - Completion │  - WPR/BAR integration  │
├─────────────────────────────────────────────────────────────┤
│                    GSP Firmware (570.144W)                   │
│  Primary Dispatcher (0x057A1AC0)  │  Secondary (0x04ED39DC) │
│  BAR Handler (0x04CB8188)         │  WPR Handlers           │
│  Migration (0x04179ED8/0x04179FBC)│  SCHED/ALLOC/PIN        │
└─────────────────────────────────────────────────────────────┘
```

## Module Structure

```
drivers/gpu/nvidia/gsp/
├── gsp.h              # Core definitions
├── gsp_rpc.h          # RPC message formats
├── gsp_vf.h           # VF definitions
├── gsp_bar.h          # BAR emulation
├── gsp_wpr.h          # WPR management
├── gsp_migration.h    # Migration protocol
├── gsp_core.c         # Probe/remove, BAR/IRQ
├── gsp_rpc.c          # RPC client implementation
├── gsp_vf.c           # VF create/destroy
├── gsp_bar.c          # BAR trap handler
├── gsp_wpr.c          # WPR protect/unprotect
├── gsp_migration.c    # Live migration
└── gsp_debugfs.c      # Debug interface
```

## Core Data Structures

### GSP Device
```c
struct gsp_device {
    struct pci_dev *pdev;
    void __iomem *bar0;           // PCIe BAR0 mapping
    void __iomem *bar1;           // PCIe BAR1 (GSP FW)
    struct gsp_firmware *fw;      // Firmware image
    struct gsp_rpc_client *rpc;   // RPC client
    struct mutex vf_lock;
    struct idr vf_idr;            // VF ID allocator
    struct list_head vf_list;
    struct gsp_debugfs *debugfs;
    spinlock_t doorbell_lock;
    uint32_t doorbell_seq;
};
```

### VF (Virtual Function)
```c
struct gsp_vf {
    struct gsp_device *gsp;
    uint32_t id;                  // VF ID (1-7)
    struct kref ref;
    struct mutex lock;
    mig_state_t mig_state;
    struct gsp_migration *migration;
    uint64_t bar1_offset;         // BAR1 aperture offset
    uint64_t bar1_size;           // BAR1 aperture size
    uint64_t pt_base;             // Page table base
    uint64_t pt_size;             // Page table size
    uint64_t cmd_base;            // Command buffer base
    uint64_t cmd_size;            // Command buffer size
    struct gsp_wpr_region *wpr_regions;
    struct list_head wpr_list;
    struct gsp_bar_state bar_state;
    atomic_t refcount;
    bool active;
};
```

## Initialization Sequence

### 1. PCI Probe
```c
static int gsp_probe(struct pci_dev *pdev, const struct pci_device_id *id) {
    struct gsp_device *gsp;
    int ret;
    
    gsp = devm_kzalloc(&pdev->dev, sizeof(*gsp), GFP_KERNEL);
    gsp->pdev = pdev;
    pci_set_drvdata(pdev, gsp);
    
    // Enable PCI device
    ret = pcim_enable_device(pdev);
    if (ret) return ret;
    
    // Map BARs
    gsp->bar0 = pcim_iomap(pdev, 0, 0);
    gsp->bar1 = pcim_iomap(pdev, 1, 0);
    if (!gsp->bar0 || !gsp->bar1) return -ENOMEM;
    
    // Load firmware
    ret = gsp_firmware_load(gsp, "gsp-570.144.fwimage.bin");
    if (ret) return ret;
    
    // Initialize RPC client
    gsp->rpc = gsp_rpc_client_init(gsp);
    if (IS_ERR(gsp->rpc)) return PTR_ERR(gsp->rpc);
    
    // Setup IRQ/MSI-X
    ret = gsp_setup_irq(gsp);
    if (ret) return ret;
    
    // Initialize debugfs
    gsp_debugfs_init(gsp);
    
    return 0;
}
```

### 2. Firmware Loading
```c
int gsp_firmware_load(struct gsp_device *gsp, const char *name) {
    const struct firmware *fw;
    int ret;
    
    ret = request_firmware(&fw, name, &gsp->pdev->dev);
    if (ret) return ret;
    
    // Verify MD5: 13ce12a392a31eeb79a2e6ebd3a4a93a
    if (!gsp_firmware_verify(fw->data, fw->size)) {
        release_firmware(fw);
        return -EINVAL;
    }
    
    // Load to GSP memory (via BAR1)
    ret = gsp_firmware_upload(gsp, fw->data, fw->size);
    release_firmware(fw);
    
    return ret;
}
```

### 3. GSP Boot
```c
int gsp_boot(struct gsp_device *gsp) {
    // 1. Reset GSP
    writel(1, gsp->bar0 + GSP_RESET_REG);
    msleep(10);
    writel(0, gsp->bar0 + GSP_RESET_REG);
    
    // 2. Wait for entry point (0x04009000) to signal ready
    ret = gsp_wait_for_ready(gsp, 5000);  // 5s timeout
    if (ret) return ret;
    
    // 3. Initialize RPC dispatchers
    ret = gsp_rpc_init_dispatchers(gsp);
    if (ret) return ret;
    
    return 0;
}
```

## RPC Client Implementation

### Doorbell Mechanism
```c
#define GSP_DOORBELL_OFFSET   0x1000
#define GSP_MSG_RING_SIZE     256
#define GSP_MSG_MAX_SIZE      256

struct gsp_rpc_client {
    struct gsp_device *gsp;
    struct mutex lock;
    uint32_t seq;
    wait_queue_head_t wq;
    struct gsp_msg_ring *ring;
    struct completion *pending[GSP_MAX_PENDING];
};

static int gsp_rpc_send_sync(struct gsp_rpc_client *rpc,
                             uint32_t rpc_type,
                             void *payload, size_t payload_size,
                             void *response, size_t *response_size) {
    struct gsp_msg msg = {
        .hdr = {
            .magic = 0x47535052,
            .version = 1,
            .rpc_type = rpc_type,
            .sequence = atomic_inc_return(&rpc->seq),
            .timestamp_ns = ktime_get_ns(),
            .payload_size = payload_size,
            .flags = MSG_SYNC,
        }
    };
    
    struct completion done;
    init_completion(&done);
    
    mutex_lock(&rpc->lock);
    rpc->pending[msg.hdr.sequence % GSP_MAX_PENDING] = &done;
    
    // Copy message to ring buffer
    memcpy(msg.hdr.payload, payload, payload_size);
    gsp_ring_write(rpc->ring, &msg, GSP_MSG_MAX_SIZE);
    
    // Ring doorbell
    writel(1, rpc->gsp->bar0 + GSP_DOORBELL_OFFSET);
    
    mutex_unlock(&rpc->lock);
    
    // Wait for completion
    if (!wait_for_completion_timeout(&done, msecs_to_jiffies(5000)))
        return -ETIMEDOUT;
    
    // Copy response
    if (response && response_size) {
        *response_size = min(*response_size, done.payload_size);
        memcpy(response, done.payload, *response_size);
    }
    
    return done.status;
}
```

### Completion Handler (ISR)
```c
static irqreturn_t gsp_isr(int irq, void *data) {
    struct gsp_device *gsp = data;
    struct gsp_msg msg;
    
    while (gsp_ring_read(gsp->rpc->ring, &msg)) {
        if (msg.hdr.flags & MSG_SYNC) {
            struct completion *done = 
                gsp->rpc->pending[msg.hdr.sequence % GSP_MAX_PENDING];
            if (done) {
                done->status = msg.payload.status;
                done->payload_size = msg.hdr.payload_size;
                memcpy(done->payload, msg.payload.data, msg.hdr.payload_size);
                complete(done);
            }
        } else {
            // Async notification
            gsp_handle_notification(gsp, &msg);
        }
    }
    
    return IRQ_HANDLED;
}
```

## VF Lifecycle

### Create VF
```c
int gsp_vf_create(struct gsp_device *gsp, uint32_t vf_id,
                  const struct gsp_vf_config *config) {
    struct gsp_vf *vf;
    int ret;
    
    mutex_lock(&gsp->vf_lock);
    
    if (idr_alloc(&gsp->vf_idr, NULL, vf_id, vf_id + 1, GFP_KERNEL) < 0) {
        ret = -EBUSY;
        goto unlock;
    }
    
    vf = kzalloc(sizeof(*vf), GFP_KERNEL);
    vf->gsp = gsp;
    vf->id = vf_id;
    kref_init(&vf->ref);
    mutex_init(&vf->lock);
    INIT_LIST_HEAD(&vf->wpr_list);
    
    // Configure VF resources
    vf->bar1_offset = config->bar1_offset;
    vf->bar1_size = config->bar1_size;
    vf->pt_base = config->pt_base;
    vf->pt_size = config->pt_size;
    vf->cmd_base = config->cmd_base;
    vf->cmd_size = config->cmd_size;
    
    // Initialize WPR for VF
    ret = gsp_vf_wpr_init(vf);
    if (ret) goto err_wpr;
    
    // Initialize BAR for VF
    ret = gsp_vf_bar_init(vf);
    if (ret) goto err_bar;
    
    list_add(&vf->list, &gsp->vf_list);
    vf->active = true;
    
    mutex_unlock(&gsp->vf_lock);
    return 0;
    
err_bar:
    gsp_vf_wpr_fini(vf);
err_wpr:
    idr_remove(&gsp->vf_idr, vf_id);
    kfree(vf);
unlock:
    mutex_unlock(&gsp->vf_lock);
    return ret;
}
```

### Destroy VF
```c
void gsp_vf_destroy(struct gsp_vf *vf) {
    struct gsp_device *gsp = vf->gsp;
    uint32_t id = vf->id;
    
    mutex_lock(&gsp->vf_lock);
    vf->active = false;
    list_del(&vf->list);
    mutex_unlock(&gsp->vf_lock);
    
    gsp_vf_bar_fini(vf);
    gsp_vf_wpr_fini(vf);
    
    if (vf->migration)
        gsp_migration_cancel(vf->migration);
    
    kref_put(&vf->ref, gsp_vf_release);
}

static void gsp_vf_release(struct kref *ref) {
    struct gsp_vf *vf = container_of(ref, struct gsp_vf, ref);
    kfree(vf);
}
```

## Memory Management (ALLOC/PIN)

### Allocate VF Memory
```c
int gsp_vf_mem_alloc(struct gsp_vf *vf, 
                     uint64_t size, uint32_t flags,
                     uint64_t *handle, uint64_t *va) {
    struct alloc_payload payload = {
        .size = size,
        .type = MEM_TYPE_VRAM,
        .alignment = 0x1000,
        .va_hint = 0,
        .flags = flags,
        .pid = current->pid,
        .vf_id = vf->id,
    };
    
    struct alloc_response resp;
    size_t resp_size = sizeof(resp);
    
    return gsp_rpc_send_sync(vf->gsp->rpc, RPC_ALLOC,
                             &payload, sizeof(payload),
                             &resp, &resp_size);
}
```

### Pin Memory for DMA
```c
int gsp_vf_mem_pin(struct gsp_vf *vf, uint64_t handle,
                   uint64_t offset, uint64_t size,
                   uint64_t *dma_addr) {
    struct pin_payload payload = {
        .handle = handle,
        .offset = offset,
        .size = size,
        .vf_id = vf->id,
        .flags = PIN_WRITE | PIN_READ,
    };
    
    struct pin_response resp;
    size_t resp_size = sizeof(resp);
    
    return gsp_rpc_send_sync(vf->gsp->rpc, RPC_PIN,
                             &payload, sizeof(payload),
                             &resp, &resp_size);
}
```

## Debugging & Testing

### Debugfs Interface
```bash
# List VFs
cat /sys/kernel/debug/gsp/vfs

# VF details
cat /sys/kernel/debug/gsp/vf/1/status

# RPC statistics
cat /sys/kernel/debug/gsp/rpc/stats

# BAR state
cat /sys/kernel/debug/gsp/vf/1/bar

# WPR regions
cat /sys/kernel/debug/gsp/vf/1/wpr

# Migration status
cat /sys/kernel/debug/gsp/vf/1/migration
```

### Trace Events
```bash
# Enable tracing
echo 1 > /sys/kernel/debug/tracing/events/gsp/enable

# Trace migration
trace-cmd record -e gsp_mig_phase -e gsp_mig_transfer

# Trace WPR
trace-cmd record -e gsp_wpr_event
```

### Unit Tests
```c
// kselftest for gsp-rpc
static void test_rpc_alloc(void) {
    struct gsp_vf *vf = create_test_vf();
    uint64_t handle, va;
    
    ASSERT_EQ(gsp_vf_mem_alloc(vf, 4096, 0, &handle, &va), 0);
    ASSERT_NE(handle, 0);
    ASSERT_NE(va, 0);
    
    gsp_vf_mem_free(vf, handle);
    destroy_test_vf(vf);
}
```

## Error Handling Checklist

| Scenario | Detection | Recovery |
|----------|-----------|----------|
| GSP firmware crash | Watchdog timeout | Reset GSP, reload FW |
| RPC timeout | Doorbell no response | Retry, then VF reset |
| BAR trap storm | Rate limit exceeded | Throttle VF |
| WPR violation | Callback storm | Quarantine VF |
| Migration fail | Commit verify fail | Rollback to source |

## Performance Tuning

### RPC Batching
```c
// Batch multiple ALLOCs in single RPC
struct batch_alloc_payload {
    uint32_t count;
    struct alloc_payload allocs[MAX_BATCH];
};
```

### Async Completion
```c
// Use async RPC for non-critical paths
gsp_rpc_send_async(rpc, RPC_NOTIFY, payload, size, callback, ctx);
```

### Memory Pool
```c
// Pre-allocate bounce buffers
struct gsp_bounce_pool {
    struct page **pages;
    int count;
    spinlock_t lock;
};
```

---

## Reference
- GSP Firmware: gsp-570.144.fwimage.bin
- Base Address: 0x04000000
- Primary Dispatcher: 0x057A1AC0
- Secondary Dispatcher: 0x04ED39DC