diff options
Diffstat (limited to 'kernel')
34 files changed, 9220 insertions, 0 deletions
diff --git a/kernel/Kconfig b/kernel/Kconfig new file mode 100644 index 0000000..5cee953 --- /dev/null +++ b/kernel/Kconfig @@ -0,0 +1,44 @@ +config NVIDIA_VGPU_VFIO + tristate "NVIDIA vGPU VFIO Mediated Device Driver" + depends on VFIO_MDEV && PCI && IOMMU_API + select VFIO_PCI_CORE + help + This driver provides VFIO mediated device support for NVIDIA GPUs, + enabling vGPU functionality through the Linux VFIO/mdev framework. + It replaces the proprietary nvidia-vgpu-vfio.ko kernel module. + + The driver handles: + - VFIO device creation and management + - BAR trapping and emulation + - GSP (GPU System Processor) communication + - Memory mapping and pinning + - IRQFD/IOEVENTFD handling + - Live migration support + + To compile this driver as a module, choose M here: the + module will be called nvidia-vgpu-vfio. + +if NVIDIA_VGPU_VFIO + +config NVIDIA_VGPU_VFIO_DEBUG + bool "Enable debug logging for NVIDIA vGPU VFIO driver" + default n + help + Enable verbose debug logging in the NVIDIA vGPU VFIO driver. + This will increase kernel log verbosity significantly. + +config NVIDIA_VGPU_VFIO_MIGRATION + bool "Enable live migration support" + default y + help + Enable live migration support for vGPU mediated devices. + This adds the migration ioctl interface and state serialization. + +config NVIDIA_VGPU_VFIO_GSP_RPC + bool "Enable GSP RPC communication" + default y + help + Enable communication with the GPU System Processor (GSP) + via RPC for scheduling and resource management. + +endif
\ No newline at end of file diff --git a/kernel/Makefile b/kernel/Makefile new file mode 100644 index 0000000..18651cf --- /dev/null +++ b/kernel/Makefile @@ -0,0 +1,33 @@ +obj-m += open-iov-vfio.o + +open-iov-vfio-y := \ + main.o \ + bar_init.o \ + bar_emulation.o \ + gsp_comm.o \ + gsp_rpc_client.o \ + gsp_rpc_integration.o \ + memory.o \ + irq.o \ + migration.o \ + ioctl.o + +ccflags-y := -I$(src)/include -Wall -Wextra -Wno-error -Wno-unused-parameter -Wno-unused-function + +# mdev dependencies +KBUILD_MODPOST_WARN := 1 + +# Use symbols from host nvidia.ko +KBUILD_EXTRA_SYMBOLS := $OPEN_IOV_ROOT/nvidia-open-gpu-kernel-modules/kernel-open/Module.symvers + +all: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules + +clean: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean + +install: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules_install + depmod -a + +.PHONY: all clean install
\ No newline at end of file diff --git a/kernel/Makefile.gsp_rpc b/kernel/Makefile.gsp_rpc new file mode 100644 index 0000000..8bbee6b --- /dev/null +++ b/kernel/Makefile.gsp_rpc @@ -0,0 +1,24 @@ +# GSP RPC Client Kernel Module Makefile +# Build: make -C /lib/modules/$(shell uname -r)/build M=$(PWD) -f Makefile.gsp_rpc + +obj-m += gsp_rpc_client.o + +gsp_rpc_client-y := gsp_rpc_client.o + +ccflags-y += -I$(src)/include +ccflags-y += -Wall -Wextra -Werror +ccflags-y += -std=gnu11 + +# For debugging +# ccflags-y += -DDEBUG + +all: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(PWD) -f $(PWD)/Makefile.gsp_rpc modules + +clean: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(PWD) -f $(PWD)/Makefile.gsp_rpc clean + +install: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(PWD) -f $(PWD)/Makefile.gsp_rpc modules_install + +.PHONY: all clean install diff --git a/kernel/bar_emulation.c b/kernel/bar_emulation.c new file mode 100644 index 0000000..17904fe --- /dev/null +++ b/kernel/bar_emulation.c @@ -0,0 +1,267 @@ +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/io.h> +#include <linux/vmalloc.h> +#include <linux/pci.h> +#include <linux/mm.h> +#include <linux/highmem.h> + +#include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" +#include "gsp_comm.h" + +#define NV_VGPU_BAR0_TRAP_REGIONS 64 + +static const struct { + u32 offset; + u32 size; + const char *name; +} nv_vgpu_bar0_traps[] = { + { 0x000000, 0x1000, "PMC" }, + { 0x001000, 0x1000, "PRAMIN" }, + { 0x002000, 0x1000, "PCR" }, + { 0x003000, 0x1000, "PRAMDAC" }, + { 0x004000, 0x1000, "PVIO" }, + { 0x005000, 0x1000, "PFIFO" }, + { 0x006000, 0x1000, "PGRAPH" }, + { 0x007000, 0x1000, "PTIMER" }, + { 0x008000, 0x1000, "PFUSE" }, + { 0x009000, 0x1000, "PBUS" }, + { 0x00A000, 0x1000, "PCOP" }, + { 0x00B000, 0x1000, "PVP" }, + { 0x00C000, 0x1000, "PCE" }, + { 0x00D000, 0x1000, "PDISP" }, + { 0x00E000, 0x1000, "PNVDEC" }, + { 0x00F000, 0x1000, "PNVENC" }, + { 0x010000, 0x1000, "PSEC2" }, + { 0x011000, 0x1000, "PMU" }, + { 0x012000, 0x1000, "PGFX" }, + { 0x013000, 0x1000, "PXVE" }, + { 0x020000, 0x10000, "FIFO" }, + { 0x040000, 0x10000, "GR" }, + { 0x050000, 0x10000, "CE" }, + { 0x060000, 0x10000, "NVDEC" }, + { 0x070000, 0x10000, "NVENC" }, + { 0x080000, 0x10000, "NVJPG" }, + { 0x090000, 0x10000, "OFA" }, + { 0x100000, 0x10000, "FB" }, + { 0x110000, 0x10000, "LTC" }, + { 0x120000, 0x10000, "GPC" }, + { 0x130000, 0x10000, "TPC" }, + { 0x140000, 0x10000, "ROP" }, + { 0x150000, 0x10000, "HUB" }, + { 0x160000, 0x10000, "NVLINK" }, + { 0x170000, 0x10000, "GSP" }, + { 0x180000, 0x10000, "RAP" }, + { 0x190000, 0x10000, "SEC" }, + { 0x1A0000, 0x10000, "PMU_RISCV" }, + { 0x1B0000, 0x10000, "FALCON" }, + { 0x1C0000, 0x10000, "XVE" }, + { 0x1D0000, 0x10000, "DMCE" }, + { 0x1E0000, 0x10000, "P2P" }, + { 0x1F0000, 0x10000, "MISC" }, +}; + +int nv_vgpu_bar_emul_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_bar_emul_ctx *ctx; + int i, ret; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + ctx->bar0_shadow_size = NV_VGPU_BAR0_SIZE; + ctx->bar0_shadow = vzalloc(ctx->bar0_shadow_size); + if (!ctx->bar0_shadow) { + ret = -ENOMEM; + goto err_free; + } + + spin_lock_init(&ctx->shadow_lock); + ctx->trap_count = 0; + + for (i = 0; i < ARRAY_SIZE(nv_vgpu_bar0_traps); i++) { + if (ctx->trap_count >= 256) + break; + ctx->trap_regs[ctx->trap_count].offset = nv_vgpu_bar0_traps[i].offset; + ctx->trap_regs[ctx->trap_count].trapped = true; + ctx->trap_count++; + } + + vgpu->bar_emul_ctx = ctx; + return 0; + +err_free: + kfree(ctx); + return ret; +} + +void nv_vgpu_bar_emul_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + + if (!ctx) + return; + + if (ctx->bar0_shadow) + vfree(ctx->bar0_shadow); + + kfree(ctx); + vgpu->bar_emul_ctx = NULL; +} + +int nv_vgpu_bar_trap_register(struct nv_vgpu_device *vgpu, int bar_idx, + u32 offset, u32 size) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + unsigned long flags; + int i; + + if (!ctx || bar_idx != 0) + return -EINVAL; + + spin_lock_irqsave(&ctx->shadow_lock, flags); + + for (i = 0; i < ctx->trap_count; i++) { + if (ctx->trap_regs[i].offset == offset) { + ctx->trap_regs[i].trapped = true; + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return 0; + } + } + + if (ctx->trap_count >= 256) { + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return -ENOSPC; + } + + ctx->trap_regs[ctx->trap_count].offset = offset; + ctx->trap_regs[ctx->trap_count].trapped = true; + ctx->trap_count++; + + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return 0; +} + +int nv_vgpu_bar_untrap_register(struct nv_vgpu_device *vgpu, int bar_idx, + u32 offset) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + unsigned long flags; + int i; + + if (!ctx || bar_idx != 0) + return -EINVAL; + + spin_lock_irqsave(&ctx->shadow_lock, flags); + + for (i = 0; i < ctx->trap_count; i++) { + if (ctx->trap_regs[i].offset == offset) { + ctx->trap_regs[i].trapped = false; + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return 0; + } + } + + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return -ENOENT; +} + +int nv_vgpu_bar_emul_read(struct nv_vgpu_device *vgpu, int bar_idx, + u64 offset, void *val, size_t size) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + unsigned long flags; + u32 *val32 = val; + u64 *val64 = val; + + if (!ctx || bar_idx != 0 || offset >= ctx->bar0_shadow_size) + return -EINVAL; + + spin_lock_irqsave(&ctx->shadow_lock, flags); + + switch (size) { + case 4: + *val32 = *(u32 *)((u8 *)ctx->bar0_shadow + offset); + break; + case 8: + *val64 = *(u64 *)((u8 *)ctx->bar0_shadow + offset); + break; + default: + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return -EINVAL; + } + + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return 0; +} + +int nv_vgpu_bar_emul_write(struct nv_vgpu_device *vgpu, int bar_idx, + u64 offset, void *val, size_t size) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + unsigned long flags; + u32 *val32 = val; + u64 *val64 = val; + bool trapped = false; + int i; + + if (!ctx || bar_idx != 0 || offset >= ctx->bar0_shadow_size) + return -EINVAL; + + spin_lock_irqsave(&ctx->shadow_lock, flags); + + for (i = 0; i < ctx->trap_count; i++) { + if (ctx->trap_regs[i].offset == offset && ctx->trap_regs[i].trapped) { + trapped = true; + ctx->trap_regs[i].value = (size == 4) ? *val32 : (u32)*val64; + ctx->trap_regs[i].is_write = true; + break; + } + } + + switch (size) { + case 4: + *(u32 *)((u8 *)ctx->bar0_shadow + offset) = *val32; + break; + case 8: + *(u64 *)((u8 *)ctx->bar0_shadow + offset) = *val64; + break; + default: + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return -EINVAL; + } + + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + + if (trapped) { + nv_vgpu_dbg(vgpu, "BAR0 trap: offset=0x%llx, size=%zu, write=0x%x", + offset, size, (size == 4) ? *val32 : (u32)*val64); + nv_vgpu_gsp_rpc_bar_trap(vgpu, offset, 1, size, *(u64 *)val, 0, NULL); + } + + return 0; +} + +int nv_vgpu_bar_emul_handle_fault(struct nv_vgpu_device *vgpu, + unsigned long addr, bool write) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + struct nv_vgpu_bar *bar = &vgpu->bars[0]; + u64 offset = addr - (unsigned long)bar->vaddr; + u32 val = 0; + int ret; + + if (!ctx || !bar->mapped || !bar->trapped) + return -EINVAL; + + if (write) { + ret = nv_vgpu_bar_emul_write(vgpu, 0, offset, &val, 4); + } else { + ret = nv_vgpu_bar_emul_read(vgpu, 0, offset, &val, 4); + } + + return ret; +}
\ No newline at end of file diff --git a/kernel/bar_init.c b/kernel/bar_init.c new file mode 100644 index 0000000..bf993ab --- /dev/null +++ b/kernel/bar_init.c @@ -0,0 +1,146 @@ +#include <linux/slab.h> +#include <linux/pci.h> +#include <linux/iommu.h> +#include <linux/vmalloc.h> + +#include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" +#include "irq.h" + +int nv_vgpu_bar_init(struct nv_vgpu_device *vgpu) +{ + struct pci_dev *pdev = vgpu->pdev; + struct nv_vgpu_bar *bar; + u64 bar_addr; + u64 bar_size; + int i, ret; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: BAR init validation"); + for (i = 0; i < 4; i++) { + bar_addr = pci_resource_start(pdev, i); + bar_size = pci_resource_len(pdev, i); + if (bar_addr && bar_size) { + nv_vgpu_info(vgpu, " BAR%d: gpa=0x%llx, size=0x%llx (%s)", + i, bar_addr, bar_size, + i == 0 ? "config" : (i == 1 ? "FB" : (i == 2 ? "I/O" : "other"))); + } + } + nv_vgpu_info(vgpu, "DRY-RUN: BAR validation passed"); + return 0; + } + + for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) { + bar = &vgpu->bars[i]; + mutex_init(&bar->lock); + bar->mapped = false; + bar->trapped = false; + } + + bar_addr = pci_resource_start(pdev, 0); + bar_size = pci_resource_len(pdev, 0); + if (bar_addr && bar_size) { + vgpu->bars[0].gpa = bar_addr; + vgpu->bars[0].size = bar_size; + vgpu->bars[0].vaddr = ioremap_wc(bar_addr, bar_size); + if (!vgpu->bars[0].vaddr) { + nv_vgpu_err(vgpu, "Failed to map BAR0"); + return -ENOMEM; + } + vgpu->bar0_config_base = bar_addr; + nv_vgpu_info(vgpu, "BAR0: gpa=0x%llx, size=0x%llx, vaddr=%p", bar_addr, bar_size, vgpu->bars[0].vaddr); + } + + bar_addr = pci_resource_start(pdev, 1); + bar_size = pci_resource_len(pdev, 1); + if (bar_addr && bar_size) { + vgpu->bars[1].gpa = bar_addr; + vgpu->bars[1].size = bar_size; + /* BAR1 (FB) is already mapped by nvidia.ko - don't remap */ + vgpu->bars[1].vaddr = NULL; + nv_vgpu_info(vgpu, "BAR1: gpa=0x%llx, size=0x%llx (managed by nvidia.ko)", bar_addr, bar_size); + } + + bar_addr = pci_resource_start(pdev, 2); + bar_size = pci_resource_len(pdev, 2); + if (bar_addr && bar_size) { + vgpu->bars[2].gpa = bar_addr; + vgpu->bars[2].size = bar_size; + vgpu->bars[2].vaddr = ioremap_wc(bar_addr, bar_size); + if (!vgpu->bars[2].vaddr) { + nv_vgpu_err(vgpu, "Failed to map BAR2"); + return -ENOMEM; + } + nv_vgpu_info(vgpu, "BAR2: gpa=0x%llx, size=0x%llx, vaddr=%p", bar_addr, bar_size, vgpu->bars[2].vaddr); + } + + bar_addr = pci_resource_start(pdev, 3); + bar_size = pci_resource_len(pdev, 3); + if (bar_addr && bar_size) { + vgpu->bars[3].gpa = bar_addr; + vgpu->bars[3].size = bar_size; + vgpu->bars[3].vaddr = ioremap_wc(bar_addr, bar_size); + if (!vgpu->bars[3].vaddr) { + nv_vgpu_err(vgpu, "Failed to map BAR3"); + return -ENOMEM; + } + nv_vgpu_info(vgpu, "BAR3: gpa=0x%llx, size=0x%llx, vaddr=%p", bar_addr, bar_size, vgpu->bars[3].vaddr); + } + + ret = nv_vgpu_bar_emul_init(vgpu); + if (ret) { + nv_vgpu_err(vgpu, "BAR emulation init failed: %d", ret); + return ret; + } + + /* MSI/MSI-X is managed by nvidia.ko - don't try to set up our own */ + nv_vgpu_info(vgpu, "MSI/MSI-X managed by nvidia.ko"); + + vgpu->fb_base = pci_resource_start(pdev, 1); + vgpu->fb_size = pci_resource_len(pdev, 1); + + return 0; +} + +void nv_vgpu_bar_cleanup(struct nv_vgpu_device *vgpu) +{ + int i; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: BAR cleanup (no hardware resources to release)"); + return; + } + + nv_vgpu_irq_teardown_msix(vgpu); + nv_vgpu_irq_teardown_msi(vgpu); + nv_vgpu_bar_emul_cleanup(vgpu); + + for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) { + if (vgpu->bars[i].vaddr) { + iounmap(vgpu->bars[i].vaddr); + vgpu->bars[i].vaddr = NULL; + } + vgpu->bars[i].mapped = false; + vgpu->bars[i].trapped = false; + vgpu->bars[i].iova = 0; + } +} + +int nv_vgpu_bar_trap(struct nv_vgpu_device *vgpu, int bar_idx) +{ + if (bar_idx < 0 || bar_idx >= NV_VGPU_MAX_REGIONS) + return -EINVAL; + + if (!vgpu->bars[bar_idx].vaddr) + return -ENODEV; + + return nv_vgpu_bar_trap_register(vgpu, bar_idx, 0, vgpu->bars[bar_idx].size); +} + +void nv_vgpu_bar_untrap(struct nv_vgpu_device *vgpu, int bar_idx) +{ + if (bar_idx < 0 || bar_idx >= NV_VGPU_MAX_REGIONS) + return; + + nv_vgpu_bar_untrap_register(vgpu, bar_idx, 0); +} diff --git a/kernel/gsp_comm.c b/kernel/gsp_comm.c new file mode 100644 index 0000000..e5f38aa --- /dev/null +++ b/kernel/gsp_comm.c @@ -0,0 +1,636 @@ +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/wait.h> +#include <linux/workqueue.h> +#include <linux/delay.h> +#include <linux/io.h> +#include <linux/pci.h> +#include <linux/interrupt.h> +#include <linux/firmware.h> +#include <linux/device.h> + +#include "nvidia-vgpu-vfio.h" +#include "gsp_comm.h" +#include "gsp_rpc_integration.h" + +#define NV_VGPU_GSP_BOOTLOADER_FILE "nvidia/tu102/gsp/bootloader-535.113.01.bin" +#define NV_VGPU_GSP_BOOTER_LOAD_FILE "nvidia/tu102/gsp/booter_load-535.113.01.bin" +#define NV_VGPU_GSP_GSP_FILE "nvidia/tu102/gsp/gsp-535.113.01.bin" + +static void nv_vgpu_gsp_rpc_work(struct work_struct *work) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = container_of(work, struct nv_vgpu_gsp_comm_ctx, rpc_work); + struct nv_vgpu_gsp_rpc_entry *entry, *tmp; + LIST_HEAD(completed); + + mutex_lock(&ctx->queue_lock); + list_splice_tail_init(&ctx->complete_queue, &completed); + mutex_unlock(&ctx->queue_lock); + + list_for_each_entry_safe(entry, tmp, &completed, list) { + list_del(&entry->list); + complete(&entry->done); + } +} + +static void nv_vgpu_gsp_poll_work(struct work_struct *work) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = container_of(work, struct nv_vgpu_gsp_comm_ctx, poll_work.work); + struct nv_vgpu_device *vgpu = ctx->vgpu; + u32 status; + + if (!ctx || !ctx->irq_enabled || !ctx->gsp_doorbell) + goto reschedule; + + status = readl(ctx->gsp_doorbell + 0x100); + if (status & 0x1) { + writel(0x1, ctx->gsp_doorbell + 0x100); + schedule_work(&ctx->rpc_work); + } + +reschedule: + if (ctx->poll_active) + schedule_delayed_work(&ctx->poll_work, msecs_to_jiffies(1)); +} + +static irqreturn_t nv_vgpu_gsp_irq_handler(int irq, void *data) +{ + struct nv_vgpu_device *vgpu = data; + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + u32 status; + + if (!ctx || !ctx->irq_enabled) + return IRQ_NONE; + + status = readl(ctx->gsp_doorbell + 0x100); + if (status & 0x1) { + writel(0x1, ctx->gsp_doorbell + 0x100); + schedule_work(&ctx->rpc_work); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int nv_vgpu_gsp_load_firmware_file(struct nv_vgpu_device *vgpu, const char *filename, void **data, size_t *size) +{ + const struct firmware *fw; + int ret; + + ret = request_firmware(&fw, filename, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "Failed to load firmware %s: %d", filename, ret); + return ret; + } + + /* Use vmalloc for large firmware files (GSP is ~23MB) */ + *data = vzalloc(fw->size); + if (!*data) { + release_firmware(fw); + return -ENOMEM; + } + + memcpy(*data, fw->data, fw->size); + *size = fw->size; + release_firmware(fw); + + nv_vgpu_info(vgpu, "Loaded firmware %s (%zu bytes)", filename, *size); + return 0; +} + +int nv_vgpu_gsp_load_firmware(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + int ret; + + if (!ctx) + return -ENODEV; + + if (ctx->firmware_loaded) + return 0; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: validating GSP firmware files"); + /* Just verify firmware files exist without loading */ + ret = request_firmware(NULL, NV_VGPU_GSP_BOOTLOADER_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTLOADER_FILE, ret); + return ret; + } + release_firmware(NULL); + + ret = request_firmware(NULL, NV_VGPU_GSP_BOOTER_LOAD_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTER_LOAD_FILE, ret); + return ret; + } + release_firmware(NULL); + + ret = request_firmware(NULL, NV_VGPU_GSP_GSP_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_GSP_FILE, ret); + return ret; + } + release_firmware(NULL); + + nv_vgpu_info(vgpu, "DRY-RUN: All GSP firmware files present"); + return 0; + } + + ret = nv_vgpu_gsp_load_firmware_file(vgpu, NV_VGPU_GSP_BOOTLOADER_FILE, + &ctx->firmware.bootloader, &ctx->firmware.bootloader_size); + if (ret) + return ret; + + ret = nv_vgpu_gsp_load_firmware_file(vgpu, NV_VGPU_GSP_BOOTER_LOAD_FILE, + &ctx->firmware.booter_load, &ctx->firmware.booter_load_size); + if (ret) + goto err_bootloader; + + ret = nv_vgpu_gsp_load_firmware_file(vgpu, NV_VGPU_GSP_GSP_FILE, + &ctx->firmware.gsp, &ctx->firmware.gsp_size); + if (ret) + goto err_booter_load; + + ctx->firmware_loaded = true; + nv_vgpu_info(vgpu, "GSP firmware loaded successfully"); + return 0; + +err_booter_load: + vfree(ctx->firmware.booter_load); + ctx->firmware.booter_load = NULL; + ctx->firmware.booter_load_size = 0; +err_bootloader: + vfree(ctx->firmware.bootloader); + ctx->firmware.bootloader = NULL; + ctx->firmware.bootloader_size = 0; + return ret; +} + +void nv_vgpu_gsp_release_firmware(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + + if (!ctx || !ctx->firmware_loaded) + return; + + vfree(ctx->firmware.bootloader); + ctx->firmware.bootloader = NULL; + ctx->firmware.bootloader_size = 0; + + vfree(ctx->firmware.booter_load); + ctx->firmware.booter_load = NULL; + ctx->firmware.booter_load_size = 0; + + vfree(ctx->firmware.gsp); + ctx->firmware.gsp = NULL; + ctx->firmware.gsp_size = 0; + + ctx->firmware_loaded = false; + nv_vgpu_info(vgpu, "GSP firmware released"); +} + +static int nv_vgpu_gsp_write_firmware(struct nv_vgpu_device *vgpu, void *data, size_t size, u64 offset) +{ + struct nv_vgpu_bar *bar = &vgpu->bars[0]; + void __iomem *dst; + + if (!bar->vaddr) + return -ENODEV; + + dst = bar->vaddr + offset; + memcpy_toio(dst, data, size); + return 0; +} + +int nv_vgpu_gsp_boot(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + int ret; + + if (!ctx || !ctx->firmware_loaded) + return -ENODEV; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: GSP boot sequence validated (would write firmware to BAR0)"); + nv_vgpu_info(vgpu, "DRY-RUN: Would write bootloader (%zu bytes) to offset 0x0", + ctx->firmware.bootloader_size); + nv_vgpu_info(vgpu, "DRY-RUN: Would write booter_load (%zu bytes) to offset 0x10000", + ctx->firmware.booter_load_size); + nv_vgpu_info(vgpu, "DRY-RUN: Would write GSP binary (%zu bytes) to offset 0x200000", + ctx->firmware.gsp_size); + nv_vgpu_info(vgpu, "DRY-RUN: Would kick GSP doorbell at offset 0x%llx", + NV_VGPU_GSP_DOORBELL_OFFSET); + return 0; + } + + nv_vgpu_info(vgpu, "Starting GSP boot sequence"); + + ret = nv_vgpu_gsp_write_firmware(vgpu, ctx->firmware.bootloader, + ctx->firmware.bootloader_size, 0x0); + if (ret) { + nv_vgpu_err(vgpu, "Failed to write bootloader: %d", ret); + return ret; + } + + ret = nv_vgpu_gsp_write_firmware(vgpu, ctx->firmware.booter_load, + ctx->firmware.booter_load_size, 0x10000); + if (ret) { + nv_vgpu_err(vgpu, "Failed to write booter_load: %d", ret); + return ret; + } + + ret = nv_vgpu_gsp_write_firmware(vgpu, ctx->firmware.gsp, + ctx->firmware.gsp_size, 0x200000); + if (ret) { + nv_vgpu_err(vgpu, "Failed to write GSP binary: %d", ret); + return ret; + } + + writel(0x1, ctx->gsp_doorbell); + + nv_vgpu_info(vgpu, "GSP boot sequence initiated"); + return 0; +} + +int nv_vgpu_gsp_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_gsp_comm_ctx *ctx; + struct nv_vgpu_bar *bar = &vgpu->bars[0]; + int ret; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: GSP init validation"); + if (!bar->vaddr) { + nv_vgpu_err(vgpu, "DRY-RUN: BAR0 not mapped, cannot validate GSP offsets"); + return -ENODEV; + } + nv_vgpu_info(vgpu, "DRY-RUN: BAR0 mapped at %p, GSP doorbell offset 0x%llx, shared mem offset 0x%llx", + bar->vaddr, NV_VGPU_GSP_DOORBELL_OFFSET, NV_VGPU_GSP_SHARED_MEM_OFFSET); + /* Validate firmware files exist */ + ret = request_firmware(NULL, NV_VGPU_GSP_BOOTLOADER_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTLOADER_FILE, ret); + return ret; + } + release_firmware(NULL); + ret = request_firmware(NULL, NV_VGPU_GSP_BOOTER_LOAD_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTER_LOAD_FILE, ret); + return ret; + } + release_firmware(NULL); + ret = request_firmware(NULL, NV_VGPU_GSP_GSP_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_GSP_FILE, ret); + return ret; + } + release_firmware(NULL); + nv_vgpu_info(vgpu, "DRY-RUN: GSP init validation passed"); + return 0; + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + mutex_init(&ctx->queue_lock); + INIT_LIST_HEAD(&ctx->pending_queue); + INIT_LIST_HEAD(&ctx->complete_queue); + init_waitqueue_head(&ctx->wait_queue); + atomic_set(&ctx->sequence_counter, 0); + spin_lock_init(&ctx->irq_lock); + + if (bar->vaddr) { + ctx->gsp_doorbell = bar->vaddr + NV_VGPU_GSP_DOORBELL_OFFSET; + ctx->gsp_shared_mem = bar->vaddr + NV_VGPU_GSP_SHARED_MEM_OFFSET; + ctx->shared_mem_size = NV_VGPU_GSP_SHARED_MEM_SIZE; + } + + ctx->rpc_wq = alloc_workqueue("nv-vgpu-gsp-rpc", WQ_UNBOUND | WQ_HIGHPRI, 1); + if (!ctx->rpc_wq) { + ret = -ENOMEM; + goto err_free; + } + + INIT_WORK(&ctx->rpc_work, nv_vgpu_gsp_rpc_work); + INIT_DELAYED_WORK(&ctx->poll_work, nv_vgpu_gsp_poll_work); + + /* Use polling instead of IRQ since nvidia.ko owns the MSI interrupt */ + ctx->irq_enabled = true; + ctx->poll_active = true; + schedule_delayed_work(&ctx->poll_work, msecs_to_jiffies(1)); + + vgpu->gsp_comm_ctx = ctx; + + /* Initialize new firmware-decompiled RPC client */ + ret = nv_vgpu_gsp_rpc_integration_init(vgpu, ctx); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to initialize GSP RPC integration: %d (non-fatal)", ret); + /* Continue anyway - legacy RPC still works */ + } + + ret = nv_vgpu_gsp_load_firmware(vgpu); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to load GSP firmware: %d", ret); + } + + ret = nv_vgpu_gsp_boot(vgpu); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to boot GSP: %d", ret); + } + + return 0; + +err_free: + kfree(ctx); + return ret; +} + +void nv_vgpu_gsp_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + struct nv_vgpu_gsp_rpc_entry *entry, *tmp; + + if (!ctx) + return; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: GSP cleanup (no hardware resources to release)"); + return; + } + + /* Cleanup new firmware-decompiled RPC client */ + nv_vgpu_gsp_rpc_integration_cleanup(vgpu); + + /* Stop polling */ + ctx->poll_active = false; + cancel_delayed_work_sync(&ctx->poll_work); + + if (ctx->rpc_wq) { + destroy_workqueue(ctx->rpc_wq); + ctx->rpc_wq = NULL; + } + + mutex_lock(&ctx->queue_lock); + list_for_each_entry_safe(entry, tmp, &ctx->pending_queue, list) { + list_del(&entry->list); + entry->result = -ENODEV; + complete(&entry->done); + } + list_for_each_entry_safe(entry, tmp, &ctx->complete_queue, list) { + list_del(&entry->list); + entry->result = -ENODEV; + complete(&entry->done); + } + mutex_unlock(&ctx->queue_lock); + + nv_vgpu_gsp_release_firmware(vgpu); + + kfree(ctx); + vgpu->gsp_comm_ctx = NULL; +} + +int nv_vgpu_gsp_send_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + struct nv_vgpu_gsp_rpc_entry *entry; + struct nv_vgpu_gsp_rpc_msg *rpc_msg; + int ret; + + if (!ctx || !ctx->gsp_doorbell || !ctx->gsp_shared_mem) + return -ENODEV; + + if (size > NV_VGPU_GSP_RPC_MAX_SIZE - sizeof(struct nv_vgpu_gsp_rpc_header)) + return -EINVAL; + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + init_completion(&entry->done); + entry->msg = kmalloc(size + sizeof(struct nv_vgpu_gsp_rpc_header), GFP_KERNEL); + if (!entry->msg) { + kfree(entry); + return -ENOMEM; + } + + rpc_msg = entry->msg; + rpc_msg->hdr.type = NV_GSP_RPC_TYPE_ALLOC; + rpc_msg->hdr.sequence = nv_vgpu_gsp_next_sequence(ctx); + rpc_msg->hdr.size = size; + rpc_msg->hdr.flags = 0; + rpc_msg->hdr.timestamp = ktime_get_ns(); + memcpy(rpc_msg->data, msg, size); + + mutex_lock(&ctx->queue_lock); + list_add_tail(&entry->list, &ctx->pending_queue); + mutex_unlock(&ctx->queue_lock); + + memcpy(ctx->gsp_shared_mem, rpc_msg, size + sizeof(struct nv_vgpu_gsp_rpc_header)); + writel(rpc_msg->hdr.sequence, ctx->gsp_doorbell); + + ret = wait_for_completion_timeout(&entry->done, + msecs_to_jiffies(NV_VGPU_GSP_RPC_TIMEOUT_MS)); + if (ret == 0) { + ret = -ETIMEDOUT; + } else { + ret = entry->result; + } + + mutex_lock(&ctx->queue_lock); + list_del(&entry->list); + mutex_unlock(&ctx->queue_lock); + + kfree(entry->msg); + kfree(entry); + + return ret; +} + +int nv_vgpu_gsp_recv_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size, int timeout_ms) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + struct nv_vgpu_gsp_rpc_msg *rpc_msg; + int ret; + + if (!ctx || !ctx->gsp_shared_mem) + return -ENODEV; + + ret = wait_event_timeout(ctx->wait_queue, + !list_empty(&ctx->complete_queue), + msecs_to_jiffies(timeout_ms)); + if (ret == 0) + return -ETIMEDOUT; + + mutex_lock(&ctx->queue_lock); + if (list_empty(&ctx->complete_queue)) { + mutex_unlock(&ctx->queue_lock); + return -ENOENT; + } + + rpc_msg = ctx->gsp_shared_mem; + if (rpc_msg->hdr.size > size) { + mutex_unlock(&ctx->queue_lock); + return -EOVERFLOW; + } + + memcpy(msg, rpc_msg->data, rpc_msg->hdr.size); + ret = rpc_msg->hdr.size; + mutex_unlock(&ctx->queue_lock); + + return ret; +} + +int nv_vgpu_gsp_alloc(struct nv_vgpu_device *vgpu, u64 size, u32 align, u64 *handle) +{ + struct { + u64 size; + u32 align; + u64 handle; + } __packed msg = { .size = size, .align = align }; + int ret; + + ret = nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); + if (ret >= 0) + *handle = msg.handle; + + return ret; +} + +int nv_vgpu_gsp_free(struct nv_vgpu_device *vgpu, u64 handle) +{ + struct { + u64 handle; + } __packed msg = { .handle = handle }; + + return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); +} + +int nv_vgpu_gsp_map(struct nv_vgpu_device *vgpu, u64 handle, u64 iova, u64 size, u32 flags) +{ + struct { + u64 handle; + u64 iova; + u64 size; + u32 flags; + } __packed msg = { .handle = handle, .iova = iova, .size = size, .flags = flags }; + + return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); +} + +int nv_vgpu_gsp_unmap(struct nv_vgpu_device *vgpu, u64 handle, u64 iova, u64 size) +{ + struct { + u64 handle; + u64 iova; + u64 size; + } __packed msg = { .handle = handle, .iova = iova, .size = size }; + + return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); +} + +int nv_vgpu_gsp_schedule(struct nv_vgpu_device *vgpu, u32 runlist_id, u32 channel_id) +{ + struct { + u32 runlist_id; + u32 channel_id; + } __packed msg = { .runlist_id = runlist_id, .channel_id = channel_id }; + + return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); +} + +int nv_vgpu_gsp_notify_bar_trap(struct nv_vgpu_device *vgpu, u64 offset, u32 size, void *val) +{ + struct { + u64 offset; + u32 size; + u64 val; + } __packed msg = { .offset = offset, .size = size, .val = *(u64 *)val }; + + return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); +} + +/* ============================================================ + * New Firmware-Decompiled RPC Operations (enhanced APIs) + * ============================================================ */ + +int nv_vgpu_gsp_rpc_alloc(struct nv_vgpu_device *vgpu, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size) +{ + return nv_vgpu_gsp_rpc_alloc_fw(vgpu, size, alignment, mem_type, flags, addr_hint, pid, + gpu_va, handle, actual_size); +} + +int nv_vgpu_gsp_rpc_free(struct nv_vgpu_device *vgpu, u64 handle) +{ + return nv_vgpu_gsp_rpc_free_fw(vgpu, handle); +} + +int nv_vgpu_gsp_rpc_map(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va) +{ + return nv_vgpu_gsp_rpc_map_fw(vgpu, handle, gpu_va, size, prot, flags, offset, mapped_va); +} + +int nv_vgpu_gsp_rpc_unmap(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size) +{ + return nv_vgpu_gsp_rpc_unmap_fw(vgpu, handle, gpu_va, size); +} + +int nv_vgpu_gsp_rpc_pin(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages) +{ + return nv_vgpu_gsp_rpc_pin_fw(vgpu, handle, offset, size, flags, dma_addr, num_pages); +} + +int nv_vgpu_gsp_rpc_unpin(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size) +{ + return nv_vgpu_gsp_rpc_unpin_fw(vgpu, handle, offset, size); +} + +int nv_vgpu_gsp_rpc_schedule(struct nv_vgpu_device *vgpu, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + return nv_vgpu_gsp_rpc_schedule_fw(vgpu, work_handle, callback, arg, priority, flags); +} + +int nv_vgpu_gsp_rpc_yield(struct nv_vgpu_device *vgpu) +{ + return nv_vgpu_gsp_rpc_yield_fw(vgpu); +} + +int nv_vgpu_gsp_rpc_bar_trap(struct nv_vgpu_device *vgpu, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value) +{ + return nv_vgpu_gsp_rpc_bar_trap_fw(vgpu, bar_addr, access_type, size, value, bar_num, ret_value); +} + +int nv_vgpu_gsp_rpc_migrate_save(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint) +{ + return nv_vgpu_gsp_rpc_migrate_save_fw(vgpu, src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, checkpoint); +} + +int nv_vgpu_gsp_rpc_migrate_restore(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint) +{ + return nv_vgpu_gsp_rpc_migrate_restore_fw(vgpu, src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, checkpoint); +}
\ No newline at end of file diff --git a/kernel/gsp_rpc_client.c b/kernel/gsp_rpc_client.c new file mode 100644 index 0000000..a1579f7 --- /dev/null +++ b/kernel/gsp_rpc_client.c @@ -0,0 +1,872 @@ +/* + * GSP RPC Client for vGPU-IO + * Based on GSP 570.144W firmware decompilation + * Firmware base: 0x04000000 + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/mm.h> +#include <linux/dma-mapping.h> +#include <linux/mutex.h> +#include <linux/spinlock.h> +#include <linux/list.h> +#include <linux/workqueue.h> +#include <linux/delay.h> +#include <asm/io.h> + +/* ============================================================ + * FIRMWARE ADDRESSES (from decompilation) + * ============================================================ */ + +#define GSP_FW_BASE 0x04000000UL + +/* Primary RPC Dispatcher */ +#define GSP_RPC_DISPATCH_PRIMARY (GSP_FW_BASE + 0x17A1AC0) /* 0x057a1ac0 */ +#define GSP_RPC_JUMP_TABLE (GSP_FW_BASE + 0x17A3AD8) /* 0x057a3ad8 - 32 entries */ + +/* Secondary RPC Dispatcher */ +#define GSP_RPC_DISPATCH_SECONDARY (GSP_FW_BASE + 0x0ED39DC) /* 0x04ed39dc */ + +/* Specialized Handlers */ +#define GSP_ALLOC_MAIN (GSP_FW_BASE + 0x17E94EC) /* 0x057e94ec */ +#define GSP_SCHEDULE_MAIN (GSP_FW_BASE + 0x11EFBF4) /* 0x051efbf4 */ +#define GSP_BAR_TRAP_HANDLER_JT (GSP_FW_BASE + 0x17675B7) /* 0x057675b7 */ +#define GSP_BAR_TRAP_HANDLER_PRIM (GSP_FW_BASE + 0x0CB8189) /* 0x04cb8189 */ +#define GSP_PIN_HANDLER_SEC (GSP_FW_BASE + 0x0ED34B8) /* 0x04ed34b8 */ + +/* Jump Table Handlers (from 0x059bfeb0) */ +#define GSP_MAP_HANDLER_A (GSP_FW_BASE + 0x1766AB7) /* 0x05766ab7 */ +#define GSP_MAP_HANDLER_B (GSP_FW_BASE + 0x1778A37) /* 0x05778a37 */ +#define GSP_UNMAP_HANDLER_A (GSP_FW_BASE + 0x17669B7) /* 0x057669b7 */ +#define GSP_UNMAP_HANDLER_B (GSP_FW_BASE + 0x1779937) /* 0x05779937 */ +#define GSP_PIN_HANDLER_A (GSP_FW_BASE + 0x17664B7) /* 0x057664b7 */ +#define GSP_PIN_HANDLER_B (GSP_FW_BASE + 0x177A3B7) /* 0x0577a3b7 */ +#define GSP_UNPIN_HANDLER_A (GSP_FW_BASE + 0x17682B7) /* 0x057682b7 */ +#define GSP_UNPIN_HANDLER_B (GSP_FW_BASE + 0x1779FB7) /* 0x05779fb7 */ +#define GSP_SCHEDULE_HANDLER_A (GSP_FW_BASE + 0x1767F37) /* 0x05767f37 */ +#define GSP_SCHEDULE_HANDLER_B (GSP_FW_BASE + 0x175AEB7) /* 0x0575aeb7 */ +#define GSP_YIELD_HANDLER_A (GSP_FW_BASE + 0x175AE37) /* 0x0575ae37 */ +#define GSP_YIELD_HANDLER_B (GSP_FW_BASE + 0x1767337) /* 0x05767337 */ +#define GSP_NOTIFY_HANDLER_A (GSP_FW_BASE + 0x177C8B7) /* 0x0577c8b7 */ +#define GSP_NOTIFY_HANDLER_B (GSP_FW_BASE + 0x1767837) /* 0x05767837 */ + +/* Entry point */ +#define GSP_ENTRY_POINT (GSP_FW_BASE + 0x0009000) /* 0x04009000 */ + +/* ============================================================ + * RPC PROTOCOL DEFINITIONS + * ============================================================ */ + +#define GSP_RPC_ALLOC 0x01 +#define GSP_RPC_FREE 0x02 +#define GSP_RPC_MAP 0x03 +#define GSP_RPC_UNMAP 0x04 +#define GSP_RPC_PIN 0x05 +#define GSP_RPC_UNPIN 0x06 +#define GSP_RPC_SCHEDULE 0x07 +#define GSP_RPC_YIELD 0x08 +#define GSP_RPC_NOTIFY 0x09 +#define GSP_RPC_BAR_TRAP 0x0A +#define GSP_RPC_MIGRATE 0x0B +#define GSP_RPC_DEBUG 0xFF + +/* ALLOC sub-commands (primary dispatcher jump table indices) */ +#define GSP_ALLOC_SUBCMD_DEFAULT 0 +#define GSP_ALLOC_SUBCMD_TYPE1 1 +#define GSP_ALLOC_SUBCMD_TYPE2 3 +#define GSP_ALLOC_SUBCMD_TYPE3 4 +#define GSP_ALLOC_SUBCMD_TYPE4 5 +#define GSP_ALLOC_SUBCMD_TYPE5 8 +#define GSP_ALLOC_SUBCMD_TYPE6 10 +#define GSP_ALLOC_SUBCMD_TYPE7 12 +#define GSP_ALLOC_SUBCMD_TYPE8 17 +#define GSP_ALLOC_SUBCMD_TYPE9 19 +#define GSP_ALLOC_SUBCMD_TYPE10 20 +#define GSP_ALLOC_SUBCMD_TYPE11 21 +#define GSP_ALLOC_SUBCMD_TYPE12 22 +#define GSP_ALLOC_SUBCMD_TYPE13 23 +#define GSP_ALLOC_SUBCMD_TYPE14 24 +#define GSP_ALLOC_SUBCMD_TYPE15 25 +#define GSP_ALLOC_SUBCMD_TYPE16 26 +#define GSP_ALLOC_SUBCMD_TYPE17 27 +#define GSP_ALLOC_SUBCMD_TYPE18 28 +#define GSP_ALLOC_SUBCMD_TYPE19 29 +#define GSP_ALLOC_SUBCMD_TYPE20 30 +#define GSP_ALLOC_SUBCMD_TYPE21 31 + +/* MIGRATE sub-commands */ +#define GSP_MIGRATE_SUBCMD_SAVE 0x04 /* From primary dispatcher branch */ +#define GSP_MIGRATE_SUBCMD_RESTORE 0x05 + +/* PIN sub-commands */ +#define GSP_PIN_SUBCMD_DEFAULT 0x05 /* From secondary dispatcher */ + +/* SCHEDULE sub-commands */ +#define GSP_SCHED_SUBCMD_DEFAULT 0x07 + +/* BAR TRAP sub-commands */ +#define GSP_BAR_TRAP_SUBCMD_DEFAULT 0x0A + +/* ============================================================ + * RPC MESSAGE STRUCTURES + * ============================================================ */ + +struct gsp_rpc_msg { + u32 type; /* RPC type (0x01-0x0B, 0xFF) */ + u32 subcmd; /* Sub-command for jump table dispatch */ + u64 payload[6]; /* Variable payload */ + u32 seq_id; /* Sequence ID for async ops */ + u32 flags; /* Flags */ + u32 reserved[2]; /* Padding to 64 bytes */ +} __packed; + +struct gsp_rpc_response { + u32 status; /* 0 = success, negative = error */ + u32 type; /* Echo of request type */ + u64 result[6]; /* Return values */ + u32 seq_id; /* Echo of request seq_id */ +} __packed; + +/* ALLOC payload */ +struct gsp_alloc_req { + u64 size; /* Allocation size in bytes */ + u64 alignment; /* Alignment requirement */ + u32 mem_type; /* Memory type (0=VRAM, 1=SYSMEM, 2=COHERENT) */ + u32 flags; /* Allocation flags */ + u64 addr_hint; /* Hint address (optional) */ + u32 pid; /* Process ID */ + u32 reserved; +}; + +struct gsp_alloc_rsp { + u64 gpu_va; /* GPU virtual address */ + u64 handle; /* Allocation handle */ + u64 size; /* Actual allocated size */ +}; + +/* MAP payload */ +struct gsp_map_req { + u64 handle; /* Allocation handle from ALLOC */ + u64 gpu_va; /* GPU virtual address to map at */ + u64 size; /* Size to map */ + u32 prot; /* Protection (R/W/X) */ + u32 flags; /* Map flags */ + u64 offset; /* Offset into allocation */ + u32 reserved[2]; +}; + +struct gsp_map_rsp { + u64 gpu_va; /* Mapped GPU VA */ +}; + +/* PIN/UNPIN payload */ +struct gsp_pin_req { + u64 handle; /* Allocation handle */ + u64 offset; /* Offset into allocation */ + u64 size; /* Size to pin */ + u32 flags; /* Pin flags (0=read, 1=write, 2=read+write) */ + u32 reserved[3]; +}; + +struct gsp_pin_rsp { + u64 dma_addr; /* DMA address for device */ + u32 num_pages; /* Number of pages pinned */ + u32 reserved; +}; + +/* SCHEDULE payload */ +struct gsp_sched_req { + u64 work_handle; /* Work descriptor handle */ + u64 callback; /* Callback function pointer */ + u64 arg; /* Callback argument */ + u32 priority; /* Priority */ + u32 flags; /* Schedule flags */ + u32 reserved[2]; +}; + +/* BAR TRAP payload */ +struct gsp_bar_trap_req { + u64 bar_addr; /* BAR register address */ + u32 access_type; /* 0=read, 1=write */ + u32 size; /* Access size (1/2/4/8) */ + u64 value; /* Value (for write) */ + u32 bar_num; /* BAR number (0-5) */ + u32 reserved; +}; + +struct gsp_bar_trap_rsp { + u64 value; /* Value (for read) */ + u32 handled; /* 1 if handled */ + u32 reserved; +}; + +/* MIGRATE payload */ +struct gsp_migrate_req { + u64 src_gpu_id; /* Source GPU ID */ + u64 dst_gpu_id; /* Destination GPU ID */ + u64 memory_bitmap; /* Memory bitmap pointer */ + u64 page_table_ptr; /* Page table pointer */ + u32 state; /* Migration state */ + u32 flags; /* Migration flags */ + u64 context; /* Migration context */ + u32 reserved; +}; + +struct gsp_migrate_rsp { + u32 status; /* Migration status */ + u32 progress; /* Progress percentage */ + u64 checkpoint; /* Checkpoint handle */ +}; + +/* ============================================================ + * GSP RPC CLIENT CONTEXT + * ============================================================ */ + +struct gsp_client { + struct device *dev; + void __iomem *gsp_fw_base; /* Mapped firmware base */ + void __iomem *rpc_doorbell; /* Doorbell register */ + spinlock_t lock; /* Submission lock */ + struct mutex rpc_mutex; /* RPC serialization */ + atomic_t seq_counter; /* Sequence ID counter */ + struct list_head pending_rpcs; /* Pending RPC list */ + wait_queue_head_t resp_wq; /* Response wait queue */ + struct workqueue_struct *wq; /* Workqueue for async */ + bool initialized; +}; + +struct gsp_pending_rpc { + struct list_head list; + struct gsp_rpc_msg msg; + struct gsp_rpc_response *rsp; + struct completion done; + u32 timeout_ms; +}; + +/* ============================================================ + * LOW-LEVEL RPC SUBMISSION + * ============================================================ */ + +static inline u32 gsp_next_seq(struct gsp_client *client) +{ + return atomic_inc_return(&client->seq_counter); +} + +static int gsp_rpc_submit_raw(struct gsp_client *client, + u32 type, u32 subcmd, + u64 *payload, u32 seq_id, + struct gsp_rpc_response *rsp, + u32 timeout_ms) +{ + struct gsp_pending_rpc *prpc; + u64 *dispatch_ptr; + int ret = 0; + + prpc = kzalloc(sizeof(*prpc), GFP_KERNEL); + if (!prpc) + return -ENOMEM; + + prpc->msg.type = type; + prpc->msg.subcmd = subcmd; + if (payload) + memcpy(prpc->msg.payload, payload, sizeof(prpc->msg.payload)); + prpc->msg.seq_id = seq_id; + prpc->rsp = rsp; + prpc->timeout_ms = timeout_ms; + init_completion(&prpc->done); + + spin_lock(&client->lock); + list_add_tail(&prpc->list, &client->pending_rpcs); + + /* Write RPC message to firmware mailbox */ + dispatch_ptr = (u64 *)(client->gsp_fw_base + 0x1000); /* Mailbox offset */ + for (int i = 0; i < 8; i++) + writeq(((u64 *)&prpc->msg)[i], &dispatch_ptr[i]); + + /* Ring doorbell */ + writel(type, client->rpc_doorbell); + spin_unlock(&client->lock); + + /* Wait for completion */ + if (!wait_for_completion_timeout(&prpc->done, + msecs_to_jiffies(timeout_ms))) { + ret = -ETIMEDOUT; + goto cleanup; + } + + if (rsp) + memcpy(rsp, prpc->rsp, sizeof(*rsp)); + +cleanup: + spin_lock(&client->lock); + list_del(&prpc->list); + spin_unlock(&client->lock); + kfree(prpc); + return ret; +} + +/* ============================================================ + * HIGH-LEVEL RPC API + * ============================================================ */ + +/* Memory Allocation */ +int gsp_rpc_alloc(struct gsp_client *client, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size) +{ + struct gsp_alloc_req req = {0}; + struct gsp_alloc_rsp rsp = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + req.size = size; + req.alignment = alignment; + req.mem_type = mem_type; + req.flags = flags; + req.addr_hint = addr_hint; + req.pid = pid; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_ALLOC, GSP_ALLOC_SUBCMD_DEFAULT, + (u64 *)&req, seq, &response, 5000); + if (ret) + return ret; + + if (response.status) { + pr_err("GSP ALLOC failed: status=%d\n", response.status); + return -EIO; + } + + memcpy(&rsp, response.result, sizeof(rsp)); + if (gpu_va) *gpu_va = rsp.gpu_va; + if (handle) *handle = rsp.handle; + if (actual_size) *actual_size = rsp.size; + + return 0; +} + +int gsp_rpc_free(struct gsp_client *client, u64 handle) +{ + struct gsp_rpc_response response = {0}; + u64 payload[6] = { handle, 0, 0, 0, 0, 0 }; + u32 seq = gsp_next_seq(client); + int ret; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_FREE, 0, + payload, seq, &response, 2000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +/* Memory Mapping */ +int gsp_rpc_map(struct gsp_client *client, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va) +{ + struct gsp_map_req req = {0}; + struct gsp_map_rsp rsp = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + req.handle = handle; + req.gpu_va = gpu_va; + req.size = size; + req.prot = prot; + req.flags = flags; + req.offset = offset; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_MAP, 0, + (u64 *)&req, seq, &response, 2000); + if (ret) + return ret; + + if (response.status) + return -EIO; + + memcpy(&rsp, response.result, sizeof(rsp)); + if (mapped_va) *mapped_va = rsp.gpu_va; + + return 0; +} + +int gsp_rpc_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size) +{ + struct gsp_rpc_response response = {0}; + u64 payload[6] = { handle, gpu_va, size, 0, 0, 0 }; + u32 seq = gsp_next_seq(client); + int ret; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_UNMAP, 0, + payload, seq, &response, 2000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +/* Page Pinning */ +int gsp_rpc_pin(struct gsp_client *client, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages) +{ + struct gsp_pin_req req = {0}; + struct gsp_pin_rsp rsp = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + req.handle = handle; + req.offset = offset; + req.size = size; + req.flags = flags; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_PIN, GSP_PIN_SUBCMD_DEFAULT, + (u64 *)&req, seq, &response, 5000); + if (ret) + return ret; + + if (response.status) + return -EIO; + + memcpy(&rsp, response.result, sizeof(rsp)); + if (dma_addr) *dma_addr = rsp.dma_addr; + if (num_pages) *num_pages = rsp.num_pages; + + return 0; +} + +int gsp_rpc_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size) +{ + struct gsp_rpc_response response = {0}; + u64 payload[6] = { handle, offset, size, 0, 0, 0 }; + u32 seq = gsp_next_seq(client); + int ret; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_UNPIN, 0, + payload, seq, &response, 2000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +/* BAR Trap Emulation */ +int gsp_rpc_bar_trap(struct gsp_client *client, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value) +{ + struct gsp_bar_trap_req req = {0}; + struct gsp_bar_trap_rsp rsp = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + req.bar_addr = bar_addr; + req.access_type = access_type; + req.size = size; + req.value = value; + req.bar_num = bar_num; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_BAR_TRAP, GSP_BAR_TRAP_SUBCMD_DEFAULT, + (u64 *)&req, seq, &response, 2000); + if (ret) + return ret; + + if (response.status) + return -EIO; + + memcpy(&rsp, response.result, sizeof(rsp)); + if (ret_value) *ret_value = rsp.value; + + return rsp.handled ? 0 : -ENODEV; +} + +/* Migration */ +int gsp_rpc_migrate_save(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint) +{ + struct gsp_migrate_req req = {0}; + struct gsp_migrate_rsp rsp = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + req.src_gpu_id = src_gpu_id; + req.dst_gpu_id = dst_gpu_id; + req.memory_bitmap = memory_bitmap; + req.page_table_ptr = page_table_ptr; + req.state = 0; /* SAVE */ + req.flags = GSP_MIGRATE_SUBCMD_SAVE; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_MIGRATE, GSP_MIGRATE_SUBCMD_SAVE, + (u64 *)&req, seq, &response, 30000); + if (ret) + return ret; + + if (response.status) + return -EIO; + + memcpy(&rsp, response.result, sizeof(rsp)); + if (checkpoint) *checkpoint = rsp.checkpoint; + + return 0; +} + +int gsp_rpc_migrate_restore(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint) +{ + struct gsp_migrate_req req = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + req.src_gpu_id = src_gpu_id; + req.dst_gpu_id = dst_gpu_id; + req.memory_bitmap = memory_bitmap; + req.page_table_ptr = page_table_ptr; + req.state = 1; /* RESTORE */ + req.flags = GSP_MIGRATE_SUBCMD_RESTORE; + req.context = checkpoint; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_MIGRATE, GSP_MIGRATE_SUBCMD_RESTORE, + (u64 *)&req, seq, &response, 30000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +/* ============================================================ + * EMULATION FALLBACKS + * ============================================================ */ + +static bool gsp_emulation_enabled = false; + +void gsp_rpc_emulation_set(bool enable) +{ + gsp_emulation_enabled = enable; +} + +bool gsp_rpc_emulation_is_enabled(void) +{ + return gsp_emulation_enabled; +} + +/* Scheduling emulation - local work queue */ +struct gsp_sched_work { + struct list_head list; + struct work_struct work; + u64 work_handle; + u64 callback; + u64 arg; + u32 priority; + u32 flags; + struct completion done; +}; + +static LIST_HEAD(gsp_sched_queue); +static DEFINE_SPINLOCK(gsp_sched_lock); +static struct workqueue_struct *gsp_sched_wq = NULL; + +static void gsp_sched_worker(struct work_struct *work) +{ + struct gsp_sched_work *sched = container_of(work, struct gsp_sched_work, work); + + /* In emulation mode, we just complete successfully */ + complete(&sched->done); +} + +int gsp_rpc_schedule(struct gsp_client *client, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + struct gsp_sched_req req = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + if (gsp_emulation_enabled) { + struct gsp_sched_work *sched = kzalloc(sizeof(*sched), GFP_KERNEL); + if (!sched) + return -ENOMEM; + + INIT_WORK(&sched->work, gsp_sched_worker); + sched->work_handle = work_handle; + sched->callback = callback; + sched->arg = arg; + sched->priority = priority; + sched->flags = flags; + init_completion(&sched->done); + + spin_lock(&gsp_sched_lock); + list_add_tail(&sched->list, &gsp_sched_queue); + spin_unlock(&gsp_sched_lock); + + if (gsp_sched_wq) + queue_work(gsp_sched_wq, &sched->work); + + wait_for_completion(&sched->done); + + spin_lock(&gsp_sched_lock); + list_del(&sched->list); + spin_unlock(&gsp_sched_lock); + + kfree(sched); + return 0; + } + + req.work_handle = work_handle; + req.callback = callback; + req.arg = arg; + req.priority = priority; + req.flags = flags; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_SCHEDULE, GSP_SCHED_SUBCMD_DEFAULT, + (u64 *)&req, seq, &response, 2000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +int gsp_rpc_yield(struct gsp_client *client) +{ + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + if (gsp_emulation_enabled) { + /* In emulation, yield is a no-op success */ + return 0; + } + + ret = gsp_rpc_submit_raw(client, GSP_RPC_YIELD, 0, + NULL, seq, &response, 1000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +/* Notification emulation - async eventfd support */ +struct gsp_notify_cb { + struct list_head list; + u32 type; + void (*callback)(struct gsp_client *, struct gsp_rpc_response *, void *); + void *ctx; +}; + +static LIST_HEAD(gsp_notify_callbacks); +static DEFINE_SPINLOCK(gsp_notify_lock); + +int gsp_rpc_notify_register(struct gsp_client *client, + u32 type, + void (*callback)(struct gsp_client *, + struct gsp_rpc_response *, + void *), + void *ctx) +{ + struct gsp_notify_cb *nc = kzalloc(sizeof(*nc), GFP_KERNEL); + if (!nc) + return -ENOMEM; + + nc->type = type; + nc->callback = callback; + nc->ctx = ctx; + + spin_lock(&gsp_notify_lock); + list_add_tail(&nc->list, &gsp_notify_callbacks); + spin_unlock(&gsp_notify_lock); + + return 0; +} + +void gsp_rpc_notify_unregister(struct gsp_client *client, u32 type) +{ + struct gsp_notify_cb *nc, *tmp; + + spin_lock(&gsp_notify_lock); + list_for_each_entry_safe(nc, tmp, &gsp_notify_callbacks, list) { + if (nc->type == type) { + list_del(&nc->list); + kfree(nc); + } + } + spin_unlock(&gsp_notify_lock); +} + +/* Handle async notifications from GSP */ +void gsp_handle_notification(struct gsp_client *client, struct gsp_rpc_msg *msg) +{ + struct gsp_notify_cb *nc; + + spin_lock(&gsp_notify_lock); + list_for_each_entry(nc, &gsp_notify_callbacks, list) { + if (nc->type == msg->type || nc->type == 0) { + /* Copy message to response format for callback */ + struct gsp_rpc_response rsp = {0}; + rsp.status = 0; + rsp.type = msg->type; + rsp.seq_id = msg->seq_id; + memcpy(rsp.result, msg->payload, sizeof(msg->payload)); + nc->callback(client, &rsp, nc->ctx); + } + } + spin_unlock(&gsp_notify_lock); +} + +/* ============================================================ + * CLIENT INITIALIZATION + * ============================================================ */ + +struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell) +{ + struct gsp_client *client; + resource_size_t fw_size = 0x2000000; /* 32MB */ + + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return ERR_PTR(-ENOMEM); + + client->dev = dev; + spin_lock_init(&client->lock); + mutex_init(&client->rpc_mutex); + INIT_LIST_HEAD(&client->pending_rpcs); + init_waitqueue_head(&client->resp_wq); + atomic_set(&client->seq_counter, 0); + + client->gsp_fw_base = ioremap(fw_base, fw_size); + if (!client->gsp_fw_base) { + kfree(client); + return ERR_PTR(-ENOMEM); + } + + client->rpc_doorbell = ioremap(doorbell, 0x1000); + if (!client->rpc_doorbell) { + iounmap(client->gsp_fw_base); + kfree(client); + return ERR_PTR(-ENOMEM); + } + + client->wq = alloc_workqueue("gsp_rpc", WQ_UNBOUND, 0); + if (!client->wq) { + iounmap(client->rpc_doorbell); + iounmap(client->gsp_fw_base); + kfree(client); + return ERR_PTR(-ENOMEM); + } + + /* Init emulation workqueue for scheduling */ + gsp_sched_wq = alloc_workqueue("gsp_sched", WQ_UNBOUND, 0); + if (!gsp_sched_wq) { + destroy_workqueue(client->wq); + iounmap(client->rpc_doorbell); + iounmap(client->gsp_fw_base); + kfree(client); + return ERR_PTR(-ENOMEM); + } + + client->initialized = true; + pr_info("GSP RPC client initialized: fw_base=%pa, doorbell=%pa\n", &fw_base, &doorbell); + + return client; +} + +void gsp_client_fini(struct gsp_client *client) +{ + if (!client) + return; + + if (client->wq) + destroy_workqueue(client->wq); + + if (gsp_sched_wq) { + destroy_workqueue(gsp_sched_wq); + gsp_sched_wq = NULL; + } + + if (client->rpc_doorbell) + iounmap(client->rpc_doorbell); + + if (client->gsp_fw_base) + iounmap(client->gsp_fw_base); + + kfree(client); +} + +/* ============================================================ + * WPR MANAGEMENT (from firmware strings) + * ============================================================ */ + +/* WPR mutex operations (from firmware: VPR_WPR_WRITE_MUTEX*) */ +int gsp_wpr_write_mutex_acquire(struct gsp_client *client, u32 wpr_id) +{ + /* Calls firmware function at VPR_WPR_WRITE_MUTEX addresses */ + /* Implementation would use RPC_DEBUG or dedicated WPR RPC */ + (void)client; + (void)wpr_id; + return 0; +} + +int gsp_wpr_write_mutex_release(struct gsp_client *client, u32 wpr_id) +{ + (void)client; + (void)wpr_id; + return 0; +} + +int gsp_wpr_scratch_mutex_acquire(struct gsp_client *client, u32 wpr_id) +{ + /* From firmware: VPR_WPR_SCRATCH_MUTEX */ + (void)client; + (void)wpr_id; + return 0; +} + +int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id) +{ + (void)client; + (void)wpr_id; + return 0; +} + +/* ============================================================ + * MODULE EXPORTS + * ============================================================ */ + +EXPORT_SYMBOL(gsp_client_init); +EXPORT_SYMBOL(gsp_client_fini); +EXPORT_SYMBOL(gsp_rpc_alloc); +EXPORT_SYMBOL(gsp_rpc_free); +EXPORT_SYMBOL(gsp_rpc_map); +EXPORT_SYMBOL(gsp_rpc_unmap); +EXPORT_SYMBOL(gsp_rpc_pin); +EXPORT_SYMBOL(gsp_rpc_unpin); +EXPORT_SYMBOL(gsp_rpc_schedule); +EXPORT_SYMBOL(gsp_rpc_yield); +EXPORT_SYMBOL(gsp_rpc_bar_trap); +EXPORT_SYMBOL(gsp_rpc_migrate_save); +EXPORT_SYMBOL(gsp_rpc_migrate_restore); +EXPORT_SYMBOL(gsp_wpr_write_mutex_acquire); +EXPORT_SYMBOL(gsp_wpr_write_mutex_release); +EXPORT_SYMBOL(gsp_wpr_scratch_mutex_acquire); +EXPORT_SYMBOL(gsp_wpr_scratch_mutex_release); +EXPORT_SYMBOL(gsp_rpc_emulation_set); +EXPORT_SYMBOL(gsp_rpc_emulation_is_enabled); +EXPORT_SYMBOL(gsp_rpc_notify_register); +EXPORT_SYMBOL(gsp_rpc_notify_unregister); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("vGPU-IO Team"); +MODULE_DESCRIPTION("GSP 570.144W RPC Client for vGPU-IO"); diff --git a/kernel/gsp_rpc_client.h b/kernel/gsp_rpc_client.h new file mode 100644 index 0000000..8702b0f --- /dev/null +++ b/kernel/gsp_rpc_client.h @@ -0,0 +1,313 @@ +/* + * GSP 570.144W RPC Client Header + * Auto-generated from firmware decompilation + */ + +#ifndef _GSP_RPC_CLIENT_H_ +#define _GSP_RPC_CLIENT_H_ + +#include <linux/types.h> +#include <linux/device.h> + +/* ============================================================ + * FIRMWARE BASE ADDRESS + * ============================================================ */ +#define GSP_FW_BASE 0x04000000UL + +/* ============================================================ + * RPC DISPATCHER ADDRESSES + * ============================================================ */ + +/* Primary Dispatcher - Jump table switch for 6 RPC types */ +#define GSP_RPC_DISPATCH_PRIMARY (GSP_FW_BASE + 0x17A1AC0) /* 0x057a1ac0 */ +#define GSP_RPC_JUMP_TABLE (GSP_FW_BASE + 0x17A3AD8) /* 0x057a3ad8 - 32 sub-commands */ + +/* Secondary Dispatcher - Full implementation for all 11 RPC types */ +#define GSP_RPC_DISPATCH_SECONDARY (GSP_FW_BASE + 0x0ED39DC) /* 0x04ed39dc */ + +/* Tertiary Dispatcher - Stack=160 bytes */ +#define GSP_RPC_DISPATCH_TERTIARY (GSP_FW_BASE + 0x0CC58F8) /* 0x04cc58f8 */ + +/* Quaternary Dispatcher - Stack=96 bytes */ +#define GSP_RPC_DISPATCH_QUATERNARY (GSP_FW_BASE + 0x1723EF4) /* 0x05723ef4 */ + +/* ============================================================ + * SPECIALIZED HANDLER ADDRESSES + * ============================================================ */ + +#define GSP_ALLOC_MAIN (GSP_FW_BASE + 0x17E94EC) /* 0x057e94ec */ +#define GSP_SCHEDULE_MAIN (GSP_FW_BASE + 0x11EFBF4) /* 0x051efbf4 */ +#define GSP_BAR_TRAP_HANDLER_JT (GSP_FW_BASE + 0x17675B7) /* 0x057675b7 */ +#define GSP_BAR_TRAP_HANDLER_PRIM (GSP_FW_BASE + 0x0CB8189) /* 0x04cb8189 */ +#define GSP_PIN_HANDLER_SEC (GSP_FW_BASE + 0x0ED34B8) /* 0x04ed34b8 */ + +/* Jump Table Handlers (from decoded table at 0x059bfeb0) */ +#define GSP_MAP_HANDLER_A (GSP_FW_BASE + 0x1766AB7) /* 0x05766ab7 */ +#define GSP_MAP_HANDLER_B (GSP_FW_BASE + 0x1778A37) /* 0x05778a37 */ +#define GSP_UNMAP_HANDLER_A (GSP_FW_BASE + 0x17669B7) /* 0x057669b7 */ +#define GSP_UNMAP_HANDLER_B (GSP_FW_BASE + 0x1779937) /* 0x05779937 */ +#define GSP_PIN_HANDLER_A (GSP_FW_BASE + 0x17664B7) /* 0x057664b7 */ +#define GSP_PIN_HANDLER_B (GSP_FW_BASE + 0x177A3B7) /* 0x0577a3b7 */ +#define GSP_UNPIN_HANDLER_A (GSP_FW_BASE + 0x17682B7) /* 0x057682b7 */ +#define GSP_UNPIN_HANDLER_B (GSP_FW_BASE + 0x1779FB7) /* 0x05779fb7 */ +#define GSP_SCHEDULE_HANDLER_A (GSP_FW_BASE + 0x1767F37) /* 0x05767f37 */ +#define GSP_SCHEDULE_HANDLER_B (GSP_FW_BASE + 0x175AEB7) /* 0x0575aeb7 */ +#define GSP_YIELD_HANDLER_A (GSP_FW_BASE + 0x175AE37) /* 0x0575ae37 */ +#define GSP_YIELD_HANDLER_B (GSP_FW_BASE + 0x1767337) /* 0x05767337 */ +#define GSP_NOTIFY_HANDLER_A (GSP_FW_BASE + 0x177C8B7) /* 0x0577c8b7 */ +#define GSP_NOTIFY_HANDLER_B (GSP_FW_BASE + 0x1767837) /* 0x05767837 */ + +/* Entry Point */ +#define GSP_ENTRY_POINT (GSP_FW_BASE + 0x0009000) /* 0x04009000 */ + +/* ============================================================ + * RPC TYPE DEFINITIONS + * ============================================================ */ +enum gsp_rpc_type { + GSP_RPC_ALLOC = 0x01, + GSP_RPC_FREE = 0x02, + GSP_RPC_MAP = 0x03, + GSP_RPC_UNMAP = 0x04, + GSP_RPC_PIN = 0x05, + GSP_RPC_UNPIN = 0x06, + GSP_RPC_SCHEDULE = 0x07, + GSP_RPC_YIELD = 0x08, + GSP_RPC_NOTIFY = 0x09, + GSP_RPC_BAR_TRAP = 0x0A, + GSP_RPC_MIGRATE = 0x0B, + GSP_RPC_DEBUG = 0xFF, +}; + +/* ============================================================ + * ALLOC SUB-COMMANDS (Primary Dispatcher Jump Table Indices) + * ============================================================ */ +enum gsp_alloc_subcmd { + GSP_ALLOC_SUBCMD_DEFAULT = 0, + GSP_ALLOC_SUBCMD_TYPE1 = 1, + GSP_ALLOC_SUBCMD_TYPE2 = 3, + GSP_ALLOC_SUBCMD_TYPE3 = 4, + GSP_ALLOC_SUBCMD_TYPE4 = 5, + GSP_ALLOC_SUBCMD_TYPE5 = 8, + GSP_ALLOC_SUBCMD_TYPE6 = 10, + GSP_ALLOC_SUBCMD_TYPE7 = 12, + GSP_ALLOC_SUBCMD_TYPE8 = 17, + GSP_ALLOC_SUBCMD_TYPE9 = 19, + GSP_ALLOC_SUBCMD_TYPE10 = 20, + GSP_ALLOC_SUBCMD_TYPE11 = 21, + GSP_ALLOC_SUBCMD_TYPE12 = 22, + GSP_ALLOC_SUBCMD_TYPE13 = 23, + GSP_ALLOC_SUBCMD_TYPE14 = 24, + GSP_ALLOC_SUBCMD_TYPE15 = 25, + GSP_ALLOC_SUBCMD_TYPE16 = 26, + GSP_ALLOC_SUBCMD_TYPE17 = 27, + GSP_ALLOC_SUBCMD_TYPE18 = 28, + GSP_ALLOC_SUBCMD_TYPE19 = 29, + GSP_ALLOC_SUBCMD_TYPE20 = 30, + GSP_ALLOC_SUBCMD_TYPE21 = 31, +}; + +/* ============================================================ + * MIGRATE SUB-COMMANDS + * ============================================================ */ +enum gsp_migrate_subcmd { + GSP_MIGRATE_SUBCMD_SAVE = 0x04, + GSP_MIGRATE_SUBCMD_RESTORE = 0x05, +}; + +/* ============================================================ + * PIN SUB-COMMANDS + * ============================================================ */ +enum gsp_pin_subcmd { + GSP_PIN_SUBCMD_DEFAULT = 0x05, +}; + +/* ============================================================ + * SCHEDULE SUB-COMMANDS + * ============================================================ */ +enum gsp_sched_subcmd { + GSP_SCHED_SUBCMD_DEFAULT = 0x07, +}; + +/* ============================================================ + * BAR TRAP SUB-COMMANDS + * ============================================================ */ +enum gsp_bar_trap_subcmd { + GSP_BAR_TRAP_SUBCMD_DEFAULT = 0x0A, +}; + +/* ============================================================ + * RPC MESSAGE STRUCTURES + * ============================================================ */ + +struct gsp_rpc_msg { + __le32 type; + __le32 subcmd; + __le64 payload[6]; + __le32 seq_id; + __le32 flags; + __le32 reserved[2]; +} __packed; + +struct gsp_rpc_response { + __le32 status; + __le32 type; + __le64 result[6]; + __le32 seq_id; +} __packed; + +/* ALLOC request/response */ +struct gsp_alloc_req { + __le64 size; + __le64 alignment; + __le32 mem_type; + __le32 flags; + __le64 addr_hint; + __le32 pid; + __le32 reserved; +} __packed; + +struct gsp_alloc_rsp { + __le64 gpu_va; + __le64 handle; + __le64 size; +} __packed; + +/* MAP request/response */ +struct gsp_map_req { + __le64 handle; + __le64 gpu_va; + __le64 size; + __le32 prot; + __le32 flags; + __le64 offset; + __le32 reserved[2]; +} __packed; + +struct gsp_map_rsp { + __le64 gpu_va; +} __packed; + +/* PIN/UNPIN request/response */ +struct gsp_pin_req { + __le64 handle; + __le64 offset; + __le64 size; + __le32 flags; + __le32 reserved[3]; +} __packed; + +struct gsp_pin_rsp { + __le64 dma_addr; + __le32 num_pages; + __le32 reserved; +} __packed; + +/* SCHEDULE request */ +struct gsp_sched_req { + __le64 work_handle; + __le64 callback; + __le64 arg; + __le32 priority; + __le32 flags; + __le32 reserved[2]; +} __packed; + +/* BAR TRAP request/response */ +struct gsp_bar_trap_req { + __le64 bar_addr; + __le32 access_type; + __le32 size; + __le64 value; + __le32 bar_num; + __le32 reserved; +} __packed; + +struct gsp_bar_trap_rsp { + __le64 value; + __le32 handled; + __le32 reserved; +} __packed; + +/* MIGRATE request/response */ +struct gsp_migrate_req { + __le64 src_gpu_id; + __le64 dst_gpu_id; + __le64 memory_bitmap; + __le64 page_table_ptr; + __le32 state; + __le32 flags; + __le64 context; + __le32 reserved; +} __packed; + +struct gsp_migrate_rsp { + __le32 status; + __le32 progress; + __le64 checkpoint; +} __packed; + +/* ============================================================ + * CLIENT OPAQUE STRUCT + * ============================================================ */ +struct gsp_client; + +/* ============================================================ + * PUBLIC API + * ============================================================ */ + +/* Client lifecycle */ +struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell); +void gsp_client_fini(struct gsp_client *client); + +/* Memory management */ +int gsp_rpc_alloc(struct gsp_client *client, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size); + +int gsp_rpc_free(struct gsp_client *client, u64 handle); + +int gsp_rpc_map(struct gsp_client *client, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va); + +int gsp_rpc_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size); + +/* Page pinning */ +int gsp_rpc_pin(struct gsp_client *client, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages); + +int gsp_rpc_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size); + +/* Scheduling */ +int gsp_rpc_schedule(struct gsp_client *client, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags); + +int gsp_rpc_yield(struct gsp_client *client); + +/* BAR trap emulation */ +int gsp_rpc_bar_trap(struct gsp_client *client, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value); + +/* Migration */ +int gsp_rpc_migrate_save(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint); + +int gsp_rpc_migrate_restore(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint); + +/* WPR Management */ +int gsp_wpr_write_mutex_acquire(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_write_mutex_release(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_scratch_mutex_acquire(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id); + +#endif /* _GSP_RPC_CLIENT_H_ */ diff --git a/kernel/gsp_rpc_integration.c b/kernel/gsp_rpc_integration.c new file mode 100644 index 0000000..4d43392 --- /dev/null +++ b/kernel/gsp_rpc_integration.c @@ -0,0 +1,535 @@ +/* + * GSP RPC Integration Layer + * Bridges the firmware-decompiled RPC client with existing gsp_comm.c + */ + +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/wait.h> +#include <linux/workqueue.h> +#include <linux/io.h> +#include <linux/pci.h> +#include <linux/interrupt.h> +#include <linux/device.h> + +#include "nvidia-vgpu-vfio.h" +#include "gsp_comm.h" +#include "gsp_rpc_client.h" +#include "gsp_rpc_integration.h" +#include "bar_emulation.h" +#include "migration.h" + +/* Integration context - wraps the new RPC client */ +struct gsp_rpc_integration { + struct nv_vgpu_device *vgpu; + struct gsp_client *rpc_client; + struct nv_vgpu_gsp_comm_ctx *comm_ctx; + struct mutex lock; + bool initialized; +}; + +/* Global integration instance (per vGPU) */ +static struct gsp_rpc_integration *g_integration = NULL; + +/* ============================================================ + * RPC Message Building Helpers + * ============================================================ */ + +static void gsp_build_alloc_msg(struct gsp_alloc_req *req, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid) +{ + req->size = cpu_to_le64(size); + req->alignment = cpu_to_le64(alignment); + req->mem_type = cpu_to_le32(mem_type); + req->flags = cpu_to_le32(flags); + req->addr_hint = cpu_to_le64(addr_hint); + req->pid = cpu_to_le32(pid); + req->reserved = 0; +} + +static void gsp_build_map_msg(struct gsp_map_req *req, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset) +{ + req->handle = cpu_to_le64(handle); + req->gpu_va = cpu_to_le64(gpu_va); + req->size = cpu_to_le64(size); + req->prot = cpu_to_le32(prot); + req->flags = cpu_to_le32(flags); + req->offset = cpu_to_le64(offset); + req->reserved[0] = 0; + req->reserved[1] = 0; +} + +static void gsp_build_pin_msg(struct gsp_pin_req *req, + u64 handle, u64 offset, u64 size, u32 flags) +{ + req->handle = cpu_to_le64(handle); + req->offset = cpu_to_le64(offset); + req->size = cpu_to_le64(size); + req->flags = cpu_to_le32(flags); + req->reserved[0] = 0; + req->reserved[1] = 0; + req->reserved[2] = 0; +} + +static void gsp_build_sched_msg(struct gsp_sched_req *req, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + req->work_handle = cpu_to_le64(work_handle); + req->callback = cpu_to_le64(callback); + req->arg = cpu_to_le64(arg); + req->priority = cpu_to_le32(priority); + req->flags = cpu_to_le32(flags); + req->reserved[0] = 0; + req->reserved[1] = 0; +} + +static void gsp_build_bar_trap_msg(struct gsp_bar_trap_req *req, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num) +{ + req->bar_addr = cpu_to_le64(bar_addr); + req->access_type = cpu_to_le32(access_type); + req->size = cpu_to_le32(size); + req->value = cpu_to_le64(value); + req->bar_num = cpu_to_le32(bar_num); + req->reserved = 0; +} + +static void gsp_build_migrate_msg(struct gsp_migrate_req *req, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u32 state, u32 flags, u64 context) +{ + req->src_gpu_id = cpu_to_le64(src_gpu_id); + req->dst_gpu_id = cpu_to_le64(dst_gpu_id); + req->memory_bitmap = cpu_to_le64(memory_bitmap); + req->page_table_ptr = cpu_to_le64(page_table_ptr); + req->state = cpu_to_le32(state); + req->flags = cpu_to_le32(flags); + req->context = cpu_to_le64(context); + req->reserved = 0; +} + +/* ============================================================ + * Integration API + * ============================================================ */ + +int nv_vgpu_gsp_rpc_integration_init(struct nv_vgpu_device *vgpu, + struct nv_vgpu_gsp_comm_ctx *comm_ctx) +{ + struct gsp_rpc_integration *integ; + phys_addr_t fw_base, doorbell; + int ret; + + if (!vgpu || !comm_ctx) + return -EINVAL; + + integ = kzalloc(sizeof(*integ), GFP_KERNEL); + if (!integ) + return -ENOMEM; + + integ->vgpu = vgpu; + integ->comm_ctx = comm_ctx; + mutex_init(&integ->lock); + + /* Calculate firmware base and doorbell from BAR0 */ + if (vgpu->bars[0].vaddr) { + fw_base = vgpu->bars[0].gpa; + doorbell = vgpu->bars[0].gpa + NV_VGPU_GSP_DOORBELL_OFFSET; + } else { + /* Default for TU102/TU104 */ + fw_base = GSP_FW_BASE; + doorbell = GSP_FW_BASE + NV_VGPU_GSP_DOORBELL_OFFSET; + } + + /* Initialize the new RPC client */ + integ->rpc_client = gsp_client_init(&vgpu->pdev->dev, fw_base, doorbell); + if (IS_ERR(integ->rpc_client)) { + ret = PTR_ERR(integ->rpc_client); + integ->rpc_client = NULL; + goto err_free; + } + + integ->initialized = true; + g_integration = integ; + + nv_vgpu_info(vgpu, "GSP RPC integration initialized (fw_base=0x%llx, doorbell=0x%llx)", + (unsigned long long)fw_base, (unsigned long long)doorbell); + + return 0; + +err_free: + kfree(integ); + return ret; +} + +void nv_vgpu_gsp_rpc_integration_cleanup(struct nv_vgpu_device *vgpu) +{ + struct gsp_rpc_integration *integ = g_integration; + + if (!integ) + return; + + mutex_lock(&integ->lock); + + if (integ->rpc_client) { + gsp_client_fini(integ->rpc_client); + integ->rpc_client = NULL; + } + + integ->initialized = false; + mutex_unlock(&integ->lock); + + kfree(integ); + g_integration = NULL; + + if (vgpu) + nv_vgpu_info(vgpu, "GSP RPC integration cleaned up"); +} + +/* ============================================================ + * Enhanced RPC Operations (using firmware-decompiled addresses) + * ============================================================ */ + +int nv_vgpu_gsp_rpc_alloc_fw(struct nv_vgpu_device *vgpu, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_alloc_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_alloc_msg(&req, size, alignment, mem_type, flags, addr_hint, pid); + + ret = gsp_rpc_alloc(integ->rpc_client, + size, alignment, mem_type, flags, addr_hint, pid, + gpu_va, handle, actual_size); + + if (ret == 0) { + nv_vgpu_dbg(vgpu, "ALLOC: size=%llu align=%llu -> gpu_va=0x%llx handle=0x%llx", + size, alignment, *gpu_va, *handle); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_free_fw(struct nv_vgpu_device *vgpu, u64 handle) +{ + struct gsp_rpc_integration *integ = g_integration; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_rpc_free(integ->rpc_client, handle); +} + +int nv_vgpu_gsp_rpc_map_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_map_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_map_msg(&req, handle, gpu_va, size, prot, flags, offset); + + ret = gsp_rpc_map(integ->rpc_client, handle, gpu_va, size, + prot, flags, offset, mapped_va); + + if (ret == 0) { + nv_vgpu_dbg(vgpu, "MAP: handle=0x%llx gpu_va=0x%llx size=%llu -> mapped_va=0x%llx", + handle, gpu_va, size, *mapped_va); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_unmap_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size) +{ + struct gsp_rpc_integration *integ = g_integration; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_rpc_unmap(integ->rpc_client, handle, gpu_va, size); +} + +int nv_vgpu_gsp_rpc_pin_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_pin_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_pin_msg(&req, handle, offset, size, flags); + + ret = gsp_rpc_pin(integ->rpc_client, handle, offset, size, + flags, dma_addr, num_pages); + + if (ret == 0) { + nv_vgpu_dbg(vgpu, "PIN: handle=0x%llx offset=%llu size=%llu -> dma_addr=0x%llx pages=%u", + handle, offset, size, *dma_addr, *num_pages); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_unpin_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size) +{ + struct gsp_rpc_integration *integ = g_integration; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_rpc_unpin(integ->rpc_client, handle, offset, size); +} + +int nv_vgpu_gsp_rpc_schedule_fw(struct nv_vgpu_device *vgpu, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_sched_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_sched_msg(&req, work_handle, callback, arg, priority, flags); + + ret = gsp_rpc_schedule(integ->rpc_client, work_handle, callback, arg, + priority, flags); + + if (ret == 0) { + nv_vgpu_dbg(vgpu, "SCHEDULE: work_handle=0x%llx priority=%u", work_handle, priority); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_yield_fw(struct nv_vgpu_device *vgpu) +{ + struct gsp_rpc_integration *integ = g_integration; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_rpc_yield(integ->rpc_client); +} + +int nv_vgpu_gsp_rpc_bar_trap_fw(struct nv_vgpu_device *vgpu, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_bar_trap_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_bar_trap_msg(&req, bar_addr, access_type, size, value, bar_num); + + ret = gsp_rpc_bar_trap(integ->rpc_client, bar_addr, access_type, size, + value, bar_num, ret_value); + + if (ret == 0) { + nv_vgpu_dbg(vgpu, "BAR_TRAP: bar=%u addr=0x%llx type=%u -> value=0x%llx", + bar_num, bar_addr, access_type, *ret_value); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_migrate_save_fw(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_migrate_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_migrate_msg(&req, src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, + GSP_MIGRATE_SUBCMD_SAVE, 0, 0); + + ret = gsp_rpc_migrate_save(integ->rpc_client, + src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, + checkpoint); + + if (ret == 0) { + nv_vgpu_info(vgpu, "MIGRATE SAVE: src=%llu dst=%llu -> checkpoint=0x%llx", + src_gpu_id, dst_gpu_id, *checkpoint); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_migrate_restore_fw(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_migrate_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_migrate_msg(&req, src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, + GSP_MIGRATE_SUBCMD_RESTORE, 0, checkpoint); + + ret = gsp_rpc_migrate_restore(integ->rpc_client, + src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, + checkpoint); + + if (ret == 0) { + nv_vgpu_info(vgpu, "MIGRATE RESTORE: src=%llu dst=%llu checkpoint=0x%llx", + src_gpu_id, dst_gpu_id, checkpoint); + } + + return ret; +} + +/* ============================================================ + * WPR Management (from firmware strings) + * ============================================================ */ + +int nv_vgpu_gsp_wpr_write_mutex_acquire(struct nv_vgpu_device *vgpu, u32 wpr_id) +{ + struct gsp_rpc_integration *integ = g_integration; + + (void)vgpu; /* integ is global, vgpu not directly used */ + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_wpr_write_mutex_acquire(integ->rpc_client, wpr_id); +} + +int nv_vgpu_gsp_wpr_write_mutex_release(struct nv_vgpu_device *vgpu, u32 wpr_id) +{ + struct gsp_rpc_integration *integ = g_integration; + + (void)vgpu; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_wpr_write_mutex_release(integ->rpc_client, wpr_id); +} + +int nv_vgpu_gsp_wpr_scratch_mutex_acquire(struct nv_vgpu_device *vgpu, u32 wpr_id) +{ + struct gsp_rpc_integration *integ = g_integration; + + (void)vgpu; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_wpr_scratch_mutex_acquire(integ->rpc_client, wpr_id); +} + +int nv_vgpu_gsp_wpr_scratch_mutex_release(struct nv_vgpu_device *vgpu, u32 wpr_id) +{ + struct gsp_rpc_integration *integ = g_integration; + + (void)vgpu; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_wpr_scratch_mutex_release(integ->rpc_client, wpr_id); +} + +/* ============================================================ + * Firmware Address Accessors (for debugging/validation) + * ============================================================ */ + +u64 nv_vgpu_gsp_get_primary_dispatcher(void) +{ + return GSP_RPC_DISPATCH_PRIMARY; +} + +u64 nv_vgpu_gsp_get_secondary_dispatcher(void) +{ + return GSP_RPC_DISPATCH_SECONDARY; +} + +u64 nv_vgpu_gsp_get_alloc_main(void) +{ + return GSP_ALLOC_MAIN; +} + +u64 nv_vgpu_gsp_get_schedule_main(void) +{ + return GSP_SCHEDULE_MAIN; +} + +u64 nv_vgpu_gsp_get_bar_trap_handler_jt(void) +{ + return GSP_BAR_TRAP_HANDLER_JT; +} + +u64 nv_vgpu_gsp_get_bar_trap_handler_primary(void) +{ + return GSP_BAR_TRAP_HANDLER_PRIM; +} + +u64 nv_vgpu_gsp_get_entry_point(void) +{ + return GSP_ENTRY_POINT; +} + +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_integration_init); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_integration_cleanup); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_alloc_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_free_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_map_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_unmap_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_pin_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_unpin_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_schedule_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_yield_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_bar_trap_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_migrate_save_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_migrate_restore_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_wpr_write_mutex_acquire); +EXPORT_SYMBOL(nv_vgpu_gsp_wpr_write_mutex_release); +EXPORT_SYMBOL(nv_vgpu_gsp_wpr_scratch_mutex_acquire); +EXPORT_SYMBOL(nv_vgpu_gsp_wpr_scratch_mutex_release); +EXPORT_SYMBOL(nv_vgpu_gsp_get_primary_dispatcher); +EXPORT_SYMBOL(nv_vgpu_gsp_get_secondary_dispatcher); +EXPORT_SYMBOL(nv_vgpu_gsp_get_alloc_main); +EXPORT_SYMBOL(nv_vgpu_gsp_get_schedule_main); +EXPORT_SYMBOL(nv_vgpu_gsp_get_bar_trap_handler_jt); +EXPORT_SYMBOL(nv_vgpu_gsp_get_bar_trap_handler_primary); +EXPORT_SYMBOL(nv_vgpu_gsp_get_entry_point); diff --git a/kernel/include/bar_emulation.h b/kernel/include/bar_emulation.h new file mode 100644 index 0000000..759fa25 --- /dev/null +++ b/kernel/include/bar_emulation.h @@ -0,0 +1,57 @@ +#ifndef _NV_VGPU_BAR_EMULATION_H_ +#define _NV_VGPU_BAR_EMULATION_H_ + +#include "nvidia-vgpu-vfio.h" + +struct nv_vgpu_bar_emul_ctx { + struct nv_vgpu_device *vgpu; + void *bar0_shadow; + size_t bar0_shadow_size; + spinlock_t shadow_lock; + + struct { + u32 offset; + u32 value; + bool is_write; + bool trapped; + } trap_regs[256]; + int trap_count; +}; + +int nv_vgpu_bar_emul_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_bar_emul_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_bar_trap_register(struct nv_vgpu_device *vgpu, int bar_idx, + u32 offset, u32 size); +int nv_vgpu_bar_untrap_register(struct nv_vgpu_device *vgpu, int bar_idx, + u32 offset); + +int nv_vgpu_bar_emul_read(struct nv_vgpu_device *vgpu, int bar_idx, + u64 offset, void *val, size_t size); +int nv_vgpu_bar_emul_write(struct nv_vgpu_device *vgpu, int bar_idx, + u64 offset, void *val, size_t size); + +int nv_vgpu_bar_emul_handle_fault(struct nv_vgpu_device *vgpu, + unsigned long addr, bool write); + +static inline u32 nv_vgpu_read_reg(void __iomem *base, u32 offset) +{ + return readl(base + offset); +} + +static inline void nv_vgpu_write_reg(void __iomem *base, u32 offset, u32 val) +{ + writel(val, base + offset); +} + +static inline u64 nv_vgpu_read_reg64(void __iomem *base, u32 offset) +{ + return readq(base + offset); +} + +static inline void nv_vgpu_write_reg64(void __iomem *base, u32 offset, u64 val) +{ + writeq(val, base + offset); +} + +#endif
\ No newline at end of file diff --git a/kernel/include/gsp_comm.h b/kernel/include/gsp_comm.h new file mode 100644 index 0000000..622f298 --- /dev/null +++ b/kernel/include/gsp_comm.h @@ -0,0 +1,155 @@ +#ifndef _NV_VGPU_GSP_COMM_H_ +#define _NV_VGPU_GSP_COMM_H_ + +#include "nvidia-vgpu-vfio.h" + +#define NV_VGPU_GSP_RPC_MAX_SIZE 4096 +#define NV_VGPU_GSP_RPC_TIMEOUT_MS 5000 +#define NV_VGPU_GSP_QUEUE_DEPTH 256 + +#define NV_VGPU_GSP_DOORBELL_OFFSET 0x100000 +#define NV_VGPU_GSP_SHARED_MEM_OFFSET 0x200000 +#define NV_VGPU_GSP_SHARED_MEM_SIZE (64 * 1024) + +#define NV_VGPU_GSP_FIRMWARE_PATH "/lib/firmware/nvidia/tu102/gsp/" + +enum nv_vgpu_gsp_rpc_type { + NV_GSP_RPC_TYPE_ALLOC = 0x01, + NV_GSP_RPC_TYPE_FREE = 0x02, + NV_GSP_RPC_TYPE_MAP = 0x03, + NV_GSP_RPC_TYPE_UNMAP = 0x04, + NV_GSP_RPC_TYPE_PIN = 0x05, + NV_GSP_RPC_TYPE_UNPIN = 0x06, + NV_GSP_RPC_TYPE_SCHEDULE = 0x07, + NV_GSP_RPC_TYPE_YIELD = 0x08, + NV_GSP_RPC_TYPE_NOTIFY = 0x09, + NV_GSP_RPC_TYPE_BAR_TRAP = 0x0A, + NV_GSP_RPC_TYPE_MIGRATE = 0x0B, + NV_GSP_RPC_TYPE_DEBUG = 0xFF, +}; + +struct nv_vgpu_gsp_rpc_header { + u32 type; + u32 sequence; + u32 size; + u32 flags; + u64 timestamp; +}; + +struct nv_vgpu_gsp_rpc_msg { + struct nv_vgpu_gsp_rpc_header hdr; + u8 data[]; +}; + +struct nv_vgpu_gsp_firmware { + void *bootloader; + size_t bootloader_size; + void *booter_load; + size_t booter_load_size; + void *gsp; + size_t gsp_size; +}; + +struct nv_vgpu_gsp_comm_ctx { + struct nv_vgpu_device *vgpu; + + void __iomem *gsp_doorbell; + void __iomem *gsp_shared_mem; + size_t shared_mem_size; + + struct mutex queue_lock; + struct list_head pending_queue; + struct list_head complete_queue; + atomic_t sequence_counter; + + wait_queue_head_t wait_queue; + + struct workqueue_struct *rpc_wq; + struct work_struct rpc_work; + + struct delayed_work poll_work; + bool poll_active; + + spinlock_t irq_lock; + bool irq_enabled; + + struct nv_vgpu_gsp_firmware firmware; + bool firmware_loaded; +}; + +struct nv_vgpu_gsp_rpc_entry { + struct list_head list; + struct nv_vgpu_gsp_rpc_msg *msg; + struct completion done; + int result; +}; + +int nv_vgpu_gsp_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_gsp_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_gsp_send_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size); +int nv_vgpu_gsp_recv_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size, int timeout_ms); + +int nv_vgpu_gsp_alloc(struct nv_vgpu_device *vgpu, u64 size, u32 align, u64 *handle); +int nv_vgpu_gsp_free(struct nv_vgpu_device *vgpu, u64 handle); +int nv_vgpu_gsp_map(struct nv_vgpu_device *vgpu, u64 handle, u64 iova, u64 size, u32 flags); +int nv_vgpu_gsp_unmap(struct nv_vgpu_device *vgpu, u64 handle, u64 iova, u64 size); +int nv_vgpu_gsp_schedule(struct nv_vgpu_device *vgpu, u32 runlist_id, u32 channel_id); +int nv_vgpu_gsp_notify_bar_trap(struct nv_vgpu_device *vgpu, u64 offset, u32 size, void *val); + +int nv_vgpu_gsp_load_firmware(struct nv_vgpu_device *vgpu); +void nv_vgpu_gsp_release_firmware(struct nv_vgpu_device *vgpu); +int nv_vgpu_gsp_boot(struct nv_vgpu_device *vgpu); + +/* ============================================================ + * New Firmware-Decompiled RPC Operations (enhanced APIs) + * ============================================================ */ + +int nv_vgpu_gsp_rpc_alloc(struct nv_vgpu_device *vgpu, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size); + +int nv_vgpu_gsp_rpc_free(struct nv_vgpu_device *vgpu, u64 handle); + +int nv_vgpu_gsp_rpc_map(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va); + +int nv_vgpu_gsp_rpc_unmap(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size); + +int nv_vgpu_gsp_rpc_pin(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages); + +int nv_vgpu_gsp_rpc_unpin(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size); + +int nv_vgpu_gsp_rpc_schedule(struct nv_vgpu_device *vgpu, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags); + +int nv_vgpu_gsp_rpc_yield(struct nv_vgpu_device *vgpu); + +int nv_vgpu_gsp_rpc_bar_trap(struct nv_vgpu_device *vgpu, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value); + +int nv_vgpu_gsp_rpc_migrate_save(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint); + +int nv_vgpu_gsp_rpc_migrate_restore(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint); + +static inline u32 nv_vgpu_gsp_next_sequence(struct nv_vgpu_gsp_comm_ctx *ctx) +{ + return atomic_inc_return(&ctx->sequence_counter); +} + +#endif
\ No newline at end of file diff --git a/kernel/include/gsp_rpc_client.h b/kernel/include/gsp_rpc_client.h new file mode 100644 index 0000000..4b48953 --- /dev/null +++ b/kernel/include/gsp_rpc_client.h @@ -0,0 +1,313 @@ +/* + * GSP 570.144W RPC Client Header + * Auto-generated from firmware decompilation + */ + +#ifndef _GSP_RPC_CLIENT_H_ +#define _GSP_RPC_CLIENT_H_ + +#include <linux/types.h> +#include <linux/device.h> +#include "nvidia-vgpu-vfio.h" + +/* ============================================================ + * FIRMWARE BASE ADDRESS + * ============================================================ */ +#define GSP_FW_BASE 0x04000000UL + +/* ============================================================ + * RPC DISPATCHER ADDRESSES + * ============================================================ */ + +/* Primary Dispatcher - Jump table switch for 6 RPC types */ +#define GSP_RPC_DISPATCH_PRIMARY (GSP_FW_BASE + 0x17A1AC0) /* 0x057a1ac0 */ +#define GSP_RPC_JUMP_TABLE (GSP_FW_BASE + 0x17A3AD8) /* 0x057a3ad8 - 32 sub-commands */ + +/* Secondary Dispatcher - Full implementation for all 11 RPC types */ +#define GSP_RPC_DISPATCH_SECONDARY (GSP_FW_BASE + 0x0ED39DC) /* 0x04ed39dc */ + +/* Tertiary Dispatcher - Stack=160 bytes */ +#define GSP_RPC_DISPATCH_TERTIARY (GSP_FW_BASE + 0x0CC58F8) /* 0x04cc58f8 */ + +/* Quaternary Dispatcher - Stack=96 bytes */ +#define GSP_RPC_DISPATCH_QUATERNARY (GSP_FW_BASE + 0x1723EF4) /* 0x05723ef4 */ + +/* ============================================================ + * SPECIALIZED HANDLER ADDRESSES + * ============================================================ */ + +#define GSP_ALLOC_MAIN (GSP_FW_BASE + 0x17E94EC) /* 0x057e94ec */ +#define GSP_SCHEDULE_MAIN (GSP_FW_BASE + 0x11EFBF4) /* 0x051efbf4 */ +#define GSP_BAR_TRAP_HANDLER_JT (GSP_FW_BASE + 0x17675B7) /* 0x057675b7 */ +#define GSP_BAR_TRAP_HANDLER_PRIM (GSP_FW_BASE + 0x0CB8189) /* 0x04cb8189 */ +#define GSP_PIN_HANDLER_SEC (GSP_FW_BASE + 0x0ED34B8) /* 0x04ed34b8 */ + +/* Jump Table Handlers (from decoded table at 0x059bfeb0) */ +#define GSP_MAP_HANDLER_A (GSP_FW_BASE + 0x1766AB7) /* 0x05766ab7 */ +#define GSP_MAP_HANDLER_B (GSP_FW_BASE + 0x1778A37) /* 0x05778a37 */ +#define GSP_UNMAP_HANDLER_A (GSP_FW_BASE + 0x17669B7) /* 0x057669b7 */ +#define GSP_UNMAP_HANDLER_B (GSP_FW_BASE + 0x1779937) /* 0x05779937 */ +#define GSP_PIN_HANDLER_A (GSP_FW_BASE + 0x17664B7) /* 0x057664b7 */ +#define GSP_PIN_HANDLER_B (GSP_FW_BASE + 0x177A3B7) /* 0x0577a3b7 */ +#define GSP_UNPIN_HANDLER_A (GSP_FW_BASE + 0x17682B7) /* 0x057682b7 */ +#define GSP_UNPIN_HANDLER_B (GSP_FW_BASE + 0x1779FB7) /* 0x05779fb7 */ +#define GSP_SCHEDULE_HANDLER_A (GSP_FW_BASE + 0x1767F37) /* 0x05767f37 */ +#define GSP_SCHEDULE_HANDLER_B (GSP_FW_BASE + 0x175AEB7) /* 0x0575aeb7 */ +#define GSP_YIELD_HANDLER_A (GSP_FW_BASE + 0x175AE37) /* 0x0575ae37 */ +#define GSP_YIELD_HANDLER_B (GSP_FW_BASE + 0x1767337) /* 0x05767337 */ +#define GSP_NOTIFY_HANDLER_A (GSP_FW_BASE + 0x177C8B7) /* 0x0577c8b7 */ +#define GSP_NOTIFY_HANDLER_B (GSP_FW_BASE + 0x1767837) /* 0x05767837 */ + +/* Entry Point */ +#define GSP_ENTRY_POINT (GSP_FW_BASE + 0x0009000) /* 0x04009000 */ + +/* ============================================================ + * RPC TYPE DEFINITIONS (match existing nv_vgpu_gsp_rpc_type) + * ============================================================ */ +enum gsp_rpc_type { + GSP_RPC_ALLOC = 0x01, + GSP_RPC_FREE = 0x02, + GSP_RPC_MAP = 0x03, + GSP_RPC_UNMAP = 0x04, + GSP_RPC_PIN = 0x05, + GSP_RPC_UNPIN = 0x06, + GSP_RPC_SCHEDULE = 0x07, + GSP_RPC_YIELD = 0x08, + GSP_RPC_NOTIFY = 0x09, + GSP_RPC_BAR_TRAP = 0x0A, + GSP_RPC_MIGRATE = 0x0B, + GSP_RPC_DEBUG = 0xFF, +}; + +/* ============================================================ + * ALLOC SUB-COMMANDS (Primary Dispatcher Jump Table Indices) + * ============================================================ */ +enum gsp_alloc_subcmd { + GSP_ALLOC_SUBCMD_DEFAULT = 0, + GSP_ALLOC_SUBCMD_TYPE1 = 1, + GSP_ALLOC_SUBCMD_TYPE2 = 3, + GSP_ALLOC_SUBCMD_TYPE3 = 4, + GSP_ALLOC_SUBCMD_TYPE4 = 5, + GSP_ALLOC_SUBCMD_TYPE5 = 8, + GSP_ALLOC_SUBCMD_TYPE6 = 10, + GSP_ALLOC_SUBCMD_TYPE7 = 12, + GSP_ALLOC_SUBCMD_TYPE8 = 17, + GSP_ALLOC_SUBCMD_TYPE9 = 19, + GSP_ALLOC_SUBCMD_TYPE10 = 20, + GSP_ALLOC_SUBCMD_TYPE11 = 21, + GSP_ALLOC_SUBCMD_TYPE12 = 22, + GSP_ALLOC_SUBCMD_TYPE13 = 23, + GSP_ALLOC_SUBCMD_TYPE14 = 24, + GSP_ALLOC_SUBCMD_TYPE15 = 25, + GSP_ALLOC_SUBCMD_TYPE16 = 26, + GSP_ALLOC_SUBCMD_TYPE17 = 27, + GSP_ALLOC_SUBCMD_TYPE18 = 28, + GSP_ALLOC_SUBCMD_TYPE19 = 29, + GSP_ALLOC_SUBCMD_TYPE20 = 30, + GSP_ALLOC_SUBCMD_TYPE21 = 31, +}; + +/* ============================================================ + * MIGRATE SUB-COMMANDS + * ============================================================ */ +enum gsp_migrate_subcmd { + GSP_MIGRATE_SUBCMD_SAVE = 0x04, + GSP_MIGRATE_SUBCMD_RESTORE = 0x05, +}; + +/* ============================================================ + * PIN SUB-COMMANDS + * ============================================================ */ +enum gsp_pin_subcmd { + GSP_PIN_SUBCMD_DEFAULT = 0x05, +}; + +/* ============================================================ + * SCHEDULE SUB-COMMANDS + * ============================================================ */ +enum gsp_sched_subcmd { + GSP_SCHED_SUBCMD_DEFAULT = 0x07, +}; + +/* ============================================================ + * BAR TRAP SUB-COMMANDS + * ============================================================ */ +enum gsp_bar_trap_subcmd { + GSP_BAR_TRAP_SUBCMD_DEFAULT = 0x0A, +}; + +/* ============================================================ + * RPC MESSAGE STRUCTURES + * ============================================================ */ + +struct gsp_rpc_msg { + __le32 type; + __le32 subcmd; + __le64 payload[6]; + __le32 seq_id; + __le32 flags; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +struct gsp_rpc_response { + __le32 status; + __le32 type; + __le64 result[6]; + __le32 seq_id; +} __attribute__((packed, aligned(8))); + +/* ALLOC request/response */ +struct gsp_alloc_req { + __le64 size; + __le64 alignment; + __le32 mem_type; + __le32 flags; + __le64 addr_hint; + __le32 pid; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +struct gsp_alloc_rsp { + __le64 gpu_va; + __le64 handle; + __le64 size; +} __attribute__((packed, aligned(8))); + +/* MAP request/response */ +struct gsp_map_req { + __le64 handle; + __le64 gpu_va; + __le64 size; + __le32 prot; + __le32 flags; + __le64 offset; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +struct gsp_map_rsp { + __le64 gpu_va; +} __attribute__((packed, aligned(8))); + +/* PIN/UNPIN request/response */ +struct gsp_pin_req { + __le64 handle; + __le64 offset; + __le64 size; + __le32 flags; + __le32 reserved[2]; +} __attribute__((packed, aligned(8))); + +struct gsp_pin_rsp { + __le64 dma_addr; + __le32 num_pages; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +/* SCHEDULE request */ +struct gsp_sched_req { + __le64 work_handle; + __le64 callback; + __le64 arg; + __le32 priority; + __le32 flags; + __le32 reserved[2]; +} __attribute__((packed, aligned(8))); + +/* BAR TRAP request/response */ +struct gsp_bar_trap_req { + __le64 bar_addr; + __le32 access_type; + __le32 size; + __le64 value; + __le32 bar_num; +} __attribute__((packed, aligned(8))); + +struct gsp_bar_trap_rsp { + __le64 value; + __le32 handled; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +/* MIGRATE request/response */ +struct gsp_migrate_req { + __le64 src_gpu_id; + __le64 dst_gpu_id; + __le64 memory_bitmap; + __le64 page_table_ptr; + __le32 state; + __le32 flags; + __le64 context; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +struct gsp_migrate_rsp { + __le32 status; + __le32 progress; + __le64 checkpoint; +} __attribute__((packed, aligned(8))); + +/* ============================================================ + * CLIENT OPAQUE STRUCT + * ============================================================ */ +struct gsp_client; + +/* ============================================================ + * PUBLIC API + * ============================================================ */ + +/* Client lifecycle */ +struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell); +void gsp_client_fini(struct gsp_client *client); + +/* Memory management */ +int gsp_rpc_alloc(struct gsp_client *client, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size); + +int gsp_rpc_free(struct gsp_client *client, u64 handle); + +int gsp_rpc_map(struct gsp_client *client, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va); + +int gsp_rpc_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size); + +/* Page pinning */ +int gsp_rpc_pin(struct gsp_client *client, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages); + +int gsp_rpc_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size); + +/* Scheduling */ +int gsp_rpc_schedule(struct gsp_client *client, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags); + +int gsp_rpc_yield(struct gsp_client *client); + +/* BAR trap emulation */ +int gsp_rpc_bar_trap(struct gsp_client *client, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value); + +/* Migration */ +int gsp_rpc_migrate_save(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint); + +int gsp_rpc_migrate_restore(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint); + +/* WPR Management */ +int gsp_wpr_write_mutex_acquire(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_write_mutex_release(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_scratch_mutex_acquire(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id); + +#endif /* _GSP_RPC_CLIENT_H_ */ diff --git a/kernel/include/gsp_rpc_integration.h b/kernel/include/gsp_rpc_integration.h new file mode 100644 index 0000000..eea3bf3 --- /dev/null +++ b/kernel/include/gsp_rpc_integration.h @@ -0,0 +1,97 @@ +/* + * GSP RPC Integration Layer Header + * Bridges the firmware-decompiled RPC client with existing gsp_comm.c + */ + +#ifndef _NV_VGPU_GSP_RPC_INTEGRATION_H_ +#define _NV_VGPU_GSP_RPC_INTEGRATION_H_ + +#include "nvidia-vgpu-vfio.h" +#include "gsp_comm.h" + +/* Initialize the RPC integration layer */ +int nv_vgpu_gsp_rpc_integration_init(struct nv_vgpu_device *vgpu, + struct nv_vgpu_gsp_comm_ctx *comm_ctx); + +/* Cleanup the RPC integration layer */ +void nv_vgpu_gsp_rpc_integration_cleanup(struct nv_vgpu_device *vgpu); + +/* ============================================================ + * Enhanced RPC Operations (using firmware-decompiled addresses) + * ============================================================ */ + +/* Memory allocation with full firmware parameter support */ +int nv_vgpu_gsp_rpc_alloc_fw(struct nv_vgpu_device *vgpu, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size); + +/* Memory free */ +int nv_vgpu_gsp_rpc_free_fw(struct nv_vgpu_device *vgpu, u64 handle); + +/* Memory mapping with full firmware parameter support */ +int nv_vgpu_gsp_rpc_map_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va); + +/* Memory unmapping */ +int nv_vgpu_gsp_rpc_unmap_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size); + +/* Page pinning for DMA */ +int nv_vgpu_gsp_rpc_pin_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages); + +/* Page unpinning */ +int nv_vgpu_gsp_rpc_unpin_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size); + +/* Work scheduling */ +int nv_vgpu_gsp_rpc_schedule_fw(struct nv_vgpu_device *vgpu, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags); + +/* Yield current work */ +int nv_vgpu_gsp_rpc_yield_fw(struct nv_vgpu_device *vgpu); + +/* BAR trap emulation */ +int nv_vgpu_gsp_rpc_bar_trap_fw(struct nv_vgpu_device *vgpu, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value); + +/* Migration - save state */ +int nv_vgpu_gsp_rpc_migrate_save_fw(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint); + +/* Migration - restore state */ +int nv_vgpu_gsp_rpc_migrate_restore_fw(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint); + +/* ============================================================ + * WPR Management (from firmware strings: VPR_WPR_WRITE_MUTEX*) + * ============================================================ */ + +int nv_vgpu_gsp_wpr_write_mutex_acquire(struct nv_vgpu_device *vgpu, u32 wpr_id); +int nv_vgpu_gsp_wpr_write_mutex_release(struct nv_vgpu_device *vgpu, u32 wpr_id); +int nv_vgpu_gsp_wpr_scratch_mutex_acquire(struct nv_vgpu_device *vgpu, u32 wpr_id); +int nv_vgpu_gsp_wpr_scratch_mutex_release(struct nv_vgpu_device *vgpu, u32 wpr_id); + +/* ============================================================ + * Firmware Address Accessors (for debugging/validation) + * ============================================================ */ + +u64 nv_vgpu_gsp_get_primary_dispatcher(void); /* 0x057a1ac0 */ +u64 nv_vgpu_gsp_get_secondary_dispatcher(void); /* 0x04ed39dc */ +u64 nv_vgpu_gsp_get_alloc_main(void); /* 0x057e94ec */ +u64 nv_vgpu_gsp_get_schedule_main(void); /* 0x051efbf4 */ +u64 nv_vgpu_gsp_get_bar_trap_handler_jt(void); /* 0x057675b7 */ +u64 nv_vgpu_gsp_get_bar_trap_handler_primary(void); /* 0x04cb8189 */ +u64 nv_vgpu_gsp_get_entry_point(void); /* 0x04009000 */ + +#endif /* _NV_VGPU_GSP_RPC_INTEGRATION_H_ */ diff --git a/kernel/include/ioctl.h b/kernel/include/ioctl.h new file mode 100644 index 0000000..4ff3df9 --- /dev/null +++ b/kernel/include/ioctl.h @@ -0,0 +1,105 @@ +#ifndef _NV_VGPU_IOCTL_H_ +#define _NV_VGPU_IOCTL_H_ + +#include "nvidia-vgpu-vfio.h" + +#define NV_VGPU_IOCTL_MAGIC 'N' + +#define NV_VGPU_IOCTL_GET_INFO \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x01, struct nv_vgpu_ioctl_get_info) + +#define NV_VGPU_IOCTL_MAP_BAR \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x02, struct nv_vgpu_ioctl_map_bar) + +#define NV_VGPU_IOCTL_UNMAP_BAR \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x03, struct nv_vgpu_ioctl_unmap_bar) + +#define NV_VGPU_IOCTL_ENABLE_IRQ \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x04, struct nv_vgpu_ioctl_irq) + +#define NV_VGPU_IOCTL_DISABLE_IRQ \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x05, struct nv_vgpu_ioctl_irq) + +#define NV_VGPU_IOCTL_GSP_RPC \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x06, struct nv_vgpu_ioctl_gsp_rpc) + +#define NV_VGPU_IOCTL_MIGRATE_SAVE \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x07, struct nv_vgpu_ioctl_migrate) + +#define NV_VGPU_IOCTL_MIGRATE_RESTORE \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x08, struct nv_vgpu_ioctl_migrate) + +#define NV_VGPU_IOCTL_SET_GSP_FIRMWARE \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x09, struct nv_vgpu_ioctl_firmware) + +struct nv_vgpu_ioctl_get_info { + u32 vgpu_type; + u32 vgpu_id; + u32 gpu_instance_id; + u32 compute_instance_id; + u32 state; + u64 fb_base; + u64 fb_size; + u64 bar0_config_base; + u32 num_bars; + struct { + u32 index; + u64 gpa; + u64 size; + u64 iova; + bool mapped; + bool trapped; + } bars[NV_VGPU_MAX_REGIONS]; +}; + +struct nv_vgpu_ioctl_map_bar { + u32 bar_index; + u64 gpa; + u64 size; + u64 iova; + u32 flags; +}; + +struct nv_vgpu_ioctl_unmap_bar { + u32 bar_index; + u64 iova; + u64 size; +}; + +struct nv_vgpu_ioctl_irq { + u32 irq_index; + int fd; +}; + +struct nv_vgpu_ioctl_gsp_rpc { + u32 type; + u32 flags; + u64 data_ptr; + u32 data_size; + u32 result; +}; + +struct nv_vgpu_ioctl_migrate { + u64 data_ptr; + u32 data_size; + u32 flags; + int result; +}; + +struct nv_vgpu_ioctl_firmware { + u32 chip_id; + u64 firmware_ptr; + u32 firmware_size; + u32 firmware_type; +}; + +int nv_vgpu_ioctl_get_info(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_map_bar(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_unmap_bar(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_enable_irq(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_disable_irq(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_gsp_rpc(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_migrate_save(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_migrate_restore(struct nv_vgpu_device *vgpu, void __user *arg); + +#endif
\ No newline at end of file diff --git a/kernel/include/irq.h b/kernel/include/irq.h new file mode 100644 index 0000000..d06ac5b --- /dev/null +++ b/kernel/include/irq.h @@ -0,0 +1,45 @@ +#ifndef _NV_VGPU_IRQ_H_ +#define _NV_VGPU_IRQ_H_ + +#include "nvidia-vgpu-vfio.h" + +#define NV_VGPU_MAX_MSI_VECTORS 32 + +struct nv_vgpu_msi_irq { + int virq; + u32 vector; + bool enabled; + struct eventfd_ctx *trigger; + struct irq_domain *domain; + struct msi_msg msg; +}; + +struct nv_vgpu_irq_ctx { + struct nv_vgpu_device *vgpu; + struct mutex lock; + struct nv_vgpu_msi_irq msis[NV_VGPU_MAX_MSI_VECTORS]; + int msi_count; + bool msi_enabled; + bool msix_enabled; +}; + +int nv_vgpu_irq_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_irq_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_irq_enable(struct nv_vgpu_device *vgpu, int irq_idx); +int nv_vgpu_irq_disable(struct nv_vgpu_device *vgpu, int irq_idx); + +int nv_vgpu_irq_setup_msi(struct nv_vgpu_device *vgpu, int num_vectors); +int nv_vgpu_irq_setup_msix(struct nv_vgpu_device *vgpu, int num_vectors); +void nv_vgpu_irq_teardown_msi(struct nv_vgpu_device *vgpu); +void nv_vgpu_irq_teardown_msix(struct nv_vgpu_device *vgpu); + +int nv_vgpu_irq_set_trigger(struct nv_vgpu_device *vgpu, int irq_idx, int fd); +int nv_vgpu_irq_get_virq(struct nv_vgpu_device *vgpu, int irq_idx); + +static inline struct nv_vgpu_irq_ctx *vgpu_to_irq_ctx(struct nv_vgpu_device *vgpu) +{ + return (struct nv_vgpu_irq_ctx *)vgpu->irq_ctx; +} + +#endif
\ No newline at end of file diff --git a/kernel/include/memory.h b/kernel/include/memory.h new file mode 100644 index 0000000..14a35e9 --- /dev/null +++ b/kernel/include/memory.h @@ -0,0 +1,45 @@ +#ifndef _NV_VGPU_MEMORY_H_ +#define _NV_VGPU_MEMORY_H_ + +#include "nvidia-vgpu-vfio.h" + +#define NV_VGPU_MAX_MAPPINGS 1024 + +struct nv_vgpu_mapping { + u64 gpa; + u64 iova; + u64 size; + struct page **pages; + int page_count; + bool pinned; + bool mapped; + struct list_head list; +}; + +struct nv_vgpu_mm_ctx { + struct nv_vgpu_device *vgpu; + struct mutex lock; + struct list_head mappings; + u32 mapping_count; + struct iommu_domain *iommu_domain; + struct dma_iommu_mapping *dma_mapping; + bool use_simple; +}; + +int nv_vgpu_memory_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_memory_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_memory_map(struct nv_vgpu_device *vgpu, u64 gpa, u64 size, u64 *iova); +int nv_vgpu_memory_unmap(struct nv_vgpu_device *vgpu, u64 iova, u64 size); + +int nv_vgpu_memory_pin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size); +int nv_vgpu_memory_unpin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size); + +int nv_vgpu_memory_mmap(struct nv_vgpu_device *vgpu, struct vm_area_struct *vma); + +static inline struct nv_vgpu_mm_ctx *vgpu_to_mm_ctx(struct nv_vgpu_device *vgpu) +{ + return vgpu->mm_ctx; +} + +#endif
\ No newline at end of file diff --git a/kernel/include/migration.h b/kernel/include/migration.h new file mode 100644 index 0000000..17cf8db --- /dev/null +++ b/kernel/include/migration.h @@ -0,0 +1,75 @@ +#ifndef _NV_VGPU_MIGRATION_H_ +#define _NV_VGPU_MIGRATION_H_ + +#include "nvidia-vgpu-vfio.h" + +#define NV_VGPU_MIGRATION_MAGIC 0x4E564750 +#define NV_VGPU_MIGRATION_VERSION 1 + +struct nv_vgpu_migration_ctx { + struct nv_vgpu_device *vgpu; + struct mutex lock; + struct nv_vgpu_migration_state state; + struct task_struct *save_thread; + struct task_struct *restore_thread; + bool in_progress; +}; + +struct nv_vgpu_migration_header { + u32 magic; + u32 version; + u32 flags; + u32 num_sections; + u64 total_size; + u64 timestamp; +}; + +enum nv_vgpu_migration_section_type { + NV_VGPU_MIG_SECTION_DEVICE_STATE = 0x01, + NV_VGPU_MIG_SECTION_BAR_STATE = 0x02, + NV_VGPU_MIG_SECTION_MEMORY_STATE = 0x03, + NV_VGPU_MIG_SECTION_IRQ_STATE = 0x04, + NV_VGPU_MIG_SECTION_GSP_STATE = 0x05, + NV_VGPU_MIG_SECTION_ENGINE_STATE = 0x06, + NV_VGPU_MIG_SECTION_CHANNEL_STATE = 0x07, + NV_VGPU_MIG_SECTION_CONTEXT_STATE = 0x08, +}; + +struct nv_vgpu_migration_section { + u32 type; + u32 flags; + u64 offset; + u64 size; + u32 checksum; +}; + +struct nv_vgpu_migration_device_state { + u32 vgpu_type; + u32 vgpu_id; + u32 gpu_instance_id; + u32 compute_instance_id; + u32 state; + u64 fb_base; + u64 fb_size; + u64 bar0_config_base; +}; + +struct nv_vgpu_migration_bar_state { + u32 bar_index; + u64 gpa; + u64 size; + u64 iova; + u8 data[]; +}; + +int nv_vgpu_migration_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_migration_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_migration_save_state(struct nv_vgpu_device *vgpu, u8 **data, size_t *size); +int nv_vgpu_migration_restore_state(struct nv_vgpu_device *vgpu, u8 *data, size_t size); + +int nv_vgpu_migration_prepare(struct nv_vgpu_device *vgpu); +int nv_vgpu_migration_complete(struct nv_vgpu_device *vgpu); +int nv_vgpu_migration_cancel(struct nv_vgpu_device *vgpu); + +#endif
\ No newline at end of file diff --git a/kernel/include/nv-hypervisor.h b/kernel/include/nv-hypervisor.h new file mode 100644 index 0000000..a2b2649 --- /dev/null +++ b/kernel/include/nv-hypervisor.h @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_HYPERVISOR_H_ +#define _NV_HYPERVISOR_H_ + +#include <nv-kernel-interface-api.h> + +// Enums for supported hypervisor types. +// New hypervisor type should be added before OS_HYPERVISOR_UNKNOWN +typedef enum _HYPERVISOR_TYPE +{ + OS_HYPERVISOR_XEN = 0, + OS_HYPERVISOR_VMWARE, + OS_HYPERVISOR_HYPERV, + OS_HYPERVISOR_KVM, + OS_HYPERVISOR_UNKNOWN +} HYPERVISOR_TYPE; + +#define CMD_VFIO_WAKE_REMOVE_GPU 1 +#define CMD_VGPU_VFIO_PRESENT 2 +#define CMD_VFIO_PCI_CORE_PRESENT 3 + +#define MAX_VF_COUNT_PER_GPU 64 + +typedef enum _VGPU_TYPE_INFO +{ + VGPU_TYPE_NAME = 0, + VGPU_TYPE_DESCRIPTION, + VGPU_TYPE_INSTANCES, +} VGPU_TYPE_INFO; + +typedef struct +{ + void *nv; + NvU32 domain; + NvU32 bus; + NvU32 device; + NvU32 return_status; +} vgpu_vfio_info; + +typedef struct +{ + NvU32 domain; + NvU8 bus; + NvU8 slot; + NvU8 function; + NvBool isNvidiaAttached; + NvBool isMdevAttached; +} vgpu_vf_pci_info; + +typedef enum VGPU_CMD_PROCESS_VF_INFO_E +{ + NV_VGPU_SAVE_VF_INFO = 0, + NV_VGPU_REMOVE_VF_PCI_INFO = 1, + NV_VGPU_REMOVE_VF_MDEV_INFO = 2, + NV_VGPU_GET_VF_INFO = 3 +} VGPU_CMD_PROCESS_VF_INFO; + +typedef enum VGPU_DEVICE_STATE_E +{ + NV_VGPU_DEV_UNUSED = 0, + NV_VGPU_DEV_OPENED = 1, + NV_VGPU_DEV_IN_USE = 2 +} VGPU_DEVICE_STATE; + +/* + * Function prototypes + */ + +HYPERVISOR_TYPE NV_API_CALL nv_get_hypervisor_type(void); + +#endif // _NV_HYPERVISOR_H_ diff --git a/kernel/include/nv-vgpu-vfio-interface.h b/kernel/include/nv-vgpu-vfio-interface.h new file mode 100644 index 0000000..2553722 --- /dev/null +++ b/kernel/include/nv-vgpu-vfio-interface.h @@ -0,0 +1,83 @@ +/* SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef _NV_VGPU_VFIO_INTERFACE_H_ +#define _NV_VGPU_VFIO_INTERFACE_H_ + +#include <linux/pci.h> +#include <linux/device.h> +#include <linux/mdev.h> +#include <linux/vfio.h> + +/* Use standard kernel types */ +typedef int NV_STATUS; +typedef unsigned char NvU8; +typedef unsigned int NvU32; +typedef unsigned short NvU16; +typedef unsigned long long NvU64; +typedef int NvS32; +typedef unsigned char NvBool; + +#define NV_OK 0 +#define NV_ERR_NOT_SUPPORTED -1 +#define NV_ERR_GENERIC -2 +#define NV_ERR_INVALID_ARGUMENT -3 +#define NV_ERR_NO_MEMORY -4 + +typedef enum { + VGPU_DEVICE_STATE_UNINITIALIZED = 0, + VGPU_DEVICE_STATE_INITIALIZING, + VGPU_DEVICE_STATE_READY, + VGPU_DEVICE_STATE_RUNNING, + VGPU_DEVICE_STATE_SUSPENDED, + VGPU_DEVICE_STATE_ERROR, + VGPU_DEVICE_STATE_MIGRATING, +} VGPU_DEVICE_STATE; + +typedef NV_STATUS (*vgpu_vfio_probe_t)(struct pci_dev *, NvU32, NvU32 *); +typedef void (*vgpu_vfio_remove_t)(struct pci_dev *); +typedef NV_STATUS (*vgpu_vfio_inject_interrupt_t)(void *); + +typedef struct +{ + vgpu_vfio_probe_t probe; + vgpu_vfio_remove_t remove; + vgpu_vfio_inject_interrupt_t inject_interrupt; +} vgpu_vfio_ops_t; + +typedef NV_STATUS (*vgpu_vfio_vgpu_create_t) (struct pci_dev *dev, const NvU8 *uuid, + NvU32 vgpu_type_id, NvU16 *vgpu_id, + NvU32 *gpu_pci_id, NvU32 gpu_pci_bdf); +typedef NV_STATUS (*vgpu_vfio_vgpu_destroy_t) (const NvU8 *uuid, NvU16 vgpu_id); +typedef NV_STATUS (*vgpu_vfio_start_t) (const NvU8 *uuid, void *wait_queue, NvS32 *return_status, + NvU8 *vm_name, NvU32 qemu_pid); +typedef NV_STATUS (*vgpu_vfio_bar_info_t) (struct pci_dev *dev, const NvU8 *, NvU64 *, + NvU32 bar_index, void *); +typedef NV_STATUS (*vgpu_vfio_get_description) (struct pci_dev *, NvU32, char *); +typedef NV_STATUS (*vgpu_vfio_get_name) (struct pci_dev *, NvU32, char *); +typedef NV_STATUS (*vgpu_vfio_get_instance) (struct pci_dev *, NvU32, char *); +typedef NV_STATUS (*vgpu_vfio_sparse_mmap_t) (struct pci_dev *, const NvU8 *, NvU64 **, NvU64 **, NvU32 *); +typedef void (*vgpu_vfio_update_request_t) (const NvU8 *, NvU64 *, NvU64 *, VGPU_DEVICE_STATE, const char *); + +typedef struct +{ + const char *version_string; + vgpu_vfio_vgpu_create_t vgpu_create; + vgpu_vfio_vgpu_destroy_t vgpu_delete; + vgpu_vfio_bar_info_t vgpu_bar_info; + vgpu_vfio_start_t vgpu_start; + vgpu_vfio_get_description get_description; + vgpu_vfio_get_instance get_instances; + vgpu_vfio_get_name get_name; + vgpu_vfio_sparse_mmap_t get_sparse_mmap; + vgpu_vfio_update_request_t update_request; +} rm_vgpu_vfio_ops_t; + +extern NV_STATUS nvidia_vgpu_vfio_set_ops(vgpu_vfio_ops_t *vgpu_vfio_ops); +extern NV_STATUS nvidia_vgpu_vfio_get_ops(rm_vgpu_vfio_ops_t *ops); + +extern NV_STATUS nvidia_vgpu_vfio_probe(struct pci_dev *); +extern void nvidia_vgpu_vfio_remove(struct pci_dev *, NvBool); + +#endif /* _NV_VGPU_VFIO_INTERFACE_H_ */
\ No newline at end of file diff --git a/kernel/include/nvidia-vgpu-vfio.h b/kernel/include/nvidia-vgpu-vfio.h new file mode 100644 index 0000000..7f778ee --- /dev/null +++ b/kernel/include/nvidia-vgpu-vfio.h @@ -0,0 +1,194 @@ +#ifndef _NVIDIA_VGPU_VFIO_H_ +#define _NVIDIA_VGPU_VFIO_H_ + +#include <linux/types.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/workqueue.h> +#include <linux/vfio.h> +#include <linux/mdev.h> +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <linux/atomic.h> + +#define OPEN_IOV_VFIO_VERSION "0.1.0" +#define OPEN_IOV_VFIO_NAME "open-iov-vfio" + +#define NVIDIA_VGPU_VFIO_VERSION OPEN_IOV_VFIO_VERSION +#define NVIDIA_VGPU_VFIO_NAME OPEN_IOV_VFIO_NAME + +#define NVIDIA_VGPU_MAX_VF_COUNT 32 +#define NVIDIA_VGPU_MAX_REGIONS 8 +#define NVIDIA_VGPU_MAX_IRQS 16 + +#define NV_VGPU_MAX_REGIONS NVIDIA_VGPU_MAX_REGIONS +#define NV_VGPU_MAX_IRQS NVIDIA_VGPU_MAX_IRQS + +#define NV_VGPU_BAR0_SIZE (16 * 1024 * 1024) +#define NV_VGPU_BAR1_SIZE (256 * 1024 * 1024) +#define NV_VGPU_BAR2_SIZE (32 * 1024 * 1024) +#define NV_VGPU_BAR3_SIZE (32 * 1024 * 1024) + +enum nv_vgpu_type { + NV_VGPU_TYPE_UNKNOWN = 0, + NV_VGPU_TYPE_AMPERE, + NV_VGPU_TYPE_HOPPER, + NV_VGPU_TYPE_TURING, + NV_VGPU_TYPE_PASCAL, + NV_VGPU_TYPE_MAXWELL, + NV_VGPU_TYPE_KEPLER, +}; + +enum nv_vgpu_state { + NV_VGPU_STATE_UNINITIALIZED = 0, + NV_VGPU_STATE_INITIALIZING, + NV_VGPU_STATE_READY, + NV_VGPU_STATE_RUNNING, + NV_VGPU_STATE_SUSPENDED, + NV_VGPU_STATE_ERROR, + NV_VGPU_STATE_MIGRATING, +}; + +struct nv_vgpu_bar { + u64 gpa; + u64 size; + u64 iova; + void *vaddr; + bool mapped; + bool trapped; + struct mutex lock; +}; + +struct nv_vgpu_irq { + int virq; + int type; + bool masked; + struct eventfd_ctx *trigger; + struct irq_domain *domain; +}; + +struct nv_vgpu_region { + u64 offset; + u64 size; + u64 iova; + int fd; + void *vaddr; + bool readonly; +}; + +struct nv_vgpu_migration_ctx; + +struct nv_vgpu_migration_state { + u32 version; + u32 flags; + u64 timestamp; + u8 *data; + size_t data_size; + struct completion done; + int result; + struct nv_vgpu_migration_ctx *ctx; +}; + +struct nv_vgpu_device { + struct mdev_device *mdev; + struct pci_dev *pdev; + struct device dev; + struct mutex lock; + enum nv_vgpu_type type; + enum nv_vgpu_state state; + u32 vgpu_id; + u32 gpu_instance_id; + u32 compute_instance_id; + + struct nv_vgpu_bar bars[NV_VGPU_MAX_REGIONS]; + struct nv_vgpu_irq irqs[NV_VGPU_MAX_IRQS]; + struct nv_vgpu_region regions[NV_VGPU_MAX_REGIONS]; + + struct nv_vgpu_migration_state migration; + + struct work_struct init_work; + struct work_struct cleanup_work; + + void *gsp_comm_ctx; + void *bar_emul_ctx; + void *mm_ctx; + void *irq_ctx; + + u64 fb_base; + u64 fb_size; + u64 bar0_config_base; + + struct list_head list; + + bool dry_run; + + /* PCI Config Space for VFIO mdev */ + u8 pci_config[256]; + bool config_space_dirty; +}; + +struct nv_vgpu_driver_data { + struct mutex lock; + struct list_head devices; + u32 device_count; + struct class *class; + struct bus_type *bus; +}; + +extern struct nv_vgpu_driver_data nv_vgpu_driver; + +int nv_vgpu_device_create(struct mdev_device *mdev); +void nv_vgpu_device_destroy(struct mdev_device *mdev); +int nv_vgpu_device_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_device_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_bar_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_bar_cleanup(struct nv_vgpu_device *vgpu); +int nv_vgpu_bar_trap(struct nv_vgpu_device *vgpu, int bar_idx); +void nv_vgpu_bar_untrap(struct nv_vgpu_device *vgpu, int bar_idx); + +int nv_vgpu_irq_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_irq_cleanup(struct nv_vgpu_device *vgpu); +int nv_vgpu_irq_enable(struct nv_vgpu_device *vgpu, int irq_idx); +int nv_vgpu_irq_disable(struct nv_vgpu_device *vgpu, int irq_idx); + +int nv_vgpu_memory_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_memory_cleanup(struct nv_vgpu_device *vgpu); +int nv_vgpu_memory_map(struct nv_vgpu_device *vgpu, u64 gpa, u64 size, u64 *iova); +int nv_vgpu_memory_unmap(struct nv_vgpu_device *vgpu, u64 iova, u64 size); + +int nv_vgpu_gsp_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_gsp_cleanup(struct nv_vgpu_device *vgpu); +int nv_vgpu_gsp_send_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size); +int nv_vgpu_gsp_recv_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size, int timeout_ms); +int nv_vgpu_gsp_load_firmware(struct nv_vgpu_device *vgpu); +void nv_vgpu_gsp_release_firmware(struct nv_vgpu_device *vgpu); +int nv_vgpu_gsp_boot(struct nv_vgpu_device *vgpu); + +int nv_vgpu_migration_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_migration_cleanup(struct nv_vgpu_device *vgpu); +int nv_vgpu_migration_save_state(struct nv_vgpu_device *vgpu, u8 **data, size_t *size); +int nv_vgpu_migration_restore_state(struct nv_vgpu_device *vgpu, u8 *data, size_t size); +int nv_vgpu_migration_prepare(struct nv_vgpu_device *vgpu); +int nv_vgpu_migration_complete(struct nv_vgpu_device *vgpu); +int nv_vgpu_migration_cancel(struct nv_vgpu_device *vgpu); +u8 *nv_vgpu_migration_get_data(struct nv_vgpu_device *vgpu, size_t *size); + +static inline struct nv_vgpu_device *mdev_to_vgpu(struct mdev_device *mdev) +{ + return dev_get_drvdata(&mdev->dev); +} + +#define nv_vgpu_dbg(vgpu, fmt, ...) \ + dev_dbg(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__) + +#define nv_vgpu_info(vgpu, fmt, ...) \ + dev_info(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__) + +#define nv_vgpu_warn(vgpu, fmt, ...) \ + dev_warn(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__) + +#define nv_vgpu_err(vgpu, fmt, ...) \ + dev_err(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__) + +#endif
\ No newline at end of file diff --git a/kernel/include/nvstatus.h b/kernel/include/nvstatus.h new file mode 100644 index 0000000..4f5284d --- /dev/null +++ b/kernel/include/nvstatus.h @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2014-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef SDK_NVSTATUS_H +#define SDK_NVSTATUS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "nvtypes.h" + +typedef NvU32 NV_STATUS; + +#define NV_STATUS_CODE( name, code, string ) name = (code), + +enum +{ + #include "nvstatuscodes.h" +}; + +#undef NV_STATUS_CODE + +/*! + * @def NV_STATUS_LEVEL_OK + * @see NV_STATUS_LEVEL + * @brief Success: No error or special condition + */ +#define NV_STATUS_LEVEL_OK 0 + +/*! + * @def NV_STATUS_LEVEL_WARN + * @see NV_STATUS_LEVEL + * @brief Success, but there is an special condition + * + * @details In general, NV_STATUS_LEVEL_WARN status codes are handled the + * same as NV_STATUS_LEVEL_OK, but are usefil to indicate that + * there is a condition that may be specially handled. + * + * Therefore, in most cases, client function should test for + * status <= NV_STATUS_LEVEL_WARN or status > NV_STATUS_LEVEL_WARN + * to determine success v. failure of a call. + */ +#define NV_STATUS_LEVEL_WARN 1 + +/*! + * @def NV_STATUS_LEVEL_ERR + * @see NV_STATUS_LEVEL + * @brief Unrecoverable error condition + */ +#define NV_STATUS_LEVEL_ERR 3 + +/*! + * @def NV_STATUS_LEVEL + * @see NV_STATUS_LEVEL_OK + * @see NV_STATUS_LEVEL_WARN + * @see NV_STATUS_LEVEL_ERR + * @brief Level of the status code + * + * @warning IMPORTANT: When comparing NV_STATUS_LEVEL(_S) against one of + * these constants, it is important to use '<=' or '>' (rather + * than '<' or '>='). + * + * For example. do: + * if (NV_STATUS_LEVEL(status) <= NV_STATUS_LEVEL_WARN) + * rather than: + * if (NV_STATUS_LEVEL(status) < NV_STATUS_LEVEL_ERR) + * + * By being consistent in this manner, it is easier to systematically + * add additional level constants. New levels are likely to lower + * (rather than raise) the severity of _ERR codes. For example, + * if we were to add NV_STATUS_LEVEL_RETRY to indicate hardware + * failures that may be recoverable (e.g. RM_ERR_TIMEOUT_RETRY + * or RM_ERR_BUSY_RETRY), it would be less severe than + * NV_STATUS_LEVEL_ERR the level to which these status codes now + * belong. Using '<=' and '>' ensures your code is not broken in + * cases like this. + */ +#define NV_STATUS_LEVEL(_S) \ + ((_S) == NV_OK? NV_STATUS_LEVEL_OK: \ + ((_S) != NV_ERR_GENERIC && (_S) & 0x00010000? NV_STATUS_LEVEL_WARN: \ + NV_STATUS_LEVEL_ERR)) + +/*! + * @def NV_STATUS_LEVEL + * @see NV_STATUS_LEVEL_OK + * @see NV_STATUS_LEVEL_WARN + * @see NV_STATUS_LEVEL_ERR + * @brief Character representing status code level + */ +#define NV_STATUS_LEVEL_CHAR(_S) \ + ((_S) == NV_OK? '0': \ + ((_S) != NV_ERR_GENERIC && (_S) & 0x00010000? 'W': \ + 'E')) + +// Function definitions +const char *nvstatusToString(NV_STATUS nvStatusIn); + +#ifdef __cplusplus +} +#endif + +#endif /* SDK_NVSTATUS_H */ diff --git a/kernel/include/nvtypes.h b/kernel/include/nvtypes.h new file mode 100644 index 0000000..7810d34 --- /dev/null +++ b/kernel/include/nvtypes.h @@ -0,0 +1,830 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVTYPES_INCLUDED +#define NVTYPES_INCLUDED + +#include "cpuopsys.h" + +#ifndef NVTYPES_USE_STDINT +#define NVTYPES_USE_STDINT 0 +#endif + +#if NVTYPES_USE_STDINT +#ifdef __cplusplus +#include <cstdint> +#include <cinttypes> +#else +#include <stdint.h> +#include <inttypes.h> +#endif // __cplusplus +#endif // NVTYPES_USE_STDINT + +#ifndef __cplusplus +// Header includes to make sure wchar_t is defined for C-file compilation +// (C++ is not affected as it is a fundamental type there) +// _MSC_VER is a hack to avoid failures for old setup of UEFI builds which are +// currently set to msvc100 but do not properly set the include paths +#if defined(NV_WINDOWS) && (!defined(_MSC_VER) || (_MSC_VER > 1600)) +#include <stddef.h> +#define NV_HAS_WCHAR_T_TYPEDEF 1 +#endif +#endif // __cplusplus + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(MAKE_NV64TYPES_8BYTES_ALIGNED) && defined(__i386__) +// ensure or force 8-bytes alignment of NV 64-bit types +#define OPTIONAL_ALIGN8_ATTR __attribute__((aligned(8))) +#else +// nothing needed +#define OPTIONAL_ALIGN8_ATTR +#endif // MAKE_NV64TYPES_8BYTES_ALIGNED && i386 + + /***************************************************************************\ +|* Typedefs *| + \***************************************************************************/ + +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +//Typedefs for MISRA COMPLIANCE +typedef unsigned long long UInt64; +typedef signed long long Int64; +typedef unsigned int UInt32; +typedef signed int Int32; +typedef unsigned short UInt16; +typedef signed short Int16; +typedef unsigned char UInt8 ; +typedef signed char Int8 ; + +typedef void Void; +typedef float float32_t; +typedef double float64_t; +#endif + + +// Floating point types +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef float32_t NvF32; /* IEEE Single Precision (S1E8M23) */ +typedef float64_t NvF64 OPTIONAL_ALIGN8_ATTR; /* IEEE Double Precision (S1E11M52) */ +#else +typedef float NvF32; /* IEEE Single Precision (S1E8M23) */ +typedef double NvF64 OPTIONAL_ALIGN8_ATTR; /* IEEE Double Precision (S1E11M52) */ +#endif + + +// 8-bit: 'char' is the only 8-bit in the C89 standard and after. +#if NVTYPES_USE_STDINT +typedef uint8_t NvV8; /* "void": enumerated or multiple fields */ +typedef uint8_t NvU8; /* 0 to 255 */ +typedef int8_t NvS8; /* -128 to 127 */ +#else +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef UInt8 NvV8; /* "void": enumerated or multiple fields */ +typedef UInt8 NvU8; /* 0 to 255 */ +typedef Int8 NvS8; /* -128 to 127 */ +#else +typedef unsigned char NvV8; /* "void": enumerated or multiple fields */ +typedef unsigned char NvU8; /* 0 to 255 */ +typedef signed char NvS8; /* -128 to 127 */ +#endif +#endif // NVTYPES_USE_STDINT + + +#if NVTYPES_USE_STDINT +typedef uint16_t NvV16; /* "void": enumerated or multiple fields */ +typedef uint16_t NvU16; /* 0 to 65535 */ +typedef int16_t NvS16; /* -32768 to 32767 */ +#else +// 16-bit: If the compiler tells us what we can use, then use it. +#ifdef __INT16_TYPE__ +typedef unsigned __INT16_TYPE__ NvV16; /* "void": enumerated or multiple fields */ +typedef unsigned __INT16_TYPE__ NvU16; /* 0 to 65535 */ +typedef signed __INT16_TYPE__ NvS16; /* -32768 to 32767 */ + +// The minimal standard for C89 and after +#else // __INT16_TYPE__ +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef UInt16 NvV16; /* "void": enumerated or multiple fields */ +typedef UInt16 NvU16; /* 0 to 65535 */ +typedef Int16 NvS16; /* -32768 to 32767 */ +#else +typedef unsigned short NvV16; /* "void": enumerated or multiple fields */ +typedef unsigned short NvU16; /* 0 to 65535 */ +typedef signed short NvS16; /* -32768 to 32767 */ +#endif +#endif // __INT16_TYPE__ +#endif // NVTYPES_USE_STDINT + +// wchar type (fixed size types consistent across Linux/Windows boundaries) +#if defined(NV_HAS_WCHAR_T_TYPEDEF) + typedef wchar_t NvWchar; +#else + typedef NvV16 NvWchar; +#endif + +// Macro to build an NvU32 from four bytes, listed from msb to lsb +#define NvU32_BUILD(a, b, c, d) \ + ((NvU32)( \ + (((NvU32)(a) & 0xff) << 24) | \ + (((NvU32)(b) & 0xff) << 16) | \ + (((NvU32)(c) & 0xff) << 8) | \ + (((NvU32)(d) & 0xff)))) + +// Macro to build an NvU64 from two DWORDS, listed from msb to lsb +#define NvU64_BUILD(a, b) \ + ((NvU64)( \ + (((NvU64)(a) & ~0U) << 32) | \ + (((NvU64)(b) & ~0U)))) + +#if NVTYPES_USE_STDINT +typedef uint32_t NvV32; /* "void": enumerated or multiple fields */ +typedef uint32_t NvU32; /* 0 to 4294967295 */ +typedef int32_t NvS32; /* -2147483648 to 2147483647 */ +#else +// 32-bit: If the compiler tells us what we can use, then use it. +#ifdef __INT32_TYPE__ +typedef unsigned __INT32_TYPE__ NvV32; /* "void": enumerated or multiple fields */ +typedef unsigned __INT32_TYPE__ NvU32; /* 0 to 4294967295 */ +typedef signed __INT32_TYPE__ NvS32; /* -2147483648 to 2147483647 */ + +// Older compilers +#else // __INT32_TYPE__ + +// For historical reasons, NvU32/NvV32 are defined to different base intrinsic +// types than NvS32 on some platforms. +// Mainly for 64-bit linux, where long is 64 bits and win9x, where int is 16 bit. +#if (defined(NV_UNIX) || defined(vxworks) || defined(NV_WINDOWS_CE) || \ + defined(__arm) || defined(__IAR_SYSTEMS_ICC__) || defined(NV_QNX) || \ + defined(NV_INTEGRITY) || defined(NV_MODS) || \ + defined(__GNUC__) || defined(__clang__) || defined(NV_MACINTOSH_64)) && \ + (!defined(NV_MACINTOSH) || defined(NV_MACINTOSH_64)) +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef UInt32 NvV32; /* "void": enumerated or multiple fields */ +typedef UInt32 NvU32; /* 0 to 4294967295 */ +#else +typedef unsigned int NvV32; /* "void": enumerated or multiple fields */ +typedef unsigned int NvU32; /* 0 to 4294967295 */ +#endif + +// The minimal standard for C89 and after +#else // (defined(NV_UNIX) || defined(vxworks) || ... +typedef unsigned long NvV32; /* "void": enumerated or multiple fields */ +typedef unsigned long NvU32; /* 0 to 4294967295 */ +#endif // (defined(NV_UNIX) || defined(vxworks) || ... + +// Mac OS 32-bit still needs this +#if defined(NV_MACINTOSH) && !defined(NV_MACINTOSH_64) +typedef signed long NvS32; /* -2147483648 to 2147483647 */ +#else +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef Int32 NvS32; /* -2147483648 to 2147483647 */ +#else +typedef signed int NvS32; /* -2147483648 to 2147483647 */ +#endif +#endif // defined(NV_MACINTOSH) && !defined(NV_MACINTOSH_64) +#endif // __INT32_TYPE__ +#endif // NVTYPES_USE_STDINT + + + +#if NVTYPES_USE_STDINT +typedef uint64_t NvU64 OPTIONAL_ALIGN8_ATTR; /* 0 to 18446744073709551615 */ +typedef int64_t NvS64 OPTIONAL_ALIGN8_ATTR; /* -9223372036854775808 to 9223372036854775807 */ + +#define NvU64_fmtX PRIX64 +#define NvU64_fmtx PRIx64 +#define NvU64_fmtu PRIu64 +#define NvU64_fmto PRIo64 +#define NvS64_fmtd PRId64 +#define NvS64_fmti PRIi64 +#else +// 64-bit types for compilers that support them, plus some obsolete variants +#if defined(__GNUC__) || defined(__clang__) || defined(__arm) || \ + defined(__IAR_SYSTEMS_ICC__) || defined(__ghs__) || defined(_WIN64) || \ + defined(__SUNPRO_C) || defined(__SUNPRO_CC) || defined (__xlC__) +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef UInt64 NvU64 OPTIONAL_ALIGN8_ATTR; /* 0 to 18446744073709551615 */ +typedef Int64 NvS64 OPTIONAL_ALIGN8_ATTR; /* -9223372036854775808 to 9223372036854775807 */ +#else +typedef unsigned long long NvU64 OPTIONAL_ALIGN8_ATTR; /* 0 to 18446744073709551615 */ +typedef long long NvS64 OPTIONAL_ALIGN8_ATTR; /* -9223372036854775808 to 9223372036854775807 */ +#endif + +#define NvU64_fmtX "llX" +#define NvU64_fmtx "llx" +#define NvU64_fmtu "llu" +#define NvU64_fmto "llo" +#define NvS64_fmtd "lld" +#define NvS64_fmti "lli" + +// Microsoft since 2003 -- https://msdn.microsoft.com/en-us/library/29dh1w7z.aspx +#else +typedef unsigned __int64 NvU64 OPTIONAL_ALIGN8_ATTR; /* 0 to 18446744073709551615 */ +typedef __int64 NvS64 OPTIONAL_ALIGN8_ATTR; /* -9223372036854775808 to 9223372036854775807 */ + +#define NvU64_fmtX "I64X" +#define NvU64_fmtx "I64x" +#define NvU64_fmtu "I64u" +#define NvU64_fmto "I64o" +#define NvS64_fmtd "I64d" +#define NvS64_fmti "I64i" + +#endif +#endif // NVTYPES_USE_STDINT + +#ifdef NV_TYPESAFE_HANDLES +/* + * Can't use opaque pointer as clients might be compiled with mismatched + * pointer sizes. TYPESAFE check will eventually be removed once all clients + * have transistioned safely to NvHandle. + * The plan is to then eventually scale up the handle to be 64-bits. + */ +typedef struct +{ + NvU32 val; +} NvHandle; +#else +/* + * For compatibility with modules that haven't moved typesafe handles. + */ +typedef NvU32 NvHandle; +#endif // NV_TYPESAFE_HANDLES + +/* Boolean type */ +typedef NvU8 NvBool; +#define NV_TRUE ((NvBool)(0 == 0)) +#define NV_FALSE ((NvBool)(0 != 0)) + +/* Tristate type: NV_TRISTATE_FALSE, NV_TRISTATE_TRUE, NV_TRISTATE_INDETERMINATE */ +typedef NvU8 NvTristate; +#define NV_TRISTATE_FALSE ((NvTristate) 0) +#define NV_TRISTATE_TRUE ((NvTristate) 1) +#define NV_TRISTATE_INDETERMINATE ((NvTristate) 2) + +/* Macros to extract the low and high parts of a 64-bit unsigned integer */ +/* Also designed to work if someone happens to pass in a 32-bit integer */ +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +#define NvU64_HI32(n) ((NvU32)((((NvU64)(n)) >> 32) & 0xffffffffU)) +#define NvU64_LO32(n) ((NvU32)(( (NvU64)(n)) & 0xffffffffU)) +#else +#define NvU64_HI32(n) ((NvU32)((((NvU64)(n)) >> 32) & 0xffffffff)) +#define NvU64_LO32(n) ((NvU32)(( (NvU64)(n)) & 0xffffffff)) +#endif +#define NvU40_HI32(n) ((NvU32)((((NvU64)(n)) >> 8) & 0xffffffffU)) +#define NvU40_HI24of32(n) ((NvU32)( (NvU64)(n) & 0xffffff00U)) + +/* Macros to get the MSB and LSB of a 32 bit unsigned number */ +#define NvU32_HI16(n) ((NvU16)((((NvU32)(n)) >> 16) & 0xffffU)) +#define NvU32_LO16(n) ((NvU16)(( (NvU32)(n)) & 0xffffU)) + + /***************************************************************************\ +|* *| +|* 64 bit type definitions for use in interface structures. *| +|* *| + \***************************************************************************/ + +#if defined(NV_64_BITS) + +typedef void* NvP64; /* 64 bit void pointer */ +typedef NvU64 NvUPtr; /* pointer sized unsigned int */ +typedef NvS64 NvSPtr; /* pointer sized signed int */ +typedef NvU64 NvLength; /* length to agree with sizeof */ + +#define NvP64_VALUE(n) (n) +#define NvP64_fmt "%p" + +#define KERNEL_POINTER_FROM_NvP64(p,v) ((p)(v)) +#define NvP64_PLUS_OFFSET(p,o) (NvP64)((NvU64)(p) + (NvU64)(o)) + +#define NvUPtr_fmtX NvU64_fmtX +#define NvUPtr_fmtx NvU64_fmtx +#define NvUPtr_fmtu NvU64_fmtu +#define NvUPtr_fmto NvU64_fmto +#define NvSPtr_fmtd NvS64_fmtd +#define NvSPtr_fmti NvS64_fmti + +#else + +typedef NvU64 NvP64; /* 64 bit void pointer */ +typedef NvU32 NvUPtr; /* pointer sized unsigned int */ +typedef NvS32 NvSPtr; /* pointer sized signed int */ +typedef NvU32 NvLength; /* length to agree with sizeof */ + +#define NvP64_VALUE(n) ((void *)(NvUPtr)(n)) +#define NvP64_fmt "0x%llx" + +#define KERNEL_POINTER_FROM_NvP64(p,v) ((p)(NvUPtr)(v)) +#define NvP64_PLUS_OFFSET(p,o) ((p) + (NvU64)(o)) + +#define NvUPtr_fmtX "X" +#define NvUPtr_fmtx "x" +#define NvUPtr_fmtu "u" +#define NvUPtr_fmto "o" +#define NvSPtr_fmtd "d" +#define NvSPtr_fmti "i" + +#endif + +#define NvP64_NULL (NvP64)0 + +/*! + * Helper macro to pack an @ref NvU64_ALIGN32 structure from a @ref NvU64. + * + * @param[out] pDst Pointer to NvU64_ALIGN32 structure to pack + * @param[in] pSrc Pointer to NvU64 with which to pack + */ +#define NvU64_ALIGN32_PACK(pDst, pSrc) \ +do { \ + (pDst)->lo = NvU64_LO32(*(pSrc)); \ + (pDst)->hi = NvU64_HI32(*(pSrc)); \ +} while (NV_FALSE) + +/*! + * Helper macro to unpack a @ref NvU64_ALIGN32 structure into a @ref NvU64. + * + * @param[out] pDst Pointer to NvU64 in which to unpack + * @param[in] pSrc Pointer to NvU64_ALIGN32 structure from which to unpack + */ +#define NvU64_ALIGN32_UNPACK(pDst, pSrc) \ +do { \ + (*(pDst)) = NvU64_ALIGN32_VAL(pSrc); \ +} while (NV_FALSE) + +/*! + * Helper macro to unpack a @ref NvU64_ALIGN32 structure as a @ref NvU64. + * + * @param[in] pSrc Pointer to NvU64_ALIGN32 structure to unpack + */ +#define NvU64_ALIGN32_VAL(pSrc) \ + ((NvU64) ((NvU64)((pSrc)->lo) | (((NvU64)(pSrc)->hi) << 32U))) + +/*! + * Helper macro to check whether the 32 bit aligned 64 bit number is zero. + * + * @param[in] _pU64 Pointer to NvU64_ALIGN32 structure. + * + * @return + * NV_TRUE _pU64 is zero. + * NV_FALSE otherwise. + */ +#define NvU64_ALIGN32_IS_ZERO(_pU64) \ + (((_pU64)->lo == 0U) && ((_pU64)->hi == 0U)) + +/*! + * Helper macro to sub two 32 aligned 64 bit numbers on 64 bit processor. + * + * @param[in] pSrc1 Pointer to NvU64_ALIGN32 source 1 structure. + * @param[in] pSrc2 Pointer to NvU64_ALIGN32 source 2 structure. + * @param[in/out] pDst Pointer to NvU64_ALIGN32 dest. structure. + */ +#define NvU64_ALIGN32_ADD(pDst, pSrc1, pSrc2) \ +do { \ + NvU64 __dst, __src1, __scr2; \ + \ + NvU64_ALIGN32_UNPACK(&__src1, (pSrc1)); \ + NvU64_ALIGN32_UNPACK(&__scr2, (pSrc2)); \ + __dst = __src1 + __scr2; \ + NvU64_ALIGN32_PACK((pDst), &__dst); \ +} while (NV_FALSE) + +/*! + * Helper macro to sub two 32 aligned 64 bit numbers on 64 bit processor. + * + * @param[in] pSrc1 Pointer to NvU64_ALIGN32 source 1 structure. + * @param[in] pSrc2 Pointer to NvU64_ALIGN32 source 2 structure. + * @param[in/out] pDst Pointer to NvU64_ALIGN32 dest. structure. + */ +#define NvU64_ALIGN32_SUB(pDst, pSrc1, pSrc2) \ +do { \ + NvU64 __dst, __src1, __scr2; \ + \ + NvU64_ALIGN32_UNPACK(&__src1, (pSrc1)); \ + NvU64_ALIGN32_UNPACK(&__scr2, (pSrc2)); \ + __dst = __src1 - __scr2; \ + NvU64_ALIGN32_PACK((pDst), &__dst); \ +} while (NV_FALSE) + +/*! + * Structure for representing 32 bit aligned NvU64 (64-bit unsigned integer) + * structures. This structure must be used because the 32 bit processor and + * 64 bit processor compilers will pack/align NvU64 differently. + * + * One use case is RM being 64 bit proc whereas PMU being 32 bit proc, this + * alignment difference will result in corrupted transactions between the RM + * and PMU. + * + * See the @ref NvU64_ALIGN32_PACK and @ref NvU64_ALIGN32_UNPACK macros for + * packing and unpacking these structures. + * + * @note The intention of this structure is to provide a datatype which will + * packed/aligned consistently and efficiently across all platforms. + * We don't want to use "NV_DECLARE_ALIGNED(NvU64, 8)" because that + * leads to memory waste on our 32-bit uprocessors (e.g. FALCONs) where + * DMEM efficiency is vital. + */ +typedef struct +{ + /*! + * Low 32 bits. + */ + NvU32 lo; + /*! + * High 32 bits. + */ + NvU32 hi; +} NvU64_ALIGN32; + +/* Useful macro to hide required double cast */ +#define NV_PTR_TO_NvP64(n) (NvP64)(NvUPtr)(n) +#define NV_SIGN_EXT_PTR_TO_NvP64(p) ((NvP64)(NvS64)(NvSPtr)(p)) +#define KERNEL_POINTER_TO_NvP64(p) ((NvP64)(uintptr_t)(p)) + + /***************************************************************************\ +|* *| +|* Limits for common types. *| +|* *| + \***************************************************************************/ + +/* Explanation of the current form of these limits: + * + * - Decimal is used, as hex values are by default positive. + * - Casts are not used, as usage in the preprocessor itself (#if) ends poorly. + * - The subtraction of 1 for some MIN values is used to get around the fact + * that the C syntax actually treats -x as NEGATE(x) instead of a distinct + * number. Since 214748648 isn't a valid positive 32-bit signed value, we + * take the largest valid positive signed number, negate it, and subtract 1. + */ +#define NV_S8_MIN (-128) +#define NV_S8_MAX (+127) +#define NV_U8_MIN (0U) +#define NV_U8_MAX (+255U) +#define NV_S16_MIN (-32768) +#define NV_S16_MAX (+32767) +#define NV_U16_MIN (0U) +#define NV_U16_MAX (+65535U) +#define NV_S32_MIN (-2147483647 - 1) +#define NV_S32_MAX (+2147483647) +#define NV_U32_MIN (0U) +#define NV_U32_MAX (+4294967295U) +#define NV_S64_MIN (-9223372036854775807LL - 1LL) +#define NV_S64_MAX (+9223372036854775807LL) +#define NV_U64_MIN (0ULL) +#define NV_U64_MAX (+18446744073709551615ULL) + +/* Aligns fields in structs so they match up between 32 and 64 bit builds */ +#if defined(__GNUC__) || defined(__clang__) || defined(NV_QNX) +#define NV_ALIGN_BYTES(size) __attribute__ ((aligned (size))) +#elif defined(__arm) +#define NV_ALIGN_BYTES(size) __align(ALIGN) +#else +// XXX This is dangerously nonportable! We really shouldn't provide a default +// version of this that doesn't do anything. +#define NV_ALIGN_BYTES(size) +#endif + +// NV_DECLARE_ALIGNED() can be used on all platforms. +// This macro form accounts for the fact that __declspec on Windows is required +// before the variable type, +// and NV_ALIGN_BYTES is required after the variable name. +#if defined(__GNUC__) || defined(__clang__) || defined(NV_QNX) +#define NV_DECLARE_ALIGNED(TYPE_VAR, ALIGN) TYPE_VAR __attribute__ ((aligned (ALIGN))) +#elif defined(_MSC_VER) +#define NV_DECLARE_ALIGNED(TYPE_VAR, ALIGN) __declspec(align(ALIGN)) TYPE_VAR +#elif defined(__arm) +#define NV_DECLARE_ALIGNED(TYPE_VAR, ALIGN) __align(ALIGN) TYPE_VAR +#endif + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +#define NV_ALIGNOF(type) _Alignof(type) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) +#define NV_ALIGNOF(type) alignof(type) +#elif defined(__GNUC__) || defined(__clang__) +#define NV_ALIGNOF(type) __alignof__(type) +#elif defined(_MSC_VER) +#define NV_ALIGNOF(type) __alignof(type) +#endif + + /***************************************************************************\ +|* Function Declaration Types *| + \***************************************************************************/ + +// stretching the meaning of "nvtypes", but this seems to least offensive +// place to re-locate these from nvos.h which cannot be included by a number +// of builds that need them + +#if defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER) + #define NV_ATTRIBUTE_UNUSED __attribute__((__unused__)) +#else + #define NV_ATTRIBUTE_UNUSED +#endif + +#if defined(_MSC_VER) + + #if _MSC_VER >= 1310 + #define NV_NOINLINE __declspec(noinline) + #else + #define NV_NOINLINE + #endif + + #define NV_INLINE __inline + + #if _MSC_VER >= 1200 + #define NV_FORCEINLINE __forceinline + #else + #define NV_FORCEINLINE __inline + #endif + + #define NV_APIENTRY __stdcall + #define NV_FASTCALL __fastcall + #define NV_CDECLCALL __cdecl + #define NV_STDCALL __stdcall + + #define NV_FORCERESULTCHECK + + #define NV_FORMAT_PRINTF(_f, _a) + +#else // ! defined(_MSC_VER) + + #if defined(__GNUC__) + #if (__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) && (__GNUC_PATCHLEVEL__ >= 1)) + #define NV_NOINLINE __attribute__((__noinline__)) + #endif + #elif defined(__clang__) + #if __has_attribute(noinline) + #define NV_NOINLINE __attribute__((__noinline__)) + #endif + #elif defined(__arm) && (__ARMCC_VERSION >= 300000) + #define NV_NOINLINE __attribute__((__noinline__)) + #elif (defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590)) ||\ + (defined(__SUNPRO_CC) && (__SUNPRO_CC >= 0x590)) + #define NV_NOINLINE __attribute__((__noinline__)) + #elif defined (__INTEL_COMPILER) + #define NV_NOINLINE __attribute__((__noinline__)) + #endif + + #if !defined(NV_NOINLINE) + #define NV_NOINLINE + #endif + + /* GreenHills compiler defines __GNUC__, but doesn't support + * __inline__ keyword. */ + #if defined(__ghs__) + #define NV_INLINE inline + #elif defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER) + #define NV_INLINE __inline__ + #elif defined (macintosh) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) + #define NV_INLINE inline + #elif defined(__arm) + #define NV_INLINE __inline + #else + #define NV_INLINE + #endif + + /* Don't force inline on DEBUG builds -- it's annoying for debuggers. */ + #if !defined(DEBUG) + /* GreenHills compiler defines __GNUC__, but doesn't support + * __attribute__ or __inline__ keyword. */ + #if defined(__ghs__) + #define NV_FORCEINLINE inline + #elif defined(__GNUC__) + // GCC 3.1 and beyond support the always_inline function attribute. + #if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) + #define NV_FORCEINLINE __attribute__((__always_inline__)) __inline__ + #else + #define NV_FORCEINLINE __inline__ + #endif + #elif defined(__clang__) + #if __has_attribute(always_inline) + #define NV_FORCEINLINE __attribute__((__always_inline__)) __inline__ + #else + #define NV_FORCEINLINE __inline__ + #endif + #elif defined(__arm) && (__ARMCC_VERSION >= 220000) + // RVDS 2.2 also supports forceinline, but ADS 1.2 does not + #define NV_FORCEINLINE __forceinline + #else /* defined(__GNUC__) */ + #define NV_FORCEINLINE NV_INLINE + #endif + #else + #define NV_FORCEINLINE NV_INLINE + #endif + + #define NV_APIENTRY + #define NV_FASTCALL + #define NV_CDECLCALL + #define NV_STDCALL + + /* + * The 'warn_unused_result' function attribute prompts GCC to issue a + * warning if the result of a function tagged with this attribute + * is ignored by a caller. In combination with '-Werror', it can be + * used to enforce result checking in RM code; at this point, this + * is only done on UNIX. + */ + #if defined(__GNUC__) && defined(NV_UNIX) + #if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4)) + #define NV_FORCERESULTCHECK __attribute__((__warn_unused_result__)) + #else + #define NV_FORCERESULTCHECK + #endif + #elif defined(__clang__) + #if __has_attribute(warn_unused_result) + #define NV_FORCERESULTCHECK __attribute__((__warn_unused_result__)) + #else + #define NV_FORCERESULTCHECK + #endif + #else /* defined(__GNUC__) */ + #define NV_FORCERESULTCHECK + #endif + + /* + * Functions decorated with NV_FORMAT_PRINTF(f, a) have a format string at + * parameter number 'f' and variadic arguments start at parameter number 'a'. + * (Note that for C++ methods, there is an implicit 'this' parameter so + * explicit parameters are numbered from 2.) + */ + #if defined(__GNUC__) + #define NV_FORMAT_PRINTF(_f, _a) __attribute__((format(printf, _f, _a))) + #else + #define NV_FORMAT_PRINTF(_f, _a) + #endif + +#endif // defined(_MSC_VER) + + /***************************************************************************\ +|* *| +|* Definitions of binary/bytes representations of floating point values *| +|* *| +|* Intended for use when passing values across kernel contexts in which *| +|* floating point use is forbidden *| +|* *| + \***************************************************************************/ +// +// Determine whether we have to turn FP32 support off based on compiler flags. +// For now, this is only done when we can detect that we're compiling for x86_64 +// with GCC/clang and SSE is disabled. +// +// This is also currently only used to control NvF32Bytes definitions. +// +// In the future, it could be possible to: +// 1.) Define this more generally +// and +// 2.) Use it in more places +// +// This would allow us to fully prevent usage of NvF32 in codebases where it is +// not supported. +// +#if !defined(NVTYPES_NV_F32_SUPPORTED) && NVCPU_IS_X86_64 && (defined(__GNUC__) || defined(__clang__)) && !defined(__SSE__) +#define NVTYPES_NV_F32_SUPPORTED 0 +#else +#define NVTYPES_NV_F32_SUPPORTED 1 +#endif + +/*! + * A struct containing the 32-bit representation of an (IEEE-754 binary32) + * floating point value, i.e., the raw underlying bytes of an @ref NvF32 + * + * This struct must be used in places where a value may be passed across a + * kernel boundary, as use of floating point within some kernels is, in general, + * banned. + */ +typedef struct +{ + NvU32 value; +} NvF32Bytes; + +#if !defined(NVDEC_1_0) +/*! + * @brief Maps the bytes of a given @ref NvF32 to an @ref NvF32Bytes + * + * @param[in] pfValue + * Pointer to value to convert + * + * @return @ref NvF32Bytes byte-mapped from @p *pfValue + */ +static NV_FORCEINLINE NvF32Bytes +nvF32BytesMapFromNvF32 +( + const NvF32 *pfValue +) +{ + union + { + NvF32 f; + NvF32Bytes b; + } value; + + value.f = *pfValue; + + return value.b; +} + +#if NVTYPES_NV_F32_SUPPORTED +/*! + * @brief Maps the bytes of a given @ref NvF32Bytes to an @ref NvF32 + * + * @param[in] pfValue + * Pointer to value to convert + * + * @return @ref NvF32 byte-mapped from @p *pfValue + */ +static NV_FORCEINLINE NvF32 +nvF32BytesMapToNvF32 +( + const NvF32Bytes *pfValue +) +{ + union + { + NvF32Bytes b; + NvF32 f; + } value; + + value.b = *pfValue; + + return value.f; +} +#endif // NVTYPES_NV_F32_SUPPORTED + +/*! + * @brief Maps the bytes of a given @ref NvU32 to an @ref NvF32Bytes + * + * @details This function is intended for first "moving" raw byte values into + * the "floating point domain." + * + * @param[in] pfValue + * Pointer to value to convert + * + * @return @ref NvF32Bytes byte-mapped from @p *pfValue + */ +static NV_FORCEINLINE NvF32Bytes +nvF32BytesMapFromNvU32 +( + const NvU32 *pfValue +) +{ + union + { + NvU32 u; + NvF32Bytes b; + } value; + + value.u = *pfValue; + + return value.b; +} + +/*! + * @brief Maps the bytes of a given @ref NvF32Bytes to an @ref NvU32 + * + * @param[in] pfValue + * Pointer to value to convert + * + * @return @ref NvU32 byte-mapped from @p *pfValue + */ +static NV_FORCEINLINE NvU32 +nvF32BytesMapToNvU32 +( + const NvF32Bytes *pfValue +) +{ + union + { + NvF32Bytes b; + NvU32 u; + } value; + + value.b = *pfValue; + + return value.u; +} +#endif // !defined(NVDEC_1_0) + +#ifdef __cplusplus +} +#endif + +#endif /* NVTYPES_INCLUDED */ diff --git a/kernel/ioctl.c b/kernel/ioctl.c new file mode 100644 index 0000000..571b209 --- /dev/null +++ b/kernel/ioctl.c @@ -0,0 +1,263 @@ +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/vmalloc.h> + +#include "nvidia-vgpu-vfio.h" +#include "ioctl.h" +#include "bar_emulation.h" +#include "gsp_comm.h" +#include "memory.h" +#include "irq.h" +#include "migration.h" + +int nv_vgpu_ioctl_get_info(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_get_info info; + int i; + + memset(&info, 0, sizeof(info)); + + mutex_lock(&vgpu->lock); + info.vgpu_type = vgpu->type; + info.vgpu_id = vgpu->vgpu_id; + info.gpu_instance_id = vgpu->gpu_instance_id; + info.compute_instance_id = vgpu->compute_instance_id; + info.state = vgpu->state; + info.fb_base = vgpu->fb_base; + info.fb_size = vgpu->fb_size; + info.bar0_config_base = vgpu->bar0_config_base; + info.num_bars = 0; + + for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) { + if (vgpu->bars[i].size > 0) { + info.bars[info.num_bars].index = i; + info.bars[info.num_bars].gpa = vgpu->bars[i].gpa; + info.bars[info.num_bars].size = vgpu->bars[i].size; + info.bars[info.num_bars].iova = vgpu->bars[i].iova; + info.bars[info.num_bars].mapped = vgpu->bars[i].mapped; + info.bars[info.num_bars].trapped = vgpu->bars[i].trapped; + info.num_bars++; + } + } + mutex_unlock(&vgpu->lock); + + if (copy_to_user(arg, &info, sizeof(info))) + return -EFAULT; + + return 0; +} + +int nv_vgpu_ioctl_map_bar(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_map_bar params; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (params.bar_index >= NV_VGPU_MAX_REGIONS) + return -EINVAL; + + mutex_lock(&vgpu->lock); + + ret = nv_vgpu_memory_pin(vgpu, params.gpa, params.size); + if (ret) { + mutex_unlock(&vgpu->lock); + return ret; + } + + ret = nv_vgpu_memory_map(vgpu, params.gpa, params.size, ¶ms.iova); + if (ret) { + nv_vgpu_memory_unpin(vgpu, params.gpa, params.size); + mutex_unlock(&vgpu->lock); + return ret; + } + + vgpu->bars[params.bar_index].gpa = params.gpa; + vgpu->bars[params.bar_index].size = params.size; + vgpu->bars[params.bar_index].iova = params.iova; + vgpu->bars[params.bar_index].mapped = true; + + if (params.flags & 0x1) { + ret = nv_vgpu_bar_trap(vgpu, params.bar_index); + if (ret) + nv_vgpu_warn(vgpu, "Failed to trap BAR %d: %d", params.bar_index, ret); + else + vgpu->bars[params.bar_index].trapped = true; + } + + mutex_unlock(&vgpu->lock); + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return 0; +} + +int nv_vgpu_ioctl_unmap_bar(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_unmap_bar params; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (params.bar_index >= NV_VGPU_MAX_REGIONS) + return -EINVAL; + + mutex_lock(&vgpu->lock); + + if (vgpu->bars[params.bar_index].trapped) { + nv_vgpu_bar_untrap(vgpu, params.bar_index); + vgpu->bars[params.bar_index].trapped = false; + } + + ret = nv_vgpu_memory_unmap(vgpu, params.iova, params.size); + if (ret) { + mutex_unlock(&vgpu->lock); + return ret; + } + + nv_vgpu_memory_unpin(vgpu, vgpu->bars[params.bar_index].gpa, params.size); + + vgpu->bars[params.bar_index].mapped = false; + vgpu->bars[params.bar_index].iova = 0; + + mutex_unlock(&vgpu->lock); + return 0; +} + +int nv_vgpu_ioctl_enable_irq(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_irq params; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + ret = nv_vgpu_irq_enable(vgpu, params.irq_index); + if (ret) + return ret; + + if (params.fd >= 0) { + ret = nv_vgpu_irq_set_trigger(vgpu, params.irq_index, params.fd); + if (ret) + nv_vgpu_irq_disable(vgpu, params.irq_index); + } + + return ret; +} + +int nv_vgpu_ioctl_disable_irq(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_irq params; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + ret = nv_vgpu_irq_set_trigger(vgpu, params.irq_index, -1); + if (ret) + return ret; + + return nv_vgpu_irq_disable(vgpu, params.irq_index); +} + +int nv_vgpu_ioctl_gsp_rpc(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_gsp_rpc params; + void *data = NULL; + int ret = 0; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (params.data_size > 0 && params.data_size <= NV_VGPU_GSP_RPC_MAX_SIZE) { + data = kmalloc(params.data_size, GFP_KERNEL); + if (!data) + return -ENOMEM; + + if (copy_from_user(data, (void __user *)params.data_ptr, params.data_size)) { + kfree(data); + return -EFAULT; + } + } + + ret = nv_vgpu_gsp_send_rpc(vgpu, data, params.data_size); + params.result = ret; + + if (data) + kfree(data); + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return 0; +} + +int nv_vgpu_ioctl_migrate_save(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_migrate params; + u8 *data = NULL; + size_t size = 0; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + ret = nv_vgpu_migration_prepare(vgpu); + if (ret) + return ret; + + ret = nv_vgpu_migration_complete(vgpu); + if (ret) + return ret; + + data = nv_vgpu_migration_get_data(vgpu, &size); + + if (params.data_ptr && params.data_size >= size) { + if (copy_to_user((void __user *)params.data_ptr, data, size)) { + ret = -EFAULT; + } + } + + params.data_size = size; + params.result = ret; + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return ret; +} + +int nv_vgpu_ioctl_migrate_restore(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_migrate params; + u8 *data = NULL; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (!params.data_ptr || params.data_size == 0) + return -EINVAL; + + data = kmalloc(params.data_size, GFP_KERNEL); + if (!data) + return -ENOMEM; + + if (copy_from_user(data, (void __user *)params.data_ptr, params.data_size)) { + kfree(data); + return -EFAULT; + } + + ret = nv_vgpu_migration_restore_state(vgpu, data, params.data_size); + params.result = ret; + + kfree(data); + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return ret; +}
\ No newline at end of file diff --git a/kernel/irq.c b/kernel/irq.c new file mode 100644 index 0000000..f4b77e2 --- /dev/null +++ b/kernel/irq.c @@ -0,0 +1,246 @@ +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/interrupt.h> +#include <linux/irqdomain.h> +#include <linux/msi.h> +#include <linux/pci.h> +#include <linux/eventfd.h> +#include <linux/vfio.h> + +#include "nvidia-vgpu-vfio.h" +#include "irq.h" + +static irqreturn_t nv_vgpu_msi_handler(int irq, void *data) +{ + struct nv_vgpu_msi_irq *msi = data; + + if (msi->trigger) + eventfd_signal(msi->trigger); + + return IRQ_HANDLED; +} + +int nv_vgpu_irq_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_irq_ctx *ctx; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: IRQ init validation"); + /* Check MSI/MSI-X capability */ + if (pci_find_capability(vgpu->pdev, PCI_CAP_ID_MSIX)) { + nv_vgpu_info(vgpu, "DRY-RUN: MSI-X capability present"); + } else if (pci_find_capability(vgpu->pdev, PCI_CAP_ID_MSI)) { + nv_vgpu_info(vgpu, "DRY-RUN: MSI capability present"); + } else { + nv_vgpu_warn(vgpu, "DRY-RUN: No MSI/MSI-X capability found"); + } + return 0; + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + mutex_init(&ctx->lock); + ctx->msi_count = 0; + ctx->msi_enabled = false; + ctx->msix_enabled = false; + + vgpu->irq_ctx = ctx; + return 0; +} + +void nv_vgpu_irq_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + + if (!ctx) + return; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: IRQ cleanup (no hardware resources to release)"); + kfree(ctx); + vgpu->irq_ctx = NULL; + return; + } + + nv_vgpu_irq_teardown_msix(vgpu); + nv_vgpu_irq_teardown_msi(vgpu); + + kfree(ctx); + vgpu->irq_ctx = NULL; +} + +int nv_vgpu_irq_setup_msi(struct nv_vgpu_device *vgpu, int num_vectors) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + struct pci_dev *pdev = vgpu->pdev; + int ret, i; + + if (!ctx || num_vectors > NV_VGPU_MAX_MSI_VECTORS) + return -EINVAL; + + ret = pci_alloc_irq_vectors(pdev, 1, num_vectors, PCI_IRQ_MSI); + if (ret < 0) + return ret; + + ctx->msi_count = ret; + ctx->msi_enabled = true; + + for (i = 0; i < ctx->msi_count; i++) { + ctx->msis[i].virq = pci_irq_vector(pdev, i); + ctx->msis[i].vector = i; + ctx->msis[i].enabled = false; + ctx->msis[i].trigger = NULL; + + ret = request_irq(ctx->msis[i].virq, nv_vgpu_msi_handler, + 0, "nv-vgpu-msi", &ctx->msis[i]); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to request MSI %d: %d", i, ret); + ctx->msis[i].virq = 0; + } + } + + return 0; +} + +void nv_vgpu_irq_teardown_msi(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + int i; + + if (!ctx || !ctx->msi_enabled) + return; + + for (i = 0; i < ctx->msi_count; i++) { + if (ctx->msis[i].virq) + free_irq(ctx->msis[i].virq, &ctx->msis[i]); + if (ctx->msis[i].trigger) + eventfd_ctx_put(ctx->msis[i].trigger); + } + + pci_free_irq_vectors(vgpu->pdev); + ctx->msi_enabled = false; + ctx->msi_count = 0; +} + +int nv_vgpu_irq_setup_msix(struct nv_vgpu_device *vgpu, int num_vectors) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + struct pci_dev *pdev = vgpu->pdev; + int ret, i; + + if (!ctx || num_vectors > NV_VGPU_MAX_MSI_VECTORS) + return -EINVAL; + + ret = pci_alloc_irq_vectors(pdev, 1, num_vectors, PCI_IRQ_MSIX); + if (ret < 0) + return ret; + + ctx->msi_count = ret; + ctx->msix_enabled = true; + + for (i = 0; i < ctx->msi_count; i++) { + ctx->msis[i].virq = pci_irq_vector(pdev, i); + ctx->msis[i].vector = i; + ctx->msis[i].enabled = false; + ctx->msis[i].trigger = NULL; + + ret = request_irq(ctx->msis[i].virq, nv_vgpu_msi_handler, + 0, "nv-vgpu-msix", &ctx->msis[i]); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to request MSIX %d: %d", i, ret); + ctx->msis[i].virq = 0; + } + } + + return 0; +} + +void nv_vgpu_irq_teardown_msix(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + int i; + + if (!ctx || !ctx->msix_enabled) + return; + + for (i = 0; i < ctx->msi_count; i++) { + if (ctx->msis[i].virq) + free_irq(ctx->msis[i].virq, &ctx->msis[i]); + if (ctx->msis[i].trigger) + eventfd_ctx_put(ctx->msis[i].trigger); + } + + pci_free_irq_vectors(vgpu->pdev); + ctx->msix_enabled = false; + ctx->msi_count = 0; +} + +int nv_vgpu_irq_enable(struct nv_vgpu_device *vgpu, int irq_idx) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + + if (!ctx || irq_idx < 0 || irq_idx >= ctx->msi_count) + return -EINVAL; + + if (!ctx->msis[irq_idx].virq) + return -ENODEV; + + ctx->msis[irq_idx].enabled = true; + enable_irq(ctx->msis[irq_idx].virq); + return 0; +} + +int nv_vgpu_irq_disable(struct nv_vgpu_device *vgpu, int irq_idx) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + + if (!ctx || irq_idx < 0 || irq_idx >= ctx->msi_count) + return -EINVAL; + + if (!ctx->msis[irq_idx].virq) + return -ENODEV; + + disable_irq(ctx->msis[irq_idx].virq); + ctx->msis[irq_idx].enabled = false; + return 0; +} + +int nv_vgpu_irq_set_trigger(struct nv_vgpu_device *vgpu, int irq_idx, int fd) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + struct eventfd_ctx *trigger; + + if (!ctx || irq_idx < 0 || irq_idx >= ctx->msi_count) + return -EINVAL; + + if (fd < 0) { + if (ctx->msis[irq_idx].trigger) { + eventfd_ctx_put(ctx->msis[irq_idx].trigger); + ctx->msis[irq_idx].trigger = NULL; + } + return 0; + } + + trigger = eventfd_ctx_fdget(fd); + if (IS_ERR(trigger)) + return PTR_ERR(trigger); + + if (ctx->msis[irq_idx].trigger) + eventfd_ctx_put(ctx->msis[irq_idx].trigger); + + ctx->msis[irq_idx].trigger = trigger; + return 0; +} + +int nv_vgpu_irq_get_virq(struct nv_vgpu_device *vgpu, int irq_idx) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + + if (!ctx || irq_idx < 0 || irq_idx >= ctx->msi_count) + return -EINVAL; + + return ctx->msis[irq_idx].virq; +}
\ No newline at end of file diff --git a/kernel/main.c b/kernel/main.c new file mode 100644 index 0000000..7d12203 --- /dev/null +++ b/kernel/main.c @@ -0,0 +1,834 @@ +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/pci.h> +#include <linux/vfio.h> +#include <linux/mdev.h> +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/workqueue.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/iommu.h> +#include <linux/vmalloc.h> +#include <linux/version.h> + +#include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" +#include "gsp_comm.h" +#include "memory.h" +#include "irq.h" +#include "migration.h" +#include "ioctl.h" + +#include "nv-vgpu-vfio-interface.h" + +#define NV_VERSION_STRING "550.163.01" + +/* ============================================================ + * Forward declarations + * ============================================================ */ + +static int nv_vgpu_mdev_create(struct mdev_device *mdev); +static void nv_vgpu_mdev_remove(struct mdev_device *mdev); +static void nv_vgpu_device_init_work(struct work_struct *work); +static void nv_vgpu_device_cleanup_work(struct work_struct *work); + +/* PCI Config Space handlers for VFIO mdev */ +static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev *pdev); +static ssize_t nv_vgpu_config_read(struct nv_vgpu_device *vgpu, char __user *buf, size_t count, loff_t *ppos); +static ssize_t nv_vgpu_config_write(struct nv_vgpu_device *vgpu, const char __user *buf, size_t count, loff_t *ppos); + +static struct mdev_type *nv_vgpu_mdev_types[4]; + +static struct mdev_driver nv_vgpu_mdev_driver = { + .device_api = VFIO_DEVICE_API_PCI_STRING, + .max_instances = 8, + .probe = nv_vgpu_mdev_create, + .remove = nv_vgpu_mdev_remove, + .driver = { + .name = "nvidia-vgpu-vfio-mdev", + }, +}; + +/* ============================================================ + * Symbols we provide to nvidia.ko + * ============================================================ */ + +static vgpu_vfio_ops_t *g_vgpu_vfio_ops = NULL; +static rm_vgpu_vfio_ops_t g_rm_vgpu_vfio_ops; + +/* mdev parent structure for our physical device */ +static struct mdev_parent *g_mdev_parent = NULL; +static struct pci_dev *g_pdev = NULL; + +/* VFIO ops implementation */ +static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, NvU32 *vgpu_id); +static void nv_vgpu_vfio_remove(struct pci_dev *pdev); +static NV_STATUS nv_vgpu_vfio_inject_interrupt(void *vgpu_dev); + +static vgpu_vfio_ops_t nv_vgpu_vfio_ops = { + .probe = nv_vgpu_vfio_probe, + .remove = nv_vgpu_vfio_remove, + .inject_interrupt = nv_vgpu_vfio_inject_interrupt, +}; + +/* RM ops implementation - callbacks from nvidia.ko */ +static NV_STATUS nv_vgpu_create_callback(struct pci_dev *pdev, const NvU8 *uuid, + NvU32 type_id, NvU16 *vgpu_id, + NvU32 *gpu_pci_id, NvU32 gpu_pci_bdf) +{ + pr_info("nvidia-vgpu-vfio: vgpu_create callback: type_id=%u\n", type_id); + if (vgpu_id) + *vgpu_id = 1; + if (gpu_pci_id) + *gpu_pci_id = 0x10de1ed3; + return NV_OK; +} + +static NV_STATUS nv_vgpu_delete_callback(const NvU8 *uuid, NvU16 vgpu_id) +{ + pr_info("nvidia-vgpu-vfio: vgpu_delete callback: vgpu_id=%u\n", vgpu_id); + return NV_OK; +} + +static NV_STATUS nv_vgpu_bar_info_callback(struct pci_dev *pdev, const NvU8 *uuid, + NvU64 *size, NvU32 bar_index, void *vgpu_dev) +{ + pr_info("nvidia-vgpu-vfio: vgpu_bar_info callback: bar_index=%u\n", bar_index); + if (size) { + switch (bar_index) { + case 0: + *size = NV_VGPU_BAR0_SIZE; /* 16MB */ + break; + case 1: + *size = NV_VGPU_BAR1_SIZE; /* 256MB */ + break; + case 2: + *size = NV_VGPU_BAR2_SIZE; /* 32MB */ + break; + case 3: + *size = NV_VGPU_BAR3_SIZE; /* 32MB */ + break; + default: + *size = 0; + break; + } + } + return NV_OK; +} + +static NV_STATUS nv_vgpu_get_name_callback(struct pci_dev *pdev, NvU32 type_id, char *buf) +{ + pr_info("nvidia-vgpu-vfio: get_name callback: type_id=%u\n", type_id); + if (buf) { + switch (type_id) { + case 0: /* Ampere */ + strcpy(buf, "NVIDIA A100"); + break; + case 1: /* Hopper */ + strcpy(buf, "NVIDIA H100"); + break; + case 2: /* Turing */ + strcpy(buf, "NVIDIA T4"); + break; + case 3: /* Pascal */ + strcpy(buf, "NVIDIA P4"); + break; + default: + strcpy(buf, "NVIDIA vGPU"); + break; + } + } + return NV_OK; +} + +static NV_STATUS nv_vgpu_get_description_callback(struct pci_dev *pdev, NvU32 type_id, char *buf) +{ + pr_info("nvidia-vgpu-vfio: get_description callback: type_id=%u\n", type_id); + if (buf) { + switch (type_id) { + case 0: + strcpy(buf, "NVIDIA Ampere Architecture vGPU"); + break; + case 1: + strcpy(buf, "NVIDIA Hopper Architecture vGPU"); + break; + case 2: + strcpy(buf, "NVIDIA Turing Architecture vGPU"); + break; + case 3: + strcpy(buf, "NVIDIA Pascal Architecture vGPU"); + break; + default: + strcpy(buf, "NVIDIA vGPU Description"); + break; + } + } + return NV_OK; +} + +static NV_STATUS nv_vgpu_get_instances_callback(struct pci_dev *pdev, NvU32 type_id, char *buf) +{ + pr_info("nvidia-vgpu-vfio: get_instances callback: type_id=%u\n", type_id); + if (buf) + strcpy(buf, "1"); + return NV_OK; +} + +static void nv_vgpu_update_request_callback(const NvU8 *uuid, NvU64 *vgpu_dev, + NvU64 *vgpu_vm, VGPU_DEVICE_STATE device_state, + const char *params) +{ + pr_info("nvidia-vgpu-vfio: update_request callback: state=%u\n", device_state); +} + +static NV_STATUS nv_vgpu_start_callback(const NvU8 *uuid, void *wait_queue, NvS32 *return_status, + NvU8 *vm_name, NvU32 qemu_pid) +{ + pr_info("nvidia-vgpu-vfio: vgpu_start callback: qemu_pid=%u\n", qemu_pid); + if (return_status) + *return_status = 0; + return NV_OK; +} + +static NV_STATUS nv_vgpu_sparse_mmap_callback(struct pci_dev *pdev, const NvU8 *uuid, + NvU64 **pages, NvU64 **offsets, NvU32 *count) +{ + pr_info("nvidia-vgpu-vfio: sparse_mmap callback\n"); + if (count) + *count = 0; + return NV_OK; +} + +/* Build RM ops structure */ +static void populate_rm_ops(void) +{ + memset(&g_rm_vgpu_vfio_ops, 0, sizeof(g_rm_vgpu_vfio_ops)); + g_rm_vgpu_vfio_ops.version_string = NV_VERSION_STRING; + g_rm_vgpu_vfio_ops.vgpu_create = nv_vgpu_create_callback; + g_rm_vgpu_vfio_ops.vgpu_delete = nv_vgpu_delete_callback; + g_rm_vgpu_vfio_ops.vgpu_bar_info = nv_vgpu_bar_info_callback; + g_rm_vgpu_vfio_ops.vgpu_start = nv_vgpu_start_callback; + g_rm_vgpu_vfio_ops.get_name = nv_vgpu_get_name_callback; + g_rm_vgpu_vfio_ops.get_description = nv_vgpu_get_description_callback; + g_rm_vgpu_vfio_ops.get_instances = nv_vgpu_get_instances_callback; + g_rm_vgpu_vfio_ops.get_sparse_mmap = nv_vgpu_sparse_mmap_callback; + g_rm_vgpu_vfio_ops.update_request = nv_vgpu_update_request_callback; +} + +/* get_ops: called by nvidia-vgpu-vfio to get RM ops */ +NV_STATUS nvidia_vgpu_vfio_get_ops(rm_vgpu_vfio_ops_t *ops) +{ + if (!ops) + return NV_ERR_INVALID_ARGUMENT; + + populate_rm_ops(); + memcpy(ops, &g_rm_vgpu_vfio_ops, sizeof(rm_vgpu_vfio_ops_t)); + pr_info("nvidia-vgpu-vfio: get_ops provided RM ops\n"); + return NV_OK; +} + +/* set_ops: called by nvidia-vgpu-vfio to provide VFIO ops */ +NV_STATUS nvidia_vgpu_vfio_set_ops(vgpu_vfio_ops_t *ops) +{ + if (!ops) + return NV_ERR_INVALID_ARGUMENT; + + g_vgpu_vfio_ops = ops; + pr_info("nvidia-vgpu-vfio: set_ops received VFIO ops\n"); + return NV_OK; +} + +extern NV_STATUS nv_vgpu_mdev_register_parent(struct pci_dev *pdev); +extern void nv_vgpu_mdev_unregister_parent(struct pci_dev *pdev); +extern int nv_vgpu_mdev_driver_register(void); +extern void nv_vgpu_mdev_driver_unregister(void); + +MODULE_SOFTDEP("pre: nvidia"); + +MODULE_AUTHOR("Open-IOV Project"); +MODULE_DESCRIPTION("Open-IOV vGPU VFIO Mediated Device Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(OPEN_IOV_VFIO_VERSION); + +struct nv_vgpu_driver_data nv_vgpu_driver = { + .lock = __MUTEX_INITIALIZER(nv_vgpu_driver.lock), + .devices = LIST_HEAD_INIT(nv_vgpu_driver.devices), + .device_count = 0, +}; + +static int nv_vgpu_mdev_create(struct mdev_device *mdev) +{ + struct nv_vgpu_device *vgpu; + struct pci_dev *pdev; + struct mdev_parent *parent; + int ret; + + vgpu = kzalloc(sizeof(*vgpu), GFP_KERNEL); + if (!vgpu) + return -ENOMEM; + + dev_set_drvdata(&mdev->dev, vgpu); + vgpu->mdev = mdev; + vgpu->state = NV_VGPU_STATE_UNINITIALIZED; + mutex_init(&vgpu->lock); + INIT_WORK(&vgpu->init_work, nv_vgpu_device_init_work); + INIT_WORK(&vgpu->cleanup_work, nv_vgpu_device_cleanup_work); + + parent = mdev->type->parent; + if (!parent || !parent->dev) { + ret = -ENODEV; + goto err_free; + } + pdev = to_pci_dev(parent->dev); + if (!pdev) { + ret = -ENODEV; + goto err_free; + } + vgpu->pdev = pci_dev_get(pdev); + + /* Initialize PCI config space for VFIO mdev */ + ret = nv_vgpu_config_space_init(vgpu, pdev); + if (ret) { + nv_vgpu_err(vgpu, "PCI config space init failed: %d", ret); + goto err_put; + } + + mutex_lock(&nv_vgpu_driver.lock); + list_add_tail(&vgpu->list, &nv_vgpu_driver.devices); + nv_vgpu_driver.device_count++; + mutex_unlock(&nv_vgpu_driver.lock); + + schedule_work(&vgpu->init_work); + + return 0; + +err_put: + pci_dev_put(vgpu->pdev); +err_free: + kfree(vgpu); + return ret; +} + +static void nv_vgpu_mdev_remove(struct mdev_device *mdev) +{ + struct nv_vgpu_device *vgpu = dev_get_drvdata(&mdev->dev); + + cancel_work_sync(&vgpu->init_work); + schedule_work(&vgpu->cleanup_work); + flush_work(&vgpu->cleanup_work); + + mutex_lock(&nv_vgpu_driver.lock); + list_del(&vgpu->list); + nv_vgpu_driver.device_count--; + mutex_unlock(&nv_vgpu_driver.lock); + + pci_dev_put(vgpu->pdev); + kfree(vgpu); +} + +static int __init nv_vgpu_init(void) +{ + int ret, i; + struct pci_dev *pdev = NULL; + + nv_vgpu_driver.class = class_create("nvidia-vgpu"); + if (IS_ERR(nv_vgpu_driver.class)) { + ret = PTR_ERR(nv_vgpu_driver.class); + pr_err("Failed to create nvidia-vgpu class: %d\n", ret); + return ret; + } + + for (i = 0; i < 4; i++) { + nv_vgpu_mdev_types[i] = kzalloc(sizeof(struct mdev_type), GFP_KERNEL); + if (!nv_vgpu_mdev_types[i]) { + ret = -ENOMEM; + goto err_types; + } + } + + nv_vgpu_mdev_types[0]->sysfs_name = "nvidia-vgpu-ampere"; + nv_vgpu_mdev_types[0]->pretty_name = "NVIDIA Ampere vGPU"; + nv_vgpu_mdev_types[1]->sysfs_name = "nvidia-vgpu-hopper"; + nv_vgpu_mdev_types[1]->pretty_name = "NVIDIA Hopper vGPU"; + nv_vgpu_mdev_types[2]->sysfs_name = "nvidia-vgpu-turing"; + nv_vgpu_mdev_types[2]->pretty_name = "NVIDIA Turing vGPU"; + nv_vgpu_mdev_types[3]->sysfs_name = "nvidia-vgpu-pascal"; + nv_vgpu_mdev_types[3]->pretty_name = "NVIDIA Pascal vGPU"; + + ret = mdev_register_driver(&nv_vgpu_mdev_driver); + if (ret) { + pr_err("Failed to register mdev driver: %d\n", ret); + goto err_types; + } + + /* Find the parent PCI device to register mdev parent */ + pdev = pci_get_device(0x10de, 0x1ed3, NULL); + if (!pdev) { + pr_err("Failed to find NVIDIA GPU (10de:1ed3) for mdev parent registration\n"); + ret = -ENODEV; + goto err_driver; + } + + /* Register mdev parent via nvidia.ko */ + ret = nv_vgpu_mdev_register_parent(pdev); + pci_dev_put(pdev); + if (ret) { + pr_err("Failed to register mdev parent: %d\n", ret); + goto err_driver; + } + + /* Provide VFIO ops to host driver (nvidia.ko) */ + ret = nvidia_vgpu_vfio_set_ops(&nv_vgpu_vfio_ops); + if (ret) { + pr_err("Failed to set VFIO ops: %d\n", ret); + goto err_driver; + } + + pr_info("NVIDIA vGPU VFIO driver loaded (version %s)\n", NVIDIA_VGPU_VFIO_VERSION); + return 0; + +err_driver: + mdev_unregister_driver(&nv_vgpu_mdev_driver); +err_types: + for (i = 0; i < 4; i++) { + kfree(nv_vgpu_mdev_types[i]); + nv_vgpu_mdev_types[i] = NULL; + } + class_destroy(nv_vgpu_driver.class); + return ret; +} + +static void __exit nv_vgpu_exit(void) +{ + int i; + struct pci_dev *pdev = NULL; + + if (nv_vgpu_driver.device_count > 0) { + struct nv_vgpu_device *vgpu; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + if (vgpu->pdev) { + pdev = vgpu->pdev; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + } + + if (!pdev) { + pdev = pci_get_device(0x10de, 0x1ed3, NULL); + } + + /* Release host vGPU ops */ + if (pdev) { + rm_vgpu_vfio_ops_t ops = {0}; + ops.version_string = NV_VERSION_STRING; + nvidia_vgpu_vfio_get_ops(&ops); + } + + /* Unregister mdev parent via nvidia.ko */ + if (pdev) { + nv_vgpu_mdev_unregister_parent(pdev); + pr_info("nvidia-vgpu-vfio: mdev parent unregistered for %s\n", dev_name(&pdev->dev)); + pci_dev_put(pdev); + } + + mdev_unregister_driver(&nv_vgpu_mdev_driver); + + class_destroy(nv_vgpu_driver.class); + + for (i = 0; i < 4; i++) { + kfree(nv_vgpu_mdev_types[i]); + nv_vgpu_mdev_types[i] = NULL; + } + + pr_info("NVIDIA vGPU VFIO driver unloaded\n"); +} + +static void nv_vgpu_device_init_work(struct work_struct *work) +{ + struct nv_vgpu_device *vgpu = container_of(work, struct nv_vgpu_device, init_work); + int ret; + + mutex_lock(&vgpu->lock); + vgpu->state = NV_VGPU_STATE_INITIALIZING; + mutex_unlock(&vgpu->lock); + + ret = nv_vgpu_bar_init(vgpu); + if (ret) { + nv_vgpu_err(vgpu, "BAR init failed: %d", ret); + goto err_state; + } + + ret = nv_vgpu_irq_init(vgpu); + if (ret) { + nv_vgpu_err(vgpu, "IRQ init failed: %d", ret); + goto err_bar; + } + + ret = nv_vgpu_memory_init(vgpu); + if (ret) { + nv_vgpu_err(vgpu, "Memory init failed: %d", ret); + goto err_irq; + } + + ret = nv_vgpu_gsp_init(vgpu); + if (ret) { + nv_vgpu_err(vgpu, "GSP init failed: %d", ret); + goto err_memory; + } + + ret = nv_vgpu_migration_init(vgpu); + if (ret) { + nv_vgpu_warn(vgpu, "Migration init failed: %d", ret); + } + + mutex_lock(&vgpu->lock); + vgpu->state = NV_VGPU_STATE_READY; + mutex_unlock(&vgpu->lock); + + nv_vgpu_info(vgpu, "vGPU device initialized successfully"); + return; + +err_memory: + nv_vgpu_memory_cleanup(vgpu); +err_irq: + nv_vgpu_irq_cleanup(vgpu); +err_bar: + nv_vgpu_bar_cleanup(vgpu); +err_state: + mutex_lock(&vgpu->lock); + vgpu->state = NV_VGPU_STATE_ERROR; + mutex_unlock(&vgpu->lock); +} + +static void nv_vgpu_device_cleanup_work(struct work_struct *work) +{ + struct nv_vgpu_device *vgpu = container_of(work, struct nv_vgpu_device, cleanup_work); + + nv_vgpu_migration_cleanup(vgpu); + nv_vgpu_gsp_cleanup(vgpu); + nv_vgpu_memory_cleanup(vgpu); + nv_vgpu_irq_cleanup(vgpu); + nv_vgpu_bar_cleanup(vgpu); + + mutex_lock(&vgpu->lock); + vgpu->state = NV_VGPU_STATE_UNINITIALIZED; + mutex_unlock(&vgpu->lock); + + nv_vgpu_info(vgpu, "vGPU device cleaned up"); +} + +static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, NvU32 *vgpu_id) +{ + pr_info("nvidia-vgpu-vfio: VFIO probe called for vGPU type %u\n", vgpu_type_id); + return NV_OK; +} + +static void nv_vgpu_vfio_remove(struct pci_dev *pdev) +{ + pr_info("nvidia-vgpu-vfio: VFIO remove called\n"); +} + +static NV_STATUS nv_vgpu_vfio_inject_interrupt(void *vgpu_dev) +{ + pr_info("nvidia-vgpu-vfio: VFIO inject interrupt called\n"); + return NV_OK; +} + +/* ============================================================ + * PCI Config Space for VFIO mdev + * ============================================================ */ + +static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev *pdev) +{ + u8 *cfg = vgpu->pci_config; + u16 vendor_id = 0x10de; /* NVIDIA */ + u16 device_id = 0x1ed3; /* TU104 vGPU */ + u16 subsys_vendor = 0x10de; + u16 subsys_device = 0x12b4; + u8 revision = 0xa1; + u8 prog_if = 0x00; + u8 subclass = 0x00; + u8 class_code = 0x03; /* Display controller */ + u8 cache_line = 0x10; + u8 latency = 0x00; + u8 header_type = 0x00; /* Normal device */ + u8 bist = 0x00; + + memset(cfg, 0, 256); + + /* PCI Header (offset 0x00-0x3F) */ + cfg[0x00] = vendor_id & 0xff; + cfg[0x01] = (vendor_id >> 8) & 0xff; + cfg[0x02] = device_id & 0xff; + cfg[0x03] = (device_id >> 8) & 0xff; + + /* Command register (0x04) - Memory + IO + BusMaster */ + cfg[0x04] = 0x07; + cfg[0x05] = 0x00; + + /* Status register (0x06) */ + cfg[0x06] = 0x10; + cfg[0x07] = 0x00; + + /* Revision ID (0x08) */ + cfg[0x08] = revision; + + /* Class code (0x09-0x0B) */ + cfg[0x09] = prog_if; + cfg[0x0a] = subclass; + cfg[0x0b] = class_code; + + /* Cache line size (0x0C) */ + cfg[0x0c] = cache_line; + + /* Latency timer (0x0D) */ + cfg[0x0d] = latency; + + /* Header type (0x0E) */ + cfg[0x0e] = header_type; + + /* BIST (0x0F) */ + cfg[0x0f] = bist; + + /* BAR0 - 16MB Memory space (prefetchable) */ + cfg[0x10] = 0x04; /* 64-bit, prefetchable */ + cfg[0x11] = 0x00; + cfg[0x12] = 0x00; + cfg[0x13] = 0x00; + cfg[0x14] = 0x00; + cfg[0x15] = 0x00; + cfg[0x16] = 0x00; + cfg[0x17] = 0x00; + + /* BAR1 - 256MB Memory space (prefetchable) */ + cfg[0x18] = 0x04; + cfg[0x19] = 0x00; + cfg[0x1a] = 0x00; + cfg[0x1b] = 0x00; + cfg[0x1c] = 0x00; + cfg[0x1d] = 0x00; + cfg[0x1e] = 0x00; + cfg[0x1f] = 0x00; + + /* BAR2 - 32MB Memory space */ + cfg[0x20] = 0x04; + cfg[0x21] = 0x00; + cfg[0x22] = 0x00; + cfg[0x23] = 0x00; + cfg[0x24] = 0x00; + cfg[0x25] = 0x00; + cfg[0x26] = 0x00; + cfg[0x27] = 0x00; + + /* BAR3 - 32MB Memory space */ + cfg[0x28] = 0x04; + cfg[0x29] = 0x00; + cfg[0x2a] = 0x00; + cfg[0x2b] = 0x00; + cfg[0x2c] = 0x00; + cfg[0x2d] = 0x00; + cfg[0x2e] = 0x00; + cfg[0x2f] = 0x00; + + /* BAR4 - Not used */ + cfg[0x30] = 0x00; + cfg[0x31] = 0x00; + cfg[0x32] = 0x00; + cfg[0x33] = 0x00; + + /* BAR5 - Not used */ + cfg[0x34] = 0x00; + cfg[0x35] = 0x00; + cfg[0x36] = 0x00; + cfg[0x37] = 0x00; + + /* CardBus CIS Pointer (0x38-0x3B) - Not used */ + cfg[0x38] = 0x00; + cfg[0x39] = 0x00; + cfg[0x3a] = 0x00; + cfg[0x3b] = 0x00; + + /* Subsystem Vendor ID (0x3C) */ + cfg[0x3c] = subsys_vendor & 0xff; + cfg[0x3d] = (subsys_vendor >> 8) & 0xff; + + /* Subsystem ID (0x3E) */ + cfg[0x3e] = subsys_device & 0xff; + cfg[0x3f] = (subsys_device >> 8) & 0xff; + + /* Expansion ROM Base Address (0x40-0x43) - Not used */ + + /* Capabilities Pointer (0x34) - Point to Power Management at 0x40 */ + cfg[0x34] = 0x40; + cfg[0x35] = 0x00; + cfg[0x36] = 0x00; + cfg[0x37] = 0x00; + + /* Interrupt Line (0x3C) - Will be set by guest */ + cfg[0x3c] = 0x00; + /* Interrupt Pin (0x3D) - INTA */ + cfg[0x3d] = 0x01; + /* Min_Gnt (0x3E) */ + cfg[0x3e] = 0x00; + /* Max_Lat (0x3F) */ + cfg[0x3f] = 0x00; + + /* Power Management Capability (0x40) */ + cfg[0x40] = 0x01; /* Capability ID: Power Management */ + cfg[0x41] = 0x50; /* Next capability: MSI at 0x50 */ + cfg[0x42] = 0x03; /* PME support from D0, D3hot, D3cold */ + cfg[0x43] = 0x00; + cfg[0x44] = 0x00; /* PM Control/Status */ + cfg[0x45] = 0x00; + cfg[0x46] = 0x00; + cfg[0x47] = 0x00; + + /* MSI Capability (0x50) */ + cfg[0x50] = 0x05; /* Capability ID: MSI */ + cfg[0x51] = 0x70; /* Next capability: MSI-X at 0x70 */ + cfg[0x52] = 0x80; /* 64-bit capable, 1 vector */ + cfg[0x53] = 0x00; + cfg[0x54] = 0x00; /* Message Address (low) - set by guest */ + cfg[0x55] = 0x00; + cfg[0x56] = 0x00; + cfg[0x57] = 0x00; + cfg[0x58] = 0x00; /* Message Address (high) - 64-bit */ + cfg[0x59] = 0x00; + cfg[0x5a] = 0x00; + cfg[0x5b] = 0x00; + cfg[0x5c] = 0x01; /* Message Data - set by guest */ + cfg[0x5d] = 0x00; + + /* MSI-X Capability (0x70) */ + cfg[0x70] = 0x11; /* Capability ID: MSI-X */ + cfg[0x71] = 0x00; /* Next capability: None */ + cfg[0x72] = 0x00; /* Table size = 0 (1 vector), 64-bit */ + cfg[0x73] = 0x00; + /* Table Offset/BIR (0x74) - Table at BAR0 offset 0x2000 */ + cfg[0x74] = 0x00; + cfg[0x75] = 0x20; + cfg[0x76] = 0x00; + cfg[0x77] = 0x00; + /* PBA Offset/BIR (0x78) - PBA at BAR0 offset 0x3000 */ + cfg[0x78] = 0x00; + cfg[0x79] = 0x30; + cfg[0x7a] = 0x00; + cfg[0x7b] = 0x00; + + /* PCI Express Capability (0x80) */ + cfg[0x80] = 0x10; /* Capability ID: PCI Express */ + cfg[0x81] = 0x00; /* Next capability: None */ + cfg[0x82] = 0x02; /* Device type: PCIe Endpoint, version 2 */ + cfg[0x83] = 0x00; + cfg[0x84] = 0x00; /* Device Capabilities */ + cfg[0x85] = 0x00; + cfg[0x86] = 0x00; + cfg[0x87] = 0x00; + cfg[0x88] = 0x00; /* Device Control */ + cfg[0x89] = 0x00; + cfg[0x8a] = 0x10; /* Device Status */ + cfg[0x8b] = 0x00; + cfg[0x8c] = 0x00; /* Link Capabilities */ + cfg[0x8d] = 0x00; + cfg[0x8e] = 0x00; + cfg[0x8f] = 0x00; + cfg[0x90] = 0x00; /* Link Control */ + cfg[0x91] = 0x00; + cfg[0x92] = 0x00; /* Link Status */ + cfg[0x93] = 0x00; + cfg[0x94] = 0x00; /* Slot Capabilities */ + cfg[0x95] = 0x00; + cfg[0x96] = 0x00; + cfg[0x97] = 0x00; + cfg[0x98] = 0x00; /* Slot Control */ + cfg[0x99] = 0x00; + cfg[0x9a] = 0x00; /* Slot Status */ + cfg[0x9b] = 0x00; + cfg[0x9c] = 0x00; /* Root Capabilities */ + cfg[0x9d] = 0x00; + cfg[0x9e] = 0x00; /* Root Control */ + cfg[0x9f] = 0x00; + + vgpu->config_space_dirty = false; + nv_vgpu_info(vgpu, "PCI config space initialized (vendor=0x%04x, device=0x%04x)", vendor_id, device_id); + return 0; +} + +static ssize_t nv_vgpu_config_read(struct nv_vgpu_device *vgpu, char __user *buf, size_t count, loff_t *ppos) +{ + size_t bytes_read = 0; + loff_t pos = *ppos; + + if (pos >= 256) + return 0; + + if (pos + count > 256) + count = 256 - pos; + + if (copy_to_user(buf, vgpu->pci_config + pos, count)) + return -EFAULT; + + *ppos += count; + return count; +} + +static ssize_t nv_vgpu_config_write(struct nv_vgpu_device *vgpu, const char __user *buf, size_t count, loff_t *ppos) +{ + loff_t pos = *ppos; + u8 val; + + if (pos >= 256) + return -EINVAL; + + if (pos + count > 256) + count = 256 - pos; + + /* Handle specific writable registers */ + while (count--) { + if (copy_from_user(&val, buf++, 1)) + return -EFAULT; + + switch (pos) { + case 0x04: /* Command register */ + /* Allow Memory, IO, BusMaster */ + vgpu->pci_config[pos] = val & 0x07; + break; + case 0x0c: /* Cache line size */ + vgpu->pci_config[pos] = val; + break; + case 0x0d: /* Latency timer */ + vgpu->pci_config[pos] = val; + break; + case 0x3c: /* Interrupt line */ + vgpu->pci_config[pos] = val; + break; + case 0x54: case 0x55: case 0x56: case 0x57: /* MSI Message Address Low */ + case 0x58: case 0x59: case 0x5a: case 0x5b: /* MSI Message Address High */ + case 0x5c: case 0x5d: /* MSI Message Data */ + vgpu->pci_config[pos] = val; + break; + case 0x88: case 0x89: /* Device Control */ + case 0x90: case 0x91: /* Link Control */ + case 0x98: case 0x99: /* Slot Control */ + case 0x9e: case 0x9f: /* Root Control */ + vgpu->pci_config[pos] = val; + break; + default: + /* Most config space is read-only for guest */ + break; + } + pos++; + } + + vgpu->config_space_dirty = true; + *ppos = pos; + return pos - *ppos + count; +} + +module_init(nv_vgpu_init); +module_exit(nv_vgpu_exit); diff --git a/kernel/memory.c b/kernel/memory.c new file mode 100644 index 0000000..710799d --- /dev/null +++ b/kernel/memory.c @@ -0,0 +1,285 @@ +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/mm.h> +#include <linux/dma-mapping.h> +#include <linux/iommu.h> +#include <linux/vmalloc.h> +#include <linux/highmem.h> +#include <linux/pfn.h> +#include <linux/scatterlist.h> + +#include "nvidia-vgpu-vfio.h" +#include "memory.h" + +struct nv_vgpu_simple_mapping { + u64 gpa; + u64 size; + u64 iova; + struct page **pages; + int page_count; + bool pinned; + bool mapped; + struct list_head list; +}; + +static LIST_HEAD(simple_mappings); +static DEFINE_MUTEX(simple_lock); + +int nv_vgpu_memory_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_mm_ctx *ctx; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: Memory init validation (simple fallback)"); + return 0; + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + mutex_init(&ctx->lock); + INIT_LIST_HEAD(&ctx->mappings); + ctx->mapping_count = 0; + ctx->iommu_domain = NULL; + ctx->use_simple = true; + + vgpu->mm_ctx = ctx; + nv_vgpu_info(vgpu, "Memory init: using simple fallback (no IOMMU)"); + return 0; +} + +void nv_vgpu_memory_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping, *tmp; + + if (!ctx) + return; + + if (ctx->use_simple) { + mutex_lock(&simple_lock); + list_for_each_entry_safe(mapping, tmp, &simple_mappings, list) { + if (mapping->mapped) { + dma_unmap_page(&vgpu->pdev->dev, mapping->iova, mapping->size, DMA_BIDIRECTIONAL); + mapping->mapped = false; + } + if (mapping->pinned) { + for (int i = 0; i < mapping->page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + mapping->pinned = false; + } + list_del(&mapping->list); + vfree(mapping->pages); + kfree(mapping); + } + mutex_unlock(&simple_lock); + } else if (ctx->iommu_domain) { + iommu_detach_device(ctx->iommu_domain, &vgpu->pdev->dev); + iommu_domain_free(ctx->iommu_domain); + } + + kfree(ctx); + vgpu->mm_ctx = NULL; +} + +int nv_vgpu_memory_pin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping; + u64 start_pfn = gpa >> PAGE_SHIFT; + u64 end_pfn = (gpa + size + PAGE_SIZE - 1) >> PAGE_SHIFT; + int page_count = end_pfn - start_pfn; + int i, ret = 0; + + if (!ctx) + return -ENODEV; + + if (!ctx->use_simple) + return -ENODEV; + + mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) + return -ENOMEM; + + mapping->gpa = gpa; + mapping->size = size; + mapping->page_count = page_count; + mapping->pages = vzalloc(page_count * sizeof(struct page *)); + if (!mapping->pages) { + kfree(mapping); + return -ENOMEM; + } + + for (i = 0; i < page_count; i++) { + u64 pfn = start_pfn + i; + struct page *page = pfn_to_page(pfn); + if (!page) { + ret = -EFAULT; + goto err_unlock; + } + get_page(page); + mapping->pages[i] = page; + } + + mapping->pinned = true; + mutex_lock(&simple_lock); + list_add_tail(&mapping->list, &simple_mappings); + mutex_unlock(&simple_lock); + + return 0; + +err_unlock: + for (i = 0; i < page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + vfree(mapping->pages); + kfree(mapping); + return ret; +} + +int nv_vgpu_memory_unpin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping; + + if (!ctx || !ctx->use_simple) + return -ENODEV; + + mutex_lock(&simple_lock); + list_for_each_entry(mapping, &simple_mappings, list) { + if (mapping->gpa == gpa && mapping->size == size) { + if (mapping->mapped) { + mutex_unlock(&simple_lock); + return -EBUSY; + } else if (mapping->pinned) { + for (int i = 0; i < mapping->page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + mapping->pinned = false; + } + list_del(&mapping->list); + vfree(mapping->pages); + kfree(mapping); + break; + } + } + mutex_unlock(&simple_lock); + + return 0; +} + +int nv_vgpu_memory_map(struct nv_vgpu_device *vgpu, u64 gpa, u64 size, u64 *iova) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping; + u64 map_iova; + int ret = 0; + + if (!ctx || !ctx->use_simple) + return -ENODEV; + + mutex_lock(&simple_lock); + list_for_each_entry(mapping, &simple_mappings, list) { + if (mapping->gpa == gpa && mapping->size == size) + break; + } + if (&mapping->list == &simple_mappings) { + mutex_unlock(&simple_lock); + return -ENOENT; + } + + if (!mapping->pinned) { + mutex_unlock(&simple_lock); + return -EINVAL; + } + + map_iova = dma_map_page(&vgpu->pdev->dev, mapping->pages[0], 0, size, DMA_BIDIRECTIONAL); + if (dma_mapping_error(&vgpu->pdev->dev, map_iova)) { + mutex_unlock(&simple_lock); + return -ENOMEM; + } + + mapping->iova = map_iova; + mapping->mapped = true; + *iova = map_iova; + mutex_unlock(&simple_lock); + + return ret; +} + +int nv_vgpu_memory_unmap(struct nv_vgpu_device *vgpu, u64 iova, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping; + int ret = 0; + + if (!ctx || !ctx->use_simple) + return -ENODEV; + + mutex_lock(&simple_lock); + list_for_each_entry(mapping, &simple_mappings, list) { + if (mapping->iova == iova && mapping->size == size) { + if (!mapping->mapped) { + mutex_unlock(&simple_lock); + return -ENOENT; + } + dma_unmap_page(&vgpu->pdev->dev, iova, size, DMA_BIDIRECTIONAL); + mapping->mapped = false; + mapping->iova = 0; + break; + } + } + mutex_unlock(&simple_lock); + + return ret; +} + +static vm_fault_t nv_vgpu_vm_fault(struct vm_fault *vmf) +{ + struct nv_vgpu_device *vgpu = vmf->vma->vm_private_data; + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping; + u64 offset = vmf->address - vmf->vma->vm_start + vmf->vma->vm_pgoff * PAGE_SIZE; + u64 pfn; + vm_fault_t ret = VM_FAULT_SIGBUS; + + if (!ctx) + return VM_FAULT_SIGBUS; + + if (!ctx->use_simple) + return VM_FAULT_SIGBUS; + + mutex_lock(&simple_lock); + list_for_each_entry(mapping, &simple_mappings, list) { + if (mapping->mapped && offset >= mapping->iova && + offset < mapping->iova + mapping->size) { + pfn = (mapping->gpa + offset - mapping->iova) >> PAGE_SHIFT; + vmf->page = pfn_to_page(pfn); + get_page(vmf->page); + ret = VM_FAULT_NOPAGE; + break; + } + } + mutex_unlock(&simple_lock); + + return ret; +} + +static const struct vm_operations_struct nv_vgpu_vm_ops = { + .fault = nv_vgpu_vm_fault, +}; + +int nv_vgpu_memory_mmap(struct nv_vgpu_device *vgpu, struct vm_area_struct *vma) +{ + vma->vm_ops = &nv_vgpu_vm_ops; + vma->vm_private_data = vgpu; + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); + return 0; +} diff --git a/kernel/memory.c.iommu b/kernel/memory.c.iommu new file mode 100644 index 0000000..335446b --- /dev/null +++ b/kernel/memory.c.iommu @@ -0,0 +1,335 @@ +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/mm.h> +#include <linux/dma-mapping.h> +#include <linux/iommu.h> +#include <linux/vmalloc.h> +#include <linux/highmem.h> +#include <linux/pfn.h> +#include <linux/scatterlist.h> + +#include "nvidia-vgpu-vfio.h" +#include "memory.h" + +int nv_vgpu_memory_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_mm_ctx *ctx; + int ret; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: Memory init validation"); + /* Check IOMMU support */ + if (iommu_present(&pci_bus_type)) { + nv_vgpu_info(vgpu, "DRY-RUN: IOMMU present"); + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + ctx->iommu_domain = iommu_domain_alloc(&pci_bus_type); + if (ctx->iommu_domain) { + nv_vgpu_info(vgpu, "DRY-RUN: IOMMU domain allocatable"); + iommu_domain_free(ctx->iommu_domain); + } else { + nv_vgpu_warn(vgpu, "DRY-RUN: IOMMU domain allocation failed"); + } + kfree(ctx); + } else { + nv_vgpu_warn(vgpu, "DRY-RUN: No IOMMU present"); + } + return 0; + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + mutex_init(&ctx->lock); + INIT_LIST_HEAD(&ctx->mappings); + ctx->mapping_count = 0; + + ctx->iommu_domain = iommu_domain_alloc(&pci_bus_type); + if (!ctx->iommu_domain) { + ret = -ENODEV; + goto err_free; + } + + ret = iommu_attach_device(ctx->iommu_domain, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to attach IOMMU domain: %d", ret); + } + + vgpu->mm_ctx = ctx; + return 0; + +err_free: + kfree(ctx); + return ret; +} + +void nv_vgpu_memory_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping, *tmp; + + if (!ctx) + return; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: Memory cleanup (no hardware resources to release)"); + if (ctx->iommu_domain) + iommu_domain_free(ctx->iommu_domain); + kfree(ctx); + vgpu->mm_ctx = NULL; + return; + } + + mutex_lock(&ctx->lock); + list_for_each_entry_safe(mapping, tmp, &ctx->mappings, list) { + if (mapping->mapped) { + iommu_unmap(ctx->iommu_domain, mapping->iova, mapping->size); + mapping->mapped = false; + } + if (mapping->pinned) { + int i; + for (i = 0; i < mapping->page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + mapping->pinned = false; + } + list_del(&mapping->list); + vfree(mapping->pages); + kfree(mapping); + ctx->mapping_count--; + } + mutex_unlock(&ctx->lock); + + if (ctx->iommu_domain) { + iommu_detach_device(ctx->iommu_domain, &vgpu->pdev->dev); + iommu_domain_free(ctx->iommu_domain); + } + + kfree(ctx); + vgpu->mm_ctx = NULL; +} + +static struct nv_vgpu_mapping *nv_vgpu_find_mapping(struct nv_vgpu_mm_ctx *ctx, u64 iova, u64 size) +{ + struct nv_vgpu_mapping *mapping; + + list_for_each_entry(mapping, &ctx->mappings, list) { + if (mapping->iova == iova && mapping->size == size) + return mapping; + } + return NULL; +} + +int nv_vgpu_memory_pin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping; + u64 start_pfn = gpa >> PAGE_SHIFT; + u64 end_pfn = (gpa + size + PAGE_SIZE - 1) >> PAGE_SHIFT; + int page_count = end_pfn - start_pfn; + int i, ret = 0; + + if (!ctx) + return -ENODEV; + + mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) + return -ENOMEM; + + mapping->gpa = gpa; + mapping->size = size; + mapping->page_count = page_count; + mapping->pages = vzalloc(page_count * sizeof(struct page *)); + if (!mapping->pages) { + kfree(mapping); + return -ENOMEM; + } + + mutex_lock(&ctx->lock); + + for (i = 0; i < page_count; i++) { + u64 pfn = start_pfn + i; + struct page *page = pfn_to_page(pfn); + if (!page) { + ret = -EFAULT; + goto err_unlock; + } + get_page(page); + mapping->pages[i] = page; + } + + mapping->pinned = true; + list_add_tail(&mapping->list, &ctx->mappings); + ctx->mapping_count++; + + mutex_unlock(&ctx->lock); + return 0; + +err_unlock: + for (i = 0; i < page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + vfree(mapping->pages); + kfree(mapping); + mutex_unlock(&ctx->lock); + return ret; +} + +int nv_vgpu_memory_unpin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping; + u64 iova = 0; + int ret = 0; + + if (!ctx) + return -ENODEV; + + mutex_lock(&ctx->lock); + list_for_each_entry(mapping, &ctx->mappings, list) { + if (mapping->gpa == gpa && mapping->size == size) { + if (mapping->mapped) { + iova = mapping->iova; + ret = -EBUSY; + } else if (mapping->pinned) { + int i; + for (i = 0; i < mapping->page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + mapping->pinned = false; + } + list_del(&mapping->list); + ctx->mapping_count--; + vfree(mapping->pages); + kfree(mapping); + break; + } + } + mutex_unlock(&ctx->lock); + + return ret; +} + +int nv_vgpu_memory_map(struct nv_vgpu_device *vgpu, u64 gpa, u64 size, u64 *iova) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping; + u64 map_iova; + int ret; + + if (!ctx || !ctx->iommu_domain) + return -ENODEV; + + mutex_lock(&ctx->lock); + mapping = nv_vgpu_find_mapping(ctx, 0, 0); + if (!mapping || mapping->gpa != gpa || mapping->size != size) { + list_for_each_entry(mapping, &ctx->mappings, list) { + if (mapping->gpa == gpa && mapping->size == size) + break; + } + if (&mapping->list == &ctx->mappings) { + mutex_unlock(&ctx->lock); + return -ENOENT; + } + } + + if (!mapping->pinned) { + mutex_unlock(&ctx->lock); + return -EINVAL; + } + + map_iova = dma_map_page(&vgpu->pdev->dev, mapping->pages[0], 0, size, DMA_BIDIRECTIONAL); + if (dma_mapping_error(&vgpu->pdev->dev, map_iova)) { + mutex_unlock(&ctx->lock); + return -ENOMEM; + } + + mapping->iova = map_iova; + mapping->mapped = true; + *iova = map_iova; + + ret = iommu_map(ctx->iommu_domain, map_iova, gpa, size, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE, GFP_KERNEL); + if (ret) { + dma_unmap_page(&vgpu->pdev->dev, map_iova, size, DMA_BIDIRECTIONAL); + mapping->mapped = false; + mapping->iova = 0; + } + + mutex_unlock(&ctx->lock); + return ret; +} + +int nv_vgpu_memory_unmap(struct nv_vgpu_device *vgpu, u64 iova, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping; + int ret = 0; + + if (!ctx || !ctx->iommu_domain) + return -ENODEV; + + mutex_lock(&ctx->lock); + mapping = nv_vgpu_find_mapping(ctx, iova, size); + if (!mapping || !mapping->mapped) { + mutex_unlock(&ctx->lock); + return -ENOENT; + } + + ret = iommu_unmap(ctx->iommu_domain, iova, size); + if (ret == 0) { + dma_unmap_page(&vgpu->pdev->dev, iova, size, DMA_BIDIRECTIONAL); + mapping->mapped = false; + mapping->iova = 0; + } + + mutex_unlock(&ctx->lock); + return ret; +} + +static vm_fault_t nv_vgpu_vm_fault(struct vm_fault *vmf) +{ + struct nv_vgpu_device *vgpu = vmf->vma->vm_private_data; + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping; + u64 offset = vmf->address - vmf->vma->vm_start + vmf->vma->vm_pgoff * PAGE_SIZE; + u64 pfn; + vm_fault_t ret = VM_FAULT_SIGBUS; + + if (!ctx) + return VM_FAULT_SIGBUS; + + mutex_lock(&ctx->lock); + list_for_each_entry(mapping, &ctx->mappings, list) { + if (mapping->mapped && offset >= mapping->iova && + offset < mapping->iova + mapping->size) { + pfn = (mapping->gpa + offset - mapping->iova) >> PAGE_SHIFT; + vmf->page = pfn_to_page(pfn); + get_page(vmf->page); + ret = VM_FAULT_NOPAGE; + break; + } + } + mutex_unlock(&ctx->lock); + + return ret; +} + +static const struct vm_operations_struct nv_vgpu_vm_ops = { + .fault = nv_vgpu_vm_fault, +}; + +int nv_vgpu_memory_mmap(struct nv_vgpu_device *vgpu, struct vm_area_struct *vma) +{ + vma->vm_ops = &nv_vgpu_vm_ops; + vma->vm_private_data = vgpu; + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); + return 0; +}
\ No newline at end of file diff --git a/kernel/migration.c b/kernel/migration.c new file mode 100644 index 0000000..c8814a2 --- /dev/null +++ b/kernel/migration.c @@ -0,0 +1,324 @@ +#include <linux/slab.h> +#include <linux/mutex.h> +#include <linux/crc32.h> +#include <linux/kthread.h> +#include <linux/completion.h> + +#include "nvidia-vgpu-vfio.h" +#include "migration.h" + +static int nv_vgpu_migration_save_thread(void *data) +{ + struct nv_vgpu_device *vgpu = data; + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + u8 *buffer; + size_t size; + int ret; + + ret = nv_vgpu_migration_save_state(vgpu, &buffer, &size); + if (ret) { + ctx->state.result = ret; + } else { + ctx->state.data = buffer; + ctx->state.data_size = size; + ctx->state.result = 0; + } + + complete(&ctx->state.done); + return 0; +} + +static int nv_vgpu_migration_restore_thread(void *data) +{ + struct nv_vgpu_device *vgpu = data; + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + int ret; + + ret = nv_vgpu_migration_restore_state(vgpu, ctx->state.data, ctx->state.data_size); + ctx->state.result = ret; + complete(&ctx->state.done); + return 0; +} + +int nv_vgpu_migration_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_migration_ctx *ctx; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + mutex_init(&ctx->lock); + init_completion(&ctx->state.done); + ctx->in_progress = false; + + vgpu->migration.ctx = ctx; + return 0; +} + +void nv_vgpu_migration_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + + if (!ctx) + return; + + if (ctx->save_thread) { + kthread_stop(ctx->save_thread); + ctx->save_thread = NULL; + } + if (ctx->restore_thread) { + kthread_stop(ctx->restore_thread); + ctx->restore_thread = NULL; + } + + if (ctx->state.data) { + kfree(ctx->state.data); + ctx->state.data = NULL; + } + + kfree(ctx); + vgpu->migration.ctx = NULL; +} + +int nv_vgpu_migration_save_state(struct nv_vgpu_device *vgpu, u8 **data, size_t *size) +{ + struct nv_vgpu_migration_header *hdr; + struct nv_vgpu_migration_section *sections; + struct nv_vgpu_migration_device_state *dev_state; + struct nv_vgpu_migration_bar_state *bar_state; + u8 *buffer; + size_t total_size = 0; + int i, num_sections = 0; + + mutex_lock(&vgpu->lock); + + total_size = sizeof(struct nv_vgpu_migration_header) + + (8 * sizeof(struct nv_vgpu_migration_section)); + + for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) { + if (vgpu->bars[i].mapped) { + total_size += sizeof(struct nv_vgpu_migration_bar_state) + vgpu->bars[i].size; + num_sections++; + } + } + + buffer = kmalloc(total_size, GFP_KERNEL); + if (!buffer) { + mutex_unlock(&vgpu->lock); + return -ENOMEM; + } + + hdr = (struct nv_vgpu_migration_header *)buffer; + hdr->magic = NV_VGPU_MIGRATION_MAGIC; + hdr->version = NV_VGPU_MIGRATION_VERSION; + hdr->flags = 0; + hdr->num_sections = num_sections + 1; + hdr->total_size = total_size; + hdr->timestamp = ktime_get_ns(); + + sections = (struct nv_vgpu_migration_section *)(buffer + sizeof(*hdr)); + + dev_state = (struct nv_vgpu_migration_device_state *)(buffer + sizeof(*hdr) + + (hdr->num_sections * sizeof(*sections))); + + dev_state->vgpu_type = vgpu->type; + dev_state->vgpu_id = vgpu->vgpu_id; + dev_state->gpu_instance_id = vgpu->gpu_instance_id; + dev_state->compute_instance_id = vgpu->compute_instance_id; + dev_state->state = vgpu->state; + dev_state->fb_base = vgpu->fb_base; + dev_state->fb_size = vgpu->fb_size; + dev_state->bar0_config_base = vgpu->bar0_config_base; + + sections[0].type = NV_VGPU_MIG_SECTION_DEVICE_STATE; + sections[0].flags = 0; + sections[0].offset = (u8 *)dev_state - buffer; + sections[0].size = sizeof(*dev_state); + sections[0].checksum = crc32(0, dev_state, sizeof(*dev_state)); + + u8 *data_ptr = (u8 *)dev_state + sizeof(*dev_state); + for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) { + if (!vgpu->bars[i].mapped) + continue; + + bar_state = (struct nv_vgpu_migration_bar_state *)data_ptr; + bar_state->bar_index = i; + bar_state->gpa = vgpu->bars[i].gpa; + bar_state->size = vgpu->bars[i].size; + bar_state->iova = vgpu->bars[i].iova; + + if (vgpu->bars[i].vaddr && vgpu->bars[i].size > 0) { + memcpy(bar_state->data, vgpu->bars[i].vaddr, vgpu->bars[i].size); + } + + sections[num_sections].type = NV_VGPU_MIG_SECTION_BAR_STATE; + sections[num_sections].flags = 0; + sections[num_sections].offset = data_ptr - buffer; + sections[num_sections].size = sizeof(*bar_state) + vgpu->bars[i].size; + sections[num_sections].checksum = crc32(0, bar_state, sections[num_sections].size); + + data_ptr += sections[num_sections].size; + num_sections++; + } + + hdr->num_sections = num_sections; + hdr->total_size = data_ptr - buffer; + + *data = buffer; + *size = hdr->total_size; + + mutex_unlock(&vgpu->lock); + return 0; +} + +int nv_vgpu_migration_restore_state(struct nv_vgpu_device *vgpu, u8 *data, size_t size) +{ + struct nv_vgpu_migration_header *hdr = (struct nv_vgpu_migration_header *)data; + struct nv_vgpu_migration_section *sections; + struct nv_vgpu_migration_device_state *dev_state; + struct nv_vgpu_migration_bar_state *bar_state; + int i; + + if (!data || size < sizeof(*hdr)) + return -EINVAL; + + if (hdr->magic != NV_VGPU_MIGRATION_MAGIC) + return -EINVAL; + + if (hdr->version > NV_VGPU_MIGRATION_VERSION) + return -EINVAL; + + sections = (struct nv_vgpu_migration_section *)(data + sizeof(*hdr)); + + mutex_lock(&vgpu->lock); + + for (i = 0; i < hdr->num_sections; i++) { + u32 checksum; + u8 *section_data = data + sections[i].offset; + + checksum = crc32(0, section_data, sections[i].size); + if (checksum != sections[i].checksum) { + nv_vgpu_err(vgpu, "Migration checksum mismatch section %d", i); + mutex_unlock(&vgpu->lock); + return -EIO; + } + + switch (sections[i].type) { + case NV_VGPU_MIG_SECTION_DEVICE_STATE: + dev_state = (struct nv_vgpu_migration_device_state *)section_data; + vgpu->type = dev_state->vgpu_type; + vgpu->vgpu_id = dev_state->vgpu_id; + vgpu->gpu_instance_id = dev_state->gpu_instance_id; + vgpu->compute_instance_id = dev_state->compute_instance_id; + vgpu->state = dev_state->state; + vgpu->fb_base = dev_state->fb_base; + vgpu->fb_size = dev_state->fb_size; + vgpu->bar0_config_base = dev_state->bar0_config_base; + break; + + case NV_VGPU_MIG_SECTION_BAR_STATE: + bar_state = (struct nv_vgpu_migration_bar_state *)section_data; + if (bar_state->bar_index < NV_VGPU_MAX_REGIONS) { + vgpu->bars[bar_state->bar_index].gpa = bar_state->gpa; + vgpu->bars[bar_state->bar_index].size = bar_state->size; + vgpu->bars[bar_state->bar_index].iova = bar_state->iova; + if (vgpu->bars[bar_state->bar_index].vaddr && bar_state->size > 0) { + memcpy(vgpu->bars[bar_state->bar_index].vaddr, bar_state->data, bar_state->size); + } + } + break; + + default: + nv_vgpu_warn(vgpu, "Unknown migration section type: %d", sections[i].type); + break; + } + } + + mutex_unlock(&vgpu->lock); + return 0; +} + +int nv_vgpu_migration_prepare(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + + if (!ctx) + return -ENODEV; + + mutex_lock(&ctx->lock); + if (ctx->in_progress) { + mutex_unlock(&ctx->lock); + return -EBUSY; + } + ctx->in_progress = true; + ctx->state.version = NV_VGPU_MIGRATION_VERSION; + ctx->state.flags = 0; + ctx->state.timestamp = ktime_get_ns(); + reinit_completion(&ctx->state.done); + mutex_unlock(&ctx->lock); + + ctx->save_thread = kthread_run(nv_vgpu_migration_save_thread, vgpu, "nv-vgpu-mig-save"); + if (IS_ERR(ctx->save_thread)) { + ctx->in_progress = false; + return PTR_ERR(ctx->save_thread); + } + + return 0; +} + +int nv_vgpu_migration_complete(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + int ret; + + if (!ctx) + return -ENODEV; + + wait_for_completion(&ctx->state.done); + ret = ctx->state.result; + + mutex_lock(&ctx->lock); + ctx->in_progress = false; + ctx->save_thread = NULL; + mutex_unlock(&ctx->lock); + + return ret; +} + +int nv_vgpu_migration_cancel(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + + if (!ctx) + return -ENODEV; + + if (ctx->save_thread) { + kthread_stop(ctx->save_thread); + ctx->save_thread = NULL; + } + if (ctx->restore_thread) { + kthread_stop(ctx->restore_thread); + ctx->restore_thread = NULL; + } + + mutex_lock(&ctx->lock); + ctx->in_progress = false; + mutex_unlock(&ctx->lock); + + return 0; +} + +u8 *nv_vgpu_migration_get_data(struct nv_vgpu_device *vgpu, size_t *size) +{ + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + + if (!ctx || !ctx->state.data) + return NULL; + + if (size) + *size = ctx->state.data_size; + + return ctx->state.data; +}
\ No newline at end of file diff --git a/kernel/tests/Makefile b/kernel/tests/Makefile new file mode 100644 index 0000000..ce44670 --- /dev/null +++ b/kernel/tests/Makefile @@ -0,0 +1,40 @@ +# GSP vGPU-IO Driver Test Makefile + +# Comprehensive test module +obj-m += gsp_vgpu_comprehensive_test.o + +# KUnit test module (requires CONFIG_KUNIT) +obj-$(CONFIG_KUNIT) += gsp_vgpu_kunit_test.o + +# Include driver headers +ccflags-y := -I$OPEN_IOV_ROOT/driver/include + +# Build against the kernel source +KDIR ?= /lib/modules/$(shell uname -r)/build + +all: + $(MAKE) -C $(KDIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(KDIR) M=$(PWD) clean + +install: + $(MAKE) -C $(KDIR) M=$(PWD) modules_install + +test: all + @echo "Loading comprehensive test module..." + sudo insmod gsp_vgpu_comprehensive_test.ko + @echo "Test module loaded, checking dmesg..." + dmesg -T | tail -50 + +test-kunit: all + @echo "Loading KUnit test module..." + sudo insmod gsp_vgpu_kunit_test.ko + @echo "KUnit test module loaded, checking dmesg..." + dmesg -T | grep -E "(kunit|KUNIT|PASS|FAIL)" | tail -50 + +test-unload: + sudo rmmod gsp_vgpu_comprehensive_test gsp_vgpu_kunit_test 2>/dev/null || true + dmesg -T | tail -20 + +.PHONY: all clean install test test-kunit test-unload
\ No newline at end of file diff --git a/kernel/tests/Module.symvers b/kernel/tests/Module.symvers new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/kernel/tests/Module.symvers diff --git a/kernel/tests/gsp_rpc_test.c b/kernel/tests/gsp_rpc_test.c new file mode 100644 index 0000000..f609066 --- /dev/null +++ b/kernel/tests/gsp_rpc_test.c @@ -0,0 +1,425 @@ +/* + * GSP RPC Client Test/Validation Harness + * Tests message encoding/decoding, RPC structure validation + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/errno.h> + +#include "../include/gsp_rpc_client.h" +#include "../include/nvidia-vgpu-vfio.h" + +#define TEST_PASS 0 +#define TEST_FAIL -1 + +static int test_count = 0; +static int pass_count = 0; +static int fail_count = 0; + +#define TEST_ASSERT(cond, msg) \ + do { \ + test_count++; \ + if (cond) { \ + pass_count++; \ + pr_info("PASS: %s\n", msg); \ + } else { \ + fail_count++; \ + pr_err("FAIL: %s\n", msg); \ + } \ + } while (0) + +#define TEST_ASSERT_EQ(a, b, msg) \ + TEST_ASSERT((a) == (b), msg " (expected " #b ", got " #a ")") + +/* ============================================================ + * Structure Size/Alignment Tests + * ============================================================ */ + +static int test_structure_sizes(void) +{ + pr_info("=== Testing Structure Sizes ===\n"); + + TEST_ASSERT_EQ(sizeof(struct gsp_rpc_msg), 72, "gsp_rpc_msg size"); + TEST_ASSERT_EQ(sizeof(struct gsp_rpc_response), 64, "gsp_rpc_response size"); + TEST_ASSERT_EQ(sizeof(struct gsp_alloc_req), 40, "gsp_alloc_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_alloc_rsp), 24, "gsp_alloc_rsp size"); + TEST_ASSERT_EQ(sizeof(struct gsp_map_req), 48, "gsp_map_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_map_rsp), 8, "gsp_map_rsp size"); + TEST_ASSERT_EQ(sizeof(struct gsp_pin_req), 40, "gsp_pin_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_pin_rsp), 16, "gsp_pin_rsp size"); + TEST_ASSERT_EQ(sizeof(struct gsp_sched_req), 40, "gsp_sched_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_bar_trap_req), 32, "gsp_bar_trap_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_bar_trap_rsp), 16, "gsp_bar_trap_rsp size"); + TEST_ASSERT_EQ(sizeof(struct gsp_migrate_req), 56, "gsp_migrate_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_migrate_rsp), 16, "gsp_migrate_rsp size"); + + return TEST_PASS; +} + +static int test_structure_alignment(void) +{ + pr_info("=== Testing Structure Alignment ===\n"); + + TEST_ASSERT_EQ(__alignof__(struct gsp_rpc_msg), 8, "gsp_rpc_msg alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_rpc_response), 8, "gsp_rpc_response alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_alloc_req), 8, "gsp_alloc_req alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_map_req), 8, "gsp_map_req alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_pin_req), 8, "gsp_pin_req alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_sched_req), 8, "gsp_sched_req alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_bar_trap_req), 8, "gsp_bar_trap_req alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_migrate_req), 8, "gsp_migrate_req alignment"); + + return TEST_PASS; +} + +/* ============================================================ + * Message Encoding/Decoding Tests + * ============================================================ */ + +static int test_rpc_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_alloc_req *alloc_req = (struct gsp_alloc_req *)msg.payload; + + pr_info("=== Testing RPC Message Encoding ===\n"); + + /* Test ALLOC message encoding */ + msg.type = cpu_to_le32(GSP_RPC_ALLOC); + msg.subcmd = cpu_to_le32(GSP_ALLOC_SUBCMD_DEFAULT); + msg.seq_id = cpu_to_le32(0x12345678); + msg.flags = cpu_to_le32(0x00000001); + + alloc_req->size = cpu_to_le64(0x100000); + alloc_req->alignment = cpu_to_le64(0x1000); + alloc_req->mem_type = cpu_to_le32(0x2); + alloc_req->flags = cpu_to_le32(0x3); + alloc_req->addr_hint = cpu_to_le64(0x0); + alloc_req->pid = cpu_to_le32(0x456); + alloc_req->reserved = 0; + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_ALLOC, "RPC type field"); + TEST_ASSERT_EQ(le32_to_cpu(msg.subcmd), GSP_ALLOC_SUBCMD_DEFAULT, "RPC subcmd field"); + TEST_ASSERT_EQ(le32_to_cpu(msg.seq_id), 0x12345678, "RPC seq_id field"); + TEST_ASSERT_EQ(le32_to_cpu(msg.flags), 0x00000001, "RPC flags field"); + TEST_ASSERT_EQ(le64_to_cpu(alloc_req->size), 0x100000, "ALLOC size field"); + TEST_ASSERT_EQ(le64_to_cpu(alloc_req->alignment), 0x1000, "ALLOC alignment field"); + TEST_ASSERT_EQ(le32_to_cpu(alloc_req->mem_type), 0x2, "ALLOC mem_type field"); + TEST_ASSERT_EQ(le32_to_cpu(alloc_req->flags), 0x3, "ALLOC flags field"); + TEST_ASSERT_EQ(le64_to_cpu(alloc_req->addr_hint), 0x0, "ALLOC addr_hint field"); + TEST_ASSERT_EQ(le32_to_cpu(alloc_req->pid), 0x456, "ALLOC pid field"); + + return TEST_PASS; +} + +static int test_map_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_map_req *map_req = (struct gsp_map_req *)msg.payload; + + pr_info("=== Testing MAP Message Encoding ===\n"); + + msg.type = cpu_to_le32(GSP_RPC_MAP); + msg.subcmd = cpu_to_le32(0); + msg.seq_id = cpu_to_le32(0xABCDEF00); + + map_req->handle = cpu_to_le64(0xDEADBEEF); + map_req->gpu_va = cpu_to_le64(0x80000000); + map_req->size = cpu_to_le64(0x200000); + map_req->prot = cpu_to_le32(0x7); + map_req->flags = cpu_to_le32(0x1); + map_req->offset = cpu_to_le64(0x0); + map_req->reserved = 0; + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_MAP, "MAP RPC type"); + TEST_ASSERT_EQ(le64_to_cpu(map_req->handle), 0xDEADBEEF, "MAP handle"); + TEST_ASSERT_EQ(le64_to_cpu(map_req->gpu_va), 0x80000000, "MAP gpu_va"); + TEST_ASSERT_EQ(le64_to_cpu(map_req->size), 0x200000, "MAP size"); + TEST_ASSERT_EQ(le32_to_cpu(map_req->prot), 0x7, "MAP prot"); + TEST_ASSERT_EQ(le32_to_cpu(map_req->flags), 0x1, "MAP flags"); + TEST_ASSERT_EQ(le64_to_cpu(map_req->offset), 0x0, "MAP offset"); + + return TEST_PASS; +} + +static int test_pin_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_pin_req *pin_req = (struct gsp_pin_req *)msg.payload; + + pr_info("=== Testing PIN Message Encoding ===\n"); + + msg.type = cpu_to_le32(GSP_RPC_PIN); + msg.subcmd = cpu_to_le32(GSP_PIN_SUBCMD_DEFAULT); + + pin_req->handle = cpu_to_le64(0xCAFEBABE); + pin_req->offset = cpu_to_le64(0x1000); + pin_req->size = cpu_to_le64(0x10000); + pin_req->flags = cpu_to_le32(0x3); + pin_req->reserved[0] = 0; + pin_req->reserved[1] = 0; + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_PIN, "PIN RPC type"); + TEST_ASSERT_EQ(le32_to_cpu(msg.subcmd), GSP_PIN_SUBCMD_DEFAULT, "PIN subcmd"); + TEST_ASSERT_EQ(le64_to_cpu(pin_req->handle), 0xCAFEBABE, "PIN handle"); + TEST_ASSERT_EQ(le64_to_cpu(pin_req->offset), 0x1000, "PIN offset"); + TEST_ASSERT_EQ(le64_to_cpu(pin_req->size), 0x10000, "PIN size"); + TEST_ASSERT_EQ(le32_to_cpu(pin_req->flags), 0x3, "PIN flags"); + + return TEST_PASS; +} + +static int test_sched_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_sched_req *sched_req = (struct gsp_sched_req *)msg.payload; + + pr_info("=== Testing SCHEDULE Message Encoding ===\n"); + + msg.type = cpu_to_le32(GSP_RPC_SCHEDULE); + msg.subcmd = cpu_to_le32(GSP_SCHED_SUBCMD_DEFAULT); + + sched_req->work_handle = cpu_to_le64(0x11111111); + sched_req->callback = cpu_to_le64(0x22222222); + sched_req->arg = cpu_to_le64(0x33333333); + sched_req->priority = cpu_to_le32(0x5); + sched_req->flags = cpu_to_le32(0x0); + sched_req->reserved[0] = 0; + sched_req->reserved[1] = 0; + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_SCHEDULE, "SCHED RPC type"); + TEST_ASSERT_EQ(le64_to_cpu(sched_req->work_handle), 0x11111111, "SCHED work_handle"); + TEST_ASSERT_EQ(le64_to_cpu(sched_req->callback), 0x22222222, "SCHED callback"); + TEST_ASSERT_EQ(le64_to_cpu(sched_req->arg), 0x33333333, "SCHED arg"); + TEST_ASSERT_EQ(le32_to_cpu(sched_req->priority), 0x5, "SCHED priority"); + + return TEST_PASS; +} + +static int test_bar_trap_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_bar_trap_req *bar_req = (struct gsp_bar_trap_req *)msg.payload; + + pr_info("=== Testing BAR_TRAP Message Encoding ===\n"); + + msg.type = cpu_to_le32(GSP_RPC_BAR_TRAP); + msg.subcmd = cpu_to_le32(GSP_BAR_TRAP_SUBCMD_DEFAULT); + + bar_req->bar_addr = cpu_to_le64(0x00100000); + bar_req->access_type = cpu_to_le32(0x1); + bar_req->size = cpu_to_le32(0x4); + bar_req->value = cpu_to_le64(0xDEADBEEF); + bar_req->bar_num = cpu_to_le32(0x1); + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_BAR_TRAP, "BAR_TRAP RPC type"); + TEST_ASSERT_EQ(le64_to_cpu(bar_req->bar_addr), 0x00100000, "BAR_TRAP bar_addr"); + TEST_ASSERT_EQ(le32_to_cpu(bar_req->access_type), 0x1, "BAR_TRAP access_type"); + TEST_ASSERT_EQ(le32_to_cpu(bar_req->size), 0x4, "BAR_TRAP size"); + TEST_ASSERT_EQ(le64_to_cpu(bar_req->value), 0xDEADBEEF, "BAR_TRAP value"); + TEST_ASSERT_EQ(le32_to_cpu(bar_req->bar_num), 0x1, "BAR_TRAP bar_num"); + + return TEST_PASS; +} + +static int test_migrate_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_migrate_req *mig_req = (struct gsp_migrate_req *)msg.payload; + + pr_info("=== Testing MIGRATE Message Encoding ===\n"); + + msg.type = cpu_to_le32(GSP_RPC_MIGRATE); + msg.subcmd = cpu_to_le32(GSP_MIGRATE_SUBCMD_SAVE); + + mig_req->src_gpu_id = cpu_to_le64(0x1); + mig_req->dst_gpu_id = cpu_to_le64(0x2); + mig_req->memory_bitmap = cpu_to_le64(0xFFFFFFFFFFFFFFFF); + mig_req->page_table_ptr = cpu_to_le64(0x80000000); + mig_req->state = cpu_to_le32(GSP_MIGRATE_SUBCMD_SAVE); + mig_req->flags = cpu_to_le32(0); + mig_req->context = cpu_to_le64(0); + mig_req->reserved = 0; + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_MIGRATE, "MIGRATE RPC type"); + TEST_ASSERT_EQ(le32_to_cpu(msg.subcmd), GSP_MIGRATE_SUBCMD_SAVE, "MIGRATE subcmd"); + TEST_ASSERT_EQ(le64_to_cpu(mig_req->src_gpu_id), 0x1, "MIGRATE src_gpu_id"); + TEST_ASSERT_EQ(le64_to_cpu(mig_req->dst_gpu_id), 0x2, "MIGRATE dst_gpu_id"); + TEST_ASSERT_EQ(le64_to_cpu(mig_req->memory_bitmap), 0xFFFFFFFFFFFFFFFF, "MIGRATE memory_bitmap"); + TEST_ASSERT_EQ(le64_to_cpu(mig_req->page_table_ptr), 0x80000000, "MIGRATE page_table_ptr"); + + /* Test RESTORE subcmd */ + msg.subcmd = cpu_to_le32(GSP_MIGRATE_SUBCMD_RESTORE); + mig_req->state = cpu_to_le32(GSP_MIGRATE_SUBCMD_RESTORE); + mig_req->flags = cpu_to_le32(GSP_MIGRATE_SUBCMD_RESTORE); + mig_req->context = cpu_to_le64(0xDEADBEEF); + + TEST_ASSERT_EQ(le32_to_cpu(msg.subcmd), GSP_MIGRATE_SUBCMD_RESTORE, "MIGRATE restore subcmd"); + TEST_ASSERT_EQ(le64_to_cpu(mig_req->context), 0xDEADBEEF, "MIGRATE checkpoint context"); + + return TEST_PASS; +} + +/* ============================================================ + * Firmware Address Validation Tests + * ============================================================ */ + +static int test_firmware_addresses(void) +{ + pr_info("=== Testing Firmware Address Constants ===\n"); + + TEST_ASSERT_EQ(GSP_FW_BASE, 0x04000000UL, "GSP_FW_BASE"); + TEST_ASSERT_EQ(GSP_RPC_DISPATCH_PRIMARY, 0x057a1ac0UL, "Primary Dispatcher"); + TEST_ASSERT_EQ(GSP_RPC_JUMP_TABLE, 0x057a3ad8UL, "Jump Table"); + TEST_ASSERT_EQ(GSP_RPC_DISPATCH_SECONDARY, 0x04ed39dcUL, "Secondary Dispatcher"); + TEST_ASSERT_EQ(GSP_RPC_DISPATCH_TERTIARY, 0x04cc58f8UL, "Tertiary Dispatcher"); + TEST_ASSERT_EQ(GSP_RPC_DISPATCH_QUATERNARY, 0x05723ef4UL, "Quaternary Dispatcher"); + TEST_ASSERT_EQ(GSP_ALLOC_MAIN, 0x057e94ecUL, "ALLOC Main"); + TEST_ASSERT_EQ(GSP_SCHEDULE_MAIN, 0x051efbf4UL, "SCHEDULE Main"); + TEST_ASSERT_EQ(GSP_BAR_TRAP_HANDLER_JT, 0x057675b7UL, "BAR_TRAP Handler (JT)"); + TEST_ASSERT_EQ(GSP_BAR_TRAP_HANDLER_PRIM, 0x04cb8189UL, "BAR_TRAP Handler (Primary)"); + TEST_ASSERT_EQ(GSP_PIN_HANDLER_SEC, 0x04ed34b8UL, "PIN Handler (Secondary)"); + TEST_ASSERT_EQ(GSP_MAP_HANDLER_A, 0x05766ab7UL, "MAP Handler A"); + TEST_ASSERT_EQ(GSP_MAP_HANDLER_B, 0x05778a37UL, "MAP Handler B"); + TEST_ASSERT_EQ(GSP_UNMAP_HANDLER_A, 0x057669b7UL, "UNMAP Handler A"); + TEST_ASSERT_EQ(GSP_UNMAP_HANDLER_B, 0x05779937UL, "UNMAP Handler B"); + TEST_ASSERT_EQ(GSP_PIN_HANDLER_A, 0x057664b7UL, "PIN Handler A"); + TEST_ASSERT_EQ(GSP_PIN_HANDLER_B, 0x0577a3b7UL, "PIN Handler B"); + TEST_ASSERT_EQ(GSP_UNPIN_HANDLER_A, 0x057682b7UL, "UNPIN Handler A"); + TEST_ASSERT_EQ(GSP_UNPIN_HANDLER_B, 0x05779fb7UL, "UNPIN Handler B"); + TEST_ASSERT_EQ(GSP_SCHEDULE_HANDLER_A, 0x05767f37UL, "SCHEDULE Handler A"); + TEST_ASSERT_EQ(GSP_SCHEDULE_HANDLER_B, 0x0575aeb7UL, "SCHEDULE Handler B"); + TEST_ASSERT_EQ(GSP_YIELD_HANDLER_A, 0x0575ae37UL, "YIELD Handler A"); + TEST_ASSERT_EQ(GSP_YIELD_HANDLER_B, 0x05767337UL, "YIELD Handler B"); + TEST_ASSERT_EQ(GSP_NOTIFY_HANDLER_A, 0x0577c8b7UL, "NOTIFY Handler A"); + TEST_ASSERT_EQ(GSP_NOTIFY_HANDLER_B, 0x05767837UL, "NOTIFY Handler B"); + TEST_ASSERT_EQ(GSP_ENTRY_POINT, 0x04009000UL, "Entry Point"); + + return TEST_PASS; +} + +/* ============================================================ + * RPC Type/Subcommand Validation Tests + * ============================================================ */ + +static int test_rpc_types(void) +{ + pr_info("=== Testing RPC Type Constants ===\n"); + + TEST_ASSERT_EQ(GSP_RPC_ALLOC, 0x01, "RPC_ALLOC"); + TEST_ASSERT_EQ(GSP_RPC_FREE, 0x02, "RPC_FREE"); + TEST_ASSERT_EQ(GSP_RPC_MAP, 0x03, "RPC_MAP"); + TEST_ASSERT_EQ(GSP_RPC_UNMAP, 0x04, "RPC_UNMAP"); + TEST_ASSERT_EQ(GSP_RPC_PIN, 0x05, "RPC_PIN"); + TEST_ASSERT_EQ(GSP_RPC_UNPIN, 0x06, "RPC_UNPIN"); + TEST_ASSERT_EQ(GSP_RPC_SCHEDULE, 0x07, "RPC_SCHEDULE"); + TEST_ASSERT_EQ(GSP_RPC_YIELD, 0x08, "RPC_YIELD"); + TEST_ASSERT_EQ(GSP_RPC_NOTIFY, 0x09, "RPC_NOTIFY"); + TEST_ASSERT_EQ(GSP_RPC_BAR_TRAP, 0x0A, "RPC_BAR_TRAP"); + TEST_ASSERT_EQ(GSP_RPC_MIGRATE, 0x0B, "RPC_MIGRATE"); + TEST_ASSERT_EQ(GSP_RPC_DEBUG, 0xFF, "RPC_DEBUG"); + + return TEST_PASS; +} + +static int test_alloc_subcmds(void) +{ + pr_info("=== Testing ALLOC Sub-commands ===\n"); + + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_DEFAULT, 0, "ALLOC_SUBCMD_DEFAULT"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE1, 1, "ALLOC_SUBCMD_TYPE1"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE2, 3, "ALLOC_SUBCMD_TYPE2"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE3, 4, "ALLOC_SUBCMD_TYPE3"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE4, 5, "ALLOC_SUBCMD_TYPE4"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE5, 8, "ALLOC_SUBCMD_TYPE5"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE6, 10, "ALLOC_SUBCMD_TYPE6"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE7, 12, "ALLOC_SUBCMD_TYPE7"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE8, 17, "ALLOC_SUBCMD_TYPE8"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE9, 19, "ALLOC_SUBCMD_TYPE9"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE10, 20, "ALLOC_SUBCMD_TYPE10"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE11, 21, "ALLOC_SUBCMD_TYPE11"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE12, 22, "ALLOC_SUBCMD_TYPE12"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE13, 23, "ALLOC_SUBCMD_TYPE13"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE14, 24, "ALLOC_SUBCMD_TYPE14"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE15, 25, "ALLOC_SUBCMD_TYPE15"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE16, 26, "ALLOC_SUBCMD_TYPE16"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE17, 27, "ALLOC_SUBCMD_TYPE17"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE18, 28, "ALLOC_SUBCMD_TYPE18"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE19, 29, "ALLOC_SUBCMD_TYPE19"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE20, 30, "ALLOC_SUBCMD_TYPE20"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE21, 31, "ALLOC_SUBCMD_TYPE21"); + + return TEST_PASS; +} + +/* ============================================================ + * Payload Size Validation + * ============================================================ */ + +static int test_payload_capacity(void) +{ + pr_info("=== Testing Payload Capacity ===\n"); + + /* gsp_rpc_msg has 6 x u64 payload = 48 bytes */ + TEST_ASSERT(sizeof(struct gsp_alloc_req) <= 48, "ALLOC req fits in payload"); + TEST_ASSERT(sizeof(struct gsp_map_req) <= 48, "MAP req fits in payload"); + TEST_ASSERT(sizeof(struct gsp_pin_req) <= 48, "PIN req fits in payload"); + TEST_ASSERT(sizeof(struct gsp_sched_req) <= 48, "SCHED req fits in payload"); + TEST_ASSERT(sizeof(struct gsp_bar_trap_req) <= 48, "BAR_TRAP req fits in payload"); + /* MIGRATE req is 56 bytes - uses extended payload format */ + TEST_ASSERT(sizeof(struct gsp_migrate_req) == 56, "MIGRATE req size is 56 bytes"); + + return TEST_PASS; +} + +/* ============================================================ + * Module Init/Exit + * ============================================================ */ + +static int __init gsp_rpc_test_init(void) +{ + pr_info("========================================\n"); + pr_info("GSP RPC Client Test Suite Starting\n"); + pr_info("========================================\n"); + + test_structure_sizes(); + test_structure_alignment(); + test_rpc_msg_encoding(); + test_map_msg_encoding(); + test_pin_msg_encoding(); + test_sched_msg_encoding(); + test_bar_trap_msg_encoding(); + test_migrate_msg_encoding(); + test_firmware_addresses(); + test_rpc_types(); + test_alloc_subcmds(); + test_payload_capacity(); + + pr_info("========================================\n"); + pr_info("Test Results: %d total, %d passed, %d failed\n", + test_count, pass_count, fail_count); + pr_info("========================================\n"); + + if (fail_count > 0) { + pr_err("TEST SUITE FAILED\n"); + return -EINVAL; + } + + pr_info("TEST SUITE PASSED\n"); + return 0; +} + +static void __exit gsp_rpc_test_exit(void) +{ + pr_info("GSP RPC Client Test Suite Exited\n"); +} + +module_init(gsp_rpc_test_init); +module_exit(gsp_rpc_test_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Open-IOV Project"); +MODULE_DESCRIPTION("GSP RPC Client Test/Validation Harness"); +MODULE_VERSION("1.0"); diff --git a/kernel/tests/gsp_vgpu_comprehensive_test.c b/kernel/tests/gsp_vgpu_comprehensive_test.c new file mode 100644 index 0000000..7918beb --- /dev/null +++ b/kernel/tests/gsp_vgpu_comprehensive_test.c @@ -0,0 +1,432 @@ +/* + * GSP vGPU-IO Driver - Comprehensive Test Suite + * Tests all driver components: RPC, VF, BAR, WPR, Migration + * Compiles against kernel headers only (no driver dependency) + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/errno.h> +#include <linux/ktime.h> +#include <linux/atomic.h> +#include <linux/list.h> +#include <linux/idr.h> +#include <linux/mutex.h> +#include <linux/kref.h> +#include <linux/completion.h> +#include <linux/wait.h> +#include <linux/spinlock.h> +#include <linux/debugfs.h> +#include <linux/vmalloc.h> + +#include <gsp_core.h> + +static int test_count = 0; +static int pass_count = 0; +static int fail_count = 0; + +#define TEST_ASSERT(cond, msg) \ + do { \ + test_count++; \ + if (cond) { \ + pass_count++; \ + pr_info("PASS: %s\n", msg); \ + } else { \ + fail_count++; \ + pr_err("FAIL: %s\n", msg); \ + } \ + } while (0) + +#define TEST_ASSERT_EQ(a, b, msg) \ + TEST_ASSERT((a) == (b), msg " (expected " #b ", got " #a ")") + +#define TEST_ASSERT_NE(a, b, msg) \ + TEST_ASSERT((a) != (b), msg " (expected not " #b ", got " #a ")") + +/* ============================================================ + * Test: Kernel API Availability + * ============================================================ */ + +static int test_kernel_apis(void) +{ + pr_info("=== Testing Kernel API Availability ===\n"); + + /* Test kref */ + struct kref *ref = kmalloc(sizeof(*ref), GFP_KERNEL); + TEST_ASSERT(ref != NULL, "kref allocation"); + kref_init(ref); + TEST_ASSERT(kref_read(ref) == 1, "kref_init sets refcount to 1"); + + kref_get(ref); + TEST_ASSERT(kref_read(ref) == 2, "kref_get increments refcount"); + + kref_put(ref, (void (*)(struct kref *))kfree); + TEST_ASSERT(kref_read(ref) == 1, "kref_put decrements refcount"); + + kref_put(ref, (void (*)(struct kref *))kfree); + TEST_ASSERT(kref_read(ref) == 0, "kref_put reaches zero"); + + /* Test completion */ + struct completion comp; + init_completion(&comp); + TEST_ASSERT(!completion_done(&comp), "completion initially not done"); + complete(&comp); + TEST_ASSERT(completion_done(&comp), "completion marked done"); + + init_completion(&comp); + TEST_ASSERT(wait_for_completion_timeout(&comp, msecs_to_jiffies(100)) > 0, + "wait_for_completion_timeout works"); + + /* Test atomic64 */ + atomic64_t a64 = ATOMIC64_INIT(0); + atomic64_inc(&a64); + TEST_ASSERT_EQ(atomic64_read(&a64), 1, "atomic64_inc"); + atomic64_add(5, &a64); + TEST_ASSERT_EQ(atomic64_read(&a64), 6, "atomic64_add"); + atomic64_sub(3, &a64); + TEST_ASSERT_EQ(atomic64_read(&a64), 3, "atomic64_sub"); + + /* Test mutex */ + struct mutex mtx; + mutex_init(&mtx); + mutex_lock(&mtx); + TEST_ASSERT(mutex_is_locked(&mtx), "mutex_is_locked after lock"); + mutex_unlock(&mtx); + TEST_ASSERT(!mutex_is_locked(&mtx), "mutex_is_locked after unlock"); + + /* Test spinlock */ + spinlock_t sl = __SPIN_LOCK_UNLOCKED(sl); + unsigned long flags; + spin_lock_irqsave(&sl, flags); + TEST_ASSERT(spin_is_locked(&sl), "spin_is_locked after lock"); + spin_unlock_irqrestore(&sl, flags); + TEST_ASSERT(!spin_is_locked(&sl), "spin_is_locked after unlock"); + + /* Test idr */ + struct idr idr; + idr_init(&idr); + int id = idr_alloc(&idr, (void *)0xdeadbeef, 1, 100, GFP_KERNEL); + TEST_ASSERT(id >= 1 && id < 100, "idr_alloc returns valid id"); + void *ptr = idr_find(&idr, id); + TEST_ASSERT(ptr == (void *)0xdeadbeef, "idr_find returns correct pointer"); + idr_remove(&idr, id); + TEST_ASSERT(idr_find(&idr, id) == NULL, "idr_remove works"); + idr_destroy(&idr); + + /* Test list */ + struct list_head head; + INIT_LIST_HEAD(&head); + TEST_ASSERT(list_empty(&head), "list_empty on new list"); + + struct list_head item; + INIT_LIST_HEAD(&item); + list_add(&item, &head); + TEST_ASSERT(!list_empty(&head), "list not empty after add"); + list_del(&item); + TEST_ASSERT(list_empty(&head), "list empty after del"); + + /* Test ktime */ + ktime_t start = ktime_get(); + ktime_t end = ktime_get(); + s64 diff = ktime_to_ns(ktime_sub(end, start)); + TEST_ASSERT(diff >= 0, "ktime_get and ktime_to_ns work"); + + return 0; +} + +/* ============================================================ + * Test: Structure Sizes (from driver header) + * ============================================================ */ + +static int test_structure_sizes(void) +{ + pr_info("=== Testing Structure Sizes ===\n"); + + /* These sizes must match driver/include/gsp_core.h */ + TEST_ASSERT_EQ(sizeof(struct gsp_rpc_header), 64, "gsp_rpc_header size"); + TEST_ASSERT_EQ(sizeof(struct gsp_rpc_message), 256, "gsp_rpc_message size"); + TEST_ASSERT_EQ(sizeof(struct gsp_rpc_response), 32, "gsp_rpc_response size"); + TEST_ASSERT_EQ(sizeof(struct gsp_alloc_payload), 40, "gsp_alloc_payload size"); + TEST_ASSERT_EQ(sizeof(struct gsp_alloc_response), 24, "gsp_alloc_response size"); + TEST_ASSERT_EQ(sizeof(struct gsp_pin_payload), 40, "gsp_pin_payload size"); + TEST_ASSERT_EQ(sizeof(struct gsp_pin_response), 8, "gsp_pin_response size"); + TEST_ASSERT_EQ(sizeof(struct gsp_bar_trap_payload), 40, "gsp_bar_trap_payload size"); + TEST_ASSERT_EQ(sizeof(struct gsp_bar_trap_response), 8, "gsp_bar_trap_response size"); + TEST_ASSERT_EQ(sizeof(struct gsp_migrate_payload), 56, "gsp_migrate_payload size"); + TEST_ASSERT_EQ(sizeof(struct gsp_subcmd_desc), 8, "gsp_subcmd_desc size"); + TEST_ASSERT_EQ(sizeof(struct gsp_vf_config), 56, "gsp_vf_config size"); + + return 0; +} + +/* ============================================================ + * Test: Enum Values (from driver header) + * ============================================================ */ + +static int test_enum_values(void) +{ + pr_info("=== Testing Enum Values ===\n"); + + TEST_ASSERT_EQ(GSP_RPC_ALLOC, 0x01, "RPC_ALLOC"); + TEST_ASSERT_EQ(GSP_RPC_FREE, 0x02, "RPC_FREE"); + TEST_ASSERT_EQ(GSP_RPC_MAP, 0x03, "RPC_MAP"); + TEST_ASSERT_EQ(GSP_RPC_UNMAP, 0x04, "RPC_UNMAP"); + TEST_ASSERT_EQ(GSP_RPC_PIN, 0x05, "RPC_PIN"); + TEST_ASSERT_EQ(GSP_RPC_UNPIN, 0x06, "RPC_UNPIN"); + TEST_ASSERT_EQ(GSP_RPC_SCHEDULE, 0x07, "RPC_SCHEDULE"); + TEST_ASSERT_EQ(GSP_RPC_YIELD, 0x08, "RPC_YIELD"); + TEST_ASSERT_EQ(GSP_RPC_NOTIFY, 0x09, "RPC_NOTIFY"); + TEST_ASSERT_EQ(GSP_RPC_BAR_TRAP, 0x0A, "RPC_BAR_TRAP"); + TEST_ASSERT_EQ(GSP_RPC_MIGRATE, 0x0B, "RPC_MIGRATE"); + TEST_ASSERT_EQ(GSP_RPC_DEBUG, 0x0C, "RPC_DEBUG"); + + TEST_ASSERT_EQ(GSP_MEM_VRAM, 0, "MEM_VRAM"); + TEST_ASSERT_EQ(GSP_MEM_SYSMEM, 1, "MEM_SYSMEM"); + TEST_ASSERT_EQ(GSP_MEM_UNCACHED, 2, "MEM_UNCACHED"); + + TEST_ASSERT_EQ(GSP_PIN_READ, (1 << 0), "PIN_READ"); + TEST_ASSERT_EQ(GSP_PIN_WRITE, (1 << 1), "PIN_WRITE"); + TEST_ASSERT_EQ(GSP_PIN_LONGTERM, (1 << 2), "PIN_LONGTERM"); + + TEST_ASSERT_EQ(GSP_MIG_QUERY, 0, "MIG_QUERY"); + TEST_ASSERT_EQ(GSP_MIG_PREPARE, 1, "MIG_PREPARE"); + TEST_ASSERT_EQ(GSP_MIG_TRANSFER, 2, "MIG_TRANSFER"); + TEST_ASSERT_EQ(GSP_MIG_COMMIT, 3, "MIG_COMMIT"); + TEST_ASSERT_EQ(GSP_MIG_CLEANUP, 4, "MIG_CLEANUP"); + + TEST_ASSERT_EQ(GSP_MIG_LIVE, (1 << 0), "MIG_LIVE"); + TEST_ASSERT_EQ(GSP_MIG_POSTCOPY, (1 << 1), "MIG_POSTCOPY"); + TEST_ASSERT_EQ(GSP_MIG_AUTO_COMMIT, (1 << 2), "MIG_AUTO_COMMIT"); + TEST_ASSERT_EQ(GSP_MIG_VERIFY, (1 << 3), "MIG_VERIFY"); + + TEST_ASSERT_EQ(GSP_VF_CREATING, 0, "VF_CREATING"); + TEST_ASSERT_EQ(GSP_VF_ACTIVE, 1, "VF_ACTIVE"); + TEST_ASSERT_EQ(GSP_VF_MIGRATING, 2, "VF_MIGRATING"); + TEST_ASSERT_EQ(GSP_VF_SUSPENDED, 3, "VF_SUSPENDED"); + TEST_ASSERT_EQ(GSP_VF_DESTROYING, 4, "VF_DESTROYING"); + + return 0; +} + +/* ============================================================ + * Test: Error Code Mapping + * ============================================================ */ + +static int test_error_codes(void) +{ + pr_info("=== Testing Error Codes ===\n"); + + TEST_ASSERT_EQ(GSP_OK, 0, "GSP_OK"); + TEST_ASSERT_EQ(GSP_EPERM, 0xFFFFFFFF, "GSP_EPERM"); + TEST_ASSERT_EQ(GSP_ENOENT, 0xFFFFFFFE, "GSP_ENOENT"); + TEST_ASSERT_EQ(GSP_EINVAL, 0xFFFFFFFD, "GSP_EINVAL"); + TEST_ASSERT_EQ(GSP_ENODEV, 0xFFFFFFFA, "GSP_ENODEV"); + TEST_ASSERT_EQ(GSP_ENOMEM, 0xFFFFFFF4, "GSP_ENOMEM"); + TEST_ASSERT_EQ(GSP_EBUSY, 0xFFFFFFEE, "GSP_EBUSY"); + TEST_ASSERT_EQ(GSP_ENOSPC, 0xFFFFFFE4, "GSP_ENOSPC"); + TEST_ASSERT_EQ(GSP_ETIMEDOUT, 0x0000001E, "GSP_ETIMEDOUT"); + TEST_ASSERT_EQ(GSP_EOVERFLOW, 0x00000025, "GSP_EOVERFLOW"); + TEST_ASSERT_EQ(GSP_EDEV, 0x0000001F, "GSP_EDEV"); + + return 0; +} + +/* ============================================================ + * Test: Firmware Constants + * ============================================================ */ + +static int test_firmware_constants(void) +{ + pr_info("=== Testing Firmware Constants ===\n"); + + TEST_ASSERT_EQ(GSP_FW_BASE_ADDR, 0x04000000ULL, "FW_BASE_ADDR"); + TEST_ASSERT_EQ(GSP_FW_SIZE, 0x02000000ULL, "FW_SIZE"); + TEST_ASSERT_EQ(GSP_ENTRY_POINT, 0x04009000ULL, "ENTRY_POINT"); + TEST_ASSERT_EQ(GSP_PRIMARY_DISPATCHER, 0x057A1AC0ULL, "PRIMARY_DISPATCHER"); + TEST_ASSERT_EQ(GSP_SECONDARY_DISPATCHER, 0x04ED39DCULL, "SECONDARY_DISPATCHER"); + TEST_ASSERT_EQ(GSP_RPC_JUMP_TABLE, 0x059BFEB0ULL, "RPC_JUMP_TABLE"); + TEST_ASSERT_EQ(GSP_BAR0_DOORBELL, 0x1000, "BAR0_DOORBELL"); + TEST_ASSERT_EQ(GSP_BAR0_MSIX, 0x2000, "BAR0_MSIX"); + TEST_ASSERT_EQ(GSP_MSG_RING_SIZE, 256, "MSG_RING_SIZE"); + TEST_ASSERT_EQ(GSP_MSG_MAX_SIZE, 256, "MSG_MAX_SIZE"); + TEST_ASSERT_EQ(GSP_MAX_PENDING, 64, "MAX_PENDING"); + + return 0; +} + +/* ============================================================ + * Test: Migration State Enum (internal) + * ============================================================ */ + +static int test_migration_states(void) +{ + pr_info("=== Testing Migration States ===\n"); + + TEST_ASSERT_EQ(GSP_MIG_S_IDLE, 0, "MIG_S_IDLE"); + TEST_ASSERT_EQ(GSP_MIG_S_QUERY, 1, "MIG_S_QUERY"); + TEST_ASSERT_EQ(GSP_MIG_S_PREPARE, 2, "MIG_S_PREPARE"); + TEST_ASSERT_EQ(GSP_MIG_S_TRANSFER, 3, "MIG_S_TRANSFER"); + TEST_ASSERT_EQ(GSP_MIG_S_COMMIT, 4, "MIG_S_COMMIT"); + TEST_ASSERT_EQ(GSP_MIG_S_CLEANUP, 5, "MIG_S_CLEANUP"); + TEST_ASSERT_EQ(GSP_MIG_S_FAILED, 6, "MIG_S_FAILED"); + TEST_ASSERT_EQ(GSP_MIG_S_COMPLETED, 7, "MIG_S_COMPLETED"); + + return 0; +} + +/* ============================================================ + * Test: WPR Flags and Events + * ============================================================ */ + +static int test_wpr_flags(void) +{ + pr_info("=== Testing WPR Flags ===\n"); + + TEST_ASSERT_EQ(GSP_WPR_WRITE, (1 << 0), "WPR_WRITE"); + TEST_ASSERT_EQ(GSP_WPR_READ, (1 << 1), "WPR_READ"); + TEST_ASSERT_EQ(GSP_WPR_EXECUTE, (1 << 2), "WPR_EXECUTE"); + + TEST_ASSERT_EQ(GSP_WPR_WRITE_COMPLETE, 0, "WPR_WRITE_COMPLETE"); + TEST_ASSERT_EQ(GSP_WPR_READ_COMPLETE, 1, "WPR_READ_COMPLETE"); + TEST_ASSERT_EQ(GSP_WPR_WRITE_TIMEOUT, 2, "WPR_WRITE_TIMEOUT"); + TEST_ASSERT_EQ(GSP_WPR_READ_TIMEOUT, 3, "WPR_READ_TIMEOUT"); + + return 0; +} + +/* ============================================================ + * Test: Message Flags + * ============================================================ */ + +static int test_message_flags(void) +{ + pr_info("=== Testing Message Flags ===\n"); + + TEST_ASSERT_EQ(GSP_MSG_SYNC, (1 << 0), "MSG_SYNC"); + TEST_ASSERT_EQ(GSP_MSG_ASYNC, (1 << 1), "MSG_ASYNC"); + TEST_ASSERT_EQ(GSP_MSG_NOREPLY, (1 << 2), "MSG_NOREPLY"); + + return 0; +} + +/* ============================================================ + * Test: BAR Trap Types + * ============================================================ */ + +static int test_bar_trap_types(void) +{ + pr_info("=== Testing BAR Trap Types ===\n"); + + TEST_ASSERT_EQ(GSP_BAR_READ, 0, "BAR_READ"); + TEST_ASSERT_EQ(GSP_BAR_WRITE, 1, "BAR_WRITE"); + TEST_ASSERT_EQ(GSP_BAR_ATOMIC, 2, "BAR_ATOMIC"); + + return 0; +} + +/* ============================================================ + * Test: Memory Allocation Patterns + * ============================================================ */ + +static int test_memory_patterns(void) +{ + pr_info("=== Testing Memory Allocation Patterns ===\n"); + + /* Test vzalloc/vfree */ + void *ptr = vzalloc(4096); + TEST_ASSERT(ptr != NULL, "vzalloc allocates memory"); + vfree(ptr); + + /* Test kmalloc/kfree */ + ptr = kmalloc(1024, GFP_KERNEL); + TEST_ASSERT(ptr != NULL, "kmalloc allocates memory"); + kfree(ptr); + + /* Test kcalloc */ + ptr = kcalloc(10, 64, GFP_KERNEL); + TEST_ASSERT(ptr != NULL, "kcalloc allocates zeroed memory"); + kfree(ptr); + + return 0; +} + +/* ============================================================ + * Test: String Operations + * ============================================================ */ + +static int test_string_ops(void) +{ + pr_info("=== Testing String Operations ===\n"); + + char buf[64]; + int len = snprintf(buf, sizeof(buf), "test_%d", 42); + TEST_ASSERT(len > 0 && len < sizeof(buf), "snprintf works"); + TEST_ASSERT(strcmp(buf, "test_42") == 0, "snprintf formats correctly"); + + char *dup = kstrdup("test_string", GFP_KERNEL); + TEST_ASSERT(dup != NULL, "kstrdup works"); + TEST_ASSERT(strcmp(dup, "test_string") == 0, "kstrdup copies correctly"); + kfree(dup); + + return 0; +} + +/* ============================================================ + * Test Suite Runner + * ============================================================ */ + +static int run_all_tests(void) +{ + pr_info("\n============================================================\n"); + pr_info("GSP vGPU-IO Comprehensive Test Suite Starting\n"); + pr_info("============================================================\n\n"); + + test_kernel_apis(); + test_structure_sizes(); + test_enum_values(); + test_error_codes(); + test_firmware_constants(); + test_migration_states(); + test_wpr_flags(); + test_message_flags(); + test_bar_trap_types(); + test_memory_patterns(); + test_string_ops(); + + pr_info("\n============================================================\n"); + pr_info("TEST RESULTS: %d total, %d passed, %d failed\n", + test_count, pass_count, fail_count); + pr_info("============================================================\n"); + + if (fail_count == 0) { + pr_info("TEST SUITE PASSED\n"); + return 0; + } else { + pr_err("TEST SUITE FAILED\n"); + return -1; + } +} + +/* ============================================================ + * Module Init/Exit + * ============================================================ */ + +static int __init gsp_test_init(void) +{ + return run_all_tests(); +} + +static void __exit gsp_test_exit(void) +{ + pr_info("GSP vGPU-IO test module unloaded\n"); +} + +module_init(gsp_test_init); +module_exit(gsp_test_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("NVIDIA vGPU-IO Team"); +MODULE_DESCRIPTION("GSP vGPU-IO Comprehensive Test Suite"); +MODULE_VERSION("1.0.0");
\ No newline at end of file diff --git a/kernel/tests/gsp_vgpu_kunit_test.c b/kernel/tests/gsp_vgpu_kunit_test.c new file mode 100644 index 0000000..c661f91 --- /dev/null +++ b/kernel/tests/gsp_vgpu_kunit_test.c @@ -0,0 +1,327 @@ +/* + * GSP vGPU-IO KUnit Tests + * In-kernel unit tests for the driver + */ + +#include <kunit/test.h> +#include <kunit/mock.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/errno.h> +#include <linux/atomic.h> +#include <linux/list.h> +#include <linux/idr.h> +#include <linux/mutex.h> +#include <linux/kref.h> +#include <linux/completion.h> +#include <linux/wait.h> +#include <linux/spinlock.h> + +#include <gsp_core.h> + +/* ============================================================ + * Test: Structure Sizes and Alignments + * ============================================================ */ + +static void test_structure_sizes(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_rpc_header), 64); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_rpc_message), 256); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_rpc_response), 32); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_subcmd_desc), 8); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_vf_config), 64); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_alloc_payload), 48); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_alloc_response), 24); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_pin_payload), 40); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_pin_response), 8); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_bar_trap_payload), 40); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_bar_trap_response), 8); + KUNIT_EXPECT_EQ(test, sizeof(struct gsp_migrate_payload), 64); +} + +static void test_structure_alignments(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_rpc_header), 8); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_rpc_message), 8); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_rpc_response), 8); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_subcmd_desc), 4); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_vf_config), 8); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_alloc_payload), 8); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_alloc_response), 8); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_pin_payload), 8); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_pin_response), 8); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_bar_trap_payload), 8); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_bar_trap_response), 8); + KUNIT_EXPECT_EQ(test, __alignof__(struct gsp_migrate_payload), 8); +} + +/* ============================================================ + * Test: Payload Encoding/Decoding + * ============================================================ */ + +static void test_alloc_payload_encoding(struct kunit *test) +{ + struct gsp_alloc_payload payload = {0}; + + payload.size = cpu_to_le64(0x100000); + payload.type = cpu_to_le32(GSP_MEM_VRAM); + payload.alignment = cpu_to_le32(0x1000); + payload.va_hint = cpu_to_le64(0x0); + payload.flags = cpu_to_le32(0x3); + payload.pid = cpu_to_le32(0x456); + payload.vf_id = cpu_to_le32(0x1); + + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.size), 0x100000); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.type), GSP_MEM_VRAM); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.alignment), 0x1000); + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.va_hint), 0x0); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.flags), 0x3); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.pid), 0x456); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.vf_id), 0x1); +} + +static void test_pin_payload_encoding(struct kunit *test) +{ + struct gsp_pin_payload payload = {0}; + + payload.handle = cpu_to_le64(0xCAFEBABE); + payload.offset = cpu_to_le64(0x1000); + payload.size = cpu_to_le64(0x10000); + payload.vf_id = cpu_to_le32(0x2); + payload.flags = cpu_to_le32(GSP_PIN_READ | GSP_PIN_WRITE); + + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.handle), 0xCAFEBABE); + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.offset), 0x1000); + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.size), 0x10000); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.vf_id), 0x2); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.flags), GSP_PIN_READ | GSP_PIN_WRITE); +} + +static void test_bar_trap_payload_encoding(struct kunit *test) +{ + struct gsp_bar_trap_payload payload = {0}; + + payload.bar_index = cpu_to_le32(1); + payload.trap_type = cpu_to_le32(GSP_BAR_WRITE); + payload.offset = cpu_to_le64(0x1000); + payload.value = cpu_to_le64(0xDEADBEEF); + payload.size = cpu_to_le32(4); + payload.vf_id = cpu_to_le32(0x1); + + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.bar_index), 1); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.trap_type), GSP_BAR_WRITE); + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.offset), 0x1000); + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.value), 0xDEADBEEF); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.size), 4); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.vf_id), 0x1); +} + +static void test_migrate_payload_encoding(struct kunit *test) +{ + struct gsp_migrate_payload payload = {0}; + + payload.phase = cpu_to_le32(GSP_MIG_PREPARE); + payload.subcmd = cpu_to_le32(3); + payload.src_vf = cpu_to_le64(0x1); + payload.dst_vf = cpu_to_le64(0x2); + payload.data_ptr = cpu_to_le64(0x80000000); + payload.data_size = cpu_to_le64(0x1000); + payload.flags = cpu_to_le32(GSP_MIG_LIVE | GSP_MIG_VERIFY); + payload.token = cpu_to_le64(0xABCDEF1234567890); + + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.phase), GSP_MIG_PREPARE); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.subcmd), 3); + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.src_vf), 0x1); + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.dst_vf), 0x2); + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.data_ptr), 0x80000000); + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.data_size), 0x1000); + KUNIT_EXPECT_EQ(test, le32_to_cpu(payload.flags), GSP_MIG_LIVE | GSP_MIG_VERIFY); + KUNIT_EXPECT_EQ(test, le64_to_cpu(payload.token), 0xABCDEF1234567890); +} + +static void test_subcmd_desc_encoding(struct kunit *test) +{ + struct gsp_subcmd_desc desc = {0}; + + desc.subcmd_id = cpu_to_le32(0x14); + desc.echo_flags_b = 0xAA; + desc.echo_flags_a = 0xBB; + desc.echo_flags_c = 0xCC; + + KUNIT_EXPECT_EQ(test, le32_to_cpu(desc.subcmd_id), 0x14); + KUNIT_EXPECT_EQ(test, desc.echo_flags_b, 0xAA); + KUNIT_EXPECT_EQ(test, desc.echo_flags_a, 0xBB); + KUNIT_EXPECT_EQ(test, desc.echo_flags_c, 0xCC); +} + +/* ============================================================ + * Test: Firmware Address Constants + * ============================================================ */ + +static void test_firmware_addresses(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, GSP_FW_BASE_ADDR, 0x04000000ULL); + KUNIT_EXPECT_EQ(test, GSP_ENTRY_POINT, 0x04009000ULL); + KUNIT_EXPECT_EQ(test, GSP_PRIMARY_DISPATCHER, 0x057A1AC0ULL); + KUNIT_EXPECT_EQ(test, GSP_SECONDARY_DISPATCHER, 0x04ED39DCULL); + KUNIT_EXPECT_EQ(test, GSP_RPC_JUMP_TABLE, 0x059BFEB0ULL); + KUNIT_EXPECT_EQ(test, 0x04179ED8ULL, 0x04179ED8ULL); /* Migration Table 1 */ + KUNIT_EXPECT_EQ(test, 0x04179FBCULL, 0x04179FBCULL); /* Migration Table 2 */ + KUNIT_EXPECT_EQ(test, 0x04CB8188ULL, 0x04CB8188ULL); /* BAR Handler */ + KUNIT_EXPECT_EQ(test, 0x04F00B0CULL, 0x04F00B0CULL); /* WPR Handler 1 */ + KUNIT_EXPECT_EQ(test, 0x04F00C78ULL, 0x04F00C78ULL); /* WPR Handler 2 */ + KUNIT_EXPECT_EQ(test, 0x057E94ECULL, 0x057E94ECULL); /* ALLOC Main */ + KUNIT_EXPECT_EQ(test, 0x051EFBF4ULL, 0x051EFBF4ULL); /* SCHEDULE Main */ +} + +/* ============================================================ + * Test: RPC Type Constants + * ============================================================ */ + +static void test_rpc_types(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, GSP_RPC_ALLOC, 0x01); + KUNIT_EXPECT_EQ(test, GSP_RPC_FREE, 0x02); + KUNIT_EXPECT_EQ(test, GSP_RPC_MAP, 0x03); + KUNIT_EXPECT_EQ(test, GSP_RPC_UNMAP, 0x04); + KUNIT_EXPECT_EQ(test, GSP_RPC_PIN, 0x05); + KUNIT_EXPECT_EQ(test, GSP_RPC_UNPIN, 0x06); + KUNIT_EXPECT_EQ(test, GSP_RPC_SCHEDULE, 0x07); + KUNIT_EXPECT_EQ(test, GSP_RPC_YIELD, 0x08); + KUNIT_EXPECT_EQ(test, GSP_RPC_NOTIFY, 0x09); + KUNIT_EXPECT_EQ(test, GSP_RPC_BAR_TRAP, 0x0A); + KUNIT_EXPECT_EQ(test, GSP_RPC_MIGRATE, 0x0B); + KUNIT_EXPECT_EQ(test, GSP_RPC_DEBUG, 0x0C); +} + +static void test_migration_constants(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, GSP_MIG_QUERY, 0); + KUNIT_EXPECT_EQ(test, GSP_MIG_PREPARE, 1); + KUNIT_EXPECT_EQ(test, GSP_MIG_TRANSFER, 2); + KUNIT_EXPECT_EQ(test, GSP_MIG_COMMIT, 3); + KUNIT_EXPECT_EQ(test, GSP_MIG_CLEANUP, 4); + + KUNIT_EXPECT_EQ(test, GSP_MIG_LIVE, (1 << 0)); + KUNIT_EXPECT_EQ(test, GSP_MIG_POSTCOPY, (1 << 1)); + KUNIT_EXPECT_EQ(test, GSP_MIG_AUTO_COMMIT, (1 << 2)); + KUNIT_EXPECT_EQ(test, GSP_MIG_VERIFY, (1 << 3)); +} + +static void test_bar_trap_types(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, GSP_BAR_READ, 0); + KUNIT_EXPECT_EQ(test, GSP_BAR_WRITE, 1); + KUNIT_EXPECT_EQ(test, GSP_BAR_ATOMIC, 2); +} + +static void test_pin_flags(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, GSP_PIN_READ, (1 << 0)); + KUNIT_EXPECT_EQ(test, GSP_PIN_WRITE, (1 << 1)); + KUNIT_EXPECT_EQ(test, GSP_PIN_LONGTERM, (1 << 2)); +} + +static void test_wpr_flags(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, GSP_WPR_WRITE, (1 << 0)); + KUNIT_EXPECT_EQ(test, GSP_WPR_READ, (1 << 1)); + KUNIT_EXPECT_EQ(test, GSP_WPR_EXECUTE, (1 << 2)); +} + +static void test_vf_states(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, GSP_VF_CREATING, 0); + KUNIT_EXPECT_EQ(test, GSP_VF_ACTIVE, 1); + KUNIT_EXPECT_EQ(test, GSP_VF_MIGRATING, 2); + KUNIT_EXPECT_EQ(test, GSP_VF_SUSPENDED, 3); + KUNIT_EXPECT_EQ(test, GSP_VF_DESTROYING, 4); +} + +static void test_msg_flags(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, GSP_MSG_SYNC, (1 << 0)); + KUNIT_EXPECT_EQ(test, GSP_MSG_ASYNC, (1 << 1)); + KUNIT_EXPECT_EQ(test, GSP_MSG_NOREPLY, (1 << 2)); +} + +/* ============================================================ + * Test: Error Code Translation + * ============================================================ */ + +static void test_error_codes(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_OK), 0); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_EPERM), -EPERM); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_ENOENT), -ENOENT); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_EINVAL), -EINVAL); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_ENODEV), -ENODEV); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_ENOMEM), -ENOMEM); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_EBUSY), -EBUSY); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_ENOSPC), -ENOSPC); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_ETIMEDOUT), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_EOVERFLOW), -EOVERFLOW); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(GSP_EDEV), -EIO); + KUNIT_EXPECT_EQ(test, gsp_errno_to_errno(0xDEADBEEF), -EIO); +} + +/* ============================================================ + * Test: Memory Types + * ============================================================ */ + +static void test_memory_types(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, GSP_MEM_VRAM, 0); + KUNIT_EXPECT_EQ(test, GSP_MEM_SYSMEM, 1); + KUNIT_EXPECT_EQ(test, GSP_MEM_UNCACHED, 2); +} + +/* ============================================================ + * Test: Payload Capacity + * ============================================================ */ + +static void test_payload_capacity(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, sizeof(struct gsp_alloc_payload) <= 192); + KUNIT_EXPECT_TRUE(test, sizeof(struct gsp_pin_payload) <= 192); + KUNIT_EXPECT_TRUE(test, sizeof(struct gsp_bar_trap_payload) <= 192); + KUNIT_EXPECT_TRUE(test, sizeof(struct gsp_migrate_payload) <= 192); + KUNIT_EXPECT_TRUE(test, sizeof(struct gsp_subcmd_desc) <= 192); +} + +/* ============================================================ + * Test Suite Registration + * ============================================================ */ + +static struct kunit_case gsp_vgpu_test_cases[] = { + KUNIT_CASE(test_structure_sizes), + KUNIT_CASE(test_structure_alignments), + KUNIT_CASE(test_alloc_payload_encoding), + KUNIT_CASE(test_pin_payload_encoding), + KUNIT_CASE(test_bar_trap_payload_encoding), + KUNIT_CASE(test_migrate_payload_encoding), + KUNIT_CASE(test_subcmd_desc_encoding), + KUNIT_CASE(test_firmware_addresses), + KUNIT_CASE(test_rpc_types), + KUNIT_CASE(test_migration_constants), + KUNIT_CASE(test_bar_trap_types), + KUNIT_CASE(test_pin_flags), + KUNIT_CASE(test_wpr_flags), + KUNIT_CASE(test_vf_states), + KUNIT_CASE(test_msg_flags), + KUNIT_CASE(test_error_codes), + KUNIT_CASE(test_memory_types), + KUNIT_CASE(test_payload_capacity), + {} +}; + +static struct kunit_suite gsp_vgpu_test_suite = { + .name = "gsp-vgpu-io", + .test_cases = gsp_vgpu_test_cases, +}; + +kunit_test_suites(&gsp_vgpu_test_suite); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Open-IOV Project"); +MODULE_DESCRIPTION("GSP vGPU-IO KUnit Tests");
\ No newline at end of file diff --git a/kernel/tests/gsp_vgpu_mock_test.c b/kernel/tests/gsp_vgpu_mock_test.c new file mode 100644 index 0000000..b9a46df --- /dev/null +++ b/kernel/tests/gsp_vgpu_mock_test.c @@ -0,0 +1,624 @@ +/* + * GSP vGPU-IO Driver Mock Test + * Tests driver logic with mocked hardware interfaces + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/errno.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/idr.h> +#include <linux/kref.h> +#include <linux/completion.h> +#include <linux/atomic.h> + +#define TEST_PASS 0 +#define TEST_FAIL -1 + +static int test_count = 0; +static int pass_count = 0; +static int fail_count = 0; + +#define TEST_ASSERT(cond, msg) \ + do { \ + test_count++; \ + if (cond) { \ + pass_count++; \ + pr_info("PASS: %s\n", msg); \ + } else { \ + fail_count++; \ + pr_err("FAIL: %s\n", msg); \ + } \ + } while (0) + +#define TEST_ASSERT_EQ(a, b, msg) \ + TEST_ASSERT((a) == (b), msg " (expected " #b ", got " #a ")") + +/* ============================================================ + * Mock Hardware Abstraction + * ============================================================ */ + +struct mock_hw { + u64 bar0_regs[256]; + u64 bar1_regs[256]; + u8 *fw_memory; + size_t fw_size; + bool doorbell_triggered; + u32 doorbell_count; + spinlock_t lock; +}; + +static struct mock_hw *mock_hw_create(void) +{ + struct mock_hw *hw = kzalloc(sizeof(*hw), GFP_KERNEL); + if (!hw) + return NULL; + + hw->fw_memory = vmalloc(32 * 1024 * 1024); // 32MB + if (!hw->fw_memory) { + kfree(hw); + return NULL; + } + hw->fw_size = 32 * 1024 * 1024; + spin_lock_init(&hw->lock); + return hw; +} + +static void mock_hw_destroy(struct mock_hw *hw) +{ + if (!hw) + return; + vfree(hw->fw_memory); + kfree(hw); +} + +static void mock_hw_write_bar0(struct mock_hw *hw, u32 offset, u32 value) +{ + unsigned long flags; + spin_lock_irqsave(&hw->lock, flags); + if (offset / 4 < 256) + hw->bar0_regs[offset / 4] = value; + if (offset == 0x1000) { // Doorbell + hw->doorbell_triggered = true; + hw->doorbell_count++; + } + spin_unlock_irqrestore(&hw->lock, flags); +} + +static u32 mock_hw_read_bar0(struct mock_hw *hw, u32 offset) +{ + unsigned long flags; + u32 value = 0; + spin_lock_irqsave(&hw->lock, flags); + if (offset / 4 < 256) + value = hw->bar0_regs[offset / 4]; + spin_unlock_irqrestore(&hw->lock, flags); + return value; +} + +static void mock_hw_write_bar1(struct mock_hw *hw, u64 offset, u64 value) +{ + unsigned long flags; + spin_lock_irqsave(&hw->lock, flags); + if (offset < hw->fw_size) + *(u64 *)(hw->fw_memory + offset) = value; + spin_unlock_irqrestore(&hw->lock, flags); +} + +static u64 mock_hw_read_bar1(struct mock_hw *hw, u64 offset) +{ + unsigned long flags; + u64 value = 0; + spin_lock_irqsave(&hw->lock, flags); + if (offset < hw->fw_size) + value = *(u64 *)(hw->fw_memory + offset); + spin_unlock_irqrestore(&hw->lock, flags); + return value; +} + +/* ============================================================ + * Mock VF Operations + * ============================================================ */ + +struct mock_vf { + u32 id; + u64 bar1_offset; + u64 bar1_size; + u64 pt_base; + u64 pt_size; + u64 cmd_base; + u64 cmd_size; + bool active; + struct { + u64 aperture_offset; + u64 aperture_size; + bool force_access; + u32 force_vf; + } bar_state; + struct list_head wpr_list; + struct mutex lock; +}; + +struct mock_wpr_region { + struct list_head list; + u64 base; + u64 size; + u32 flags; +}; + +static struct mock_vf *mock_vf_create(u32 id, u64 bar1_offset, u64 bar1_size, + u64 pt_base, u64 pt_size, + u64 cmd_base, u64 cmd_size) +{ + struct mock_vf *vf = kzalloc(sizeof(*vf), GFP_KERNEL); + if (!vf) + return NULL; + + vf->id = id; + vf->bar1_offset = bar1_offset; + vf->bar1_size = bar1_size; + vf->pt_base = pt_base; + vf->pt_size = pt_size; + vf->cmd_base = cmd_base; + vf->cmd_size = cmd_size; + vf->active = true; + INIT_LIST_HEAD(&vf->wpr_list); + mutex_init(&vf->lock); + + vf->bar_state.aperture_offset = bar1_offset; + vf->bar_state.aperture_size = bar1_size; + vf->bar_state.force_access = false; + vf->bar_state.force_vf = 0; + + return vf; +} + +static void mock_vf_destroy(struct mock_vf *vf) +{ + if (!vf) + return; + + struct mock_wpr_region *rgn, *tmp; + mutex_lock(&vf->lock); + list_for_each_entry_safe(rgn, tmp, &vf->wpr_list, list) { + list_del(&rgn->list); + kfree(rgn); + } + mutex_unlock(&vf->lock); + + kfree(vf); +} + +static int mock_vf_wpr_add(struct mock_vf *vf, u64 base, u64 size, u32 flags) +{ + struct mock_wpr_region *rgn = kzalloc(sizeof(*rgn), GFP_KERNEL); + if (!rgn) + return -ENOMEM; + + rgn->base = base; + rgn->size = size; + rgn->flags = flags; + + mutex_lock(&vf->lock); + list_add(&rgn->list, &vf->wpr_list); + mutex_unlock(&vf->lock); + + return 0; +} + +static int mock_vf_wpr_remove(struct mock_vf *vf, u64 base) +{ + struct mock_wpr_region *rgn, *tmp; + bool found = false; + + mutex_lock(&vf->lock); + list_for_each_entry_safe(rgn, tmp, &vf->wpr_list, list) { + if (rgn->base == base) { + list_del(&rgn->list); + kfree(rgn); + found = true; + break; + } + } + mutex_unlock(&vf->lock); + + return found ? 0 : -ENOENT; +} + +/* ============================================================ + * Mock RPC Client + * ============================================================ */ + +struct mock_rpc_client { + u32 seq; + u32 pending_seq[64]; + u32 pending_count; + struct mutex lock; +}; + +static struct mock_rpc_client *mock_rpc_create(void) +{ + struct mock_rpc_client *rpc = kzalloc(sizeof(*rpc), GFP_KERNEL); + if (!rpc) + return NULL; + mutex_init(&rpc->lock); + return rpc; +} + +static void mock_rpc_destroy(struct mock_rpc_client *rpc) +{ + if (!rpc) + return; + kfree(rpc); +} + +static int mock_rpc_send(struct mock_rpc_client *rpc, u32 rpc_type, + void *payload, size_t payload_size, + void *response, size_t *response_size) +{ + if (!rpc) + return -EINVAL; + + mutex_lock(&rpc->lock); + rpc->seq++; + rpc->pending_seq[rpc->pending_count++] = rpc->seq; + mutex_unlock(&rpc->lock); + + /* Simulate successful response */ + if (response && response_size) { + struct gsp_rpc_response *resp = response; + resp->status = 0; + resp->rpc_type = rpc_type; + resp->sequence = rpc->seq; + *response_size = sizeof(*resp); + } + + return 0; +} + +/* ============================================================ + * Driver Logic Tests with Mocks + * ============================================================ */ + +static int test_mock_hardware(void) +{ + struct mock_hw *hw = mock_hw_create(); + TEST_ASSERT(hw != NULL, "Mock HW create"); + + /* Test BAR0 write/read */ + mock_hw_write_bar0(hw, 0x1000, 0x1); + TEST_ASSERT(hw->doorbell_triggered, "Doorbell triggered"); + TEST_ASSERT_EQ(hw->doorbell_count, 1, "Doorbell count"); + + u32 val = mock_hw_read_bar0(hw, 0x1000); + TEST_ASSERT_EQ(val, 0x1, "BAR0 read back"); + + /* Test BAR1 write/read */ + mock_hw_write_bar1(hw, 0x1000, 0xDEADBEEFCAFEBABE); + u64 val64 = mock_hw_read_bar1(hw, 0x1000); + TEST_ASSERT_EQ(val64, 0xDEADBEEFCAFEBABE, "BAR1 read back"); + + mock_hw_destroy(hw); + return TEST_PASS; +} + +static int test_mock_vf_lifecycle(void) +{ + struct mock_vf *vf = mock_vf_create(1, 0x10000000, 0x100000, + 0x20000000, 0x100000, + 0x30000000, 0x10000); + TEST_ASSERT(vf != NULL, "Mock VF create"); + + TEST_ASSERT(vf->active, "VF active after create"); + TEST_ASSERT_EQ(vf->id, 1, "VF ID"); + TEST_ASSERT_EQ(vf->bar1_offset, 0x10000000, "BAR1 offset"); + TEST_ASSERT_EQ(vf->bar1_size, 0x100000, "BAR1 size"); + TEST_ASSERT_EQ(vf->pt_base, 0x20000000, "Page table base"); + TEST_ASSERT_EQ(vf->pt_size, 0x100000, "Page table size"); + + /* Test BAR state */ + TEST_ASSERT_EQ(vf->bar_state.aperture_offset, 0x10000000, "Aperture offset"); + TEST_ASSERT_EQ(vf->bar_state.aperture_size, 0x100000, "Aperture size"); + TEST_ASSERT(!vf->bar_state.force_access, "Force access initially false"); + + /* Test WPR list empty */ + TEST_ASSERT(list_empty(&vf->wpr_list), "WPR list empty initially"); + + mock_vf_destroy(vf); + return TEST_PASS; +} + +static int test_mock_vf_wpr(void) +{ + struct mock_vf *vf = mock_vf_create(1, 0x10000000, 0x100000, + 0x20000000, 0x100000, + 0x30000000, 0x10000); + TEST_ASSERT(vf != NULL, "Mock VF create"); + + /* Add WPR region */ + int ret = mock_vf_wpr_add(vf, 0x10000000, 0x10000, 3); + TEST_ASSERT_EQ(ret, 0, "WPR add"); + + ret = mock_vf_wpr_add(vf, 0x20000000, 0x20000, 1); + TEST_ASSERT_EQ(ret, 0, "WPR add second"); + + /* Verify list */ + int count = 0; + struct mock_wpr_region *rgn; + mutex_lock(&vf->lock); + list_for_each_entry(rgn, &vf->wpr_list, list) { + count++; + } + mutex_unlock(&vf->lock); + TEST_ASSERT_EQ(count, 2, "Two WPR regions"); + + /* Remove one */ + ret = mock_vf_wpr_remove(vf, 0x10000000); + TEST_ASSERT_EQ(ret, 0, "WPR remove"); + + count = 0; + mutex_lock(&vf->lock); + list_for_each_entry(rgn, &vf->wpr_list, list) { + count++; + } + mutex_unlock(&vf->lock); + TEST_ASSERT_EQ(count, 1, "One WPR region remaining"); + + /* Remove non-existent */ + ret = mock_vf_wpr_remove(vf, 0x30000000); + TEST_ASSERT_EQ(ret, -ENOENT, "WPR remove non-existent returns ENOENT"); + + mock_vf_destroy(vf); + return TEST_PASS; +} + +static int test_mock_rpc_client(void) +{ + struct mock_rpc_client *rpc = mock_rpc_create(); + TEST_ASSERT(rpc != NULL, "Mock RPC create"); + + struct gsp_rpc_response resp; + size_t resp_size = sizeof(resp); + struct gsp_alloc_payload payload = { + .size = cpu_to_le64(0x100000), + .type = cpu_to_le32(GSP_MEM_VRAM), + }; + + int ret = mock_rpc_send(rpc, GSP_RPC_ALLOC, &payload, sizeof(payload), &resp, &resp_size); + TEST_ASSERT_EQ(ret, 0, "RPC send"); + + TEST_ASSERT_EQ(resp.status, 0, "RPC response status"); + TEST_ASSERT_EQ(resp.rpc_type, GSP_RPC_ALLOC, "RPC response type"); + TEST_ASSERT_EQ(resp.sequence, 1, "RPC response sequence"); + + /* Second call */ + ret = mock_rpc_send(rpc, GSP_RPC_PIN, &payload, sizeof(payload), &resp, &resp_size); + TEST_ASSERT_EQ(ret, 0, "RPC send second"); + TEST_ASSERT_EQ(resp.sequence, 2, "RPC response sequence 2"); + + mock_rpc_destroy(rpc); + return TEST_PASS; +} + +static int test_driver_vf_creation_logic(void) +{ + /* Simulate driver VF creation logic */ + struct mock_hw *hw = mock_hw_create(); + struct mock_rpc_client *rpc = mock_rpc_create(); + struct mock_vf *vfs[8] = {0}; + + /* Create VFs 1-4 */ + for (int i = 1; i <= 4; i++) { + vfs[i] = mock_vf_create(i, 0x10000000 * i, 0x100000, + 0x20000000 * i, 0x100000, + 0x30000000 * i, 0x10000); + TEST_ASSERT(vfs[i] != NULL, "VF creation in loop"); + TEST_ASSERT(vfs[i]->active, "VF active in loop"); + } + + /* Initialize BAR for each VF */ + for (int i = 1; i <= 4; i++) { + vfs[i]->bar_state.aperture_offset = vfs[i]->bar1_offset; + vfs[i]->bar_state.aperture_size = vfs[i]->bar1_size; + TEST_ASSERT_EQ(vfs[i]->bar_state.aperture_offset, 0x10000000 * i, "VF BAR offset"); + TEST_ASSERT_EQ(vfs[i]->bar_state.aperture_size, 0x100000, "VF BAR size"); + } + + /* Add WPR for each VF */ + for (int i = 1; i <= 4; i++) { + int ret = mock_vf_wpr_add(vfs[i], vfs[i]->pt_base, vfs[i]->pt_size, 3); + TEST_ASSERT_EQ(ret, 0, "VF WPR add in loop"); + } + + /* Simulate RPC calls for each VF */ + for (int i = 1; i <= 4; i++) { + struct gsp_rpc_response resp; + size_t resp_size = sizeof(resp); + struct gsp_alloc_payload payload = { + .size = cpu_to_le64(0x100000), + .type = cpu_to_le32(GSP_MEM_VRAM), + .vf_id = cpu_to_le32(i), + }; + int ret = mock_rpc_send(rpc, GSP_RPC_ALLOC, &payload, sizeof(payload), &resp, &resp_size); + TEST_ASSERT_EQ(ret, 0, "VF RPC alloc"); + TEST_ASSERT_EQ(resp.status, 0, "VF RPC response OK"); + } + + /* Cleanup */ + for (int i = 1; i <= 4; i++) { + mock_vf_destroy(vfs[i]); + } + mock_rpc_destroy(rpc); + mock_hw_destroy(hw); + + return TEST_PASS; +} + +static int test_migration_progress_tracking(void) +{ + struct { + u64 pages_transferred; + u64 bytes_transferred; + u64 dirty_pages; + u64 *page_list; + u32 page_count; + } progress = {0}; + + progress.page_count = 1000; + progress.page_list = kcalloc(progress.page_count, sizeof(u64), GFP_KERNEL); + TEST_ASSERT(progress.page_list != NULL, "Progress page list alloc"); + + /* Simulate transfer loop */ + for (int iter = 0; iter < 10; iter++) { + progress.pages_transferred += 100; + progress.bytes_transferred += 100 * 4096; + progress.dirty_pages = 50 - (iter * 5); + + TEST_ASSERT(progress.pages_transferred <= 1000, "Pages not exceeded"); + TEST_ASSERT(progress.bytes_transferred <= 1000 * 4096, "Bytes not exceeded"); + } + + TEST_ASSERT_EQ(progress.pages_transferred, 1000, "Final pages"); + TEST_ASSERT_EQ(progress.bytes_transferred, 1000 * 4096, "Final bytes"); + TEST_ASSERT_EQ(progress.dirty_pages, 0, "Final dirty pages"); + + kfree(progress.page_list); + return TEST_PASS; +} + +static int test_bar_state_machine(void) +{ + struct { + u64 aperture_offset; + u64 aperture_size; + bool force_access; + u32 force_vf; + } bar_state = {0}; + + /* Initial state */ + bar_state.aperture_offset = 0x10000000; + bar_state.aperture_size = 0x100000; + TEST_ASSERT_EQ(bar_state.aperture_offset, 0x10000000, "Initial aperture offset"); + TEST_ASSERT(!bar_state.force_access, "Force access initially false"); + + /* Enable force access */ + bar_state.force_access = true; + bar_state.force_vf = 1; + TEST_ASSERT(bar_state.force_access, "Force access enabled"); + TEST_ASSERT_EQ(bar_state.force_vf, 1, "Force VF set"); + + /* Disable force access */ + bar_state.force_access = false; + bar_state.force_vf = 0; + TEST_ASSERT(!bar_state.force_access, "Force access disabled"); + TEST_ASSERT_EQ(bar_state.force_vf, 0, "Force VF cleared"); + + return TEST_PASS; +} + +static int test_concurrent_vf_operations(void) +{ + struct mock_vf *vf1 = mock_vf_create(1, 0x10000000, 0x100000, 0x20000000, 0x100000, 0x30000000, 0x10000); + struct mock_vf *vf2 = mock_vf_create(2, 0x20000000, 0x100000, 0x40000000, 0x100000, 0x60000000, 0x10000); + struct mock_rpc_client *rpc = mock_rpc_create(); + + TEST_ASSERT(vf1 && vf2 && rpc, "Create two VFs and RPC"); + + /* Simulate concurrent operations */ + struct gsp_rpc_response resp; + size_t resp_size = sizeof(resp); + + /* VF1 alloc */ + struct gsp_alloc_payload payload1 = { + .size = cpu_to_le64(0x100000), + .type = cpu_to_le32(GSP_MEM_VRAM), + .vf_id = cpu_to_le32(1), + }; + int ret = mock_rpc_send(rpc, GSP_RPC_ALLOC, &payload1, sizeof(payload1), &resp, &resp_size); + TEST_ASSERT_EQ(ret, 0, "VF1 alloc"); + TEST_ASSERT_EQ(resp.status, 0, "VF1 alloc response"); + + /* VF2 alloc */ + struct gsp_alloc_payload payload2 = { + .size = cpu_to_le64(0x200000), + .type = cpu_to_le32(GSP_MEM_VRAM), + .vf_id = cpu_to_le32(2), + }; + ret = mock_rpc_send(rpc, GSP_RPC_ALLOC, &payload2, sizeof(payload2), &resp, &resp_size); + TEST_ASSERT_EQ(ret, 0, "VF2 alloc"); + TEST_ASSERT_EQ(resp.status, 0, "VF2 alloc response"); + + /* VF1 pin */ + struct gsp_pin_payload pin_payload = { + .handle = cpu_to_le64(0x1000), + .vf_id = cpu_to_le32(1), + }; + ret = mock_rpc_send(rpc, GSP_RPC_PIN, &pin_payload, sizeof(pin_payload), &resp, &resp_size); + TEST_ASSERT_EQ(ret, 0, "VF1 pin"); + + /* VF2 BAR trap */ + struct gsp_bar_trap_payload bar_payload = { + .vf_id = cpu_to_le32(2), + .bar_index = cpu_to_le32(1), + }; + ret = mock_rpc_send(rpc, GSP_RPC_BAR_TRAP, &bar_payload, sizeof(bar_payload), &resp, &resp_size); + TEST_ASSERT_EQ(ret, 0, "VF2 BAR trap"); + + /* Verify sequence numbers */ + TEST_ASSERT_EQ(resp.sequence, 4, "Sequence number 4"); + + mock_vf_destroy(vf1); + mock_vf_destroy(vf2); + mock_rpc_destroy(rpc); + + return TEST_PASS; +} + +/* ============================================================ + * Module Init/Exit + * ============================================================ */ + +static int __init gsp_mock_test_init(void) +{ + pr_info("========================================\n"); + pr_info("GSP vGPU-IO Mock Driver Test Suite\n"); + pr_info("========================================\n"); + + test_mock_hardware(); + test_mock_vf_lifecycle(); + test_mock_vf_wpr(); + test_mock_rpc_client(); + test_driver_vf_creation_logic(); + test_migration_progress_tracking(); + test_bar_state_machine(); + test_concurrent_vf_operations(); + + pr_info("========================================\n"); + pr_info("Mock Test Results: %d total, %d passed, %d failed\n", + test_count, pass_count, fail_count); + pr_info("========================================\n"); + + if (fail_count > 0) { + pr_err("MOCK TEST SUITE FAILED\n"); + return -EINVAL; + } + + pr_info("MOCK TEST SUITE PASSED\n"); + return 0; +} + +static void __exit gsp_mock_test_exit(void) +{ + pr_info("GSP vGPU-IO Mock Test Suite Exited\n"); +} + +module_init(gsp_mock_test_init); +module_exit(gsp_mock_test_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Open-IOV Project"); +MODULE_DESCRIPTION("GSP vGPU-IO Mock Driver Test"); +MODULE_VERSION("1.0");
\ No newline at end of file |