From 2302fc09080bad51c73f293294e19fd4db464eb0 Mon Sep 17 00:00:00 2001 From: Open-IOV Date: Wed, 23 Sep 2026 05:44:08 -0400 Subject: Open-IOV: initial public release Open-IOV virtualizes NVIDIA GPUs so that unmodified guest drivers run with real GPU functionality - CUDA first, 3D and video to follow - on unmodified hosts, without a proprietary vGPU license and without patching any driver, firmware or hypervisor. Core principle: emulate the interface, pass through the work. kernel/ open-iov-vfio.ko - host VFIO mdev driver: virtual GPU device (PCI config space, BAR shadows, MSI-X), GSP boot emulation, GSP-RM message-queue/RPC emulation, memory model (GPU-VA to guest-PA discovery) and the semaphore-completion service that advances guest UVM progress. userspace/ libiov.so management API and examples. tools/ guest diagnostics, firmware analysis wrappers, RM probes. scripts/ stack manager, VM harness, build/load/test automation. config/ hardware/version profiles (the portability surface). docs/ documentation tree - start at docs/README.md. wiki/ long-form firmware and protocol deep dives. Status: initialization works end to end for a stock 570.144 Linux guest on Turing (TU104): driver probe, nvidia-smi, the full GSP-RM control battery, object creation and the early CUDA/UVM start-up path. GPU work forwarding - the step that makes CUDA kernels execute on real hardware - is the active work item; see docs/04-implementation/01-known-knowns-and-unknowns.md. License: BSD-3-Clause for the project; GPL-2.0 for kernel modules that link GPL-only kernel interfaces. See LICENSE and docs/09-reference/legal-and-licensing.md. --- kernel/Kconfig | 44 + kernel/Makefile | 43 + kernel/Makefile.gsp_rpc | 24 + kernel/architecture/architecture.c | 176 +++ kernel/architecture/architecture.h | 49 + kernel/bar_emulation.c | 578 +++++++ kernel/bar_init.c | 175 +++ kernel/dynamic_rpc_dispatcher.c | 438 ++++++ kernel/dynamic_rpc_dispatcher.h | 66 + kernel/firmware_analyzer.c | 224 +++ kernel/firmware_analyzer.h | 36 + kernel/gsp_comm.c | 638 ++++++++ kernel/gsp_emulation.c | 936 ++++++++++++ kernel/gsp_emulation.h | 206 +++ kernel/gsp_rm_emulation.c | 2501 ++++++++++++++++++++++++++++++ kernel/gsp_rm_emulation.h | 187 +++ kernel/gsp_rm_tu104.h | 1009 ++++++++++++ kernel/gsp_rpc_client.c | 1174 ++++++++++++++ kernel/gsp_rpc_client.h | 355 +++++ kernel/gsp_rpc_integration.c | 535 +++++++ kernel/gsp_sema_service.c | 418 +++++ kernel/gsp_sema_service.h | 16 + kernel/include/bar_emulation.h | 79 + kernel/include/gsp_comm.h | 155 ++ kernel/include/gsp_rpc_client.h | 313 ++++ kernel/include/gsp_rpc_integration.h | 97 ++ kernel/include/ioctl.h | 126 ++ kernel/include/irq.h | 45 + kernel/include/memory.h | 56 + kernel/include/migration.h | 75 + kernel/include/nv-hypervisor.h | 93 ++ kernel/include/nv-vgpu-vfio-interface.h | 83 + kernel/include/nvidia-vgpu-vfio.h | 232 +++ kernel/include/nvstatus.h | 123 ++ kernel/include/nvtypes.h | 830 ++++++++++ kernel/ioctl.c | 307 ++++ kernel/irq.c | 246 +++ kernel/main.c | 2532 +++++++++++++++++++++++++++++++ kernel/memory.c | 274 ++++ kernel/memory.c.iommu | 335 ++++ kernel/migration.c | 324 ++++ kernel/tests/Makefile | 24 + kernel/tests/gsp_emulation_test.c | 350 +++++ kernel/tests/gsp_rpc_test.c | 425 ++++++ kernel/version/version.c | 129 ++ kernel/version/version.h | 35 + 46 files changed, 17116 insertions(+) create mode 100644 kernel/Kconfig create mode 100644 kernel/Makefile create mode 100644 kernel/Makefile.gsp_rpc create mode 100644 kernel/architecture/architecture.c create mode 100644 kernel/architecture/architecture.h create mode 100644 kernel/bar_emulation.c create mode 100644 kernel/bar_init.c create mode 100644 kernel/dynamic_rpc_dispatcher.c create mode 100644 kernel/dynamic_rpc_dispatcher.h create mode 100644 kernel/firmware_analyzer.c create mode 100644 kernel/firmware_analyzer.h create mode 100644 kernel/gsp_comm.c create mode 100644 kernel/gsp_emulation.c create mode 100644 kernel/gsp_emulation.h create mode 100644 kernel/gsp_rm_emulation.c create mode 100644 kernel/gsp_rm_emulation.h create mode 100644 kernel/gsp_rm_tu104.h create mode 100644 kernel/gsp_rpc_client.c create mode 100644 kernel/gsp_rpc_client.h create mode 100644 kernel/gsp_rpc_integration.c create mode 100644 kernel/gsp_sema_service.c create mode 100644 kernel/gsp_sema_service.h create mode 100644 kernel/include/bar_emulation.h create mode 100644 kernel/include/gsp_comm.h create mode 100644 kernel/include/gsp_rpc_client.h create mode 100644 kernel/include/gsp_rpc_integration.h create mode 100644 kernel/include/ioctl.h create mode 100644 kernel/include/irq.h create mode 100644 kernel/include/memory.h create mode 100644 kernel/include/migration.h create mode 100644 kernel/include/nv-hypervisor.h create mode 100644 kernel/include/nv-vgpu-vfio-interface.h create mode 100644 kernel/include/nvidia-vgpu-vfio.h create mode 100644 kernel/include/nvstatus.h create mode 100644 kernel/include/nvtypes.h create mode 100644 kernel/ioctl.c create mode 100644 kernel/irq.c create mode 100644 kernel/main.c create mode 100644 kernel/memory.c create mode 100644 kernel/memory.c.iommu create mode 100644 kernel/migration.c create mode 100644 kernel/tests/Makefile create mode 100644 kernel/tests/gsp_emulation_test.c create mode 100644 kernel/tests/gsp_rpc_test.c create mode 100644 kernel/version/version.c create mode 100644 kernel/version/version.h (limited to 'kernel') diff --git a/kernel/Kconfig b/kernel/Kconfig new file mode 100644 index 0000000..5cee953 --- /dev/null +++ b/kernel/Kconfig @@ -0,0 +1,44 @@ +config NVIDIA_VGPU_VFIO + tristate "NVIDIA vGPU VFIO Mediated Device Driver" + depends on VFIO_MDEV && PCI && IOMMU_API + select VFIO_PCI_CORE + help + This driver provides VFIO mediated device support for NVIDIA GPUs, + enabling vGPU functionality through the Linux VFIO/mdev framework. + It replaces the proprietary nvidia-vgpu-vfio.ko kernel module. + + The driver handles: + - VFIO device creation and management + - BAR trapping and emulation + - GSP (GPU System Processor) communication + - Memory mapping and pinning + - IRQFD/IOEVENTFD handling + - Live migration support + + To compile this driver as a module, choose M here: the + module will be called nvidia-vgpu-vfio. + +if NVIDIA_VGPU_VFIO + +config NVIDIA_VGPU_VFIO_DEBUG + bool "Enable debug logging for NVIDIA vGPU VFIO driver" + default n + help + Enable verbose debug logging in the NVIDIA vGPU VFIO driver. + This will increase kernel log verbosity significantly. + +config NVIDIA_VGPU_VFIO_MIGRATION + bool "Enable live migration support" + default y + help + Enable live migration support for vGPU mediated devices. + This adds the migration ioctl interface and state serialization. + +config NVIDIA_VGPU_VFIO_GSP_RPC + bool "Enable GSP RPC communication" + default y + help + Enable communication with the GPU System Processor (GSP) + via RPC for scheduling and resource management. + +endif \ No newline at end of file diff --git a/kernel/Makefile b/kernel/Makefile new file mode 100644 index 0000000..8562430 --- /dev/null +++ b/kernel/Makefile @@ -0,0 +1,43 @@ +obj-m += open-iov-vfio.o + +open-iov-vfio-y := \ + main.o \ + bar_init.o \ + bar_emulation.o \ + gsp_comm.o \ + gsp_emulation.o \ + gsp_rm_emulation.o \ + gsp_sema_service.o \ + gsp_rpc_client.o \ + gsp_rpc_integration.o \ + memory.o \ + irq.o \ + migration.o \ + ioctl.o \ + firmware_analyzer.o \ + dynamic_rpc_dispatcher.o \ + architecture/architecture.o \ + version/version.o + +ccflags-y := -I$(src)/include -I$(src)/architecture -I$(src)/version -Wall -Wextra -Wno-error -Wno-unused-parameter -Wno-unused-function + +# mdev dependencies +KBUILD_MODPOST_WARN := 1 + +# Use symbols from host nvidia.ko (path resolved relative to this repo) +KBUILD_EXTRA_SYMBOLS := $(abspath $(src)/../nvidia-open-gpu-kernel-modules/kernel-open/Module.symvers) + +# Use $(CURDIR), NOT $(PWD): when invoked via the top-level Makefile +# (make -C kernel), $(PWD) still points at the caller's directory and kbuild +# silently builds nothing (the "stale .ko" trap). +all: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(CURDIR) modules + +clean: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(CURDIR) clean + +install: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(CURDIR) modules_install + depmod -a + +.PHONY: all clean install \ No newline at end of file diff --git a/kernel/Makefile.gsp_rpc b/kernel/Makefile.gsp_rpc new file mode 100644 index 0000000..8bbee6b --- /dev/null +++ b/kernel/Makefile.gsp_rpc @@ -0,0 +1,24 @@ +# GSP RPC Client Kernel Module Makefile +# Build: make -C /lib/modules/$(shell uname -r)/build M=$(PWD) -f Makefile.gsp_rpc + +obj-m += gsp_rpc_client.o + +gsp_rpc_client-y := gsp_rpc_client.o + +ccflags-y += -I$(src)/include +ccflags-y += -Wall -Wextra -Werror +ccflags-y += -std=gnu11 + +# For debugging +# ccflags-y += -DDEBUG + +all: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(PWD) -f $(PWD)/Makefile.gsp_rpc modules + +clean: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(PWD) -f $(PWD)/Makefile.gsp_rpc clean + +install: + $(MAKE) -C /lib/modules/$(shell uname -r)/build M=$(PWD) -f $(PWD)/Makefile.gsp_rpc modules_install + +.PHONY: all clean install diff --git a/kernel/architecture/architecture.c b/kernel/architecture/architecture.c new file mode 100644 index 0000000..bc85f85 --- /dev/null +++ b/kernel/architecture/architecture.c @@ -0,0 +1,176 @@ +/* + * Hardware Architecture Abstraction Layer + * Defines hardware-specific configurations for different GPU architectures + */ + +#include +#include +#include +#include + +#include "architecture.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Open-IOV"); +MODULE_DESCRIPTION("Hardware Architecture Abstraction"); + +/* Turing architecture (TU102, TU104, TU106) */ +static struct hw_architecture arch_turing = { + .name = "turing", + .pci_vendor_id = 0x10DE, + .pci_device_ids = (u32[]){0x1E82, 0x1E83, 0x1E84, 0x1E85, 0x1E86, 0x1E87, 0x1E88, 0x1E89, 0x1E8A, 0x1E8B, 0x1E8C, 0x1E8D, 0x1E8E, 0x1E8F, 0x1ED0, 0x1ED1, 0x1ED2, 0x1ED3, 0x1ED4, 0x1ED5, 0x1ED6, 0x1ED7, 0x1ED8, 0x1ED9, 0x1EDA, 0x1EDB, 0x1EDC, 0x1EDD, 0x1EDE, 0x1EDF, 0}, + .num_device_ids = 30, + .bar_sizes = { + .bar0 = 16 * 1024 * 1024, /* 16MB */ + .bar1 = 256 * 1024 * 1024, /* 256MB */ + .bar2 = 32 * 1024 * 1024, /* 32MB */ + .bar3 = 32 * 1024 * 1024, /* 32MB */ + }, + .doorbell_offset = 0x110C00, + .shared_mem_offset = 0x1000, + .shared_mem_size = 4096, + .msix_table_offset = 0x2000, + .msix_pba_offset = 0x3000, + .msix_vectors = 32, + .max_vfs = 8, + .vram_size = 16 * 1024 * 1024 * 1024ULL, /* 16GB */ + .page_size = 4096, + .supports_mig = false, + .supports_sriov = true, + .gsp_version = "570.144W", +}; + +/* Ampere architecture (GA100, GA102, GA104) */ +static struct hw_architecture arch_ampere = { + .name = "ampere", + .pci_vendor_id = 0x10DE, + .pci_device_ids = (u32[]){0x20B0, 0x20B1, 0x20B2, 0x20B3, 0x20B4, 0x20B5, 0x20B6, 0x20B7, 0x20B8, 0x20B9, 0x20BA, 0x20BB, 0x20BC, 0x20BD, 0x20BE, 0x20BF, 0x20C0, 0x20C1, 0x20C2, 0x20C3, 0x20C4, 0x20C5, 0x20C6, 0x20C7, 0x20C8, 0x20C9, 0x20CA, 0x20CB, 0x20CC, 0x20CD, 0x20CE, 0x20CF, 0x20D0, 0x20D1, 0x20D2, 0x20D3, 0x20D4, 0x20D5, 0x20D6, 0x20D7, 0x20D8, 0x20D9, 0x20DA, 0x20DB, 0x20DC, 0x20DD, 0x20DE, 0x20DF, 0x20E0, 0x20E1, 0x20E2, 0x20E3, 0x20E4, 0x20E5, 0x20E6, 0x20E7, 0x20E8, 0x20E9, 0x20EA, 0x20EB, 0x20EC, 0x20ED, 0x20EE, 0x20EF, 0}, + .num_device_ids = 96, + .bar_sizes = { + .bar0 = 16 * 1024 * 1024, /* 16MB */ + .bar1 = 512 * 1024 * 1024, /* 512MB */ + .bar2 = 32 * 1024 * 1024, /* 32MB */ + .bar3 = 32 * 1024 * 1024, /* 32MB */ + }, + .doorbell_offset = 0x110C00, + .shared_mem_offset = 0x1000, + .shared_mem_size = 4096, + .msix_table_offset = 0x2000, + .msix_pba_offset = 0x3000, + .msix_vectors = 32, + .max_vfs = 16, + .vram_size = 80 * 1024 * 1024 * 1024ULL, /* 80GB (A100) */ + .page_size = 4096, + .supports_mig = true, + .supports_sriov = true, + .gsp_version = "570.144W", +}; + +/* Blackwell architecture (GB100, GB202, GB203) */ +static struct hw_architecture arch_blackwell = { + .name = "blackwell", + .pci_vendor_id = 0x10DE, + .pci_device_ids = (u32[]){0x2300, 0x2301, 0x2302, 0x2303, 0x2304, 0x2305, 0x2306, 0x2307, 0x2308, 0x2309, 0x230A, 0x230B, 0x230C, 0x230D, 0x230E, 0x230F, 0}, + .num_device_ids = 16, + .bar_sizes = { + .bar0 = 16 * 1024 * 1024, /* 16MB */ + .bar1 = 512 * 1024 * 1024, /* 512MB */ + .bar2 = 64 * 1024 * 1024, /* 64MB */ + .bar3 = 64 * 1024 * 1024, /* 64MB */ + }, + .doorbell_offset = 0x110C00, + .shared_mem_offset = 0x1000, + .shared_mem_size = 4096, + .msix_table_offset = 0x2000, + .msix_pba_offset = 0x3000, + .msix_vectors = 64, + .max_vfs = 32, + .vram_size = 192 * 1024 * 1024 * 1024ULL, /* 192GB (B200) */ + .page_size = 4096, + .supports_mig = true, + .supports_sriov = true, + .gsp_version = "580.XX", +}; + +static struct hw_architecture *architectures[] = { + &arch_turing, + &arch_ampere, + &arch_blackwell, + NULL +}; + +struct hw_architecture *arch_get_by_name(const char *name) +{ + for (int i = 0; architectures[i]; i++) { + if (strcmp(architectures[i]->name, name) == 0) + return architectures[i]; + } + return NULL; +} + +struct hw_architecture *arch_get_by_pci_id(u32 vendor_id, u32 device_id) +{ + for (int i = 0; architectures[i]; i++) { + if (architectures[i]->pci_vendor_id == vendor_id) { + for (int j = 0; j < architectures[i]->num_device_ids; j++) { + if (architectures[i]->pci_device_ids[j] == device_id) + return architectures[i]; + } + } + } + return NULL; +} + +struct hw_architecture *arch_get_default(void) +{ + return &arch_turing; +} + +const struct hw_bar_config *arch_get_bar_config(struct hw_architecture *arch) +{ + return arch ? &arch->bar_sizes : NULL; +} + +u64 arch_get_doorbell_offset(struct hw_architecture *arch) +{ + return arch ? arch->doorbell_offset : 0x110C00; +} + +u64 arch_get_shared_mem_offset(struct hw_architecture *arch) +{ + return arch ? arch->shared_mem_offset : 0x1000; +} + +u32 arch_get_msix_vectors(struct hw_architecture *arch) +{ + return arch ? arch->msix_vectors : 32; +} + +bool arch_supports_mig(struct hw_architecture *arch) +{ + return arch ? arch->supports_mig : false; +} + +bool arch_supports_sriov(struct hw_architecture *arch) +{ + return arch ? arch->supports_sriov : false; +} + +const char *arch_get_gsp_version(struct hw_architecture *arch) +{ + return arch ? arch->gsp_version : "unknown"; +} + +EXPORT_SYMBOL_GPL(arch_get_by_name); +EXPORT_SYMBOL_GPL(arch_get_by_pci_id); +EXPORT_SYMBOL_GPL(arch_get_default); +EXPORT_SYMBOL_GPL(arch_get_bar_config); +EXPORT_SYMBOL_GPL(arch_get_doorbell_offset); +EXPORT_SYMBOL_GPL(arch_get_shared_mem_offset); +EXPORT_SYMBOL_GPL(arch_get_msix_vectors); +EXPORT_SYMBOL_GPL(arch_supports_mig); +EXPORT_SYMBOL_GPL(arch_supports_sriov); +EXPORT_SYMBOL_GPL(arch_get_gsp_version); + +MODULE_DESCRIPTION("Hardware Architecture Abstraction"); +MODULE_LICENSE("GPL"); \ No newline at end of file diff --git a/kernel/architecture/architecture.h b/kernel/architecture/architecture.h new file mode 100644 index 0000000..3745bf8 --- /dev/null +++ b/kernel/architecture/architecture.h @@ -0,0 +1,49 @@ +#ifndef _ARCHITECTURE_H_ +#define _ARCHITECTURE_H_ + +#include + +/* BAR configuration */ +struct hw_bar_config { + u64 bar0; + u64 bar1; + u64 bar2; + u64 bar3; +}; + +/* Hardware architecture definition */ +struct hw_architecture { + const char *name; + u32 pci_vendor_id; + u32 *pci_device_ids; + int num_device_ids; + struct hw_bar_config bar_sizes; + u64 doorbell_offset; + u64 shared_mem_offset; + u32 shared_mem_size; + u64 msix_table_offset; + u64 msix_pba_offset; + u32 msix_vectors; + u32 max_vfs; + u64 vram_size; + u32 page_size; + bool supports_mig; + bool supports_sriov; + const char *gsp_version; +}; + +/* Architecture registry */ +struct hw_architecture *arch_get_by_name(const char *name); +struct hw_architecture *arch_get_by_pci_id(u32 vendor_id, u32 device_id); +struct hw_architecture *arch_get_default(void); + +/* Architecture queries */ +const struct hw_bar_config *arch_get_bar_config(struct hw_architecture *arch); +u64 arch_get_doorbell_offset(struct hw_architecture *arch); +u64 arch_get_shared_mem_offset(struct hw_architecture *arch); +u32 arch_get_msix_vectors(struct hw_architecture *arch); +bool arch_supports_mig(struct hw_architecture *arch); +bool arch_supports_sriov(struct hw_architecture *arch); +const char *arch_get_gsp_version(struct hw_architecture *arch); + +#endif /* _ARCHITECTURE_H_ */ \ No newline at end of file diff --git a/kernel/bar_emulation.c b/kernel/bar_emulation.c new file mode 100644 index 0000000..685831a --- /dev/null +++ b/kernel/bar_emulation.c @@ -0,0 +1,578 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" +#include "gsp_comm.h" +#include "gsp_emulation.h" + +#define NV_VGPU_BAR0_TRAP_REGIONS 64 + +/* VBIOS (PROM window) emulation - see nv_vgpu_bar_emul_init */ +#define NV_VGPU_VBIOS_FW_NAME "tu104_vbios.bin" +#define NV_VGPU_PROM_WINDOW_OFFSET 0x300000 +#define NV_VGPU_VBIOS_MAX_SIZE 0x100000 /* 1MB (s_getBaseBiosMaxSize_TU102) */ + +/* MSI-X Table entry size (16 bytes: addr_lo, addr_hi, data, vector_control) */ +#define NV_VGPU_MSIX_ENTRY_SIZE 16 +#define NV_VGPU_MSIX_TABLE_ENTRIES 32 +#define NV_VGPU_MSIX_TABLE_SIZE (NV_VGPU_MSIX_ENTRY_SIZE * NV_VGPU_MSIX_TABLE_ENTRIES) +#define NV_VGPU_MSIX_PBA_SIZE 8 /* 64 bits for 32 vectors */ + +static const struct { + u32 offset; + u32 size; + const char *name; +} nv_vgpu_bar0_traps[] = { + { 0x000000, 0x1000, "PMC" }, + { 0x001000, 0x1000, "PRAMIN" }, + { 0x002000, 0x1000, "PCR" }, + { 0x003000, 0x1000, "PRAMDAC" }, + { 0x004000, 0x1000, "PVIO" }, + { 0x005000, 0x1000, "PFIFO" }, + { 0x006000, 0x1000, "PGRAPH" }, + { 0x007000, 0x1000, "PTIMER" }, + { 0x008000, 0x1000, "PFUSE" }, + { 0x009000, 0x1000, "PBUS" }, + { 0x00A000, 0x1000, "PCOP" }, + { 0x00B000, 0x1000, "PVP" }, + { 0x00C000, 0x1000, "PCE" }, + { 0x00D000, 0x1000, "PDISP" }, + { 0x00E000, 0x1000, "PNVDEC" }, + { 0x00F000, 0x1000, "PNVENC" }, + { 0x010000, 0x1000, "PSEC2" }, + { 0x011000, 0x1000, "PMU" }, + { 0x012000, 0x1000, "PGFX" }, + { 0x013000, 0x1000, "PXVE" }, + { 0x020000, 0x10000, "FIFO" }, + { 0x040000, 0x10000, "GR" }, + { 0x050000, 0x10000, "CE" }, + { 0x060000, 0x10000, "NVDEC" }, + { 0x070000, 0x10000, "NVENC" }, + { 0x080000, 0x10000, "NVJPG" }, + { 0x090000, 0x10000, "OFA" }, + { 0x100000, 0x10000, "FB" }, + { 0x110000, 0x10000, "LTC" }, + { 0x120000, 0x10000, "GPC" }, + { 0x130000, 0x10000, "TPC" }, + { 0x140000, 0x10000, "ROP" }, + { 0x150000, 0x10000, "HUB" }, + { 0x160000, 0x10000, "NVLINK" }, + { 0x170000, 0x10000, "GSP" }, + { 0x180000, 0x10000, "RAP" }, + { 0x190000, 0x10000, "SEC" }, + { 0x1A0000, 0x10000, "PMU_RISCV" }, + { 0x1B0000, 0x10000, "FALCON" }, + { 0x1C0000, 0x10000, "XVE" }, + { 0x1D0000, 0x10000, "DMCE" }, + { 0x1E0000, 0x10000, "P2P" }, + { 0x1F0000, 0x10000, "MISC" }, +}; + +static void nv_vgpu_msix_init(struct nv_vgpu_bar_emul_ctx *ctx) +{ + memset(ctx->msix_table, 0, sizeof(ctx->msix_table)); + ctx->msix_pba = 0; + spin_lock_init(&ctx->msix_lock); + ctx->msix_enabled = false; +} + +static void nv_vgpu_msix_write_table(struct nv_vgpu_bar_emul_ctx *ctx, + u32 offset, u64 val, u32 size) +{ + u32 entry_idx = offset / NV_VGPU_MSIX_ENTRY_SIZE; + u32 entry_offset = offset % NV_VGPU_MSIX_ENTRY_SIZE; + + if (entry_idx >= NV_VGPU_MSIX_TABLE_ENTRIES) + return; + + spin_lock(&ctx->msix_lock); + + switch (entry_offset) { + case 0x0: /* Message Address Low */ + ctx->msix_table[entry_idx].addr = (ctx->msix_table[entry_idx].addr & 0xFFFFFFFF00000000ULL) | val; + break; + case 0x4: /* Message Address High */ + ctx->msix_table[entry_idx].addr = (ctx->msix_table[entry_idx].addr & 0x00000000FFFFFFFFULL) | (val << 32); + break; + case 0x8: /* Message Data */ + ctx->msix_table[entry_idx].data = (u32)val; + break; + case 0xC: /* Vector Control */ + ctx->msix_table[entry_idx].vector_control = (u32)val; + break; + } + + spin_unlock(&ctx->msix_lock); +} + +static u64 nv_vgpu_msix_read_table(struct nv_vgpu_bar_emul_ctx *ctx, + u32 offset, u32 size) +{ + u32 entry_idx = offset / NV_VGPU_MSIX_ENTRY_SIZE; + u32 entry_offset = offset % NV_VGPU_MSIX_ENTRY_SIZE; + u64 val = 0; + + if (entry_idx >= NV_VGPU_MSIX_TABLE_ENTRIES) + return 0; + + spin_lock(&ctx->msix_lock); + + switch (entry_offset) { + case 0x0: /* Message Address Low */ + val = ctx->msix_table[entry_idx].addr & 0xFFFFFFFF; + break; + case 0x4: /* Message Address High */ + val = (ctx->msix_table[entry_idx].addr >> 32) & 0xFFFFFFFF; + break; + case 0x8: /* Message Data */ + val = ctx->msix_table[entry_idx].data; + break; + case 0xC: /* Vector Control */ + val = ctx->msix_table[entry_idx].vector_control; + break; + } + + spin_unlock(&ctx->msix_lock); + return val; +} + +static u64 nv_vgpu_msix_read_pba(struct nv_vgpu_bar_emul_ctx *ctx, u32 size) +{ + u64 val; + spin_lock(&ctx->msix_lock); + val = ctx->msix_pba; + spin_unlock(&ctx->msix_lock); + return val; +} + +void nv_vgpu_msix_set_pending(struct nv_vgpu_device *vgpu, int vector) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + + if (!ctx || vector < 0 || vector >= NV_VGPU_MSIX_TABLE_ENTRIES) + return; + + spin_lock(&ctx->msix_lock); + ctx->msix_pba |= (1ULL << vector); + spin_unlock(&ctx->msix_lock); +} + +void nv_vgpu_msix_clear_pending(struct nv_vgpu_device *vgpu, int vector) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + + if (!ctx || vector < 0 || vector >= NV_VGPU_MSIX_TABLE_ENTRIES) + return; + + spin_lock(&ctx->msix_lock); + ctx->msix_pba &= ~(1ULL << vector); + spin_unlock(&ctx->msix_lock); +} + +int nv_vgpu_bar_emul_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_bar_emul_ctx *ctx; + int i, ret; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + ctx->bar0_shadow_size = NV_VGPU_BAR0_SIZE; + ctx->bar0_shadow = vzalloc(ctx->bar0_shadow_size); + if (!ctx->bar0_shadow) { + ret = -ENOMEM; + goto err_free; + } + + spin_lock_init(&ctx->shadow_lock); + ctx->trap_count = 0; + + for (i = 0; i < ARRAY_SIZE(nv_vgpu_bar0_traps); i++) { + if (ctx->trap_count >= 256) + break; + ctx->trap_regs[ctx->trap_count].offset = nv_vgpu_bar0_traps[i].offset; + ctx->trap_regs[ctx->trap_count].trapped = true; + ctx->trap_count++; + } + + /* Initialize MSI-X emulation */ + nv_vgpu_msix_init(ctx); + + /* + * Seed PMC boot registers so the guest driver sees a REAL, + * PHYSICAL, non-virtual TU104 GPU. Values captured from the host's + * physical TU104 (RTX 2080 SUPER, PCI 0000:01:00.0): + * NV_PMC_BOOT_0 (BAR0+0x000) = 0x164000a1 (arch TU104, rev A1) + * NV_PMC_BOOT_1 (BAR0+0x004) = 0x00000000 (VGPU field = REAL, + * no PV/VF - the guest + * must take the physical + * probe path, never a + * para-virtual vGPU path) + * NV_PMC_BOOT_42 (BAR0+0xa00) = 0x164a1000 (chip id / revision) + * + * The guest reads these via the VFIO-mmap'd shadow (and via VFIO + * region reads, which hit the same shadow). Without them the open + * driver's halmgrGetHalForGpu() fails and probe aborts with the + * "does not include the required GPU System Processor (GSP)" error. + */ + *(u32 *)((u8 *)ctx->bar0_shadow + 0x000) = 0x164000a1; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x004) = 0x00000000; + *(u32 *)((u8 *)ctx->bar0_shadow + 0xa00) = 0x164a1000; + + /* + * Pre-set the GSP secure-boot (GFW) completion state. + * + * During RmInitAdapter the guest driver waits for the GPU's + * post-FLR secure boot (kgspWaitForGfwBootOk -> gpuWaitForGfwBootComplete + * -> _gpuIsGfwBootCompleted_TU102 + kflcnWaitForHalt). On real hardware + * the boot ROM runs this automatically after reset; the driver only + * *waits* for it (it never kicks the falcon here). We present the + * "already booted" state so the wait succeeds immediately: + * 0x118128 NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK + * bit0 READ_PROTECTION_LEVEL0 = ENABLE (FWSEC lowered PLM) + * 0x118234 NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT + * bits7:0 PROGRESS = 0xFF (COMPLETED) + * 0x110100 GSP falcon CPUCTL bit4 = HALTED + * + * Driver-kicked falcon runs later (FWSEC/Booter) clear CPUCTL via + * STARTCPU; the GSP emulation thread re-completes those (see + * gsp_emu_poll_boot in gsp_emulation.c). + */ + *(u32 *)((u8 *)ctx->bar0_shadow + 0x118128) = 0x00000001; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x118234) = 0x000000ff; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x110100) = 0x00000010; + + /* + * More post-GFW-boot state the driver validates before/while booting + * GSP-RM: + * 0x1100f4 GSP falcon HWCFG2 bit10 RISCV = ENABLE. Checked by + * kflcnIsRiscvCpuEnabled in kgspPrepareForBootstrap; + * without it probe dies with "RISC-V core is not enabled". + * 0x009430 NV_PTIMER_TIME_PRIV_LEVEL_MASK bit4 + * WRITE_PROTECTION_LEVEL0 = ENABLE. FWSEC lowers PLMs + * after GFW boot; driver asserts "Write to PTIMER + * attempted even though Level 0 PLM is disabled" when + * setting the GPU clock otherwise. + */ + *(u32 *)((u8 *)ctx->bar0_shadow + 0x1100f4) = 0x00000400; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x009430) = 0x00000010; + + /* + * 0x100ce0 NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE - usable FB size. + * Encoded as fbSize = LOWER_MAG << (LOWER_SCALE + 20). Set to + * 256MB (matches our BAR1): mag=1, scale=8 -> 1 << 28. On real + * hardware VBIOS devinit programs this after GFW boot; the guest + * driver's kgspPopulateWprMeta reads it via kmemsysGetUsableFbSize + * and uses it to compute the FRTS offset (fbSize - reservedFromEnd). + * With it 0, the FRTS offset underflows and FWSEC verification + * fails with "WPR2 initialized at an unexpected location". + */ + *(u32 *)((u8 *)ctx->bar0_shadow + 0x100ce0) = 0x00000018; + + /* + * NV_PCFG (BAR0+0x88000) - PCI config space mirror the driver reads + * via gpuReadBusConfigReg_HAL (GPU_BUS_CFG_RD32): + * 0x88084 NV_XVE_LINK_CAPABILITIES: MAX_LINK_SPEED 3:0 = 3 (8 GT/s, + * Gen3), MAX_LINK_WIDTH 9:4 = 0x10 (x16). + * 0x88088 NV_XVE_LINK_CONTROL_STATUS: negotiated speed 19:16 = 3, + * negotiated width 25:20 = 0x10 (x16). + * UVM's nvGpuOpsDeviceCreate -> getPCIELinkRateMBps -> + * kbifControlGetPCIEInfo -> kbifGetGpuLinkCapabilities reads 0x88084; + * zeros make calculatePCIELinkRateMBps log "Unknown PCIe speed" and + * return NV_ERR_INVALID_STATE, which fails cuInit + * (CUDA_ERROR_NOT_INITIALIZED in the guest). + * + * DEVICE_BASE(NV_PCFG) expands to 0 in this driver (nvmisc.h: + * "#define DEVICE_BASE(d) (0?d)"), so the read is actually + * gpuNvAddr + 0x84 - mirror the values at the low alias as well. + */ + *(u32 *)((u8 *)ctx->bar0_shadow + 0x88084) = 0x00000103; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x88088) = 0x01030000; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x00084) = 0x00000103; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x00088) = 0x01030000; + /* Same values under the VF register window aliases, in case this + * driver reaches the config mirror through virtualRegPhysOffset. */ + *(u32 *)((u8 *)ctx->bar0_shadow + 0xB80084) = 0x00000103; + *(u32 *)((u8 *)ctx->bar0_shadow + 0xB80088) = 0x01030000; + *(u32 *)((u8 *)ctx->bar0_shadow + 0xB88084) = 0x00000103; + *(u32 *)((u8 *)ctx->bar0_shadow + 0xB88088) = 0x01030000; + *(u32 *)((u8 *)ctx->bar0_shadow + 0xC08084) = 0x00000103; + *(u32 *)((u8 *)ctx->bar0_shadow + 0xC08088) = 0x01030000; + + /* + * Per-VF static sizes the driver reads through the VF register window + * (GPU_VREG_RD32 = BAR0 + sriovState.virtualRegPhysOffset, 0xB80000 + * for a non-SRIOV virtual GPU). Both are read-only SIZE registers on + * real hardware; we pre-fill them so the local (non-RPC) paths agree + * with the GSP_RM_CONTROL replies in gsp_rm_emulation.c. + * + * 0x3010/0x3030 NV_VIRTUAL_FUNCTION_PRIV_MMU_FAULT_BUFFER_SIZE(0/1): + * _VAL is in NVC369_BUF_SIZE (32B) units. 0x800 -> 64KB + * replayable, 0x80 -> 4KB legacy; matches TU104_GMMU_*_SIZE. + * kgmmuSetAndGetDefaultFaultBufferSize_TU102 writes SET_DEFAULT + * (bit30) and reads _VAL back, so a pre-filled _VAL is all it + * needs. + * 0x3110 NV_VIRTUAL_FUNCTION_PRIV_ACCESS_COUNTER_NOTIFY_BUFFER_SIZE: + * UVM's uvmGetAccessCounterBufferSize_TU102 multiplies it by + * NV_ACCESS_COUNTER_NOTIFY_BUF_SIZE (32B). 0x1000 -> 128KB, the + * value the host's physical TU104 reports. Zero makes the + * access-counter buffer a 0-byte allocation and cuInit fails in + * uvmInitializeAccessCntrBuffer. + */ + *(u32 *)((u8 *)ctx->bar0_shadow + 0xB83010) = 0x00000800; + *(u32 *)((u8 *)ctx->bar0_shadow + 0xB83030) = 0x00000080; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x003010) = 0x00000800; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x003030) = 0x00000080; + *(u32 *)((u8 *)ctx->bar0_shadow + 0xB83110) = 0x00001000; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x003110) = 0x00001000; + *(u32 *)((u8 *)ctx->bar0_shadow + 0xB8311C) = 0x00000000; + *(u32 *)((u8 *)ctx->bar0_shadow + 0xB83118) = 0x00000000; + + + /* + * Serve a valid VBIOS at the PROM window (BAR0+0x300000). + * + * The guest driver's physical GSP boot (kgspInitRm -> TU102 path) + * reads the VBIOS via NV_PROM_DATA (BAR0+0x300000) to extract the + * FWSEC ucode (mandatory on Turing: kgspGetFrtsSize is hardwired + * to 1MB, so FWSEC/FRTS cannot be skipped). Without a valid ROM + * signature the probe aborts with: + * "kgspExtractVbiosFromRom_TU102: did not find valid ROM signature" + * + * The image is a dump of the host's physical TU104 VBIOS (read via + * its own PROM window), loaded from /lib/firmware/tu104_vbios.bin. + * Non-fatal if absent: probe then fails later in FWSEC, not here. + */ + { + const struct firmware *vbios_fw; + int fw_ret = request_firmware(&vbios_fw, NV_VGPU_VBIOS_FW_NAME, + &vgpu->pdev->dev); + if (fw_ret == 0) { + u32 copy_size = min_t(u32, vbios_fw->size, + NV_VGPU_VBIOS_MAX_SIZE); + + if (NV_VGPU_PROM_WINDOW_OFFSET + copy_size <= ctx->bar0_shadow_size) { + memcpy((u8 *)ctx->bar0_shadow + NV_VGPU_PROM_WINDOW_OFFSET, + vbios_fw->data, copy_size); + nv_vgpu_info(vgpu, "VBIOS loaded at BAR0+0x%x (%u bytes)", + NV_VGPU_PROM_WINDOW_OFFSET, copy_size); + } else { + nv_vgpu_warn(vgpu, "VBIOS too large for PROM window (%u bytes)", + copy_size); + } + release_firmware(vbios_fw); + } else { + nv_vgpu_warn(vgpu, "VBIOS firmware %s not found (%d); " + "guest GSP boot will fail at ROM parse", + NV_VGPU_VBIOS_FW_NAME, fw_ret); + } + } + + vgpu->bar_emul_ctx = ctx; + return 0; + +err_free: + kfree(ctx); + return ret; +} + +void nv_vgpu_bar_emul_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + + if (!ctx) + return; + + if (ctx->bar0_shadow) + vfree(ctx->bar0_shadow); + + kfree(ctx); + vgpu->bar_emul_ctx = NULL; +} + +int nv_vgpu_bar_trap_register(struct nv_vgpu_device *vgpu, int bar_idx, + u32 offset, u32 size) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + unsigned long flags; + int i; + + if (!ctx || bar_idx != 0) + return -EINVAL; + + spin_lock_irqsave(&ctx->shadow_lock, flags); + + for (i = 0; i < ctx->trap_count; i++) { + if (ctx->trap_regs[i].offset == offset) { + ctx->trap_regs[i].trapped = true; + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return 0; + } + } + + if (ctx->trap_count >= 256) { + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return -ENOSPC; + } + + ctx->trap_regs[ctx->trap_count].offset = offset; + ctx->trap_regs[ctx->trap_count].trapped = true; + ctx->trap_count++; + + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return 0; +} + +int nv_vgpu_bar_untrap_register(struct nv_vgpu_device *vgpu, int bar_idx, + u32 offset) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + unsigned long flags; + int i; + + if (!ctx || bar_idx != 0) + return -EINVAL; + + spin_lock_irqsave(&ctx->shadow_lock, flags); + + for (i = 0; i < ctx->trap_count; i++) { + if (ctx->trap_regs[i].offset == offset) { + ctx->trap_regs[i].trapped = false; + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return 0; + } + } + + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return -ENOENT; +} + +int nv_vgpu_bar_emul_read(struct nv_vgpu_device *vgpu, int bar_idx, + u64 offset, void *val, size_t size) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + unsigned long flags; + u32 *val32 = val; + u64 *val64 = val; + + if (!ctx || bar_idx != 0 || offset >= ctx->bar0_shadow_size) + return -EINVAL; + + /* Handle MSI-X Table reads (at BAR0+0x2000) */ + if (offset >= GSP_EMU_MSIX_TABLE_OFFSET && + offset < GSP_EMU_MSIX_TABLE_OFFSET + NV_VGPU_MSIX_TABLE_SIZE) { + u32 msix_offset = offset - GSP_EMU_MSIX_TABLE_OFFSET; + *val64 = nv_vgpu_msix_read_table(ctx, msix_offset, size); + return 0; + } + + /* Handle MSI-X PBA reads (at BAR0+0x3000) */ + if (offset >= GSP_EMU_MSIX_PBA_OFFSET && + offset < GSP_EMU_MSIX_PBA_OFFSET + NV_VGPU_MSIX_PBA_SIZE) { + *val64 = nv_vgpu_msix_read_pba(ctx, size); + return 0; + } + + spin_lock_irqsave(&ctx->shadow_lock, flags); + + switch (size) { + case 4: + *val32 = *(u32 *)((u8 *)ctx->bar0_shadow + offset); + break; + case 8: + *val64 = *(u64 *)((u8 *)ctx->bar0_shadow + offset); + break; + default: + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return -EINVAL; + } + + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return 0; +} + +int nv_vgpu_bar_emul_write(struct nv_vgpu_device *vgpu, int bar_idx, + u64 offset, void *val, size_t size) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + unsigned long flags; + u32 *val32 = val; + u64 *val64 = val; + bool trapped = false; + int i; + + if (!ctx || bar_idx != 0 || offset >= ctx->bar0_shadow_size) + return -EINVAL; + + /* Handle MSI-X Table writes (at BAR0+0x2000) */ + if (offset >= GSP_EMU_MSIX_TABLE_OFFSET && + offset < GSP_EMU_MSIX_TABLE_OFFSET + NV_VGPU_MSIX_TABLE_SIZE) { + u32 msix_offset = offset - GSP_EMU_MSIX_TABLE_OFFSET; + nv_vgpu_msix_write_table(ctx, msix_offset, *val64, size); + return 0; + } + + spin_lock_irqsave(&ctx->shadow_lock, flags); + + for (i = 0; i < ctx->trap_count; i++) { + if (ctx->trap_regs[i].offset == offset && ctx->trap_regs[i].trapped) { + trapped = true; + ctx->trap_regs[i].value = (size == 4) ? *val32 : (u32)*val64; + ctx->trap_regs[i].is_write = true; + break; + } + } + + switch (size) { + case 4: + *(u32 *)((u8 *)ctx->bar0_shadow + offset) = *val32; + break; + case 8: + *(u64 *)((u8 *)ctx->bar0_shadow + offset) = *val64; + break; + default: + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + return -EINVAL; + } + + spin_unlock_irqrestore(&ctx->shadow_lock, flags); + + if (trapped) { + nv_vgpu_dbg(vgpu, "BAR0 trap: offset=0x%llx, size=%zu, write=0x%x", + offset, size, (size == 4) ? *val32 : (u32)*val64); + nv_vgpu_gsp_rpc_bar_trap(vgpu, offset, 1, size, *(u64 *)val, 0, NULL); + } + + return 0; +} + +int nv_vgpu_bar_emul_handle_fault(struct nv_vgpu_device *vgpu, + unsigned long addr, bool write) +{ + struct nv_vgpu_bar_emul_ctx *ctx = vgpu->bar_emul_ctx; + struct nv_vgpu_bar *bar = &vgpu->bars[0]; + u64 offset = addr - (unsigned long)bar->vaddr; + u32 val = 0; + int ret; + + if (!ctx || !bar->mapped || !bar->trapped) + return -EINVAL; + + if (write) { + ret = nv_vgpu_bar_emul_write(vgpu, 0, offset, &val, 4); + } else { + ret = nv_vgpu_bar_emul_read(vgpu, 0, offset, &val, 4); + } + + return ret; +} \ No newline at end of file diff --git a/kernel/bar_init.c b/kernel/bar_init.c new file mode 100644 index 0000000..9926e2b --- /dev/null +++ b/kernel/bar_init.c @@ -0,0 +1,175 @@ +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" +#include "irq.h" + +int nv_vgpu_bar_init(struct nv_vgpu_device *vgpu) +{ + struct pci_dev *pdev = vgpu->pdev; + struct nv_vgpu_bar *bar; + u64 bar_addr; + u64 bar_size; + int i, ret; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: BAR init validation"); + for (i = 0; i < 4; i++) { + bar_addr = pci_resource_start(pdev, i); + bar_size = pci_resource_len(pdev, i); + if (bar_addr && bar_size) { + nv_vgpu_info(vgpu, " BAR%d: gpa=0x%llx, size=0x%llx (%s)", + i, bar_addr, bar_size, + i == 0 ? "config" : (i == 1 ? "FB" : (i == 2 ? "I/O" : "other"))); + } + } + nv_vgpu_info(vgpu, "DRY-RUN: BAR validation passed"); + return 0; + } + + for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) { + bar = &vgpu->bars[i]; + mutex_init(&bar->lock); + bar->mapped = false; + bar->trapped = false; + } + + bar_addr = pci_resource_start(pdev, 0); + bar_size = pci_resource_len(pdev, 0); + if (bar_addr && bar_size) { + vgpu->bars[0].gpa = bar_addr; + vgpu->bars[0].size = bar_size; + vgpu->bars[0].vaddr = ioremap_wc(bar_addr, bar_size); + if (!vgpu->bars[0].vaddr) { + nv_vgpu_err(vgpu, "Failed to map BAR0"); + return -ENOMEM; + } + vgpu->bar0_config_base = bar_addr; + nv_vgpu_info(vgpu, "BAR0: gpa=0x%llx, size=0x%llx, vaddr=%p", bar_addr, bar_size, vgpu->bars[0].vaddr); + } + + bar_addr = pci_resource_start(pdev, 1); + bar_size = pci_resource_len(pdev, 1); + if (bar_addr && bar_size) { + vgpu->bars[1].gpa = bar_addr; + vgpu->bars[1].size = bar_size; + /* Emulated vidmem: a dedicated host-RAM buffer backing the guest's + * VRAM aperture. Do NOT map the host GPU's real BAR1 - guest VRAM + * accesses must hit our own clean buffer, not the (broken) host + * GPU's VRAM. Allocated lazily on first use to keep mdev create + * cheap; 256MB to match the GSCI fb size. */ + vgpu->bars[1].vaddr = vzalloc(NV_VGPU_BAR1_SIZE); + if (!vgpu->bars[1].vaddr) { + nv_vgpu_err(vgpu, "Failed to allocate emulated vidmem (256MB)"); + return -ENOMEM; + } + nv_vgpu_info(vgpu, "BAR1: gpa=0x%llx, size=0x%llx, emulated vidmem=%p", + bar_addr, bar_size, vgpu->bars[1].vaddr); + } + + bar_addr = pci_resource_start(pdev, 2); + bar_size = pci_resource_len(pdev, 2); + if (bar_addr && bar_size) { + vgpu->bars[2].gpa = bar_addr; + vgpu->bars[2].size = bar_size; + vgpu->bars[2].vaddr = ioremap_wc(bar_addr, bar_size); + if (!vgpu->bars[2].vaddr) { + nv_vgpu_err(vgpu, "Failed to map BAR2"); + return -ENOMEM; + } + nv_vgpu_info(vgpu, "BAR2: gpa=0x%llx, size=0x%llx, vaddr=%p", bar_addr, bar_size, vgpu->bars[2].vaddr); + } + + bar_addr = pci_resource_start(pdev, 3); + bar_size = pci_resource_len(pdev, 3); + if (bar_addr && bar_size) { + vgpu->bars[3].gpa = bar_addr; + vgpu->bars[3].size = bar_size; + /* Emulated BAR3: dedicated host-RAM shadow. Do NOT ioremap the + * host GPU's real BAR3 - guest writes through this region must + * hit our own clean buffer, not the (running) host GPU's + * instance memory window. 32MB to match the GSCI-reported BAR3. */ + vgpu->bars[3].vaddr = vzalloc(bar_size); + if (!vgpu->bars[3].vaddr) { + nv_vgpu_err(vgpu, "Failed to allocate emulated BAR3 (%llu bytes)", + (unsigned long long)bar_size); + return -ENOMEM; + } + nv_vgpu_info(vgpu, "BAR3: gpa=0x%llx, size=0x%llx, emulated shadow=%p", + bar_addr, bar_size, vgpu->bars[3].vaddr); + } + + ret = nv_vgpu_bar_emul_init(vgpu); + if (ret) { + nv_vgpu_err(vgpu, "BAR emulation init failed: %d", ret); + return ret; + } + + /* MSI/MSI-X is managed by nvidia.ko - don't try to set up our own */ + nv_vgpu_info(vgpu, "MSI/MSI-X managed by nvidia.ko"); + + vgpu->fb_base = pci_resource_start(pdev, 1); + vgpu->fb_size = pci_resource_len(pdev, 1); + + return 0; +} + +void nv_vgpu_bar_cleanup(struct nv_vgpu_device *vgpu) +{ + int i; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: BAR cleanup (no hardware resources to release)"); + return; + } + + nv_vgpu_irq_teardown_msix(vgpu); + nv_vgpu_irq_teardown_msi(vgpu); + nv_vgpu_bar_emul_cleanup(vgpu); + + /* Free the emulated vidmem buffer (vzalloc'd, not ioremap'd) */ + if (vgpu->bars[1].vaddr) { + vfree(vgpu->bars[1].vaddr); + vgpu->bars[1].vaddr = NULL; + } + + /* Free the emulated BAR3 shadow (vzalloc'd, not ioremap'd) */ + if (vgpu->bars[3].vaddr) { + vfree(vgpu->bars[3].vaddr); + vgpu->bars[3].vaddr = NULL; + } + + for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) { + if (i == 1 || i == 3) + continue; /* shadow buffers already freed above */ + if (vgpu->bars[i].vaddr) { + iounmap(vgpu->bars[i].vaddr); + vgpu->bars[i].vaddr = NULL; + } + vgpu->bars[i].mapped = false; + vgpu->bars[i].trapped = false; + vgpu->bars[i].iova = 0; + } +} + +int nv_vgpu_bar_trap(struct nv_vgpu_device *vgpu, int bar_idx) +{ + if (bar_idx < 0 || bar_idx >= NV_VGPU_MAX_REGIONS) + return -EINVAL; + + if (!vgpu->bars[bar_idx].vaddr) + return -ENODEV; + + return nv_vgpu_bar_trap_register(vgpu, bar_idx, 0, vgpu->bars[bar_idx].size); +} + +void nv_vgpu_bar_untrap(struct nv_vgpu_device *vgpu, int bar_idx) +{ + if (bar_idx < 0 || bar_idx >= NV_VGPU_MAX_REGIONS) + return; + + nv_vgpu_bar_untrap_register(vgpu, bar_idx, 0); +} diff --git a/kernel/dynamic_rpc_dispatcher.c b/kernel/dynamic_rpc_dispatcher.c new file mode 100644 index 0000000..ca4268d --- /dev/null +++ b/kernel/dynamic_rpc_dispatcher.c @@ -0,0 +1,438 @@ +/* + * Dynamic RPC Dispatcher - Version-aware RPC dispatch + * Uses discovered firmware addresses for native handlers, + * falls back to emulation for missing handlers + */ + +#include +#include +#include +#include +#include +#include + +#include "dynamic_rpc_dispatcher.h" +#include "gsp_rpc_client.h" +#include "firmware_analyzer.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Open-IOV"); +MODULE_DESCRIPTION("Dynamic RPC Dispatcher"); + +/* Dispatcher context */ +struct dynamic_dispatcher { + struct rpc_interface *current_interface; + struct fw_analysis_result fw_analysis; + spinlock_t lock; + bool initialized; +}; + +static struct dynamic_dispatcher g_dispatcher = { + .lock = __SPIN_LOCK_UNLOCKED(g_dispatcher.lock), + .initialized = false, +}; + +/* Version 570 interface (native firmware handlers) */ +static int rpc_570_alloc(struct gsp_client *client, u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, u64 *gpu_va, u64 *handle, u64 *actual_size) +{ + return gsp_rpc_alloc(client, size, alignment, mem_type, flags, addr_hint, pid, gpu_va, handle, actual_size); +} + +static int rpc_570_free(struct gsp_client *client, u64 handle) +{ + return gsp_rpc_free(client, handle); +} + +static int rpc_570_map(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, u64 *mapped_va) +{ + return gsp_rpc_map(client, handle, gpu_va, size, prot, flags, offset, mapped_va); +} + +static int rpc_570_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size) +{ + return gsp_rpc_unmap(client, handle, gpu_va, size); +} + +static int rpc_570_pin(struct gsp_client *client, u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages) +{ + return gsp_rpc_pin(client, handle, offset, size, flags, dma_addr, num_pages); +} + +static int rpc_570_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size) +{ + return gsp_rpc_unpin(client, handle, offset, size); +} + +static int rpc_570_schedule(struct gsp_client *client, u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + return gsp_rpc_schedule(client, work_handle, callback, arg, priority, flags); +} + +static int rpc_570_yield(struct gsp_client *client) +{ + return gsp_rpc_yield(client); +} + +static int rpc_570_bar_trap(struct gsp_client *client, u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value) +{ + return gsp_rpc_bar_trap(client, bar_addr, access_type, size, value, bar_num, ret_value); +} + +static int rpc_570_migrate_save(struct gsp_client *client, u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, u64 *checkpoint) +{ + return gsp_rpc_migrate_save(client, src_gpu_id, dst_gpu_id, memory_bitmap, page_table_ptr, checkpoint); +} + +static int rpc_570_migrate_restore(struct gsp_client *client, u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, u64 checkpoint) +{ + return gsp_rpc_migrate_restore(client, src_gpu_id, dst_gpu_id, memory_bitmap, page_table_ptr, checkpoint); +} + +static int rpc_570_notify_register(struct gsp_client *client, u32 type, + void (*callback)(struct gsp_client *, struct gsp_rpc_response *, void *), + void *ctx) +{ + return gsp_rpc_notify_register(client, type, callback, ctx); +} + +static void rpc_570_notify_unregister(struct gsp_client *client, u32 type) +{ + gsp_rpc_notify_unregister(client, type); +} + +static struct rpc_interface interface_570 = { + .version = "570.144W", + .alloc = rpc_570_alloc, + .free = rpc_570_free, + .map = rpc_570_map, + .unmap = rpc_570_unmap, + .pin = rpc_570_pin, + .unpin = rpc_570_unpin, + .schedule = rpc_570_schedule, + .yield = rpc_570_yield, + .bar_trap = rpc_570_bar_trap, + .migrate_save = rpc_570_migrate_save, + .migrate_restore = rpc_570_migrate_restore, + .notify_register = rpc_570_notify_register, + .notify_unregister = rpc_570_notify_unregister, + .use_emulation = {0}, +}; + +/* Emulation interface (fallback) */ +static int rpc_emu_alloc(struct gsp_client *client, u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, u64 *gpu_va, u64 *handle, u64 *actual_size) +{ + return gsp_rpc_alloc_emul(client, size, alignment, mem_type, flags, addr_hint, pid, gpu_va, handle, actual_size); +} + +static int rpc_emu_free(struct gsp_client *client, u64 handle) +{ + return gsp_rpc_free_emul(client, handle); +} + +static int rpc_emu_map(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, u64 *mapped_va) +{ + return gsp_rpc_map_emul(client, handle, gpu_va, size, prot, flags, offset, mapped_va); +} + +static int rpc_emu_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size) +{ + return gsp_rpc_unmap_emul(client, handle, gpu_va, size); +} + +static int rpc_emu_pin(struct gsp_client *client, u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages) +{ + return gsp_rpc_pin_emul(client, handle, offset, size, flags, dma_addr, num_pages); +} + +static int rpc_emu_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size) +{ + return gsp_rpc_unpin_emul(client, handle, offset, size); +} + +static int rpc_emu_schedule(struct gsp_client *client, u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + return 0; /* Emulation always succeeds */ +} + +static int rpc_emu_yield(struct gsp_client *client) +{ + return 0; +} + +static int rpc_emu_bar_trap(struct gsp_client *client, u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value) +{ + return gsp_rpc_bar_trap(client, bar_addr, access_type, size, value, bar_num, ret_value); +} + +static int rpc_emu_migrate_save(struct gsp_client *client, u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, u64 *checkpoint) +{ + return 0; +} + +static int rpc_emu_migrate_restore(struct gsp_client *client, u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, u64 checkpoint) +{ + return 0; +} + +static int rpc_emu_notify_register(struct gsp_client *client, u32 type, + void (*callback)(struct gsp_client *, struct gsp_rpc_response *, void *), + void *ctx) +{ + return gsp_rpc_notify_register(client, type, callback, ctx); +} + +static void rpc_emu_notify_unregister(struct gsp_client *client, u32 type) +{ + gsp_rpc_notify_unregister(client, type); +} + +static struct rpc_interface interface_emu = { + .version = "emulation", + .alloc = rpc_emu_alloc, + .free = rpc_emu_free, + .map = rpc_emu_map, + .unmap = rpc_emu_unmap, + .pin = rpc_emu_pin, + .unpin = rpc_emu_unpin, + .schedule = rpc_emu_schedule, + .yield = rpc_emu_yield, + .bar_trap = rpc_emu_bar_trap, + .migrate_save = rpc_emu_migrate_save, + .migrate_restore = rpc_emu_migrate_restore, + .notify_register = rpc_emu_notify_register, + .notify_unregister = rpc_emu_notify_unregister, + .use_emulation = {1}, +}; + +int dynamic_dispatcher_init(const char *fw_path) +{ + struct fw_analysis_result result; + int ret; + + spin_lock(&g_dispatcher.lock); + + if (g_dispatcher.initialized) { + spin_unlock(&g_dispatcher.lock); + return 0; + } + + /* Analyze firmware */ + ret = fw_analyzer_load_and_analyze(fw_path, &result); + if (ret) { + pr_warn("DYN_DISPATCH: Firmware analysis failed, using emulation: %d\n", ret); + g_dispatcher.current_interface = &interface_emu; + g_dispatcher.initialized = true; + spin_unlock(&g_dispatcher.lock); + return 0; + } + + g_dispatcher.fw_analysis = result; + + /* Select interface based on firmware version */ + if (result.matched && strcmp(result.version, "570.144W") == 0) { + interface_570.fw_addrs = result.addresses; + g_dispatcher.current_interface = &interface_570; + pr_info("DYN_DISPATCH: Using native 570.144W interface\n"); + } else { + g_dispatcher.current_interface = &interface_emu; + pr_info("DYN_DISPATCH: Using emulation interface (version: %s)\n", result.version); + } + + g_dispatcher.initialized = true; + spin_unlock(&g_dispatcher.lock); + + fw_analyzer_print_result(&result); + return 0; +} + +void dynamic_dispatcher_fini(void) +{ + spin_lock(&g_dispatcher.lock); + g_dispatcher.initialized = false; + g_dispatcher.current_interface = NULL; + spin_unlock(&g_dispatcher.lock); +} + +struct rpc_interface *dynamic_dispatcher_get_interface(void) +{ + struct rpc_interface *iface; + + spin_lock(&g_dispatcher.lock); + iface = g_dispatcher.current_interface; + spin_unlock(&g_dispatcher.lock); + + return iface; +} + +bool dynamic_dispatcher_use_emulation(enum gsp_rpc_type type) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface) + return true; + + if (type >= GSP_RPC_MAX_TYPE) + return true; + + return iface->use_emulation[type]; +} + +int dynamic_dispatcher_alloc(struct gsp_client *client, u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, u64 *gpu_va, u64 *handle, u64 *actual_size) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->alloc) + return -ENODEV; + return iface->alloc(client, size, alignment, mem_type, flags, addr_hint, pid, gpu_va, handle, actual_size); +} + +int dynamic_dispatcher_free(struct gsp_client *client, u64 handle) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->free) + return -ENODEV; + return iface->free(client, handle); +} + +int dynamic_dispatcher_map(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, u64 *mapped_va) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->map) + return -ENODEV; + return iface->map(client, handle, gpu_va, size, prot, flags, offset, mapped_va); +} + +int dynamic_dispatcher_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->unmap) + return -ENODEV; + return iface->unmap(client, handle, gpu_va, size); +} + +int dynamic_dispatcher_pin(struct gsp_client *client, u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->pin) + return -ENODEV; + return iface->pin(client, handle, offset, size, flags, dma_addr, num_pages); +} + +int dynamic_dispatcher_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->unpin) + return -ENODEV; + return iface->unpin(client, handle, offset, size); +} + +int dynamic_dispatcher_schedule(struct gsp_client *client, u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->schedule) + return -ENODEV; + return iface->schedule(client, work_handle, callback, arg, priority, flags); +} + +int dynamic_dispatcher_yield(struct gsp_client *client) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->yield) + return -ENODEV; + return iface->yield(client); +} + +int dynamic_dispatcher_bar_trap(struct gsp_client *client, u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->bar_trap) + return -ENODEV; + return iface->bar_trap(client, bar_addr, access_type, size, value, bar_num, ret_value); +} + +int dynamic_dispatcher_migrate_save(struct gsp_client *client, u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, u64 *checkpoint) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->migrate_save) + return -ENODEV; + return iface->migrate_save(client, src_gpu_id, dst_gpu_id, memory_bitmap, page_table_ptr, checkpoint); +} + +int dynamic_dispatcher_migrate_restore(struct gsp_client *client, u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, u64 checkpoint) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->migrate_restore) + return -ENODEV; + return iface->migrate_restore(client, src_gpu_id, dst_gpu_id, memory_bitmap, page_table_ptr, checkpoint); +} + +int dynamic_dispatcher_notify_register(struct gsp_client *client, u32 type, + void (*callback)(struct gsp_client *, struct gsp_rpc_response *, void *), + void *ctx) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (!iface || !iface->notify_register) + return -ENODEV; + return iface->notify_register(client, type, callback, ctx); +} + +void dynamic_dispatcher_notify_unregister(struct gsp_client *client, u32 type) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + if (iface && iface->notify_unregister) + iface->notify_unregister(client, type); +} + +const char *dynamic_dispatcher_get_version(void) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + return iface ? iface->version : "unknown"; +} + +const struct fw_addresses *dynamic_dispatcher_get_fw_addresses(void) +{ + struct rpc_interface *iface = dynamic_dispatcher_get_interface(); + return iface ? &iface->fw_addrs : NULL; +} + +EXPORT_SYMBOL_GPL(dynamic_dispatcher_init); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_fini); +// EXPORT_SYMBOL_GPL(dynamic_dispatcher_get_interface); // Internal function, not exported +EXPORT_SYMBOL_GPL(dynamic_dispatcher_use_emulation); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_alloc); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_free); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_map); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_unmap); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_pin); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_unpin); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_schedule); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_yield); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_bar_trap); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_migrate_save); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_migrate_restore); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_notify_register); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_notify_unregister); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_get_version); +EXPORT_SYMBOL_GPL(dynamic_dispatcher_get_fw_addresses); + +MODULE_DESCRIPTION("Dynamic RPC Dispatcher"); +MODULE_LICENSE("GPL"); \ No newline at end of file diff --git a/kernel/dynamic_rpc_dispatcher.h b/kernel/dynamic_rpc_dispatcher.h new file mode 100644 index 0000000..2abecb5 --- /dev/null +++ b/kernel/dynamic_rpc_dispatcher.h @@ -0,0 +1,66 @@ +#ifndef _DYNAMIC_RPC_DISPATCHER_H_ +#define _DYNAMIC_RPC_DISPATCHER_H_ + +#include +#include "gsp_rpc_client.h" +#include "firmware_analyzer.h" + +/* Version-specific RPC interface */ +struct rpc_interface { + char version[32]; + struct fw_addresses fw_addrs; + + /* Native firmware handlers */ + int (*alloc)(struct gsp_client *, u64, u64, u32, u32, u64, u32, u64 *, u64 *, u64 *); + int (*free)(struct gsp_client *, u64); + int (*map)(struct gsp_client *, u64, u64, u64, u32, u32, u64, u64 *); + int (*unmap)(struct gsp_client *, u64, u64, u64); + int (*pin)(struct gsp_client *, u64, u64, u64, u32, u64 *, u32 *); + int (*unpin)(struct gsp_client *, u64, u64, u64); + int (*schedule)(struct gsp_client *, u64, u64, u64, u32, u32); + int (*yield)(struct gsp_client *); + int (*bar_trap)(struct gsp_client *, u64, u32, u32, u64, u32, u64 *); + int (*migrate_save)(struct gsp_client *, u64, u64, u64, u64, u64 *); + int (*migrate_restore)(struct gsp_client *, u64, u64, u64, u64, u64); + int (*notify_register)(struct gsp_client *, u32, void (*)(struct gsp_client *, struct gsp_rpc_response *, void *), void *); + void (*notify_unregister)(struct gsp_client *, u32); + + /* Emulation fallback flags */ + bool use_emulation[GSP_RPC_MAX_TYPE]; +}; + +/* Public API */ +int dynamic_dispatcher_init(const char *fw_path); +void dynamic_dispatcher_fini(void); +struct rpc_interface *dynamic_dispatcher_get_interface(void); +bool dynamic_dispatcher_use_emulation(enum gsp_rpc_type type); + +/* Dispatcher functions */ +int dynamic_dispatcher_alloc(struct gsp_client *client, u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, u64 *gpu_va, u64 *handle, u64 *actual_size); +int dynamic_dispatcher_free(struct gsp_client *client, u64 handle); +int dynamic_dispatcher_map(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, u64 *mapped_va); +int dynamic_dispatcher_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size); +int dynamic_dispatcher_pin(struct gsp_client *client, u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages); +int dynamic_dispatcher_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size); +int dynamic_dispatcher_schedule(struct gsp_client *client, u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags); +int dynamic_dispatcher_yield(struct gsp_client *client); +int dynamic_dispatcher_bar_trap(struct gsp_client *client, u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value); +int dynamic_dispatcher_migrate_save(struct gsp_client *client, u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, u64 *checkpoint); +int dynamic_dispatcher_migrate_restore(struct gsp_client *client, u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, u64 checkpoint); +int dynamic_dispatcher_notify_register(struct gsp_client *client, u32 type, + void (*callback)(struct gsp_client *, struct gsp_rpc_response *, void *), + void *ctx); +void dynamic_dispatcher_notify_unregister(struct gsp_client *client, u32 type); + +/* Info */ +const char *dynamic_dispatcher_get_version(void); +const struct fw_addresses *dynamic_dispatcher_get_fw_addresses(void); + +#endif /* _DYNAMIC_RPC_DISPATCHER_H_ */ \ No newline at end of file diff --git a/kernel/firmware_analyzer.c b/kernel/firmware_analyzer.c new file mode 100644 index 0000000..2d8d84d --- /dev/null +++ b/kernel/firmware_analyzer.c @@ -0,0 +1,224 @@ +/* + * Firmware Analyzer - Runtime GSP firmware analysis + * Auto-discovers RPC handler addresses, struct layouts, and firmware signatures + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "firmware_analyzer.h" +#include "gsp_rpc_client.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Open-IOV"); +MODULE_DESCRIPTION("GSP Firmware Analyzer"); + +/* Firmware signature database */ +struct fw_signature { + char version[32]; + u32 crc32; + u8 sha256[32]; + struct fw_addresses addrs; +}; + +static struct fw_signature known_signatures[] = { + { + .version = "570.144W", + .crc32 = 0x0, /* Will be computed */ + .sha256 = {0}, /* Will be computed */ + .addrs = { + .rpc_dispatch_primary = 0x057A1AC0, + .rpc_jump_table = 0x059BFEB0, + .alloc_main = 0x057E94EC, + .schedule_main = 0x051EFBF4, + .bar_trap_handler = 0x04CB8188, + .wpr_handler1 = 0x04F00B0C, + .wpr_handler2 = 0x04F00C78, + .migration_table1 = 0x04179ED8, + .migration_table2 = 0x04179FBC, + .entry_point = 0x04009000, + } + }, +}; + +/* Pattern signatures for auto-discovery */ +static const u8 pattern_rpc_dispatch[] = { + 0x67, 0x05, 0x00, 0x00, /* auipc a0, 0x5 */ + 0x03, 0x85, 0x05, 0x00, /* ld a1, 0x500(a0) */ +}; + +static const u8 pattern_jump_table[] = { + 0x00, 0x00, 0x00, 0x00, /* .dword entries */ +}; + +static const u8 pattern_alloc[] = { + 0x73, 0x22, 0x00, 0x00, /* sd s0, 0(sp) */ + 0x73, 0x62, 0x08, 0x00, /* sd s1, 8(sp) */ +}; + +static int compute_firmware_crc32(const u8 *data, size_t len, u32 *crc_out) +{ + *crc_out = crc32_le(0, data, len); + return 0; +} + +static int compute_firmware_sha256(const u8 *data, size_t len, u8 *sha_out) +{ + struct crypto_shash *tfm; + struct shash_desc *desc; + int ret; + + tfm = crypto_alloc_shash("sha256", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL); + if (!desc) { + crypto_free_shash(tfm); + return -ENOMEM; + } + + desc->tfm = tfm; + ret = crypto_shash_init(desc); + if (!ret) + ret = crypto_shash_update(desc, data, len); + if (!ret) + ret = crypto_shash_final(desc, sha_out); + + kfree(desc); + crypto_free_shash(tfm); + return ret; +} + +static int find_pattern(const u8 *data, size_t data_len, const u8 *pattern, size_t pattern_len, u64 *offset) +{ + for (size_t i = 0; i + pattern_len <= data_len; i++) { + if (memcmp(data + i, pattern, pattern_len) == 0) { + *offset = i; + return 0; + } + } + return -ENOENT; +} + +static int analyze_rpc_dispatcher(const u8 *fw, size_t fw_len, struct fw_addresses *addrs) +{ + u64 offset; + int ret; + + ret = find_pattern(fw, fw_len, pattern_rpc_dispatch, sizeof(pattern_rpc_dispatch), &offset); + if (ret == 0) { + addrs->rpc_dispatch_primary = GSP_FW_BASE + offset; + pr_info("FW_ANALYZER: Found RPC dispatcher at 0x%llx\n", addrs->rpc_dispatch_primary); + } + + ret = find_pattern(fw, fw_len, pattern_jump_table, sizeof(pattern_jump_table), &offset); + if (ret == 0) { + addrs->rpc_jump_table = GSP_FW_BASE + offset; + pr_info("FW_ANALYZER: Found RPC jump table at 0x%llx\n", addrs->rpc_jump_table); + } + + ret = find_pattern(fw, fw_len, pattern_alloc, sizeof(pattern_alloc), &offset); + if (ret == 0) { + addrs->alloc_main = GSP_FW_BASE + offset; + pr_info("FW_ANALYZER: Found ALLOC handler at 0x%llx\n", addrs->alloc_main); + } + + return 0; +} + +int fw_analyzer_analyze(const u8 *fw_data, size_t fw_len, struct fw_analysis_result *result) +{ + int ret; + + if (!fw_data || !result || fw_len == 0) + return -EINVAL; + + memset(result, 0, sizeof(*result)); + result->fw_size = fw_len; + + /* Compute signatures */ + ret = compute_firmware_crc32(fw_data, fw_len, &result->crc32); + if (ret) + return ret; + + ret = compute_firmware_sha256(fw_data, fw_len, result->sha256); + if (ret) + return ret; + + pr_info("FW_ANALYZER: Firmware size=%zu, CRC32=0x%08x\n", fw_len, result->crc32); + pr_info("FW_ANALYZER: SHA256: %*phN\n", 32, result->sha256); + + /* Match against known signatures */ + for (size_t i = 0; i < ARRAY_SIZE(known_signatures); i++) { + if (known_signatures[i].crc32 == result->crc32 && + memcmp(known_signatures[i].sha256, result->sha256, 32) == 0) { + strncpy(result->version, known_signatures[i].version, sizeof(result->version) - 1); + result->addresses = known_signatures[i].addrs; + result->matched = true; + pr_info("FW_ANALYZER: Matched known firmware version: %s\n", result->version); + return 0; + } + } + + /* Auto-discover addresses if no match */ + pr_info("FW_ANALYZER: No known signature match, attempting auto-discovery\n"); + ret = analyze_rpc_dispatcher(fw_data, fw_len, &result->addresses); + if (ret == 0) { + result->matched = false; + snprintf(result->version, sizeof(result->version), "unknown-%08x", result->crc32); + } + + return ret; +} + +int fw_analyzer_load_and_analyze(const char *fw_path, struct fw_analysis_result *result) +{ + const struct firmware *fw; + int ret; + + ret = request_firmware(&fw, fw_path, NULL); + if (ret) { + pr_err("FW_ANALYZER: Failed to load firmware %s: %d\n", fw_path, ret); + return ret; + } + + ret = fw_analyzer_analyze(fw->data, fw->size, result); + + release_firmware(fw); + return ret; +} + +void fw_analyzer_print_result(const struct fw_analysis_result *result) +{ + pr_info("=== Firmware Analysis Result ===\n"); + pr_info("Version: %s\n", result->version); + pr_info("Size: %zu bytes\n", result->fw_size); + pr_info("CRC32: 0x%08x\n", result->crc32); + pr_info("SHA256: %*phN\n", 32, result->sha256); + pr_info("Matched: %s\n", result->matched ? "yes" : "no"); + pr_info("Addresses:\n"); + pr_info(" RPC Dispatch Primary: 0x%llx\n", result->addresses.rpc_dispatch_primary); + pr_info(" RPC Jump Table: 0x%llx\n", result->addresses.rpc_jump_table); + pr_info(" ALLOC Main: 0x%llx\n", result->addresses.alloc_main); + pr_info(" SCHEDULE Main: 0x%llx\n", result->addresses.schedule_main); + pr_info(" BAR Trap Handler: 0x%llx\n", result->addresses.bar_trap_handler); + pr_info(" WPR Handler 1: 0x%llx\n", result->addresses.wpr_handler1); + pr_info(" WPR Handler 2: 0x%llx\n", result->addresses.wpr_handler2); + pr_info(" Migration Table 1: 0x%llx\n", result->addresses.migration_table1); + pr_info(" Migration Table 2: 0x%llx\n", result->addresses.migration_table2); + pr_info(" Entry Point: 0x%llx\n", result->addresses.entry_point); +} +EXPORT_SYMBOL_GPL(fw_analyzer_analyze); +EXPORT_SYMBOL_GPL(fw_analyzer_load_and_analyze); +EXPORT_SYMBOL_GPL(fw_analyzer_print_result); + +MODULE_DESCRIPTION("GSP Firmware Analyzer"); +MODULE_LICENSE("GPL"); \ No newline at end of file diff --git a/kernel/firmware_analyzer.h b/kernel/firmware_analyzer.h new file mode 100644 index 0000000..69a4ec3 --- /dev/null +++ b/kernel/firmware_analyzer.h @@ -0,0 +1,36 @@ +#ifndef _FW_ANALYZER_H_ +#define _FW_ANALYZER_H_ + +#include +#include "gsp_rpc_client.h" + +/* Firmware addresses discovered from analysis */ +struct fw_addresses { + u64 rpc_dispatch_primary; + u64 rpc_jump_table; + u64 alloc_main; + u64 schedule_main; + u64 bar_trap_handler; + u64 wpr_handler1; + u64 wpr_handler2; + u64 migration_table1; + u64 migration_table2; + u64 entry_point; +}; + +/* Analysis result */ +struct fw_analysis_result { + char version[32]; + size_t fw_size; + u32 crc32; + u8 sha256[32]; + struct fw_addresses addresses; + bool matched; +}; + +/* Public API */ +int fw_analyzer_analyze(const u8 *fw_data, size_t fw_len, struct fw_analysis_result *result); +int fw_analyzer_load_and_analyze(const char *fw_path, struct fw_analysis_result *result); +void fw_analyzer_print_result(const struct fw_analysis_result *result); + +#endif /* _FW_ANALYZER_H_ */ \ No newline at end of file diff --git a/kernel/gsp_comm.c b/kernel/gsp_comm.c new file mode 100644 index 0000000..094fcfb --- /dev/null +++ b/kernel/gsp_comm.c @@ -0,0 +1,638 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "gsp_comm.h" +#include "gsp_rpc_integration.h" + +#define NV_VGPU_GSP_BOOTLOADER_FILE "nvidia/tu102/gsp/bootloader-535.113.01.bin" +#define NV_VGPU_GSP_BOOTER_LOAD_FILE "nvidia/tu102/gsp/booter_load-535.113.01.bin" +#define NV_VGPU_GSP_GSP_FILE "nvidia/tu102/gsp/gsp-535.113.01.bin" + +static void nv_vgpu_gsp_rpc_work(struct work_struct *work) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = container_of(work, struct nv_vgpu_gsp_comm_ctx, rpc_work); + struct nv_vgpu_gsp_rpc_entry *entry, *tmp; + LIST_HEAD(completed); + + mutex_lock(&ctx->queue_lock); + list_splice_tail_init(&ctx->complete_queue, &completed); + mutex_unlock(&ctx->queue_lock); + + list_for_each_entry_safe(entry, tmp, &completed, list) { + list_del(&entry->list); + complete(&entry->done); + } +} + +static void nv_vgpu_gsp_poll_work(struct work_struct *work) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = container_of(work, struct nv_vgpu_gsp_comm_ctx, poll_work.work); + u32 status; + + if (!ctx || !ctx->irq_enabled || !ctx->gsp_doorbell) + goto reschedule; + + status = readl(ctx->gsp_doorbell + 0x100); + if (status & 0x1) { + writel(0x1, ctx->gsp_doorbell + 0x100); + schedule_work(&ctx->rpc_work); + } + +reschedule: + if (ctx->poll_active) + schedule_delayed_work(&ctx->poll_work, msecs_to_jiffies(1)); +} + +static irqreturn_t nv_vgpu_gsp_irq_handler(int irq, void *data) +{ + struct nv_vgpu_device *vgpu = data; + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + u32 status; + + if (!ctx || !ctx->irq_enabled) + return IRQ_NONE; + + status = readl(ctx->gsp_doorbell + 0x100); + if (status & 0x1) { + writel(0x1, ctx->gsp_doorbell + 0x100); + schedule_work(&ctx->rpc_work); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int nv_vgpu_gsp_load_firmware_file(struct nv_vgpu_device *vgpu, const char *filename, void **data, size_t *size) +{ + const struct firmware *fw; + int ret; + + ret = request_firmware(&fw, filename, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "Failed to load firmware %s: %d", filename, ret); + return ret; + } + + /* Use vmalloc for large firmware files (GSP is ~23MB) */ + *data = vzalloc(fw->size); + if (!*data) { + release_firmware(fw); + return -ENOMEM; + } + + memcpy(*data, fw->data, fw->size); + *size = fw->size; + release_firmware(fw); + + nv_vgpu_info(vgpu, "Loaded firmware %s (%zu bytes)", filename, *size); + return 0; +} + +int nv_vgpu_gsp_load_firmware(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + int ret; + + if (!ctx) + return -ENODEV; + + if (ctx->firmware_loaded) + return 0; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: validating GSP firmware files"); + /* Just verify firmware files exist without loading */ + ret = request_firmware(NULL, NV_VGPU_GSP_BOOTLOADER_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTLOADER_FILE, ret); + return ret; + } + release_firmware(NULL); + + ret = request_firmware(NULL, NV_VGPU_GSP_BOOTER_LOAD_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTER_LOAD_FILE, ret); + return ret; + } + release_firmware(NULL); + + ret = request_firmware(NULL, NV_VGPU_GSP_GSP_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_GSP_FILE, ret); + return ret; + } + release_firmware(NULL); + + nv_vgpu_info(vgpu, "DRY-RUN: All GSP firmware files present"); + return 0; + } + + ret = nv_vgpu_gsp_load_firmware_file(vgpu, NV_VGPU_GSP_BOOTLOADER_FILE, + &ctx->firmware.bootloader, &ctx->firmware.bootloader_size); + if (ret) + return ret; + + ret = nv_vgpu_gsp_load_firmware_file(vgpu, NV_VGPU_GSP_BOOTER_LOAD_FILE, + &ctx->firmware.booter_load, &ctx->firmware.booter_load_size); + if (ret) + goto err_bootloader; + + ret = nv_vgpu_gsp_load_firmware_file(vgpu, NV_VGPU_GSP_GSP_FILE, + &ctx->firmware.gsp, &ctx->firmware.gsp_size); + if (ret) + goto err_booter_load; + + ctx->firmware_loaded = true; + nv_vgpu_info(vgpu, "GSP firmware loaded successfully"); + return 0; + +err_booter_load: + vfree(ctx->firmware.booter_load); + ctx->firmware.booter_load = NULL; + ctx->firmware.booter_load_size = 0; +err_bootloader: + vfree(ctx->firmware.bootloader); + ctx->firmware.bootloader = NULL; + ctx->firmware.bootloader_size = 0; + return ret; +} + +void nv_vgpu_gsp_release_firmware(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + + if (!ctx || !ctx->firmware_loaded) + return; + + vfree(ctx->firmware.bootloader); + ctx->firmware.bootloader = NULL; + ctx->firmware.bootloader_size = 0; + + vfree(ctx->firmware.booter_load); + ctx->firmware.booter_load = NULL; + ctx->firmware.booter_load_size = 0; + + vfree(ctx->firmware.gsp); + ctx->firmware.gsp = NULL; + ctx->firmware.gsp_size = 0; + + ctx->firmware_loaded = false; + nv_vgpu_info(vgpu, "GSP firmware released"); +} + +static int nv_vgpu_gsp_write_firmware(struct nv_vgpu_device *vgpu, void *data, size_t size, u64 offset) +{ + struct nv_vgpu_bar *bar = &vgpu->bars[0]; + void __iomem *dst; + + if (!bar->vaddr) + return -ENODEV; + + dst = bar->vaddr + offset; + memcpy_toio(dst, data, size); + return 0; +} + +int nv_vgpu_gsp_boot(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + int ret; + + if (!ctx || !ctx->firmware_loaded) + return -ENODEV; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: GSP boot sequence validated (would write firmware to BAR0)"); + nv_vgpu_info(vgpu, "DRY-RUN: Would write bootloader (%zu bytes) to offset 0x0", + ctx->firmware.bootloader_size); + nv_vgpu_info(vgpu, "DRY-RUN: Would write booter_load (%zu bytes) to offset 0x10000", + ctx->firmware.booter_load_size); + nv_vgpu_info(vgpu, "DRY-RUN: Would write GSP binary (%zu bytes) to offset 0x200000", + ctx->firmware.gsp_size); + nv_vgpu_info(vgpu, "DRY-RUN: Would kick GSP doorbell at offset 0x%llx", + (u64)NV_VGPU_GSP_DOORBELL_OFFSET); + return 0; + } + + nv_vgpu_info(vgpu, "Starting GSP boot sequence"); + + ret = nv_vgpu_gsp_write_firmware(vgpu, ctx->firmware.bootloader, + ctx->firmware.bootloader_size, 0x0); + if (ret) { + nv_vgpu_err(vgpu, "Failed to write bootloader: %d", ret); + return ret; + } + + ret = nv_vgpu_gsp_write_firmware(vgpu, ctx->firmware.booter_load, + ctx->firmware.booter_load_size, 0x10000); + if (ret) { + nv_vgpu_err(vgpu, "Failed to write booter_load: %d", ret); + return ret; + } + + ret = nv_vgpu_gsp_write_firmware(vgpu, ctx->firmware.gsp, + ctx->firmware.gsp_size, 0x200000); + if (ret) { + nv_vgpu_err(vgpu, "Failed to write GSP binary: %d", ret); + return ret; + } + + writel(0x1, ctx->gsp_doorbell); + + nv_vgpu_info(vgpu, "GSP boot sequence initiated"); + return 0; +} + +int nv_vgpu_gsp_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_gsp_comm_ctx *ctx; + struct nv_vgpu_bar *bar = &vgpu->bars[0]; + int ret; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: GSP init validation"); + if (!bar->vaddr) { + nv_vgpu_err(vgpu, "DRY-RUN: BAR0 not mapped, cannot validate GSP offsets"); + return -ENODEV; + } + nv_vgpu_info(vgpu, "DRY-RUN: BAR0 mapped at %p, GSP doorbell offset 0x%llx, shared mem offset 0x%llx", + bar->vaddr, (u64)NV_VGPU_GSP_DOORBELL_OFFSET, (u64)NV_VGPU_GSP_SHARED_MEM_OFFSET); + /* Validate firmware files exist */ + ret = request_firmware(NULL, NV_VGPU_GSP_BOOTLOADER_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTLOADER_FILE, ret); + return ret; + } + release_firmware(NULL); + ret = request_firmware(NULL, NV_VGPU_GSP_BOOTER_LOAD_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_BOOTER_LOAD_FILE, ret); + return ret; + } + release_firmware(NULL); + ret = request_firmware(NULL, NV_VGPU_GSP_GSP_FILE, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_err(vgpu, "DRY-RUN: Firmware %s not found: %d", NV_VGPU_GSP_GSP_FILE, ret); + return ret; + } + release_firmware(NULL); + nv_vgpu_info(vgpu, "DRY-RUN: GSP init validation passed"); + return 0; + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + mutex_init(&ctx->queue_lock); + INIT_LIST_HEAD(&ctx->pending_queue); + INIT_LIST_HEAD(&ctx->complete_queue); + init_waitqueue_head(&ctx->wait_queue); + atomic_set(&ctx->sequence_counter, 0); + spin_lock_init(&ctx->irq_lock); + + if (bar->vaddr) { + ctx->gsp_doorbell = bar->vaddr + NV_VGPU_GSP_DOORBELL_OFFSET; + ctx->gsp_shared_mem = bar->vaddr + NV_VGPU_GSP_SHARED_MEM_OFFSET; + ctx->shared_mem_size = NV_VGPU_GSP_SHARED_MEM_SIZE; + } + + ctx->rpc_wq = alloc_workqueue("nv-vgpu-gsp-rpc", WQ_UNBOUND | WQ_HIGHPRI, 1); + if (!ctx->rpc_wq) { + ret = -ENOMEM; + goto err_free; + } + + INIT_WORK(&ctx->rpc_work, nv_vgpu_gsp_rpc_work); + INIT_DELAYED_WORK(&ctx->poll_work, nv_vgpu_gsp_poll_work); + + /* Use polling instead of IRQ since nvidia.ko owns the MSI interrupt */ + ctx->irq_enabled = true; + ctx->poll_active = true; + schedule_delayed_work(&ctx->poll_work, msecs_to_jiffies(1)); + + vgpu->gsp_comm_ctx = ctx; + + /* Initialize new firmware-decompiled RPC client */ + ret = nv_vgpu_gsp_rpc_integration_init(vgpu, ctx); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to initialize GSP RPC integration: %d (non-fatal)", ret); + /* Continue anyway - legacy RPC still works */ + } + + ret = nv_vgpu_gsp_load_firmware(vgpu); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to load GSP firmware: %d", ret); + } + + ret = nv_vgpu_gsp_boot(vgpu); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to boot GSP: %d", ret); + } + + return 0; + +err_free: + kfree(ctx); + return ret; +} + +void nv_vgpu_gsp_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + struct nv_vgpu_gsp_rpc_entry *entry, *tmp; + + if (!ctx) + return; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: GSP cleanup (no hardware resources to release)"); + return; + } + + /* Cleanup new firmware-decompiled RPC client */ + nv_vgpu_gsp_rpc_integration_cleanup(vgpu); + + /* Stop polling */ + ctx->poll_active = false; + cancel_delayed_work_sync(&ctx->poll_work); + + /* Cancel RPC work */ + cancel_work_sync(&ctx->rpc_work); + + if (ctx->rpc_wq) { + destroy_workqueue(ctx->rpc_wq); + ctx->rpc_wq = NULL; + } + + mutex_lock(&ctx->queue_lock); + list_for_each_entry_safe(entry, tmp, &ctx->pending_queue, list) { + list_del(&entry->list); + entry->result = -ENODEV; + complete(&entry->done); + } + list_for_each_entry_safe(entry, tmp, &ctx->complete_queue, list) { + list_del(&entry->list); + entry->result = -ENODEV; + complete(&entry->done); + } + mutex_unlock(&ctx->queue_lock); + + nv_vgpu_gsp_release_firmware(vgpu); + + kfree(ctx); + vgpu->gsp_comm_ctx = NULL; +} + +int nv_vgpu_gsp_send_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + struct nv_vgpu_gsp_rpc_entry *entry; + struct nv_vgpu_gsp_rpc_msg *rpc_msg; + int ret; + + if (!ctx || !ctx->gsp_doorbell || !ctx->gsp_shared_mem) + return -ENODEV; + + if (size > NV_VGPU_GSP_RPC_MAX_SIZE - sizeof(struct nv_vgpu_gsp_rpc_header)) + return -EINVAL; + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + init_completion(&entry->done); + entry->msg = kmalloc(size + sizeof(struct nv_vgpu_gsp_rpc_header), GFP_KERNEL); + if (!entry->msg) { + kfree(entry); + return -ENOMEM; + } + + rpc_msg = entry->msg; + rpc_msg->hdr.type = NV_GSP_RPC_TYPE_ALLOC; + rpc_msg->hdr.sequence = nv_vgpu_gsp_next_sequence(ctx); + rpc_msg->hdr.size = size; + rpc_msg->hdr.flags = 0; + rpc_msg->hdr.timestamp = ktime_get_ns(); + memcpy(rpc_msg->data, msg, size); + + mutex_lock(&ctx->queue_lock); + list_add_tail(&entry->list, &ctx->pending_queue); + mutex_unlock(&ctx->queue_lock); + + memcpy(ctx->gsp_shared_mem, rpc_msg, size + sizeof(struct nv_vgpu_gsp_rpc_header)); + writel(rpc_msg->hdr.sequence, ctx->gsp_doorbell); + + ret = wait_for_completion_timeout(&entry->done, + msecs_to_jiffies(NV_VGPU_GSP_RPC_TIMEOUT_MS)); + if (ret == 0) { + ret = -ETIMEDOUT; + } else { + ret = entry->result; + } + + mutex_lock(&ctx->queue_lock); + list_del(&entry->list); + mutex_unlock(&ctx->queue_lock); + + kfree(entry->msg); + kfree(entry); + + return ret; +} + +int nv_vgpu_gsp_recv_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size, int timeout_ms) +{ + struct nv_vgpu_gsp_comm_ctx *ctx = vgpu->gsp_comm_ctx; + struct nv_vgpu_gsp_rpc_msg *rpc_msg; + int ret; + + if (!ctx || !ctx->gsp_shared_mem) + return -ENODEV; + + ret = wait_event_timeout(ctx->wait_queue, + !list_empty(&ctx->complete_queue), + msecs_to_jiffies(timeout_ms)); + if (ret == 0) + return -ETIMEDOUT; + + mutex_lock(&ctx->queue_lock); + if (list_empty(&ctx->complete_queue)) { + mutex_unlock(&ctx->queue_lock); + return -ENOENT; + } + + rpc_msg = ctx->gsp_shared_mem; + if (rpc_msg->hdr.size > size) { + mutex_unlock(&ctx->queue_lock); + return -EOVERFLOW; + } + + memcpy(msg, rpc_msg->data, rpc_msg->hdr.size); + ret = rpc_msg->hdr.size; + mutex_unlock(&ctx->queue_lock); + + return ret; +} + +int nv_vgpu_gsp_alloc(struct nv_vgpu_device *vgpu, u64 size, u32 align, u64 *handle) +{ + struct { + u64 size; + u32 align; + u64 handle; + } __packed msg = { .size = size, .align = align }; + int ret; + + ret = nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); + if (ret >= 0) + *handle = msg.handle; + + return ret; +} + +int nv_vgpu_gsp_free(struct nv_vgpu_device *vgpu, u64 handle) +{ + struct { + u64 handle; + } __packed msg = { .handle = handle }; + + return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); +} + +int nv_vgpu_gsp_map(struct nv_vgpu_device *vgpu, u64 handle, u64 iova, u64 size, u32 flags) +{ + struct { + u64 handle; + u64 iova; + u64 size; + u32 flags; + } __packed msg = { .handle = handle, .iova = iova, .size = size, .flags = flags }; + + return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); +} + +int nv_vgpu_gsp_unmap(struct nv_vgpu_device *vgpu, u64 handle, u64 iova, u64 size) +{ + struct { + u64 handle; + u64 iova; + u64 size; + } __packed msg = { .handle = handle, .iova = iova, .size = size }; + + return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); +} + +int nv_vgpu_gsp_schedule(struct nv_vgpu_device *vgpu, u32 runlist_id, u32 channel_id) +{ + struct { + u32 runlist_id; + u32 channel_id; + } __packed msg = { .runlist_id = runlist_id, .channel_id = channel_id }; + + return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); +} + +int nv_vgpu_gsp_notify_bar_trap(struct nv_vgpu_device *vgpu, u64 offset, u32 size, void *val) +{ + struct { + u64 offset; + u32 size; + u64 val; + } __packed msg = { .offset = offset, .size = size, .val = *(u64 *)val }; + + return nv_vgpu_gsp_send_rpc(vgpu, &msg, sizeof(msg)); +} + +/* ============================================================ + * New Firmware-Decompiled RPC Operations (enhanced APIs) + * ============================================================ */ + +int nv_vgpu_gsp_rpc_alloc(struct nv_vgpu_device *vgpu, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size) +{ + return nv_vgpu_gsp_rpc_alloc_fw(vgpu, size, alignment, mem_type, flags, addr_hint, pid, + gpu_va, handle, actual_size); +} + +int nv_vgpu_gsp_rpc_free(struct nv_vgpu_device *vgpu, u64 handle) +{ + return nv_vgpu_gsp_rpc_free_fw(vgpu, handle); +} + +int nv_vgpu_gsp_rpc_map(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va) +{ + return nv_vgpu_gsp_rpc_map_fw(vgpu, handle, gpu_va, size, prot, flags, offset, mapped_va); +} + +int nv_vgpu_gsp_rpc_unmap(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size) +{ + return nv_vgpu_gsp_rpc_unmap_fw(vgpu, handle, gpu_va, size); +} + +int nv_vgpu_gsp_rpc_pin(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages) +{ + return nv_vgpu_gsp_rpc_pin_fw(vgpu, handle, offset, size, flags, dma_addr, num_pages); +} + +int nv_vgpu_gsp_rpc_unpin(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size) +{ + return nv_vgpu_gsp_rpc_unpin_fw(vgpu, handle, offset, size); +} + +int nv_vgpu_gsp_rpc_schedule(struct nv_vgpu_device *vgpu, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + return nv_vgpu_gsp_rpc_schedule_fw(vgpu, work_handle, callback, arg, priority, flags); +} + +int nv_vgpu_gsp_rpc_yield(struct nv_vgpu_device *vgpu) +{ + return nv_vgpu_gsp_rpc_yield_fw(vgpu); +} + +int nv_vgpu_gsp_rpc_bar_trap(struct nv_vgpu_device *vgpu, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value) +{ + return nv_vgpu_gsp_rpc_bar_trap_fw(vgpu, bar_addr, access_type, size, value, bar_num, ret_value); +} + +int nv_vgpu_gsp_rpc_migrate_save(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint) +{ + return nv_vgpu_gsp_rpc_migrate_save_fw(vgpu, src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, checkpoint); +} + +int nv_vgpu_gsp_rpc_migrate_restore(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint) +{ + return nv_vgpu_gsp_rpc_migrate_restore_fw(vgpu, src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, checkpoint); +} \ No newline at end of file diff --git a/kernel/gsp_emulation.c b/kernel/gsp_emulation.c new file mode 100644 index 0000000..833ce42 --- /dev/null +++ b/kernel/gsp_emulation.c @@ -0,0 +1,936 @@ +/* + * GSP Emulation Engine Implementation + * Host-side GSP emulation thread that processes RPCs from guest via shared memory + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "gsp_emulation.h" +#include "gsp_rm_emulation.h" +#include "gsp_rpc_client.h" +#include "bar_emulation.h" +#include "irq.h" +#include "memory.h" + +/* ============================================================ + * Constants + * ============================================================ */ + +#define GSP_EMU_DOORBELL_OFFSET 0x110C00 +#define GSP_EMU_SHARED_MEM_OFFSET 0x1000 +#define GSP_EMU_SHARED_MEM_SIZE 4096 +#define GSP_EMU_MSIX_TABLE_OFFSET 0x2000 +#define GSP_EMU_MSIX_PBA_OFFSET 0x3000 +#define GSP_EMU_POLL_BUSY_US 100 +#define GSP_EMU_POLL_SLEEP_MIN_US 50 +#define GSP_EMU_POLL_SLEEP_MAX_US 100 + +/* ============================================================ + * Helper Functions + * ============================================================ */ + +static inline u32 gsp_emu_read_doorbell(struct gsp_emu_context *ctx) +{ + return readl(ctx->doorbell_reg); +} + +static inline void gsp_emu_write_doorbell(struct gsp_emu_context *ctx, u32 val) +{ + writel(val, ctx->doorbell_reg); +} + +static inline void gsp_emu_copy_request_from_shared(struct gsp_emu_context *ctx, + struct gsp_rpc_msg *req) +{ + memcpy(req, ctx->shared_mem_virt, sizeof(*req)); +} + +static inline void gsp_emu_copy_response_to_shared(struct gsp_emu_context *ctx, + struct gsp_rpc_response *rsp) +{ + memcpy(ctx->shared_mem_virt + 0x80, rsp, sizeof(*rsp)); +} + +/* ============================================================ + * GSP Falcon Boot-Watch (observability for physical GSP boot) + * ============================================================ */ + +/* + * The guest's physical GSP boot (FWSEC -> Booter -> RISCV) pokes the GSP + * falcon block via the mmap'd shadow, which the host cannot trap. Poll these + * BAR0 offsets and log changes so the boot sequence is visible in host dmesg. + */ +static const struct { + u32 offset; + const char *name; +} gsp_emu_falcon_watch[GSP_EMU_FALCON_WATCH_COUNT] = { + { 0x110040, "GSP_MAILBOX0" }, + { 0x110044, "GSP_MAILBOX1" }, + { 0x110080, "GSP_FALCON_OS" }, + { 0x110100, "GSP_CPUCTL" }, + { 0x110180, "GSP_IMEMC" }, + { 0x110184, "GSP_IMEMD" }, + { 0x1101c0, "GSP_DMEMC" }, + { 0x1101c4, "GSP_DMEMD" }, + { 0x1103c0, "GSP_ENGINE" }, + { 0x110c04, "GSP_QUEUE_HEAD1" }, + { 0x110c08, "GSP_QUEUE_TAIL0" }, + { 0x110c0c, "GSP_QUEUE_TAIL1" }, + { 0x118128, "PGC6_PLM_GFW" }, + { 0x118234, "PGC6_GFW_BOOT" }, + { 0x1fa824, "WPR2_ADDR_LO" }, + { 0x1fa828, "WPR2_ADDR_HI" }, + { 0x001438, "VBIOS_SCRATCH_0E" }, + { 0x840040, "SEC2_MAILBOX0" }, + { 0x840100, "SEC2_CPUCTL" }, + { 0x8403c0, "SEC2_ENGINE" }, +}; + +static void gsp_emu_poll_falcon_watch(struct gsp_emu_context *ctx) +{ + int i; + + for (i = 0; i < GSP_EMU_FALCON_WATCH_COUNT; i++) { + u32 val = readl(ctx->bar0_vaddr + gsp_emu_falcon_watch[i].offset); + + if (!ctx->falcon_watch_valid) { + ctx->falcon_watch_last[i] = val; + continue; + } + + if (val != ctx->falcon_watch_last[i]) { + nv_vgpu_info(ctx->vgpu, "FALCON %s (0x%x): 0x%08x -> 0x%08x", + gsp_emu_falcon_watch[i].name, + gsp_emu_falcon_watch[i].offset, + ctx->falcon_watch_last[i], val); + ctx->falcon_watch_last[i] = val; + } + } + ctx->falcon_watch_valid = true; +} + +/* ============================================================ + * Physical GSP Boot Emulation (FWSEC / Booter falcon auto-complete) + * ============================================================ */ + +/* Register offsets + CPUCTL bits live in gsp_emulation.h (shared with + * gsp_rm_emulation.c), as are gsp_emu_shadow_read32/write32. */ + +enum gsp_emu_boot_stage { + GSP_BOOT_STAGE_IDLE = 0, /* waiting for first kick (FWSEC) */ + GSP_BOOT_STAGE_FWSEC_DONE, /* FWSEC FRTS completed */ + GSP_BOOT_STAGE_BOOTER_DONE, /* Booter Load completed */ +}; + +/* + * gsp_emu_clear_boot_state - return the emulated GSP to a cold state. + * + * On real hardware, driver unload tears down WPR2 (Booter Unload) so the + * next probe's _kgspBootGspRm WPR2-up check passes. Turing guests produce + * no observable BAR0 writes on unload, so callers must trigger this on + * indirect death signals (ENGINE reset, GSP-RM queue going stale). + * Clears WPR2, RISCV active status and the falcon stage tracker, and + * drops the parsed GSP-RM queue state. + */ +/* + * gsp_emu_clear_boot_state - return the emulated GSP to a cold state. + * + * On real hardware, driver unload tears down WPR2 (Booter Unload) so the + * next probe's _kgspBootGspRm WPR2-up check passes. Turing guests produce + * no observable BAR0 writes on unload, so callers must trigger this on + * indirect death signals (ENGINE reset, GSP-RM queue going stale). + * + * IMPORTANT: on the Booter-unload kick we clear the FALCON boot state + * (WPR2/RISCV/stages) but PRESERVE the GSP-RM queue state. The driver + * sends cleanup FREEs AFTER the Booter-unload kick; if we reset the queue + * here, those frees time out (fn-10). The queue state is only cleared when + * the queue is genuinely gone (garbage elements, insane writePtr, ENGINE + * reset, or a new probe writing fresh libos args to MAILBOX0/1). + */ +void gsp_emu_clear_boot_state(struct gsp_emu_context *ctx, const char *reason) +{ + nv_vgpu_info(ctx->vgpu, "GSP-BOOT: clearing boot state (%s)", reason); + + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_LO, 0); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_HI, 0); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_RISCV_STATUS, 0); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_MAILBOX0, 0); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_MAILBOX1, 0); + + ctx->boot_stage = GSP_BOOT_STAGE_IDLE; + ctx->flcn_kick_count = 0; + + /* Do NOT reset the GSP-RM queue state here - the driver is still + * sending cleanup frees on it. Only reset on genuine queue death + * (garbage elements, insane writePtr) or ENGINE reset. */ +} + +/* + * gsp_emu_poll_boot - drive the guest's physical GSP boot to completion. + * + * The guest driver kicks falcon ucodes (FWSEC for FRTS, then Booter Load for + * GSP-RM) by writing STARTCPU to the GSP falcon CPUCTL, then polls for HALTED + * plus stage-specific result registers. The auto-boot waits (GFW) are already + * satisfied by the state pre-set in nv_vgpu_bar_emul_init. Here we watch for + * STARTCPU kicks and complete each stage so the guest believes its falcon + * ucodes ran successfully. + */ +static void gsp_emu_poll_boot(struct gsp_emu_context *ctx) +{ + u32 gsp_cpuctl = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_CPUCTL); + u32 sec2_cpuctl = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_SEC2_CPUCTL); + u32 engine = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_ENGINE); + u32 sw_vector; + + /* Software-interrupt trigger: the driver writes a vector to + * CPU_INTR_LEAF_TRIGGER to fire it (the osVerifyInterrupts sanity + * test uses the doorbell vector). Set the pending bits in the LEAF + * and TOP interrupt-tree registers and inject MSI-X so the guest's + * ISR runs - without this the sanity test times out and + * osVerifySystemEnvironment fails, bailing RmInitAdapter. */ + sw_vector = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER); + if (sw_vector) { + u32 leaf_idx = sw_vector / 32; + u32 leaf_bit = sw_vector % 32; + u32 subtree = sw_vector / 64; + u32 top_idx = subtree / 32; + u32 top_bit = subtree % 32; + u32 leaf_val = gsp_emu_shadow_read32(ctx, + GSP_EMU_REG_CPU_INTR_LEAF_BASE + leaf_idx * 4); + u32 top_val = gsp_emu_shadow_read32(ctx, + GSP_EMU_REG_CPU_INTR_TOP_BASE + top_idx * 4); + + gsp_emu_shadow_write32(ctx, + GSP_EMU_REG_CPU_INTR_LEAF_BASE + leaf_idx * 4, + leaf_val | (1U << leaf_bit)); + gsp_emu_shadow_write32(ctx, + GSP_EMU_REG_CPU_INTR_TOP_BASE + top_idx * 4, + top_val | (1U << top_bit)); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER, 0); + + nv_vgpu_info(ctx->vgpu, "GSP-BOOT: SW interrupt vector %u -> LEAF[%u]|=0x%x TOP[%u]|=0x%x, MSI-X", + sw_vector, leaf_idx, 1U << leaf_bit, top_idx, + 1U << top_bit); + gsp_emu_inject_completion_interrupt(ctx, 0); + } + + /* TLB invalidate auto-complete: the driver sets TRIGGER then spins + * polling for it to clear (kgmmuCheckPendingInvalidates_TU102 reads + * GPU_VREG_RD32(NV_VIRTUAL_FUNCTION_PRIV_MMU_INVALIDATE) = + * BAR0 + 0xB80000 + 0x30B0; the physical path uses the plain alias). + * Report instant completion or the driver soft-locks spinning inside + * osDevReadReg032. */ + { + static const u32 tlb_inv_regs[] = { + GSP_EMU_REG_MMU_INVALIDATE, + GSP_EMU_REG_MMU_INVALIDATE_WIN, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(tlb_inv_regs); i++) { + u32 val = gsp_emu_shadow_read32(ctx, tlb_inv_regs[i]); + + if (!(val & GSP_EMU_MMU_INVALIDATE_TRIGGER)) + continue; + gsp_emu_shadow_write32(ctx, tlb_inv_regs[i], + val & ~GSP_EMU_MMU_INVALIDATE_TRIGGER); + pr_info_ratelimited("GSP-BOOT: MMU_INVALIDATE[0x%x] trigger auto-cleared (val=0x%x)\n", + tlb_inv_regs[i], val); + } + } + + /* L2 cache-op auto-complete: kmemsysDoCacheOp_GM107 writes PENDING + * (bit 0) into the NV_UFLUSH_* / VF L2 invalidate registers and spins + * until the polled bits read back 0 (FB flush, L2 sysmem/peermem + * invalidate, L2 clean, L2 flush-dirty, L2 wait-for-sys-reads). On + * Turing the sysmem/peermem invalidate registers go through the + * GPU_VREG_* window (BAR0 + 0xB80000 + NV_VIRTUAL_FUNCTION_PRIV_L2_*), + * so cover both the plain and windowed aliases. The emulated L2 has + * nothing to do, so clear them as soon as they are observed. */ + { + static const u32 uflush_regs[] = { + GSP_EMU_REG_UFLUSH_FB_FLUSH, + GSP_EMU_REG_UFLUSH_L2_SYSMEM_INV, + GSP_EMU_REG_UFLUSH_L2_PEERMEM_INV, + GSP_EMU_REG_UFLUSH_L2_CLEAN_COMPTAGS, + GSP_EMU_REG_UFLUSH_L2_FLUSH_DIRTY, + GSP_EMU_REG_UFLUSH_L2_WAIT_SYS_READS, + GSP_EMU_REG_VF_L2_SYSMEM_INV, + GSP_EMU_REG_VF_L2_PEERMEM_INV, + GSP_EMU_REG_VF_L2_SYSMEM_INV_WIN, + GSP_EMU_REG_VF_L2_PEERMEM_INV_WIN, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(uflush_regs); i++) { + u32 val = gsp_emu_shadow_read32(ctx, + uflush_regs[i]); + + if (!val) + continue; + gsp_emu_shadow_write32(ctx, uflush_regs[i], 0); + pr_info_ratelimited("GSP-BOOT: UFLUSH[0x%x] 0x%x auto-cleared\n", + uflush_regs[i], val); + } + } + + /* CrashCat wayfinder suppression: if the GSP falcon's DEBUGINFO + * register holds a valid wayfinder-L0 signature (0xdead), the driver's + * kgspHealthCheck (run on every RPC drain) reads "reports" from + * garbage and loops forever in nv_open_q, which then can't poll RPCs + * (fn-10 FREE timeouts). Force it invalid - our emulated GSP has no + * crashcat reports to offer. */ + { + u32 wfl0 = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_FALCON_DEBUGINFO); + + if ((wfl0 & 0xffff) == 0xdead) { + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_FALCON_DEBUGINFO, 0); + nv_vgpu_info(ctx->vgpu, "GSP-BOOT: cleared CrashCat wayfinder (was 0x%x)", + wfl0); + } + } + + /* Temporary: watch PCFG / link-cap mirror offsets for guest writes */ + { + static const u32 watch[] = { + 0x000004, 0x000084, 0x880000, 0x880004, 0x880080, + 0x880084, 0x880088, 0x88008c, 0xb80084, 0xb88084, + 0xc08084, + }; + static u32 last[ARRAY_SIZE(watch)]; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(watch); i++) { + u32 v = gsp_emu_shadow_read32(ctx, watch[i]); + + if (v != last[i]) { + nv_vgpu_info(ctx->vgpu, "GSP-BOOT: PCFG[0x%x] = 0x%x (was 0x%x)", + watch[i], v, last[i]); + last[i] = v; + } + } + } + + /* GSP ENGINE reset asserted -> falcon rebooted, restart stage tracking */ + if ((engine & 0x1) && ctx->boot_stage != GSP_BOOT_STAGE_IDLE) { + nv_vgpu_info(ctx->vgpu, "GSP-BOOT: falcon reset, back to IDLE"); + /* ENGINE reset is a full GPU reset: the GSP-RM queue state is + * stale, reset it too. */ + gsp_emu_clear_boot_state(ctx, "ENGINE reset"); + gsp_rm_emu_reset(ctx); + return; + } + + /* + * PRAMIN window translation: the guest programs NV_PBUS_BAR0_WINDOW + * (BAR0+0x1700) to select which vidmem page the PRAMIN window + * (BAR0+0x700000) maps to. On real hardware the PRAMIN window is a + * CPU aperture into vidmem at (BASE << 16) + offset. In our shadow + * the PRAMIN window is just part of the shadow buffer - it does NOT + * alias the emulated BAR1 (vidmem). + * + * The BAR2 verify test writes through the vBar2 aperture (PRAMIN) + * and reads back through the BAR0 window (also PRAMIN), expecting + * the same data. Both paths hit the same shadow location, so the + * test data IS consistent. We do NOT sync the PRAMIN window from + * the emulated BAR1 - the sync would overwrite the test data + * written by the driver through the vBar2 aperture. + * + * The BAR0_WINDOW_BASE register is captured for observability only. + */ + { + u32 bar0_win = gsp_emu_shadow_read32(ctx, 0x1700); + u32 base = bar0_win & 0xFFFFFF; + u32 target = (bar0_win >> 24) & 0x3; + + if (!ctx->bar0_window_valid || + base != ctx->bar0_window_base || + target != ctx->bar0_window_target) { + static u32 last_pramin_val; + u32 pramin_val = gsp_emu_shadow_read32(ctx, 0x70e000); + + ctx->bar0_window_base = base; + ctx->bar0_window_target = target; + ctx->bar0_window_valid = true; + nv_vgpu_info(ctx->vgpu, + "GSP-BOOT: BAR0_WINDOW base=0x%x target=%u, PRAMIN[0xe000]=0x%08x (was 0x%08x)", + base, target, pramin_val, last_pramin_val); + last_pramin_val = pramin_val; + } + } + + /* + * FWSEC kick on the GSP falcon (FRTS setup). Complete it: + * - CPUCTL: clear STARTCPU, set HALTED + * - VBIOS_SCRATCH[0x0E] FRTS_ERR_CODE = NONE (0) + * - WPR2_ADDR_HI != 0, WPR2_ADDR_LO = frtsOffset >> 0xc (in _VAL bits31:4) + */ + if ((gsp_cpuctl & GSP_EMU_CPUCTL_STARTCPU) && + ctx->boot_stage == GSP_BOOT_STAGE_IDLE) { + /* A fresh FWSEC kick means a new GSP boot: drop any GSP-RM + * queue state left from the previous probe (the driver has + * already torn down its old queue). Without this the next + * Booter completion would skip the chain re-parse + * (rm->parsed still set) and GspStatusQueueInit would time + * out on re-probe. */ + gsp_rm_emu_reset(ctx); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_VBIOS_SCRATCH_0E, 0); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_HI, 0x100000); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_WPR2_ADDR_LO, + ctx->wpr2_addr_lo); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_CPUCTL, + GSP_EMU_CPUCTL_HALTED); + ctx->boot_stage = GSP_BOOT_STAGE_FWSEC_DONE; + ctx->flcn_kick_count++; + nv_vgpu_info(ctx->vgpu, "GSP-BOOT: FWSEC kick completed " + "(WPR2_LO=0x%x)", ctx->wpr2_addr_lo); + return; + } + + /* + * Booter Load kick on the SEC2 falcon (loads/authenticates the GSP-RM + * image and starts the GSP RISCV). Complete it: + * - SEC2 CPUCTL: clear STARTCPU, set HALTED + * - SEC2 MAILBOX0 = 0 (booter error code, 0 = success) + * - GSP RISCV_STATUS = ACTIVE (Booter started the RISCV; + * kflcnIsRiscvActive passes) + */ + if ((sec2_cpuctl & GSP_EMU_CPUCTL_STARTCPU) && + ctx->boot_stage == GSP_BOOT_STAGE_FWSEC_DONE) { + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_MAILBOX0, 0); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_RISCV_STATUS, 0x1); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_CPUCTL, + GSP_EMU_CPUCTL_HALTED); + ctx->boot_stage = GSP_BOOT_STAGE_BOOTER_DONE; + ctx->flcn_kick_count++; + nv_vgpu_info(ctx->vgpu, "GSP-BOOT: Booter kick completed, " + "RISCV active"); + /* The driver has programmed the libos init args address into + * GSP MAILBOX0/1 by now; parse the queue chain, link the + * status queue and emit GSP_INIT_DONE. */ + gsp_rm_emu_on_booter_done(ctx); + return; + } + + /* Any other STARTCPU kick (resume/unload/etc.): complete it. + * SEC2 kicks report their ucode result in SEC2_MAILBOX0 (0=success) - + * e.g. Booter Unload during GPU shutdown. Without clearing it the + * driver reads a stale/garbage error code (0xff) and the unload + * fails, killing the subsequent re-init. A SEC2 kick after a + * completed boot is a GSP shutdown/reload: drop the boot state + * (GSP no longer active, queue dead). Clear boot state FIRST so + * WPR2 is already down when the driver sees CPUCTL=HALTED and + * checks kgspIsWpr2Up (avoids "Booter Unload: WPR2 is still up"). */ + if (gsp_cpuctl & GSP_EMU_CPUCTL_STARTCPU) + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_GSP_CPUCTL, + GSP_EMU_CPUCTL_HALTED); + if (sec2_cpuctl & GSP_EMU_CPUCTL_STARTCPU) { + if (ctx->boot_stage == GSP_BOOT_STAGE_BOOTER_DONE) + gsp_emu_clear_boot_state(ctx, "Booter unload/reload kick"); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_MAILBOX0, 0); + gsp_emu_shadow_write32(ctx, GSP_EMU_REG_SEC2_CPUCTL, + GSP_EMU_CPUCTL_HALTED); + } +} + +/* ============================================================ + * RPC Processing + * ============================================================ */ + +static int gsp_emu_dispatch_rpc(struct gsp_emu_context *ctx, + struct gsp_rpc_msg *req, + struct gsp_rpc_response *rsp) +{ + int ret = 0; + ktime_t start = ktime_get(); + + memset(rsp, 0, sizeof(*rsp)); + rsp->type = req->type; + rsp->seq_id = req->seq_id; + + switch (req->type) { + case GSP_RPC_ALLOC: { + struct gsp_alloc_req *areq = (struct gsp_alloc_req *)req->payload; + struct gsp_alloc_rsp arsp = {0}; + u64 gpu_va = 0, handle = 0, actual_size = 0; + + ret = gsp_rpc_alloc(ctx->rpc_client, + areq->size, areq->alignment, areq->mem_type, + areq->flags, areq->addr_hint, areq->pid, + &gpu_va, &handle, &actual_size); + if (!ret) { + arsp.gpu_va = gpu_va; + arsp.handle = handle; + arsp.size = actual_size; + memcpy(rsp->result, &arsp, sizeof(arsp)); + } + break; + } + + case GSP_RPC_FREE: { + u64 handle = req->payload[0]; + ret = gsp_rpc_free(ctx->rpc_client, handle); + break; + } + + case GSP_RPC_MAP: { + struct gsp_map_req *mreq = (struct gsp_map_req *)req->payload; + struct gsp_map_rsp mrsp = {0}; + u64 mapped_va = 0; + + ret = gsp_rpc_map(ctx->rpc_client, + mreq->handle, mreq->gpu_va, mreq->size, + mreq->prot, mreq->flags, mreq->offset, + &mapped_va); + if (!ret) { + mrsp.gpu_va = mapped_va; + memcpy(rsp->result, &mrsp, sizeof(mrsp)); + } + break; + } + + case GSP_RPC_UNMAP: { + u64 handle = req->payload[0]; + u64 gpu_va = req->payload[1]; + u64 size = req->payload[2]; + ret = gsp_rpc_unmap(ctx->rpc_client, handle, gpu_va, size); + break; + } + + case GSP_RPC_PIN: { + struct gsp_pin_req *preq = (struct gsp_pin_req *)req->payload; + struct gsp_pin_rsp prsp = {0}; + u64 dma_addr = 0; + u32 num_pages = 0; + + ret = gsp_rpc_pin(ctx->rpc_client, + preq->handle, preq->offset, preq->size, + preq->flags, &dma_addr, &num_pages); + if (!ret) { + prsp.dma_addr = dma_addr; + prsp.num_pages = num_pages; + memcpy(rsp->result, &prsp, sizeof(prsp)); + } + break; + } + + case GSP_RPC_UNPIN: { + u64 handle = req->payload[0]; + u64 offset = req->payload[1]; + u64 size = req->payload[2]; + ret = gsp_rpc_unpin(ctx->rpc_client, handle, offset, size); + break; + } + + case GSP_RPC_SCHEDULE: { + struct gsp_sched_req *sreq = (struct gsp_sched_req *)req->payload; + ret = gsp_rpc_schedule(ctx->rpc_client, + sreq->work_handle, sreq->callback, sreq->arg, + sreq->priority, sreq->flags); + break; + } + + case GSP_RPC_YIELD: { + ret = gsp_rpc_yield(ctx->rpc_client); + break; + } + + case GSP_RPC_NOTIFY: { + /* NOTIFY is handled via registration, not direct RPC */ + nv_vgpu_info(ctx->vgpu, "NOTIFY RPC received (type=0x%x)", req->subcmd); + ret = 0; + break; + } + + case GSP_RPC_BAR_TRAP: { + struct gsp_bar_trap_req *breq = (struct gsp_bar_trap_req *)req->payload; + struct gsp_bar_trap_rsp brsp = {0}; + u64 ret_value = 0; + + ret = gsp_rpc_bar_trap(ctx->rpc_client, + breq->bar_addr, breq->access_type, breq->size, + breq->value, breq->bar_num, &ret_value); + if (!ret) { + brsp.value = ret_value; + brsp.handled = 1; + memcpy(rsp->result, &brsp, sizeof(brsp)); + } + break; + } + + case GSP_RPC_MIGRATE: { + struct gsp_migrate_req *mreq = (struct gsp_migrate_req *)req->payload; + struct gsp_migrate_rsp mrsp = {0}; + u64 checkpoint = 0; + + if (mreq->flags == GSP_MIGRATE_SUBCMD_SAVE) { + ret = gsp_rpc_migrate_save(ctx->rpc_client, + mreq->src_gpu_id, mreq->dst_gpu_id, + mreq->memory_bitmap, mreq->page_table_ptr, + &checkpoint); + if (!ret) { + mrsp.checkpoint = checkpoint; + memcpy(rsp->result, &mrsp, sizeof(mrsp)); + } + } else if (mreq->flags == GSP_MIGRATE_SUBCMD_RESTORE) { + ret = gsp_rpc_migrate_restore(ctx->rpc_client, + mreq->src_gpu_id, mreq->dst_gpu_id, + mreq->memory_bitmap, mreq->page_table_ptr, + mreq->context); + } + break; + } + + case GSP_RPC_DEBUG: { + nv_vgpu_info(ctx->vgpu, "DEBUG RPC received (seq=%u)", req->seq_id); + ret = 0; + break; + } + + default: + nv_vgpu_warn(ctx->vgpu, "Unknown RPC type: 0x%x", req->type); + rsp->status = -ENOTSUPP; + ret = -ENOTSUPP; + break; + } + + if (ret) + rsp->status = ret; + + ctx->total_latency_ns += ktime_to_ns(ktime_sub(ktime_get(), start)); + return ret; +} + +/* ============================================================ + * MSI-X Interrupt Injection + * ============================================================ */ + +void gsp_emu_inject_completion_interrupt(struct gsp_emu_context *ctx, u32 seq_id) +{ + struct eventfd_ctx *trigger = NULL; + + /* Write to MSI-X PBA (Pending Bit Array) at BAR0+0x3000 */ + if (ctx->bar0_vaddr) { + u32 *pba = (u32 *)(ctx->bar0_vaddr + GSP_EMU_MSIX_PBA_OFFSET); + writel(1, pba); /* Set pending bit for vector 0 */ + } + + /* Look up the vector-0 trigger LIVE: QEMU wires eventfds via + * VFIO_DEVICE_SET_IRQS after gsp_emu_init runs, so an init-time copy + * would be stale (NULL). */ + if (ctx->vgpu) + trigger = ctx->vgpu->irqs[0].trigger; + + /* Signal eventfd for VFIO interrupt delivery */ + if (trigger) + eventfd_signal(trigger); +} + +/* ============================================================ + * Emulation Thread + * ============================================================ */ + +static int gsp_emu_thread_fn(void *data) +{ + struct gsp_emu_context *ctx = data; + struct gsp_rpc_msg req; + struct gsp_rpc_response rsp; + u32 doorbell_val; + int ret; + + nv_vgpu_info(ctx->vgpu, "GSP emulation thread started"); + + while (!kthread_should_stop()) { + /* Read doorbell */ + doorbell_val = gsp_emu_read_doorbell(ctx); + + /* Watch GSP falcon boot registers (physical GSP boot tracing) */ + gsp_emu_poll_falcon_watch(ctx); + + /* Drive the guest's physical GSP boot to completion */ + gsp_emu_poll_boot(ctx); + + /* GSP-RM msgq emulation (physical path: sysmem cmd/status queues) */ + gsp_rm_emu_poll(ctx); + + if (doorbell_val != 0) { + /* ACK doorbell immediately */ + gsp_emu_write_doorbell(ctx, 0); + + /* Copy request from shared memory */ + gsp_emu_copy_request_from_shared(ctx, &req); + + /* Validate sequence ID */ + if (req.seq_id == 0) { + nv_vgpu_warn(ctx->vgpu, "Invalid seq_id=0, ignoring RPC"); + continue; + } + + /* Process RPC */ + ret = gsp_emu_dispatch_rpc(ctx, &req, &rsp); + + /* Write response to shared memory */ + gsp_emu_copy_response_to_shared(ctx, &rsp); + + /* Inject completion interrupt */ + gsp_emu_inject_completion_interrupt(ctx, req.seq_id); + + if (ret) + ctx->failed_rpcs++; + else + ctx->processed_rpcs++; + } else { + /* Adaptive polling: busy-wait first, then sleep (interruptible for kthread_stop) */ + static int poll_count = 0; + if (poll_count < GSP_EMU_POLL_BUSY_US) { + udelay(1); + poll_count++; + } else { + /* Use interruptible sleep so kthread_stop can wake us */ + schedule_timeout_interruptible(usecs_to_jiffies(GSP_EMU_POLL_SLEEP_MAX_US)); + poll_count = 0; + } + } + } + + nv_vgpu_info(ctx->vgpu, "GSP emulation thread stopped (processed=%llu, failed=%llu)", + ctx->processed_rpcs, ctx->failed_rpcs); + complete(&ctx->thread_stopped); + return 0; +} + +/* ============================================================ + * Initialization & Cleanup + * ============================================================ */ + +int gsp_emu_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_bar_emul_ctx *bar_ctx; + struct gsp_emu_context *ctx; + int ret = 0; + + nv_vgpu_info(vgpu, "Initializing GSP emulation engine"); + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + ctx->running = false; + ctx->initialized = false; + ctx->stopping = false; + init_completion(&ctx->thread_stopped); + spin_lock_init(&ctx->lock); + + /* Physical GSP boot emulation state. wpr2_addr_lo matches what the + * guest driver expects WPR2_ADDR_LO to read after FWSEC FRTS: + * frtsOffset = fbSize - reservedFromEnd. With fbSize = 256MB + * (LOCAL_MEMORY_RANGE) and the 570.144 image's 2MB reservation, + * frtsOffset = 0x10000000 - 0x200000 = 0xfe00000; the register's + * _VAL field (bits31:4) must equal frtsOffset >> 12 = 0xfe00, so the + * raw register value is 0xfe000. */ + ctx->boot_stage = 0; /* GSP_BOOT_STAGE_IDLE */ + ctx->flcn_kick_count = 0; + ctx->wpr2_addr_lo = 0x000fe000; + + /* The guest's BAR0 is the emulation shadow (vzalloc'd by + * nv_vgpu_bar_emul_init, mapped to the guest via VFIO mmap) - NOT the + * host physical BAR0 (vgpu->bars[0].vaddr, which belongs to the real GPU + * and the host nvidia driver). All guest doorbell writes, RPC messages, + * MSI-X table writes and PBA reads land in the shadow, so the emulation + * engine must operate on the shadow exclusively. Polling the physical + * BAR0 would never see guest traffic and would corrupt host GPU state. */ + bar_ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx; + if (!bar_ctx || !bar_ctx->bar0_shadow) { + nv_vgpu_err(vgpu, "BAR0 emulation shadow not available"); + ret = -ENODEV; + goto err_free_ctx; + } + + ctx->bar0_vaddr = (void __iomem *)bar_ctx->bar0_shadow; + + /* Doorbell register at BAR0+0x110C00 (guest-visible shadow) */ + ctx->doorbell_reg = ctx->bar0_vaddr + GSP_EMU_DOORBELL_OFFSET; + + /* RPC shared memory at BAR0+0x1000. This ALIASES the guest-visible + * shadow - it is NOT a separate allocation. shared_mem_dma stays 0 to + * mark it as aliased so cleanup does not dma_free it. */ + ctx->shared_mem_size = GSP_EMU_SHARED_MEM_SIZE; + ctx->shared_mem = ctx->bar0_vaddr + GSP_EMU_SHARED_MEM_OFFSET; + ctx->shared_mem_virt = (void *)ctx->shared_mem; + ctx->shared_mem_dma = 0; + + /* Initialize RPC client with the shadow BAR0 base and doorbell */ + ctx->rpc_client = gsp_client_init_emulation(&vgpu->dev, ctx->bar0_vaddr, ctx->doorbell_reg); + if (IS_ERR(ctx->rpc_client)) { + ret = PTR_ERR(ctx->rpc_client); + nv_vgpu_err(vgpu, "Failed to init GSP RPC client: %d", ret); + goto err_free_ctx; + } + + /* Enable emulation mode in RPC client */ + gsp_rpc_emulation_set(true); + + /* Initialize GSP-RM msgq emulation (physical path) */ + ret = gsp_rm_emu_init(vgpu, ctx); + if (ret) { + nv_vgpu_err(vgpu, "Failed to init GSP-RM emulation: %d", ret); + goto err_fini_client; + } + + /* NOTE: MSI-X eventfd triggers are NOT copied here - QEMU wires them + * via VFIO_DEVICE_SET_IRQS later, so gsp_emu_inject_completion_interrupt + * looks them up live from vgpu->irqs[]. */ + + /* Start emulation thread */ + ctx->thread = kthread_run(gsp_emu_thread_fn, ctx, "gsp_emu_%u", vgpu->vgpu_id); + if (IS_ERR(ctx->thread)) { + ret = PTR_ERR(ctx->thread); + nv_vgpu_err(vgpu, "Failed to start GSP emulation thread: %d", ret); + goto err_fini_client; + } + /* Hold a reference on the thread's task_struct so it stays valid + * even if the thread crashes (e.g. fortify_panic). Without this, + * cleanup could dereference a freed task_struct. */ + get_task_struct(ctx->thread); + + ctx->running = true; + ctx->initialized = true; + vgpu->gsp_comm_ctx = ctx; /* Reuse existing field for emulation context */ + + nv_vgpu_info(vgpu, "GSP emulation engine initialized successfully"); + return 0; + +err_fini_client: + gsp_client_fini(ctx->rpc_client); +err_free_ctx: + /* shared_mem_virt aliases the BAR0 shadow (owned by bar_emulation), + * nothing to free here */ + kfree(ctx); + return ret; +} + +void gsp_emu_cleanup(struct nv_vgpu_device *vgpu) +{ + struct gsp_emu_context *ctx = vgpu->gsp_comm_ctx; + + if (!ctx) + return; + + nv_vgpu_info(vgpu, "Cleaning up GSP emulation engine"); + + /* Stop emulation thread with timeout */ + if (ctx->thread) { + ctx->stopping = true; + /* Only call kthread_stop() if the thread is still alive. + * If the thread crashed (e.g. fortify_panic), the kernel already + * killed it and kthread_stop() would hit a refcount saturation + * warning on the dead task_struct. */ + if (pid_alive(ctx->thread)) { + kthread_stop(ctx->thread); + /* Wait for thread to actually stop with timeout (5 seconds) */ + if (!wait_for_completion_timeout(&ctx->thread_stopped, msecs_to_jiffies(5000))) { + nv_vgpu_warn(vgpu, "GSP emulation thread did not stop in time, forcing cleanup"); + } + } else { + nv_vgpu_warn(vgpu, "GSP emulation thread already dead (crashed), skipping kthread_stop"); + } + put_task_struct(ctx->thread); + ctx->thread = NULL; + } + ctx->running = false; + + /* Teardown GSP-RM msgq emulation */ + gsp_rm_emu_fini(ctx); + + /* Disable emulation mode */ + gsp_rpc_emulation_set(false); + + /* Cleanup RPC client */ + if (ctx->rpc_client) { + gsp_client_fini(ctx->rpc_client); + ctx->rpc_client = NULL; + } + + /* Shared memory aliases the BAR0 shadow (owned by bar_emulation) - + * only free it if it was a standalone DMA allocation (dma != 0) */ + if (ctx->shared_mem_virt && ctx->shared_mem_dma) { + dma_free_wc(&vgpu->pdev->dev, ctx->shared_mem_size, + ctx->shared_mem_virt, ctx->shared_mem_dma); + } + ctx->shared_mem_virt = NULL; + + /* BAR0 shadow is managed by nv_vgpu_bar_emul_init/cleanup */ + ctx->bar0_vaddr = NULL; + + vgpu->gsp_comm_ctx = NULL; + kfree(ctx); + + nv_vgpu_info(vgpu, "GSP emulation engine cleaned up"); +} + +/* ============================================================ + * Public RPC Processing (for external callers) + * ============================================================ */ + +int gsp_emu_process_rpc(struct gsp_emu_context *ctx) +{ + struct gsp_rpc_msg req; + struct gsp_rpc_response rsp; + int ret; + + if (!ctx || !ctx->initialized) + return -ENODEV; + + gsp_emu_copy_request_from_shared(ctx, &req); + + if (req.seq_id == 0) + return -EINVAL; + + ret = gsp_emu_dispatch_rpc(ctx, &req, &rsp); + + gsp_emu_copy_response_to_shared(ctx, &rsp); + gsp_emu_inject_completion_interrupt(ctx, req.seq_id); + + return ret; +} + +/* ============================================================ + * Statistics + * ============================================================ */ + +void gsp_emu_get_stats(struct gsp_emu_context *ctx, + u64 *processed, u64 *failed, u64 *total_latency_ns) +{ + if (!ctx) + return; + + if (processed) + *processed = ctx->processed_rpcs; + if (failed) + *failed = ctx->failed_rpcs; + if (total_latency_ns) + *total_latency_ns = ctx->total_latency_ns; +} + +EXPORT_SYMBOL(gsp_emu_process_rpc); +EXPORT_SYMBOL(gsp_emu_get_stats); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Open-IOV Project"); +MODULE_DESCRIPTION("GSP Emulation Engine for Host-Side vGPU"); \ No newline at end of file diff --git a/kernel/gsp_emulation.h b/kernel/gsp_emulation.h new file mode 100644 index 0000000..9a8f241 --- /dev/null +++ b/kernel/gsp_emulation.h @@ -0,0 +1,206 @@ +/* + * GSP Emulation Engine for Host-Side GSP Emulation + * Replaces guest-side GSP communication with host kernel emulation + */ + +#ifndef _GSP_EMULATION_H_ +#define _GSP_EMULATION_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "gsp_rpc_client.h" +#include "bar_emulation.h" +#include "irq.h" +#include "memory.h" + +/* ============================================================ + * GSP Emulation Constants + * ============================================================ */ + +#define GSP_EMU_DOORBELL_OFFSET 0x110C00 +#define GSP_EMU_SHARED_MEM_OFFSET 0x1000 +#define GSP_EMU_SHARED_MEM_SIZE 4096 +#define GSP_EMU_MSIX_TABLE_OFFSET 0x2000 +#define GSP_EMU_MSIX_PBA_OFFSET 0x3000 +#define GSP_EMU_POLL_BUSY_US 100 +#define GSP_EMU_POLL_SLEEP_MIN_US 50 +#define GSP_EMU_POLL_SLEEP_MAX_US 100 + +/* GSP falcon boot-watch: registers polled for change logging so the guest's + * physical GSP boot sequence is visible on the host (guest BAR0 access is + * mmap-based and otherwise invisible). Offsets are BAR0-relative, GSP falcon + * block at 0x110000. */ +#define GSP_EMU_FALCON_WATCH_COUNT 20 + +/* BAR0 shadow offsets used by the guest's physical GSP boot */ +#define GSP_EMU_REG_GSP_CPUCTL 0x110100 /* bit1 STARTCPU, bit4 HALTED */ +#define GSP_EMU_REG_GSP_ENGINE 0x1103c0 /* bit0 RESET */ +#define GSP_EMU_REG_GSP_MAILBOX0 0x110040 +#define GSP_EMU_REG_GSP_MAILBOX1 0x110044 +#define GSP_EMU_REG_VBIOS_SCRATCH_0E 0x001438 /* FRTS err code (0=NONE) */ +#define GSP_EMU_REG_WPR2_ADDR_LO 0x1fa824 +#define GSP_EMU_REG_WPR2_ADDR_HI 0x1fa828 +#define GSP_EMU_REG_RISCV_STATUS 0x111240 /* bit0 ACTIVE (NV_PRISCV_RISCV + CORE_SWITCH_RISCV_STATUS=0x240 at + GSP riscvRegisterBase = + NV_FALCON2_GSP_BASE=0x111000) */ + +/* SEC2 falcon (Booter Load/Unload runs here, not on the GSP falcon) */ +#define GSP_EMU_REG_SEC2_CPUCTL 0x840100 +#define GSP_EMU_REG_SEC2_MAILBOX0 0x840040 +#define GSP_EMU_REG_SEC2_ENGINE 0x8403c0 + +/* Base of the virtual-function register window (NV_VIRTUAL_FUNCTION_FULL_PHYS_ + * OFFSET). The client RM reaches per-VF registers through GPU_VREG_* = + * BAR0 + sriovState.virtualRegPhysOffset + offset; for a non-SRIOV virtual GPU + * that offset is 0xB80000, and 0 for a VF, hence the plain + windowed aliases + * below. */ +#define GSP_EMU_REG_FULL_PHYS_OFFSET 0xB80000 + +/* NV_VIRTUAL_FUNCTION_PRIV_MMU_INVALIDATE: driver sets TRIGGER(bit31) to + * start a TLB invalidate, then polls for bit31==0 (done). We auto-clear it. + * The client RM reaches it through the GPU_VREG_* window (BAR0 + 0xB80000 + + * 0x30B0) and the physical path pokes the plain 0x30B0 alias, so watch both. */ +#define GSP_EMU_REG_MMU_INVALIDATE 0x0030b0 +#define GSP_EMU_MMU_INVALIDATE_TRIGGER (1U << 31) +#define GSP_EMU_REG_MMU_INVALIDATE_WIN (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x0030b0) + +/* L2 cache-op completion registers (NV_UFLUSH_*): the driver writes PENDING + * (bit 0) to issue FB flush / L2 invalidate / L2 clean+dirty flush and then + * spins in kmemsysDoCacheOp_GM107 until the polled bits read back 0. On real + * hardware the L2 clears them when the operation completes; the emulated L2 + * completes instantly, so auto-clear any pending bits. + * + * On Turing the sysmem/peermem invalidate registers are reached through the + * virtual-register window (GPU_VREG_*: BAR0 + sriovState.virtualRegPhysOffset + * + NV_VIRTUAL_FUNCTION_PRIV_L2_*); the offset is 0xB80000 for a non-SRIOV + * virtual GPU and 0 for a VF, so watch both the plain and windowed aliases. */ +#define GSP_EMU_REG_UFLUSH_BASE 0x070000 +#define GSP_EMU_REG_UFLUSH_FB_FLUSH (GSP_EMU_REG_UFLUSH_BASE + 0x00) +#define GSP_EMU_REG_UFLUSH_L2_SYSMEM_INV (GSP_EMU_REG_UFLUSH_BASE + 0x04) +#define GSP_EMU_REG_UFLUSH_L2_PEERMEM_INV (GSP_EMU_REG_UFLUSH_BASE + 0x08) +#define GSP_EMU_REG_UFLUSH_L2_CLEAN_COMPTAGS (GSP_EMU_REG_UFLUSH_BASE + 0x0c) +#define GSP_EMU_REG_UFLUSH_L2_FLUSH_DIRTY (GSP_EMU_REG_UFLUSH_BASE + 0x10) +#define GSP_EMU_REG_UFLUSH_L2_WAIT_SYS_READS (GSP_EMU_REG_UFLUSH_BASE + 0x14) + +/* NV_VIRTUAL_FUNCTION_PRIV_L2_{SYSMEM,PEERMEM}_INVALIDATE (dev_vm.h) */ +#define GSP_EMU_REG_VF_L2_SYSMEM_INV 0x000f00 +#define GSP_EMU_REG_VF_L2_PEERMEM_INV 0x000f04 + +/* NV_PFALCON_FALCON_DEBUGINFO (GSP falcon BAR0+0x110094): holds the CrashCat + * wayfinder-L0 descriptor. If its low 16 bits == 0xdead (NV_CRASHCAT_SIGNATURE), + * the driver's kgspHealthCheck reads crashcat reports from garbage and loops + * forever in nv_open_q (which then can't poll RPCs -> fn-10 FREE timeouts). + * Force it to 0 (invalid wayfinder -> crashcatEngineGetNextCrashReport + * returns NULL) so the health check always passes. */ +#define GSP_EMU_REG_FALCON_DEBUGINFO 0x110094 + +/* CPU interrupt tree (Turing, physical-GPU path): the driver accesses the + * VF window registers at virtualRegPhysOffset = NV_VIRTUAL_FUNCTION_FULL_PHYS_OFFSET + * (0xB80000) + offset. + * LEAF_TRIGGER 0xB80000+0x1640 - write a vector to fire its software interrupt + * LEAF[i] 0xB80000+0x1000+i*4 - per-32-vector pending bits + * TOP[i] 0xB81600+i*4 - per-32-subtree pending bits + * _osVerifyInterrupts (osVerifySystemEnvironment) fires a doorbell software + * interrupt and waits for its ISR; if we don't set the pending state and + * inject MSI-X, the ISR never runs and RmInitAdapter bails. */ +#define GSP_EMU_REG_CPU_INTR_LEAF_TRIGGER (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x1640) +#define GSP_EMU_REG_CPU_INTR_LEAF_BASE (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x1000) +#define GSP_EMU_REG_CPU_INTR_TOP_BASE (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x1600) + +/* Windowed aliases of the VF L2 invalidate registers (GPU_VREG_* path). */ +#define GSP_EMU_REG_VF_L2_SYSMEM_INV_WIN (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x000f00) +#define GSP_EMU_REG_VF_L2_PEERMEM_INV_WIN (GSP_EMU_REG_FULL_PHYS_OFFSET + 0x000f04) + +#define GSP_EMU_CPUCTL_STARTCPU 0x2 +#define GSP_EMU_CPUCTL_HALTED 0x10 + +/* ============================================================ + * GSP Emulation Context + * ============================================================ */ + +struct gsp_emu_context { + struct nv_vgpu_device *vgpu; + struct task_struct *thread; + struct completion thread_stopped; + void __iomem *bar0_vaddr; + void __iomem *doorbell_reg; + void __iomem *shared_mem; + struct gsp_client *rpc_client; + spinlock_t lock; + bool running; + bool initialized; + bool stopping; + + /* Shared memory for RPC communication (4KB at BAR0+0x1000) */ + void *shared_mem_virt; + dma_addr_t shared_mem_dma; + size_t shared_mem_size; + + /* Statistics */ + u64 processed_rpcs; + u64 failed_rpcs; + u64 total_latency_ns; + + /* MSI-X injection */ + struct eventfd_ctx *msix_trigger[NV_VGPU_MAX_IRQS]; + + /* Falcon boot-watch: last-sampled values for change detection */ + u32 falcon_watch_last[GSP_EMU_FALCON_WATCH_COUNT]; + bool falcon_watch_valid; + + /* Physical GSP boot emulation state (gsp_emu_poll_boot) */ + u32 boot_stage; /* enum gsp_emu_boot_stage */ + u32 flcn_kick_count; /* STARTCPU kicks seen since last ENGINE reset */ + u32 wpr2_addr_lo; /* WPR2_ADDR_LO value to report after FWSEC */ + + /* GSP-RM msgq emulation (physical path, guest-sysmem queues) */ + struct gsp_rm_emu *rm_emu; + + /* PRAMIN window translation state (BAR0 window -> emulated vidmem) */ + u32 bar0_window_base; /* last-seen NV_PBUS_BAR0_WINDOW BASE (page) */ + u32 bar0_window_target; /* last-seen TARGET (0=vidmem, 2/3=sysmem) */ + bool bar0_window_valid; /* base/target captured at least once */ +}; + +/* Shadow accessors shared with gsp_rm_emulation.c */ +static inline void gsp_emu_shadow_write32(struct gsp_emu_context *ctx, + u32 offset, u32 val) +{ + writel(val, ctx->bar0_vaddr + offset); +} + +static inline u32 gsp_emu_shadow_read32(struct gsp_emu_context *ctx, u32 offset) +{ + return readl(ctx->bar0_vaddr + offset); +} + +/* ============================================================ + * Public API + * ============================================================ */ + +int gsp_emu_init(struct nv_vgpu_device *vgpu); +void gsp_emu_cleanup(struct nv_vgpu_device *vgpu); + +/* Clear emulated GSP boot state (WPR2/RISCV/stages/RM queues) on + * ENGINE reset or detected GSP death (stale msgq). */ +void gsp_emu_clear_boot_state(struct gsp_emu_context *ctx, const char *reason); +int gsp_emu_process_rpc(struct gsp_emu_context *ctx); +void gsp_emu_inject_completion_interrupt(struct gsp_emu_context *ctx, u32 seq_id); + +/* ============================================================ + * Debug/Statistics + * ============================================================ */ + +void gsp_emu_get_stats(struct gsp_emu_context *ctx, + u64 *processed, u64 *failed, u64 *total_latency_ns); + +#endif /* _GSP_EMULATION_H_ */ \ No newline at end of file diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c new file mode 100644 index 0000000..b9f8417 --- /dev/null +++ b/kernel/gsp_rm_emulation.c @@ -0,0 +1,2501 @@ +/* + * GSP-RM Emulation (physical-path) for Host-Side vGPU + * + * Emulates the GSP-RM msgq RPC channel in guest system memory. + * See gsp_rm_emulation.h for the design overview. + */ + +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "gsp_emulation.h" +#include "gsp_rm_emulation.h" +#include "gsp_rm_tu104.h" + +/* ============================================================ + * Guest-memory access ("virtual DMA" on behalf of the device) + * ============================================================ */ + +static int gsp_rm_read(struct gsp_emu_context *ctx, u64 gpa, void *buf, + size_t len) +{ + struct nv_vgpu_device *vgpu = ctx->vgpu; + + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + return -ENODEV; + + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); +} + +static int gsp_rm_read_dev(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, + size_t len) +{ + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + return -ENODEV; + + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); +} + +static int gsp_rm_write(struct gsp_emu_context *ctx, u64 gpa, void *buf, + size_t len) +{ + struct nv_vgpu_device *vgpu = ctx->vgpu; + + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + return -ENODEV; + + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); +} + +/* + * Debug helper: scan guest RAM for a 64-bit value (8-byte aligned) and/or a + * 32-bit value (4-byte aligned), reporting the GPAs where they are found. + * Used to locate guest structures (e.g. the UVM pushbuffer) without knowing + * the guest's GPU page tables. Triggered via the module parameter + * scan_guest_ram=. + */ +int gsp_rm_scan_guest_ram(struct nv_vgpu_device *vgpu, u64 needle, + u64 start, u64 end, u32 max_hits) +{ + u8 *buf; + u64 gpa; + u32 hits = 0; + + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + return -ENODEV; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) { + u32 off; + + if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE)) + continue; + + for (off = 0; off + 8 <= PAGE_SIZE; off += 4) { + u64 v; + u32 v32; + + memcpy(&v, buf + off, 8); + memcpy(&v32, buf + off, 4); + + if (v == needle) { + pr_info("gsp-resman-vfio: scan: u64 0x%llx at gpa 0x%llx (page off 0x%x)\n", + needle, gpa + off, off); + if (++hits >= max_hits) + break; + } + if (v32 == (u32)needle && needle <= 0xffffffffULL) { + pr_info("gsp-resman-vfio: scan: u32 0x%x at gpa 0x%llx (page off 0x%x)\n", + (u32)needle, gpa + off, off); + if (++hits >= max_hits) + break; + } + } + } + + kfree(buf); + pr_info("gsp-resman-vfio: scan: needle=0x%llx range=0x%llx..0x%llx hits=%u\n", + needle, start, end, hits); + return hits; +} + +/* + * Debug helper: scan guest RAM for values matching (v & mask) == value. + * Used to find a stored page-base VA when only "base + index*size" is known. + */ +int gsp_rm_scan_guest_ram_masked(struct nv_vgpu_device *vgpu, u64 value, + u64 mask, u64 start, u64 end, u32 max_hits) +{ + u8 *buf; + u64 gpa; + u32 hits = 0; + + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + return -ENODEV; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) { + u32 off; + + if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE)) + continue; + + for (off = 0; off + 8 <= PAGE_SIZE; off += 4) { + u64 v64; + u32 v32; + + memcpy(&v64, buf + off, 8); + memcpy(&v32, buf + off, 4); + if ((v64 & mask) == value) { + pr_info("nvidia-vgpu-vfio: masked scan: u64 0x%llx at gpa 0x%llx (off 0x%x)\n", + v64, gpa + off, off); + if (++hits >= max_hits) + break; + } + if ((v32 & (u32)mask) == (u32)value) { + pr_info("nvidia-vgpu-vfio: masked scan: u32 0x%x at gpa 0x%llx (off 0x%x)\n", + v32, gpa + off, off); + if (++hits >= max_hits) + break; + } + } + } + + kfree(buf); + pr_info("nvidia-vgpu-vfio: masked scan: value=0x%llx mask=0x%llx hits=%u\n", + value, mask, hits); + return hits; +} + +/* + * Debug helper: scan guest RAM for page-table entries pointing at a given + * physical address (matches v & 0xfffff000 == pa for u32 and the 64-bit + * equivalent). Used to locate the guest's GPU page tables. + */ +int gsp_rm_scan_guest_ram_pt(struct nv_vgpu_device *vgpu, u64 pa, + u64 start, u64 end, u32 max_hits) +{ + u8 *buf; + u64 gpa; + u32 hits = 0; + u64 pa_page = pa & ~0xfffULL; + + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + return -ENODEV; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) { + u32 off; + + if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE)) + continue; + + for (off = 0; off + 8 <= PAGE_SIZE; off += 4) { + u64 v64; + u32 v32; + + memcpy(&v64, buf + off, 8); + memcpy(&v32, buf + off, 4); + if ((v64 & 0xfffffffffffff000ULL) == pa_page) { + pr_info("gsp-resman-vfio: pt scan: u64 pte 0x%llx at gpa 0x%llx (off 0x%x)\n", + v64, gpa + off, off); + if (++hits >= max_hits) + break; + } + if ((v32 & 0xfffff000U) == (u32)pa_page) { + pr_info("gsp-resman-vfio: pt scan: u32 pte 0x%x at gpa 0x%llx (off 0x%x)\n", + v32, gpa + off, off); + if (++hits >= max_hits) + break; + } + } + } + + kfree(buf); + pr_info("gsp-resman-vfio: pt scan: pa=0x%llx hits=%u\n", pa, hits); + return hits; +} + +/* + * Debug helper: scan guest RAM for a sequence of up to 4 u32 values. + * Used to locate a CE semaphore-release method sequence in a pushbuffer. + */ +int gsp_rm_scan_guest_ram_seq(struct nv_vgpu_device *vgpu, const u32 *pat, + u32 npat, u64 start, u64 end, u32 max_hits) +{ + u8 *buf; + u64 gpa; + u32 hits = 0; + + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) + return -ENODEV; + if (!npat || npat > 4) + return -EINVAL; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + for (gpa = start; gpa < end && hits < max_hits; gpa += PAGE_SIZE) { + u32 off; + + if (gsp_rm_read_dev(vgpu, gpa, buf, PAGE_SIZE)) + continue; + + for (off = 0; off + npat * 4 <= PAGE_SIZE; off += 4) { + u32 i, ok = 1; + + for (i = 0; i < npat; i++) { + u32 v; + + memcpy(&v, buf + off + i * 4, 4); + if (v != pat[i]) { + ok = 0; + break; + } + } + if (ok) { + pr_info("gsp-resman-vfio: seq scan: hit in guest RAM at gpa 0x%llx (page off 0x%x)\n", + gpa + off, off); + if (++hits >= max_hits) + break; + } + } + } + + kfree(buf); + pr_info("gsp-resman-vfio: seq scan: range=0x%llx..0x%llx hits=%u\n", + start, end, hits); + return hits; +}static u64 gsp_rm_off_to_gpa(struct gsp_rm_emu *rm, u64 off) +{ + u64 idx = off >> 12; + + if (idx >= rm->mq.pageTableEntryCount) + return 0; + return rm->ptes[idx] + (off & 0xfff); +} + +/* + * Page-aware shared-buffer I/O. The queue buffer is NV_MEMORY_NONCONTIGUOUS: + * its 4K pages are independent guest allocations, so a range crossing a page + * boundary is NOT contiguous in guest physical memory. Translating the start + * and blasting a multi-page range would scribble into arbitrary guest RAM + * (observed as guest userspace segfaults). Split at page boundaries and + * translate each page separately. + */ +static int gsp_rm_write_buf(struct gsp_emu_context *ctx, u64 off, + const void *buf, size_t len) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + const u8 *p = buf; + int ret; + + while (len) { + u64 gpa = gsp_rm_off_to_gpa(rm, off); + size_t chunk = min_t(size_t, len, 0x1000 - (off & 0xfff)); + + if (!gpa) + return -EINVAL; + ret = gsp_rm_write(ctx, gpa, (void *)p, chunk); + if (ret) + return ret; + p += chunk; + off += chunk; + len -= chunk; + } + return 0; +} + +static int gsp_rm_read_buf(struct gsp_emu_context *ctx, u64 off, + void *buf, size_t len) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + u8 *p = buf; + int ret; + + while (len) { + u64 gpa = gsp_rm_off_to_gpa(rm, off); + size_t chunk = min_t(size_t, len, 0x1000 - (off & 0xfff)); + + if (!gpa) + return -EINVAL; + ret = gsp_rm_read(ctx, gpa, p, chunk); + if (ret) + return ret; + p += chunk; + off += chunk; + len -= chunk; + } + return 0; +} + +/* ============================================================ + * Status queue element construction + * ============================================================ */ + +/* XOR-fold checksum over u64s (message_queue_priv.h _checkSum32) */ +static u32 gsp_rm_checksum32(const void *data, u32 len) +{ + const u64 *p = data; + const u64 *end = (const u64 *)((const u8 *)data + len); + u64 sum = 0; + + while (p < end) + sum ^= *p++; + + return (u32)(sum >> 32) ^ (u32)sum; +} + +/* + * 570.144 GSP_MSG_QUEUE_ELEMENT wire layout (guest driver is 570.144 stock): + * 0x00 authTagBuffer[16] (zero - no CC) + * 0x10 aadBuffer[16] (zero - no CC) + * 0x20 checkSum (XOR-fold over msgLen, makes total 0) + * 0x24 seqNum (per-record sequence) + * 0x28 elemCount (elements this record spans) + * 0x30 rpc_message_header_v (32 bytes) + payload + * GSP_MSG_QUEUE_ELEMENT_HDR_SIZE = offsetof(rpc) = 0x30. + */ +#define GSPRM_ELEM_HDR_SIZE 0x30 + +/* + * Max RPC data that fits in one queue element: + * 4096 (element) - 0x30 (element hdr) - 32 (rpc hdr) = 4016 bytes. + */ +#define GSPRM_MAX_SINGLE_ELEM_DATA (GSPRM_QUEUE_ELEMENT_SIZE - \ + GSPRM_ELEM_HDR_SIZE - \ + sizeof(struct gsprm_rpc_header)) + +/* + * Build a multi-element record into a contiguous buffer. + * + * The record spans elem_count consecutive queue elements: + * [0]: element header (0x30) + rpc header (32) + data[0..4015] + * [1..N-1]: continuation data only (no headers) + * + * Returns the total record size in bytes (elem_count * 4096). + * Sets *out_elem_count to the number of elements used. + */ +static u32 gsp_rm_build_record(struct gsp_rm_emu *rm, u8 *buf, u32 buf_size, + u32 function, u32 sequence, u32 result, + const void *data, u32 data_len, + u32 *out_elem_count) +{ + struct gsprm_rpc_header *hdr = + (struct gsprm_rpc_header *)(buf + GSPRM_ELEM_HDR_SIZE); + u32 rpc_len = sizeof(*hdr) + data_len; + u32 msg_len = GSPRM_ELEM_HDR_SIZE + rpc_len; + u32 aligned_len; + u32 elem_count; + u32 first_data; + u32 remaining; + u32 i; + + /* How many elements does this record span? */ + elem_count = (msg_len + GSPRM_QUEUE_ELEMENT_SIZE - 1) / + GSPRM_QUEUE_ELEMENT_SIZE; + if (elem_count == 0) + elem_count = 1; + + if (elem_count * GSPRM_QUEUE_ELEMENT_SIZE > buf_size) { + pr_err("GSPRM: build_record overflow: need %u have %u\n", + elem_count * GSPRM_QUEUE_ELEMENT_SIZE, buf_size); + return 0; + } + + memset(buf, 0, elem_count * GSPRM_QUEUE_ELEMENT_SIZE); + + /* First element: header + rpc header + start of data */ + memcpy(buf + 0x24, &rm->stat_seq, 4); /* seqNum */ + memcpy(buf + 0x28, &elem_count, 4); /* elemCount */ + + hdr->header_version = GSPRM_RPC_HEADER_VERSION; + hdr->signature = GSPRM_RPC_SIGNATURE; + hdr->length = rpc_len; + hdr->function = function; + hdr->rpc_result = result; + hdr->rpc_result_private = result; + hdr->sequence = sequence; + hdr->spare = 0; + + /* Copy data across element boundaries */ + first_data = min(data_len, (u32)GSPRM_MAX_SINGLE_ELEM_DATA); + if (data && first_data) + memcpy((u8 *)(hdr + 1), data, first_data); + + remaining = data_len - first_data; + if (remaining > 0 && data) { + const u8 *src = (const u8 *)data + first_data; + u8 *dst = buf + GSPRM_QUEUE_ELEMENT_SIZE; + + for (i = 1; i < elem_count && remaining > 0; i++) { + u32 chunk = min(remaining, + (u32)GSPRM_QUEUE_ELEMENT_SIZE); + + memcpy(dst, src, chunk); + src += chunk; + dst += GSPRM_QUEUE_ELEMENT_SIZE; + remaining -= chunk; + } + } + + /* checksum covers msg_len zero-padded to 8 (buffer already zeroed) */ + aligned_len = (msg_len + 7) & ~7U; + *(u32 *)(buf + 0x20) = gsp_rm_checksum32(buf, aligned_len); + + *out_elem_count = elem_count; + return elem_count * GSPRM_QUEUE_ELEMENT_SIZE; +} + +/* + * Write a status queue record (1+ elements) and advance the writePtr. + */ +static int gsp_rm_send(struct gsp_emu_context *ctx, u32 function, + u32 sequence, u32 result, const void *data, u32 data_len) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + u8 *buf; + u32 slot; + u32 new_wp; + u32 elem_count; + u32 record_size; + u32 buf_size; + int ret; + + /* Allocate buffer for worst-case multi-element record */ + buf_size = GSPRM_QUEUE_ELEMENT_SIZE * + ((data_len + GSPRM_MAX_SINGLE_ELEM_DATA + + GSPRM_QUEUE_ELEMENT_SIZE - 1) / + GSPRM_QUEUE_ELEMENT_SIZE + 1); + if (buf_size < GSPRM_QUEUE_ELEMENT_SIZE) + buf_size = GSPRM_QUEUE_ELEMENT_SIZE; + + buf = kzalloc(buf_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + record_size = gsp_rm_build_record(rm, buf, buf_size, function, + sequence, result, data, data_len, + &elem_count); + if (!record_size) { + kfree(buf); + return -EOVERFLOW; + } + + /* Check we don't overflow the status queue */ + if (elem_count > rm->stat_msgcount) { + nv_vgpu_err(ctx->vgpu, "GSPRM: record too large: %u elements > queue %u", + elem_count, rm->stat_msgcount); + kfree(buf); + return -EOVERFLOW; + } + + /* Write all elements to consecutive queue slots. Page-aware: the + * queue buffer is non-contiguous, so translate per page. */ + slot = rm->stat_write % rm->stat_msgcount; + + /* Handle wrap-around: write may span the end of the circular queue */ + { + u32 elems_to_end = rm->stat_msgcount - slot; + u32 first_chunk = min(elem_count, elems_to_end); + u32 second_chunk = elem_count - first_chunk; + u64 rec_off = rm->stat_off + rm->entry_off + + (u64)slot * GSPRM_QUEUE_ELEMENT_SIZE; + + ret = gsp_rm_write_buf(ctx, rec_off, buf, + first_chunk * GSPRM_QUEUE_ELEMENT_SIZE); + if (ret) { + nv_vgpu_err(ctx->vgpu, "GSPRM: status write failed off=0x%llx ret=%d", + rec_off, ret); + kfree(buf); + return ret; + } + + if (second_chunk > 0) { + ret = gsp_rm_write_buf(ctx, rm->stat_off + rm->entry_off, + buf + first_chunk * GSPRM_QUEUE_ELEMENT_SIZE, + second_chunk * GSPRM_QUEUE_ELEMENT_SIZE); + if (ret) { + nv_vgpu_err(ctx->vgpu, "GSPRM: status wrap write failed ret=%d", + ret); + kfree(buf); + return ret; + } + } + } + + kfree(buf); + + /* Advance the write pointer with the same wrap semantics as the + * driver's msgq (slot index in [0, msgCount) - the peer's free-space + * and slot math breaks on unbounded values). stat_seq stays + * monotonic per-record. */ + rm->stat_write = (rm->stat_write + elem_count) % rm->stat_msgcount; + rm->stat_seq++; + + /* bump status TX header writePtr (offset 0x10) */ + new_wp = rm->stat_write; + ret = gsp_rm_write_buf(ctx, rm->stat_off + 0x10, + &new_wp, sizeof(new_wp)); + if (ret) { + nv_vgpu_err(ctx->vgpu, "GSPRM: status writePtr update failed ret=%d", ret); + return ret; + } + + /* Notify the guest (MSI-X vector 0). The driver's bottom-half drains + * events under the GPU lock; the rpcRecvPoll poller holds the same + * lock from send to poll, so a reply can't be consumed by the event + * drain while a poller waits for it (same as real GSP-RM). */ + gsp_emu_inject_completion_interrupt(ctx, rm->stat_seq); + + return 0; +} + +/* ============================================================ + * Queue-chain discovery (libos args -> RMARGS -> GSP_ARGUMENTS_CACHED) + * ============================================================ */ + +static int gsp_rm_parse_queue_chain(struct gsp_emu_context *ctx) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + struct gsprm_libos_region *regions; + int ret, i; + u64 gsp_args_gpa = 0; + u32 total_size; + + regions = kmalloc_array(GSPRM_MAX_LIBOS_REGIONS, sizeof(*regions), + GFP_KERNEL); + if (!regions) + return -ENOMEM; + + /* 1. Read the libos init args array */ + ret = gsp_rm_read(ctx, rm->libos_args_gpa, regions, + GSPRM_MAX_LIBOS_REGIONS * sizeof(*regions)); + if (ret) { + nv_vgpu_err(ctx->vgpu, "GSPRM: libos args read failed gpa=0x%llx ret=%d", + rm->libos_args_gpa, ret); + goto out; + } + + /* 2. Find the RMARGS region -> GSP_ARGUMENTS_CACHED */ + for (i = 0; i < GSPRM_MAX_LIBOS_REGIONS; i++) { + if (regions[i].id8 == 0) + break; + nv_vgpu_info(ctx->vgpu, "GSPRM: libos region[%d] id8=0x%llx pa=0x%llx size=0x%llx kind=%u loc=%u", + i, regions[i].id8, regions[i].pa, regions[i].size, + regions[i].kind, regions[i].loc); + if (regions[i].id8 == GSPRM_LIBOS_ID_RMARGS) + gsp_args_gpa = regions[i].pa; + } + if (!gsp_args_gpa) { + nv_vgpu_err(ctx->vgpu, "GSPRM: RMARGS region not found"); + ret = -ENOENT; + goto out; + } + rm->gsp_args_gpa = gsp_args_gpa; + + /* 3. Read GSP_ARGUMENTS_CACHED (MESSAGE_QUEUE_INIT_ARGUMENTS first) */ + ret = gsp_rm_read(ctx, gsp_args_gpa, &rm->mq, sizeof(rm->mq)); + if (ret) { + nv_vgpu_err(ctx->vgpu, "GSPRM: GSP_ARGUMENTS_CACHED read failed ret=%d", ret); + goto out; + } + + nv_vgpu_info(ctx->vgpu, "GSPRM: queue args sharedPA=0x%llx ptes=%u cmdOff=0x%llx statOff=0x%llx hdrSize=0x%llx elemMin=0x%llx elemMax=0x%llx hdrAlign=%u elemAlign=%u", + rm->mq.sharedMemPhysAddr, rm->mq.pageTableEntryCount, + rm->mq.cmdQueueOffset, rm->mq.statQueueOffset, + rm->mq.queueElementHdrSize, rm->mq.queueElementSizeMin, + rm->mq.queueElementSizeMax, rm->mq.queueHeaderAlign, + rm->mq.queueElementAlign); + print_hex_dump(KERN_INFO, "GSPRM args raw: ", DUMP_PREFIX_OFFSET, + 16, 8, &rm->mq, sizeof(rm->mq), false); + + if (!rm->mq.sharedMemPhysAddr || !rm->mq.pageTableEntryCount || + rm->mq.pageTableEntryCount > 1024) { + nv_vgpu_err(ctx->vgpu, "GSPRM: invalid queue args"); + ret = -EINVAL; + goto out; + } + + /* 4. Read the page table (RmPhysAddr per 4K page of the shared buffer) */ + rm->ptes = kmalloc_array(rm->mq.pageTableEntryCount, sizeof(u64), + GFP_KERNEL); + if (!rm->ptes) { + ret = -ENOMEM; + goto out; + } + ret = gsp_rm_read(ctx, rm->mq.sharedMemPhysAddr, rm->ptes, + rm->mq.pageTableEntryCount * sizeof(u64)); + if (ret) { + nv_vgpu_err(ctx->vgpu, "GSPRM: page table read failed ret=%d", ret); + goto out; + } + + /* 5. Compute queue geometry. NOTE: the shared buffer is + * NV_MEMORY_NONCONTIGUOUS - only queue-offsets are linear, GPAs are + * not. Every access must translate through rm->ptes (gsp_rm_off_to_gpa). */ + rm->cmd_off = rm->mq.cmdQueueOffset; + rm->stat_off = rm->mq.statQueueOffset; + rm->cmd_gpa = gsp_rm_off_to_gpa(rm, rm->cmd_off); + rm->stat_gpa = gsp_rm_off_to_gpa(rm, rm->stat_off); + rm->cmd_size = (u32)(rm->mq.statQueueOffset - rm->mq.cmdQueueOffset); + total_size = rm->mq.pageTableEntryCount << 12; + rm->stat_size = total_size - (u32)rm->mq.statQueueOffset; + rm->entry_off = 0x1000; /* ALIGN_UP(0x20 + 4, 1 << elemAlign(12)) */ + rm->cmd_msgcount = (rm->cmd_size - rm->entry_off) / GSPRM_QUEUE_ELEMENT_SIZE; + rm->stat_msgcount = (rm->stat_size - rm->entry_off) / GSPRM_QUEUE_ELEMENT_SIZE; + + if (!rm->cmd_gpa || !rm->stat_gpa || !rm->cmd_msgcount || !rm->stat_msgcount) { + nv_vgpu_err(ctx->vgpu, "GSPRM: bad queue geometry cmd_gpa=0x%llx stat_gpa=0x%llx cmd_mc=%u stat_mc=%u", + rm->cmd_gpa, rm->stat_gpa, rm->cmd_msgcount, + rm->stat_msgcount); + ret = -EINVAL; + goto out; + } + + nv_vgpu_info(ctx->vgpu, "GSPRM: cmd queue gpa=0x%llx size=0x%x mc=%u; status queue gpa=0x%llx size=0x%x mc=%u", + rm->cmd_gpa, rm->cmd_size, rm->cmd_msgcount, + rm->stat_gpa, rm->stat_size, rm->stat_msgcount); + + ret = 0; +out: + kfree(regions); + return ret; +} + +/* + * Write the status queue TX header (what GSP-RM's msgqTxCreate produces). + */ +static int gsp_rm_link_status_queue(struct gsp_emu_context *ctx) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + struct gsprm_msgq_tx_header tx; + u32 zero = 0; + int ret; + + memset(&tx, 0, sizeof(tx)); + tx.version = GSPRM_MSGQ_VERSION; + tx.size = rm->stat_size; + tx.msgSize = GSPRM_QUEUE_ELEMENT_SIZE; + tx.msgCount = rm->stat_msgcount; + tx.writePtr = 0; + tx.flags = GSPRM_MSGQ_FLAGS_SWAP_RX; + tx.rxHdrOff = 0x20; /* ALIGN_UP(sizeof(msgqTxHeader)=32, 1<entry_off; + + ret = gsp_rm_write_buf(ctx, rm->stat_off, &tx, sizeof(tx)); + if (ret) { + nv_vgpu_err(ctx->vgpu, "GSPRM: status TX header write failed ret=%d", ret); + return ret; + } + + /* status queue RX header (our command-queue read position, swap mode) */ + ret = gsp_rm_write_buf(ctx, rm->stat_off + tx.rxHdrOff, + &zero, sizeof(zero)); + if (ret) { + nv_vgpu_err(ctx->vgpu, "GSPRM: status RX header write failed ret=%d", ret); + return ret; + } + + nv_vgpu_info(ctx->vgpu, "GSPRM: status queue linked (size=0x%x msgCount=%u)", + tx.size, tx.msgCount); + return 0; +} + +/* ============================================================ + * Command processing + * ============================================================ */ + +static const char *gsp_rm_rpc_name(u32 function) +{ + switch (function) { + case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO: return "SET_GUEST_SYSTEM_INFO"; + case GSPRM_RPC_FUNC_ALLOC_ROOT: return "ALLOC_ROOT"; + case GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO: return "GET_GSP_STATIC_INFO"; + case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO: return "GSP_SET_SYSTEM_INFO"; + case GSPRM_RPC_FUNC_SET_REGISTRY: return "SET_REGISTRY"; + case GSPRM_RPC_FUNC_CONTINUATION_RECORD: return "CONTINUATION_RECORD"; + case GSPRM_RPC_FUNC_GSP_RM_CONTROL: return "GSP_RM_CONTROL"; + default: return "UNKNOWN"; + } +} + +/* + * SET_GUEST_SYSTEM_INFO: the driver negotiates the RPC interface version + * from the reply's vgxVersionMajorNum/MinorNum fields. Without them the + * version-gated HALs stay stubs and every subsequent RPC fails with + * NV_VGPU_MSG_RESULT_RPC_UNKNOWN_FUNCTION (0xff100001). Echo the driver's + * own version back (no host-guest version skew). + */ +static void gsp_rm_handle_set_guest_system_info(struct gsp_emu_context *ctx, + const u8 *elem, + u32 sequence) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + const u8 *req_data = elem + GSPRM_ELEM_HDR_SIZE + + sizeof(struct gsprm_rpc_header); + u8 *reply; + u32 major, minor; + + memcpy(&major, req_data + 0, 4); + memcpy(&minor, req_data + 4, 4); + if (!major) + major = GSPRM_VGX_VERSION_MAJOR; + if (!minor && major != GSPRM_VGX_VERSION_MAJOR) + minor = GSPRM_VGX_VERSION_MINOR; + + reply = kzalloc(GSPRM_SET_GUEST_INFO_SIZE, GFP_KERNEL); + if (!reply) { + nv_vgpu_err(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO reply alloc failed"); + return; + } + memcpy(reply + 0, &major, 4); + memcpy(reply + 4, &minor, 4); + + nv_vgpu_info(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO seq=%u -> VGX %u.%u", + sequence, major, minor); + + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO, sequence, 0, + reply, GSPRM_SET_GUEST_INFO_SIZE)) + nv_vgpu_err(ctx->vgpu, "GSPRM: SET_GUEST_SYSTEM_INFO reply send failed"); + else + rm->rpcs_replied++; + + kfree(reply); +} + +/* ============================================================ + * GET_GSP_STATIC_INFO (function 65) + * + * Builds the 1656-byte GspStaticConfigInfo for TU104. + * ============================================================ */ + +static void gsp_rm_build_static_info(u8 *buf) +{ + u64 val64; + u32 val32; + u8 *p; + + memset(buf, 0, TU104_GSCI_SIZE); + + /* grCapsBits (offset 0, 23 bytes) - basic GR caps */ + buf[0] = 0x01; /* GR present */ + + /* gidInfo (offset 24, 268 bytes): index(4) + flags(4) + length(4) + data(256) */ + p = buf + TU104_GSCI_OFF_gidInfo; + val32 = 0; memcpy(p + 0, &val32, 4); /* index */ + val32 = 0x80000000; memcpy(p + 4, &val32, 4); /* flags: valid */ + val32 = 16; memcpy(p + 8, &val32, 4); /* length = 16 (UUID) */ + /* GPU UUID - deterministic fake for TU104 */ + memcpy(p + 12, "GPU-9da5e221-18", 15); + + /* SKUInfo (offset 292, 48 bytes) */ + p = buf + TU104_GSCI_OFF_SKUInfo; + memcpy(p + 4, "TU104", 5); /* chipSKU */ + memcpy(p + 13, "400", 3); /* chipSKUMod */ + memcpy(p + 20, "N18E", 4); /* project */ + memcpy(p + 24, "G3", 2); /* projectSKU */ + + /* fbRegionInfoParams (offset 344, 776 bytes) */ + p = buf + TU104_GSCI_OFF_fbRegionInfoParams; + val32 = 1; memcpy(p, &val32, 4); /* numFBRegions = 1 */ + /* Region 0: base=0, limit=256MB-1, reserved=0 */ + { + struct tu104_fb_region *region = (void *)(p + 8); /* 8-aligned */ + val64 = 0; memcpy(®ion->base, &val64, 8); + val64 = 0x0FFFFFFF; memcpy(®ion->limit, &val64, 8); + val64 = 0; memcpy(®ion->reserved, &val64, 8); + val32 = 0; memcpy(®ion->performance, &val32, 4); + region->supportCompressed = 0; + /* memmgrCalcReservedFbSpace asserts that SOME region supports + * ISO "even if we have no display" - without it the reserved + * space split fails and kdisp can't allocate its channel + * table. A single ISO-capable region satisfies fast/slow/iso + * assignment. */ + region->supportISO = 1; + region->bProtected = 0; + memset(region->blackList, 0, sizeof(region->blackList)); + } + + /* sriovCaps (offset 1120, 80 bytes) - all zeros (not SR-IOV) */ + + /* sriovMaxGfid (offset 1200, 4 bytes) */ + val32 = 0; + memcpy(buf + TU104_GSCI_OFF_sriovMaxGfid, &val32, 4); + + /* engineCaps (offset 1204, 12 bytes) */ + val32 = 0x60003E02; /* GR0|CE0-4|NVDEC0|NVDEC1 */ + memcpy(buf + TU104_GSCI_OFF_engineCaps, &val32, 4); + val32 = 0x00362020; /* NVENC0|SW|SEC2|HOST|PMU|FBFLCN */ + memcpy(buf + TU104_GSCI_OFF_engineCaps + 4, &val32, 4); + val32 = 0; + memcpy(buf + TU104_GSCI_OFF_engineCaps + 8, &val32, 4); + + /* poisonFuseEnabled (offset 1216, 1 byte) */ + buf[TU104_GSCI_OFF_poisonFuseEnabled] = 0; + + /* fb_length (offset 1224, 8 bytes) - 256MB (512MB broke the FWSEC + * WPR2-location check below). */ + val64 = 0x10000000ULL; + memcpy(buf + TU104_GSCI_OFF_fb_length, &val64, 8); + + /* fbio_mask (offset 1232, 8 bytes) - 8 FBIOs for 256-bit bus */ + val64 = 0xFF; + memcpy(buf + TU104_GSCI_OFF_fbio_mask, &val64, 8); + + /* fb_bus_width (offset 1240, 4 bytes) - 256-bit */ + val32 = 256; + memcpy(buf + TU104_GSCI_OFF_fb_bus_width, &val32, 4); + + /* fb_ram_type (offset 1244, 4 bytes) - GDDR6 = 6 */ + val32 = 6; + memcpy(buf + TU104_GSCI_OFF_fb_ram_type, &val32, 4); + + /* fbp_mask (offset 1248, 8 bytes) - 6 FBPs for 6 GPCs */ + val64 = 0x3F; + memcpy(buf + TU104_GSCI_OFF_fbp_mask, &val64, 8); + + /* l2_cache_size (offset 1256, 4 bytes) - 4MB */ + val32 = 0x400000; + memcpy(buf + TU104_GSCI_OFF_l2_cache_size, &val32, 4); + + /* gpuNameString (offset 1260, 64 bytes) */ + strncpy((char *)(buf + TU104_GSCI_OFF_gpuNameString), + "NVIDIA GeForce RTX 2080 SUPER", 64); + + /* gpuShortNameString (offset 1324, 64 bytes) */ + strncpy((char *)(buf + TU104_GSCI_OFF_gpuShortNameString), + "TU104", 64); + + /* gpuNameString_Unicode (offset 1388, 128 bytes) - skip for now */ + + /* Branding booleans */ + buf[TU104_GSCI_OFF_bIsMobile] = 1; /* Mobile/Max-Q */ + + /* bar1PdeBase (offset 1536, 8 bytes) */ + val64 = 0; + memcpy(buf + TU104_GSCI_OFF_bar1PdeBase, &val64, 8); + + /* bar2PdeBase (offset 1544, 8 bytes) */ + val64 = 0; + memcpy(buf + TU104_GSCI_OFF_bar2PdeBase, &val64, 8); + + /* bVbiosValid (offset 1552, 1 byte) */ + buf[TU104_GSCI_OFF_bVbiosValid] = 1; + + /* vbiosSubVendor (offset 1556, 4 bytes) - ASUS */ + val32 = 0x1043; + memcpy(buf + TU104_GSCI_OFF_vbiosSubVendor, &val32, 4); + + /* vbiosSubDevice (offset 1560, 4 bytes) */ + val32 = 0x1F01; + memcpy(buf + TU104_GSCI_OFF_vbiosSubDevice, &val32, 4); + + /* hInternalClient/Device/Subdevice */ + val32 = TU104_H_INTERNAL_CLIENT; + memcpy(buf + TU104_GSCI_OFF_hInternalClient, &val32, 4); + val32 = TU104_H_INTERNAL_DEVICE; + memcpy(buf + TU104_GSCI_OFF_hInternalDevice, &val32, 4); + val32 = TU104_H_INTERNAL_SUBDEVICE; + memcpy(buf + TU104_GSCI_OFF_hInternalSubdevice, &val32, 4); + + /* ecidInfo (offset 1616, 24 bytes) - leave zeroed */ + + /* fwWprLayoutOffset (offset 1640, 16 bytes): nonWprHeapOffset, frtsOffset */ + val64 = 0; + memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset, &val64, 8); + val64 = 0xFE00000; /* frtsOffset = fbSize - 2MB (256MB) */ + memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset + 8, &val64, 8); +} + +static void gsp_rm_handle_get_static_info(struct gsp_emu_context *ctx, + u32 sequence) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + u8 *reply; + + reply = kzalloc(TU104_GSCI_SIZE, GFP_KERNEL); + if (!reply) { + nv_vgpu_err(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO reply alloc failed"); + return; + } + + gsp_rm_build_static_info(reply); + + nv_vgpu_info(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO seq=%u -> %u bytes (fb=%lluMB, engines=0x%x/0x%x)", + sequence, TU104_GSCI_SIZE, + 0x10000000ULL >> 20, 0x60003E02, 0x362020); + + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO, sequence, 0, + reply, TU104_GSCI_SIZE)) + nv_vgpu_err(ctx->vgpu, "GSPRM: GET_GSP_STATIC_INFO reply send failed"); + else + rm->rpcs_replied++; + + kfree(reply); +} + +/* ============================================================ + * GSP_RM_CONTROL (function 76) - sub-command dispatch + * + * rpc_gsp_rm_control_v03_00: hClient(4) + hObject(4) + cmd(4) + status(4) + * + paramsSize(4) + flags(4) + params[] + * ============================================================ */ + +struct gsp_rm_control_req { + u32 hClient; + u32 hObject; + u32 cmd; + u32 status; + u32 paramsSize; + u32 flags; + /* params[] follows */ +}; + +#define GSP_RM_CTRL_HDR_SIZE 24 +static void gsp_rm_handle_fifo_device_table(struct gsp_emu_context *ctx, + u32 sequence, + struct gsp_rm_control_req *req, + const u8 *req_params, + u32 *out_status, + u8 *out_params, u32 *out_params_size) +{ + struct tu104_fifo_get_device_info_params *fifo_params; + u32 base, count, i, copy_count; + + fifo_params = (struct tu104_fifo_get_device_info_params *)out_params; + base = 0; + /* baseIndex is the first field of the request params, which live in + * the queue element buffer (NOT after the local header struct!). Only + * readable when the whole request fits in the first element. */ + if (req->paramsSize >= 4 && + req->paramsSize + GSP_RM_CTRL_HDR_SIZE <= GSPRM_MAX_SINGLE_ELEM_DATA) + memcpy(&base, req_params, 4); + if (base >= TU104_FIFO_TABLE_MAX_DEVICES) + base = 0; + + /* How many entries to return in this page */ + count = TU104_ENGINE_TABLE_SIZE - base; + if (count > TU104_FIFO_TABLE_MAX_ENTRIES) + count = TU104_FIFO_TABLE_MAX_ENTRIES; + + copy_count = count; + if (base >= TU104_ENGINE_TABLE_SIZE) + copy_count = 0; + + fifo_params->baseIndex = base; + fifo_params->numEntries = copy_count; + fifo_params->bMore = (base + copy_count) < TU104_ENGINE_TABLE_SIZE; + + for (i = 0; i < copy_count; i++) + memcpy(&fifo_params->entries[i], &tu104_engine_table[base + i], + sizeof(struct tu104_fifo_device_entry)); + + *out_params_size = offsetof(struct tu104_fifo_get_device_info_params, + entries) + + copy_count * sizeof(struct tu104_fifo_device_entry); + *out_status = 0; /* NV_OK */ + + nv_vgpu_info(ctx->vgpu, "GSPRM: FIFO_GET_DEVICE_INFO_TABLE base=%u count=%u more=%u", + base, copy_count, fifo_params->bMore); +} + +static void gsp_rm_handle_intr_kernel_table(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, u32 *out_params_size) +{ + struct tu104_intr_get_kernel_table_params *intr_params; + u32 copy_size; + + intr_params = (struct tu104_intr_get_kernel_table_params *)out_params; + + intr_params->tableLen = TU104_INTR_KERNEL_TABLE_SIZE; + copy_size = TU104_INTR_KERNEL_TABLE_SIZE * + sizeof(struct tu104_intr_table_entry); + memcpy(intr_params->table, tu104_intr_kernel_table, copy_size); + + /* Subtree map - must match intrInitSubtreeMap_TU102 (the values + * physical-path guests receive from real GSP-RM): + * ESCHED_DRIVEN_ENGINE = {3,3} (STALL_SUBTREE) + * ESCHED_DRIVEN_ENGINE_NOTIF = {0,0} + * UVM_OWNED = {1,1} (fault 64, access cntr 72) + * UVM_SHARED = {2,2} (doorbells/GSP/DISP) + * DEFAULT/RUNLIST/RUNLIST_NOTIF = INVALID (unset on Turing) + * intrCacheIntrFields requires accessCntrSubtree(=1) within + * UVM_OWNED and UVM_SHARED to be a single subtree. */ + { + u32 c; + + for (c = 0; c < TU104_INTR_CATEGORY_ENUM_COUNT; c++) { + intr_params->subtreeMap[c].subtreeStart = TU104_INTR_INVALID_SUBTREE; + intr_params->subtreeMap[c].subtreeEnd = TU104_INTR_INVALID_SUBTREE; + } + intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE].subtreeStart = 3; + intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE].subtreeEnd = 3; + intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION].subtreeStart = 0; + intr_params->subtreeMap[TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION].subtreeEnd = 0; + intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_OWNED].subtreeStart = 1; + intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_OWNED].subtreeEnd = 1; + intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_SHARED].subtreeStart = 2; + intr_params->subtreeMap[TU104_INTR_CATEGORY_UVM_SHARED].subtreeEnd = 2; + } + + /* The driver reads subtreeMap at offset 2052 (after the full + * 128-entry array), so the reply must span the whole struct - + * not just the used entries. */ + *out_params_size = offsetof(struct tu104_intr_get_kernel_table_params, + subtreeMap) + + sizeof(intr_params->subtreeMap); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: INTR_GET_KERNEL_TABLE tableLen=%u", + TU104_INTR_KERNEL_TABLE_SIZE); +} + +static void gsp_rm_handle_constructed_falcon_info(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + struct tu104_get_constructed_falcon_info_params *falcon_params; + + falcon_params = (struct tu104_get_constructed_falcon_info_params *)out_params; + + falcon_params->numConstructedFalcons = TU104_FALCON_TABLE_SIZE; + memcpy(falcon_params->constructedFalconsTable, tu104_falcon_table, + TU104_FALCON_TABLE_SIZE * + sizeof(struct tu104_constructed_falcon_info)); + + *out_params_size = offsetof(struct tu104_get_constructed_falcon_info_params, + constructedFalconsTable) + + TU104_FALCON_TABLE_SIZE * + sizeof(struct tu104_constructed_falcon_info); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: GET_CONSTRUCTED_FALCON_INFO numFalcons=%u", + TU104_FALCON_TABLE_SIZE); +} + +static void gsp_rm_handle_internal_device_table(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + struct tu104_internal_get_device_info_params *dev_params; + u32 copy_size; + + dev_params = (struct tu104_internal_get_device_info_params *)out_params; + + dev_params->numEntries = TU104_INTERNAL_DEVICE_TABLE_SIZE; + copy_size = TU104_INTERNAL_DEVICE_TABLE_SIZE * + sizeof(struct tu104_internal_device_info); + memcpy(dev_params->deviceInfoTable, tu104_internal_device_table, + copy_size); + + *out_params_size = offsetof(struct tu104_internal_get_device_info_params, + deviceInfoTable) + copy_size; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: INTERNAL_GET_DEVICE_INFO_TABLE numEntries=%u", + TU104_INTERNAL_DEVICE_TABLE_SIZE); +} + +/* + * INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP (0x20800a41) + * + * Returns a compressed register access map telling the guest which BAR0 + * registers are userspace-accessible. For our emulated GPU, return an + * empty map (no compressed data, no profiling ranges). + */ +static void gsp_rm_handle_user_register_access_map(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + /* NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS: + * userRegisterAccessMapSize(4) + compressedSize(4) + + * compressedData[4096] + profilingRangesSize(4) + profilingRanges[4096] + * Total: 8204 bytes + */ + u32 val32 = 0; + + memset(out_params, 0, 8204); + memcpy(out_params + 0, &val32, 4); /* userRegisterAccessMapSize */ + memcpy(out_params + 4, &val32, 4); /* compressedSize */ + /* compressedData[4096] = zeros */ + memcpy(out_params + 4104, &val32, 4); /* profilingRangesSize */ + /* profilingRanges[4096] = zeros */ + + *out_params_size = 8204; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: USER_REGISTER_ACCESS_MAP -> empty map"); +} + +/* + * INTERNAL_FIFO_GET_NUM_CHANNELS (0x20800a61) + * + * The guest asks GSP-RM how many channels each runlist supports + * (kfifoRunlistQueryNumChannels). Returning 0 trips the driver's + * NV_ASSERT(numChannels > 0) and kdisp can't allocate its + * clientChannelTable. Report the consumer-Turing standard 64. + */ +static void gsp_rm_handle_fifo_get_num_channels(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + u32 val32; + + /* params: runlistId(4, in) + numChannels(4, out) */ + val32 = TU104_FIFO_NUM_CHANNELS_PER_RUNLIST; + memcpy(out_params + 4, &val32, 4); + + *out_params_size = 8; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: FIFO_GET_NUM_CHANNELS -> %u", val32); +} + +/* + * BUS_GET_PCI_BAR_INFO (0x20801803) + * + * The guest caches pciBarSizes[] from this call (kern_bus.c). Reporting + * zeros leaves BAR1/BAR2 aperture lengths at 0. Report our emulated BAR + * layout; index 2 is the instance-memory window convention + * (BAR0+0x700000, 1MB - gpuPhysInstAddr fallback when PCI BAR2 is absent). + */ +static void gsp_rm_handle_bus_get_pci_bar_info(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + struct nv_vgpu_device *vgpu = ctx->vgpu; + u32 count = 4; + u32 i; + u64 sizes[4]; + u64 offsets[4]; + + sizes[0] = vgpu->bars[0].size; offsets[0] = vgpu->bars[0].gpa; + sizes[1] = vgpu->bars[1].size; offsets[1] = vgpu->bars[1].gpa; + sizes[2] = 0x100000; offsets[2] = vgpu->bars[0].gpa + 0x700000; + sizes[3] = vgpu->bars[3].size; offsets[3] = vgpu->bars[3].gpa; + + memset(out_params, 0, 8 + 8 * 24); + memcpy(out_params + 0, &count, 4); + + for (i = 0; i < count; i++) { + u8 *e = out_params + 8 + i * 24; + u32 flags = 0; + u32 barSizeMB = (u32)(sizes[i] >> 20); + + memcpy(e + 0, &flags, 4); + memcpy(e + 4, &barSizeMB, 4); + memcpy(e + 8, &sizes[i], 8); + memcpy(e + 16, &offsets[i], 8); + } + + *out_params_size = 8 + count * 24; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: BUS_GET_PCI_BAR_INFO -> %u BARs (BAR1=%uMB)", + count, (u32)(sizes[1] >> 20)); +} + +/* + * INTERNAL_DISPLAY_GET_STATIC_INFO (0x20800a01) + * + * kdispStateInitLocked caches numHeads/numDispChannels from this call and + * allocates clientChannelTable[numDispChannels] - a zero reply makes the + * allocation fail and kills RmInitNvDevice. The driver's wire struct is + * 36 bytes (observed paramsSize): feHwSysCap(0) windowPresentMask(4) + * bFbRemapperEnabled(8) numHeads(12) i2cPort(16) internalDispActiveMask(20) + * embeddedDisplayPortMask(24) muxSupported(28) numDispChannels(32). + * Report a minimal headless display: 1 head, 1 (core) channel. + */ +static void gsp_rm_handle_display_get_static_info(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + u32 val32; + + memset(out_params, 0, 36); + + val32 = 1; /* numHeads */ + memcpy(out_params + 12, &val32, 4); + val32 = 1; /* numDispChannels (core channel) */ + memcpy(out_params + 32, &val32, 4); + + *out_params_size = 36; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: DISPLAY_GET_STATIC_INFO -> heads=1 dispChannels=1"); +} + +/* + * INTERNAL_DISPLAY_GET_IP_VERSION (0x20800a4b) + * + * kdispStatePreInitLocked passes an UNINITIALIZED ctrlParams and feeds the + * reply straight into gpuInitDispIpHal - it must be a real IP version. + * Turing display is DISP v04.00 (DispIpHal halspec idx 10). + */ +static void gsp_rm_handle_display_get_ip_version(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + u32 val32 = TU104_DISPLAY_IP_VERSION; + + memcpy(out_params, &val32, 4); + *out_params_size = 4; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: DISPLAY_GET_IP_VERSION -> 0x%x", val32); +} + +/* + * CE_GET_FAULT_METHOD_BUFFER_SIZE (0x20802a08) + * + * kchangrpInit allocates one fault method buffer per runqueue of this + * size; a zero reply makes the allocation fail (NV_ERR_NO_MEMORY) and + * kills channel-group creation. GV100 formula inherited by Turing: + * 135 * (13*totalCes + 2) page-aligned; TU104 = 5 PCEs -> 12288. + */ +static void gsp_rm_handle_ce_get_fault_method_buffer_size( + struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + u32 val32 = TU104_CE_FAULT_METHOD_BUFFER_SIZE; + + memcpy(out_params, &val32, 4); + *out_params_size = 4; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_FAULT_METHOD_BUFFER_SIZE -> %u", val32); +} + +/* + * CE_GET_PHYSICAL_CAPS (0x20802a07) and friends + * + * kceGetDeviceCaps (kernel_ce.c) fills the CE caps the client hands to UVM + * from NV2080_CTRL_CMD_CE_GET_PHYSICAL_CAPS, which reaches GSP-RM. UVM's + * channel manager rejects any CE without the SYSMEM cap for its + * CPU_TO_GPU/GPU_TO_CPU channels ("Failed to initialize the channel manager: + * NV_ERR_NOT_SUPPORTED" in uvm_gpu.c) -- with the previous zeroed reply every + * CE was unusable and UVM_REGISTER_GPU failed with 0x56. + * + * params: { ceEngineType(4), capsTbl[2] } (NV2080_CTRL_CE_GET_CAPS_V2_PARAMS + * layout). Turing LCEs are sysmem-capable, P2P-capable and not GRCE. + */ +static void gsp_rm_handle_ce_physical_caps(struct gsp_emu_context *ctx, + const u8 *req_params, u32 req_size, + u32 *out_status, u8 *out_params, + u32 *out_params_size) +{ + u32 ce_type = 0; + + if (req_size >= 4) + memcpy(&ce_type, req_params, 4); + + memset(out_params, 0, 8); + memcpy(out_params + 0, &ce_type, 4); + + if (ce_type >= TU104_ENGINE_TYPE_COPY0 && + ce_type < TU104_ENGINE_TYPE_COPY0 + TU104_NUM_CES) { + out_params[4] = TU104_CE_CAPS0; + out_params[5] = TU104_CE_CAPS1; + } else { + *out_status = 0x56; /* NV_ERR_NOT_SUPPORTED */ + *out_params_size = req_size; + nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_PHYSICAL_CAPS type=0x%x -> NOT_SUPPORTED", + ce_type); + return; + } + + *out_params_size = req_size ? req_size : 8; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_PHYSICAL_CAPS type=0x%x -> caps=%02x %02x", + ce_type, out_params[4], out_params[5]); +} + +/* + * CE_GET_ALL_PHYSICAL_CAPS (0x20802a0b) / CE_GET_ALL_CAPS (0x20802a0a) + * + * params: NvU8 capsTbl[64][2] + NvU64 present (136 bytes). Report the five + * Turing CEs as present with the same caps as the per-CE query. + */ +static void gsp_rm_handle_ce_all_caps(struct gsp_emu_context *ctx, + u32 *out_status, u8 *out_params, + u32 *out_params_size) +{ + u64 present = 0; + u32 i; + + memset(out_params, 0, 136); + for (i = 0; i < TU104_NUM_CES; i++) { + out_params[i * 2 + 0] = TU104_CE_CAPS0; + out_params[i * 2 + 1] = TU104_CE_CAPS1; + present |= 1ULL << i; + } + memcpy(out_params + 128, &present, 8); + + *out_params_size = 136; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_ALL(_PHYSICAL)_CAPS -> %u CEs present=0x%llx", + TU104_NUM_CES, present); +} + +/* + * CE_GET_CE_PCE_MASK (0x20802a02) + * + * params: { ceEngineType(4), pceMask(4) }. Each Turing LCE has its own PCE, + * so every CE maps to exactly one PCE bit. UVM's queryCopyEngines stores this + * in UvmGpuCopyEngineCaps.cePceMask. + */ +static void gsp_rm_handle_ce_pce_mask(struct gsp_emu_context *ctx, + const u8 *req_params, u32 req_size, + u32 *out_status, u8 *out_params, + u32 *out_params_size) +{ + u32 ce_type = 0, mask = 0; + + if (req_size >= 4) + memcpy(&ce_type, req_params, 4); + + if (ce_type >= TU104_ENGINE_TYPE_COPY0 && + ce_type < TU104_ENGINE_TYPE_COPY0 + TU104_NUM_CES) + mask = 1u << (ce_type - TU104_ENGINE_TYPE_COPY0); + + memset(out_params, 0, 8); + memcpy(out_params + 0, &ce_type, 4); + memcpy(out_params + 4, &mask, 4); + + *out_params_size = req_size ? req_size : 8; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: CE_GET_CE_PCE_MASK type=0x%x -> 0x%x", + ce_type, mask); +} + +/* + * INTERNAL_GMMU_GET_STATIC_INFO (0x20800a59) + * + * The guest asks GSP-RM (us) for the GMMU static info; kgmmu. pStaticInfo-> + * replayableFaultBufferSize feeds kgmmuFaultBufferReplayableAllocate (called + * from faultbufConstruct when UVM opens the device). Zero -> 0-page alloc -> + * NV_ERR_NO_MEMORY -> faultbufConstruct fails and cuInit aborts. + * params: replayableFaultBufferSize(4) + replayableShadowFaultBufferMetadata + * Size(4) + nonReplayableFaultBufferSize(4) + nonReplayableShadowFaultBuffer + * MetadataSize(4). Shadow metadata is only used with confidential computing, + * so it stays 0. + */ +static void gsp_rm_handle_gmmu_get_static_info(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + u32 replay = TU104_GMMU_REPLAYABLE_FAULT_BUFFER_SIZE; + u32 nonreplay = TU104_GMMU_NONREPLAYABLE_FAULT_BUFFER_SIZE; + u32 zero = 0; + + memcpy(out_params + 0, &replay, 4); + memcpy(out_params + 4, &zero, 4); + memcpy(out_params + 8, &nonreplay, 4); + memcpy(out_params + 12, &zero, 4); + + *out_params_size = 16; + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: GMMU_GET_STATIC_INFO -> replayFaultBuf=0x%x nonReplayFaultBuf=0x%x", + replay, nonreplay); +} + +/* + * STATIC_KGR_GET_GLOBAL_SM_ORDER (0x20800a22) + * + * kgrStaticInit copies globalSmOrder[grIdx] into the GR engine's static + * info - init-critical (NV_CHECK_OK_OR_GOTO, cannot fail). Zeros leave + * numSm=0 and the driver/nvidia-smi spin on empty topology. Report the + * TU104 layout: 6 GPCs x 4 TPCs x 2 SMs = 48 SMs on GR0, engines 1-7 empty. + * params: globalSmOrder[8] x (globalSmId[240]x12B + numSm(2) + numTpc(2)). + */ +static void gsp_rm_handle_kgr_get_global_sm_order(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + struct tu104_gr_global_sm_order *order = + (struct tu104_gr_global_sm_order *)out_params; /* engine 0 */ + u32 idx = 0; + u32 gpc, tpc, sm; + + memset(out_params, 0, TU104_GR_MAX_ENGINES * + sizeof(struct tu104_gr_global_sm_order)); + + for (gpc = 0; gpc < TU104_NUM_GPC; gpc++) { + for (tpc = 0; tpc < TU104_TPC_PER_GPC; tpc++) { + for (sm = 0; sm < TU104_SM_PER_TPC; sm++) { + struct tu104_gr_sm_id *e = &order->globalSmId[idx++]; + + e->gpcId = gpc; + e->localTpcId = tpc; + e->localSmId = sm; + e->globalTpcId = gpc * TU104_TPC_PER_GPC + tpc; + e->virtualGpcId = gpc; + e->migratableTpcId = e->globalTpcId; + } + } + } + order->numSm = TU104_NUM_SM; + order->numTpc = TU104_NUM_TPC; + + *out_params_size = TU104_GR_MAX_ENGINES * + sizeof(struct tu104_gr_global_sm_order); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_GLOBAL_SM_ORDER -> %u SMs, %u TPCs on GR0", + TU104_NUM_SM, TU104_NUM_TPC); +} + +/* + * STATIC_KGR_GET_INFO (0x20800a2a) + * + * Returns the GR infoList the driver caches as pGrInfo and reads by index - + * critically MAX_SUBCONTEXT_COUNT (0x2c): zero makes the channel group's + * numMax==0 and channel allocation asserts (numMax == numFree && numMax != 0 + * -> NV_ERR_INVALID_STATE -> "No devices"). Fill TU104 litter topology + + * subcontext counts (MAX_VEID_COUNT_INIT = 64). engineInfo[8] x infoList[58] + * of {index,data} = 3712 bytes; engine 0 = GR0. + */ +static void gsp_rm_handle_kgr_get_info(struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + struct tu104_gr_static_info *info = + (struct tu104_gr_static_info *)out_params; /* engine 0 */ + u32 i; + + memset(out_params, 0, TU104_GR_INFO_MAX_ENGINES * + sizeof(struct tu104_gr_static_info)); + + /* positional array: index field = position for every entry */ + for (i = 0; i < TU104_GR_INFO_MAX_SIZE; i++) + info->infoList[i].index = i; + + /* TU104 litter topology: 6 GPC x 4 TPC x 2 SM = 48 SMs */ + info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_GPCS].data = TU104_NUM_GPC; + info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_TPC_PER_GPC].data = TU104_TPC_PER_GPC; + info->infoList[TU104_GR_INFO_IDX_GPU_CORE_COUNT].data = TU104_NUM_SM; + info->infoList[TU104_GR_INFO_IDX_LITTER_NUM_SM_PER_TPC].data = TU104_SM_PER_TPC; + info->infoList[TU104_GR_INFO_IDX_FAMILY_MAX_TPC_PER_GPC].data = TU104_TPC_PER_GPC; + + /* subcontext (VEID) counts: 64 per hw */ + info->infoList[TU104_GR_INFO_IDX_MAX_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT; + info->infoList[TU104_GR_INFO_IDX_MAX_LEGACY_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT; + info->infoList[TU104_GR_INFO_IDX_MAX_PER_ENGINE_SUBCONTEXT_COUNT].data = TU104_MAX_SUBCONTEXT_COUNT; + + *out_params_size = TU104_GR_INFO_MAX_ENGINES * + sizeof(struct tu104_gr_static_info); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_INFO -> GR0 litter %uGPC/%uTPC/%uSM, maxSubctx=%u", + TU104_NUM_GPC, TU104_TPC_PER_GPC, TU104_NUM_SM, + TU104_MAX_SUBCONTEXT_COUNT); +} + +/* + * STATIC_KGR_GET_FLOORSPEEPING_MASKS (0x20800a26) + * + * Driver computes numGpcs from floorsweepingMasks.gpcMask; zero kills + * krcWatchdogInit ("numGpcs > 0" assert -> RmInitAdapter bails). Fill + * TU104's present-GPC/TPC masks: 6 GPCs (bits 0-5), 4 TPCs per GPC. + */ +static void gsp_rm_handle_kgr_get_floorsweeping_masks( + struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + struct tu104_gr_floorsweeping_masks *m = + (struct tu104_gr_floorsweeping_masks *)out_params; /* engine 0 */ + u32 g; + + memset(out_params, 0, TU104_GR_INFO_MAX_ENGINES * + sizeof(struct tu104_gr_floorsweeping_masks)); + + m->gpcMask = (1U << TU104_NUM_GPC) - 1; /* bits 0-5 = 6 GPCs */ + m->physGpcMask = m->gpcMask; + m->physGfxGpcMask = m->gpcMask; + m->numGfxTpc = TU104_NUM_TPC; + for (g = 0; g < TU104_NUM_GPC; g++) { + m->tpcMask[g] = (1U << TU104_TPC_PER_GPC) - 1; /* 4 TPCs */ + m->tpcCount[g] = TU104_TPC_PER_GPC; + m->mmuPerGpc[g] = 1; + m->numPesPerGpc[g] = 1; + m->zcullMask[g] = (1U << TU104_TPC_PER_GPC) - 1; + } + + *out_params_size = TU104_GR_INFO_MAX_ENGINES * + sizeof(struct tu104_gr_floorsweeping_masks); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_FLOORSWEEPING_MASKS -> gpcMask=0x%x tpc/gpc=%u", + m->gpcMask, TU104_TPC_PER_GPC); +} + +/* + * STATIC_KGR_GET_CONTEXT_BUFFERS_INFO (0x20800a32) + * + * kgraphicsInitializeDeferredStaticData() asks the physical RM (GSP-RM) for + * the per-engine context buffer sizes; kgrctxAllocMainCtxBuffer() then sizes + * the golden image ctx buffer from engine[GRAPHICS].size. The zeroed generic + * reply left size == 0 -> a 0-page allocation -> NV_ERR_NO_MEMORY at + * mem_desc.c:1353 (kgrctxAllocMainCtxBuffer -> kgraphicsCreateGoldenImageChannel). + * + * The real physical RM probes these from the ctxsw ucode + * (grSetupInitialCtxState -> grDiscoverCtxBufferSize) and reports NV_U32_MAX + * for unsupported engine IDs. The client skips NV_U32_MAX entries in + * kgraphicsCreateGoldenImageChannel's VA reservation, so the emulated layout + * below mirrors that: page-aligned scratch sizes for the buffers the client + * allocates, NV_U32_MAX for the engine IDs Turing does not support. + */ +static void gsp_rm_handle_kgr_get_context_buffers_info( + struct gsp_emu_context *ctx, + u32 sequence, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + static const struct tu104_ctx_buffer_info sizes[TU104_CTX_ENGINE_ID_COUNT] = { + [TU104_CTX_ENG_GRAPHICS] = { TU104_CTX_SIZE_MAIN, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_ZCULL] = { TU104_CTX_SIZE_ZCULL, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PM] = { TU104_CTX_SIZE_PM, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PREEMPT] = { TU104_CTX_SIZE_PREEMPT, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_SPILL] = { TU104_CTX_SIZE_SPILL, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PAGEPOOL] = { TU104_CTX_SIZE_PAGEPOOL, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_BETACB] = { TU104_CTX_SIZE_BETACB, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_RTV] = { TU104_CTX_SIZE_RTV, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PATCH] = { TU104_CTX_SIZE_PATCH, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_BUNDLE_CB] = { TU104_CTX_SIZE_BUNDLE_CB, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PAGEPOOL_GLOBAL] = { TU104_CTX_SIZE_PAGEPOOL, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_ATTRIBUTE_CB] = { TU104_CTX_SIZE_ATTRIBUTE_CB, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_RTV_CB_GLOBAL] = { TU104_CTX_SIZE_RTV_CB_GLOBAL, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_FECS_EVENT] = { TU104_CTX_SIZE_FECS_EVENT, 0x1000 }, + [TU104_CTX_ENG_GRAPHICS_PRIV_ACCESS_MAP] = { TU104_CTX_SIZE_PRIV_ACCESS_MAP, 0x1000 }, + }; + struct tu104_gr_get_context_buffers_info_params *info = + (struct tu104_gr_get_context_buffers_info_params *)out_params; + u32 e, i; + + memset(out_params, 0, sizeof(*info)); + + for (e = 0; e < TU104_CTX_GPU_MAX_ENGINES; e++) { + for (i = 0; i < TU104_CTX_ENGINE_ID_COUNT; i++) { + /* engine IDs the chip does not support report + * NV_U32_MAX (the client skips those entries) */ + if (sizes[i].size == 0) { + info->engineContextBuffersInfo[e].engine[i].size = 0xFFFFFFFF; + info->engineContextBuffersInfo[e].engine[i].alignment = 0xFFFFFFFF; + continue; + } + info->engineContextBuffersInfo[e].engine[i].size = sizes[i].size; + info->engineContextBuffersInfo[e].engine[i].alignment = sizes[i].alignment; + } + } + + *out_params_size = sizeof(*info); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: KGR_GET_CONTEXT_BUFFERS_INFO -> main ctx 0x%x, patch 0x%x, pm 0x%x (engines=%u x %u)", + TU104_CTX_SIZE_MAIN, TU104_CTX_SIZE_PATCH, TU104_CTX_SIZE_PM, + TU104_CTX_GPU_MAX_ENGINES, TU104_CTX_ENGINE_ID_COUNT); +} + +/* + * FB_GET_INFO_V2 (0x20801303) + * + * The client (kern_mem_sys_ctrl.c _fbGetFbInfos) sends a list of FB info + * indices to the physical RM and expects the same indices back with data + * populated. A zeroed reply trips "pFbInfos[i].index == ..." asserts and + * leaves all FB info zero - CUDA/UVM read total/usable memory, BAR1 size, + * L2/LTC topology from here. Echo the requested indices and fill in the + * emulated TU104 values. + */ +static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index) +{ + struct nv_vgpu_device *vgpu = ctx->vgpu; + u64 fb = vgpu->fb_size ? vgpu->fb_size : NV_VGPU_BAR1_SIZE; + + switch (index) { + case TU104_FB_INFO_IDX_TILE_REGION_COUNT: + return 0; + case TU104_FB_INFO_IDX_COMPRESSION_SIZE: + return 0; + case TU104_FB_INFO_IDX_PARTITION_COUNT: + return 1; + case TU104_FB_INFO_IDX_BAR1_SIZE: + return (u32)fb; + case TU104_FB_INFO_IDX_RAM_SIZE: + case TU104_FB_INFO_IDX_TOTAL_RAM_SIZE: + case TU104_FB_INFO_IDX_USABLE_RAM_SIZE: + return (u32)fb; + case TU104_FB_INFO_IDX_HEAP_SIZE: + case TU104_FB_INFO_IDX_MAPPABLE_HEAP_SIZE: + return (u32)fb; + case TU104_FB_INFO_IDX_BUS_WIDTH: + return 256; + case TU104_FB_INFO_IDX_RAM_TYPE: + return TU104_RAM_TYPE_GDDR6; + case TU104_FB_INFO_IDX_BANK_COUNT: + return 8; + case TU104_FB_INFO_IDX_FB_TAX_SIZE_KB: + case TU104_FB_INFO_IDX_LARGEST_FREE_REGION_SIZE_KB: + return (u32)(fb >> 10); + case TU104_FB_INFO_IDX_PARTITION_MASK: + return 1; + case TU104_FB_INFO_IDX_HEAP_FREE: + return (u32)fb; + case TU104_FB_INFO_IDX_RAM_LOCATION: + return TU104_RAM_LOCATION_GPU_DEDICATED; + case TU104_FB_INFO_IDX_FBP_COUNT: + return 1; + case TU104_FB_INFO_IDX_FBP_MASK: + return 1; + case TU104_FB_INFO_IDX_L2CACHE_SIZE: + return TU104_L2CACHE_SIZE; + case TU104_FB_INFO_IDX_MEMORYINFO_VENDOR_ID: + return TU104_MEMORYINFO_VENDOR_ID_SAMSUNG; + case TU104_FB_INFO_IDX_BAR1_AVAIL_SIZE: + case TU104_FB_INFO_IDX_BAR1_MAX_CONTIG_AVAIL_SIZE: + return (u32)fb; + case TU104_FB_INFO_IDX_LTC_COUNT: + return TU104_LTC_COUNT; + case TU104_FB_INFO_IDX_LTS_COUNT: + return TU104_LTS_COUNT; + case TU104_FB_INFO_IDX_PSEUDO_CHANNEL_MODE: + return TU104_PSEUDO_CHANNEL_MODE_DISABLED; + case TU104_FB_INFO_IDX_ALLOW_PAGE_RETIREMENT: + return 0; + case TU104_FB_INFO_IDX_LTC_MASK: + return (1U << TU104_LTC_COUNT) - 1; + default: + return 0; + } +} + +static void gsp_rm_handle_fb_get_info_v2(struct gsp_emu_context *ctx, + const u8 *req_params, + u32 req_params_size, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + struct tu104_fb_get_info_v2_params *reply = + (struct tu104_fb_get_info_v2_params *)out_params; + u32 count = 0; + u32 i; + + if (req_params_size >= sizeof(count)) + memcpy(&count, req_params, sizeof(count)); + if (count > TU104_FB_INFO_MAX_LIST_SIZE) + count = TU104_FB_INFO_MAX_LIST_SIZE; + + memset(out_params, 0, sizeof(*reply)); + reply->fbInfoListSize = count; + + for (i = 0; i < count; i++) { + const u8 *in = req_params + sizeof(count) + + i * sizeof(struct tu104_fb_info_entry); + u32 index = 0; + + if (sizeof(count) + (i + 1) * sizeof(struct tu104_fb_info_entry) <= + req_params_size) + memcpy(&index, in, sizeof(index)); + + reply->fbInfoList[i].index = index; + reply->fbInfoList[i].data = tu104_fb_info_data(ctx, index); + } + + *out_params_size = sizeof(*reply); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: FB_GET_INFO_V2 -> %u entries populated (first idx=0x%x)", + count, count ? reply->fbInfoList[0].index : 0); +} + +/* + * BUS_GET_INFO_V2 (0x20801823) + * + * Same flat index/data list pattern as FB_GET_INFO_V2. UVM's + * gpuGetSysmemLinkInfo() asks for SYSMEM_CONNECTION_TYPE and then + * PCIE_GPU_LINK_CAPS to compute the sysmem link bandwidth; a zeroed reply + * makes getPCIELinkRateMBps() return NV_ERR_INVALID_STATE ("Unsupported + * sysmem connection type"). Report a PCIe Gen3 x16 link for the emulated + * vGPU and echo every requested index. + */ +static u32 tu104_bus_info_data(u32 index) +{ + switch (index) { + case TU104_BUS_INFO_IDX_TYPE: + return TU104_BUS_INFO_TYPE_PCI_EXPRESS; + case TU104_BUS_INFO_IDX_INTLINE: + return 0; + case TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CAPS: + case TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CAPS: + case TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CAPS: + case TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CAPS: + case TU104_BUS_INFO_IDX_PCIE_GEN_INFO: + return TU104_PCIE_LINK_CAP_GEN3_X16; + case TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CTRL_STATUS: + case TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CTRL_STATUS: + case TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CTRL: + case TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CTRL: + return TU104_PCIE_LINK_CTRL_STATUS_GEN3_X16; + case TU104_BUS_INFO_IDX_SYSMEM_CONNECTION_TYPE: + return TU104_BUS_INFO_SYSMEM_CONN_PCIE; + default: + return 0; + } +} + +static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx, + const u8 *req_params, + u32 req_params_size, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + struct tu104_bus_get_info_v2_params *reply = + (struct tu104_bus_get_info_v2_params *)out_params; + u32 count = 0; + u32 i; + + if (req_params_size >= sizeof(count)) + memcpy(&count, req_params, sizeof(count)); + if (count > TU104_BUS_INFO_MAX_LIST_SIZE) + count = TU104_BUS_INFO_MAX_LIST_SIZE; + + memset(out_params, 0, sizeof(*reply)); + reply->busInfoListSize = count; + + for (i = 0; i < count; i++) { + const u8 *in = req_params + sizeof(count) + + i * sizeof(struct tu104_bus_info_entry); + u32 index = 0; + + if (sizeof(count) + (i + 1) * sizeof(struct tu104_bus_info_entry) <= + req_params_size) + memcpy(&index, in, sizeof(index)); + + reply->busInfoList[i].index = index; + reply->busInfoList[i].data = tu104_bus_info_data(index); + } + + *out_params_size = sizeof(*reply); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: BUS_GET_INFO_V2 -> %u entries populated (first idx=0x%x)", + count, count ? reply->busInfoList[0].index : 0); + for (i = 0; i < count; i++) + nv_vgpu_info(ctx->vgpu, "GSPRM: bus[%u] idx=0x%x data=0x%x", + i, reply->busInfoList[i].index, + reply->busInfoList[i].data); +} + +/* + * NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES (0x90f10106) + * + * The client tells the server (us) the physical addresses of its GPU page + * table levels together with the VA range they cover: + * + * struct { + * NvHandle hSubDevice; // +0 + * NvU32 subDeviceId; // +4 + * NvU64 pageSize; // +8 (coverage of the reserved level) + * NvU64 virtAddrLo; // +16 + * NvU64 virtAddrHi; // +24 + * NvU32 numLevelsToCopy; // +32 + * NvU32 pad; // +36 + * struct { // +40, 24 bytes each, 6 max + * NvU64 physAddress; + * NvU64 size; + * NvU32 aperture; // 1=sysmem, 2=vidmem + * NvU8 pageShift; + * } levels[6]; + * } // 184 bytes + * + * This is the client's own page table geometry: remember it so client GPU VAs + * can be translated to guest physical addresses (e.g. to complete emulated + * GPU semaphore releases). + */ +#define GSPRM_COPY_PDES_LEVELS_OFF 40 +#define GSPRM_COPY_PDES_LEVEL_SIZE 24 +#define GSPRM_COPY_PDES_MAX_LEVELS 6 + +static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx, + const u8 *params, u32 params_size) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + u64 va_lo = 0, va_hi = 0, page_size = 0; + u32 num_levels = 0, i; + + if (params_size < GSPRM_COPY_PDES_LEVELS_OFF) + return; + + memcpy(&page_size, params + 8, 8); + memcpy(&va_lo, params + 16, 8); + memcpy(&va_hi, params + 24, 8); + memcpy(&num_levels, params + 32, 4); + if (num_levels > GSPRM_COPY_PDES_MAX_LEVELS) + num_levels = GSPRM_COPY_PDES_MAX_LEVELS; + + rm->pt.va_lo = va_lo; + rm->pt.va_hi = va_hi; + rm->pt.page_size = page_size; + rm->pt.num_levels = num_levels; + + nv_vgpu_info(ctx->vgpu, + "GSPRM: COPY_SERVER_PDES va=[0x%llx..0x%llx] pageSize=0x%llx levels=%u", + va_lo, va_hi, page_size, num_levels); + + for (i = 0; i < num_levels; i++) { + const u8 *lv = params + GSPRM_COPY_PDES_LEVELS_OFF + + i * GSPRM_COPY_PDES_LEVEL_SIZE; + u64 phys = 0, size = 0; + u32 aperture = 0; + u8 page_shift = 0; + + if (GSPRM_COPY_PDES_LEVELS_OFF + + (i + 1) * GSPRM_COPY_PDES_LEVEL_SIZE > params_size) + break; + + memcpy(&phys, lv + 0, 8); + memcpy(&size, lv + 8, 8); + memcpy(&aperture, lv + 16, 4); + memcpy(&page_shift, lv + 20, 1); + + rm->pt.levels[i].phys = phys; + rm->pt.levels[i].size = size; + rm->pt.levels[i].aperture = aperture; + rm->pt.levels[i].page_shift = page_shift; + + nv_vgpu_info(ctx->vgpu, + "GSPRM: pt level[%u] phys=0x%llx size=0x%llx aperture=%u pageShift=%u", + i, phys, size, aperture, page_shift); + } +} + +static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, + const u8 *elem, u32 sequence) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + const u8 *req_data = elem + GSPRM_ELEM_HDR_SIZE + + sizeof(struct gsprm_rpc_header); + struct gsp_rm_control_req req; + u32 reply_data_len; + u8 *reply; + u32 out_status = 0; + u32 out_params_size = 0; + + memcpy(&req, req_data, GSP_RM_CTRL_HDR_SIZE); + + nv_vgpu_info(ctx->vgpu, "GSPRM: GSP_RM_CONTROL cmd=0x%x hClient=0x%x hObject=0x%x paramsSize=%u flags=0x%x", + req.cmd, req.hClient, req.hObject, req.paramsSize, + req.flags); + + /* Allocate reply: control header + max possible params */ + reply_data_len = GSP_RM_CTRL_HDR_SIZE + max(req.paramsSize, (u32)4096); + reply = kzalloc(reply_data_len, GFP_KERNEL); + if (!reply) { + nv_vgpu_err(ctx->vgpu, "GSPRM: RM_CONTROL reply alloc failed"); + return; + } + + switch (req.cmd) { + case TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE: + gsp_rm_handle_fifo_device_table(ctx, sequence, &req, + req_data + GSP_RM_CTRL_HDR_SIZE, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case 0x90f10106: /* NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES */ + gsp_rm_handle_copy_server_pdes(ctx, req_data + GSP_RM_CTRL_HDR_SIZE, + req.paramsSize); + out_status = 0; + out_params_size = 0; + break; + + case TU104_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE: + gsp_rm_handle_intr_kernel_table(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO: + gsp_rm_handle_constructed_falcon_info(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE: + gsp_rm_handle_internal_device_table(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case 0x20800a41: /* INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP */ + gsp_rm_handle_user_register_access_map(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS: + gsp_rm_handle_fifo_get_num_channels(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_BUS_GET_PCI_BAR_INFO: + gsp_rm_handle_bus_get_pci_bar_info(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO: + gsp_rm_handle_display_get_static_info(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION: + gsp_rm_handle_display_get_ip_version(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE: + gsp_rm_handle_ce_get_fault_method_buffer_size(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_CE_GET_PHYSICAL_CAPS: + gsp_rm_handle_ce_physical_caps(ctx, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_CE_GET_CE_PCE_MASK: + gsp_rm_handle_ce_pce_mask(ctx, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_CE_GET_ALL_CAPS: + case TU104_CTRL_CE_GET_ALL_PHYSICAL_CAPS: + gsp_rm_handle_ce_all_caps(ctx, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO: + gsp_rm_handle_gmmu_get_static_info(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_STATIC_KGR_GET_GLOBAL_SM_ORDER: + gsp_rm_handle_kgr_get_global_sm_order(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_STATIC_KGR_GET_INFO: + gsp_rm_handle_kgr_get_info(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO: + gsp_rm_handle_kgr_get_context_buffers_info(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_FB_GET_INFO_V2: + gsp_rm_handle_fb_get_info_v2(ctx, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_BUS_GET_INFO_V2: + gsp_rm_handle_bus_get_info_v2(ctx, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_STATIC_KGR_GET_FLOORSWEEPING_MASKS: + gsp_rm_handle_kgr_get_floorsweeping_masks(ctx, sequence, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + default: + /* Unknown control command - return success with zeroed params. + * Do NOT echo the request: pure queries pass uninitialized + * stack structs as input (e.g. DISPLAY_GET_IP_VERSION), so an + * echo hands the driver its own stack garbage as the answer. */ + nv_vgpu_info(ctx->vgpu, "GSPRM: RM_CONTROL unknown cmd=0x%x -> generic SUCCESS", + req.cmd); + /* Dump small request params so mapping/alloc controls that + * carry PAs can be identified (rate-limited). */ + if (req.paramsSize && req.paramsSize <= 96 && + rm->ctrl_param_dumps < 60) { + rm->ctrl_param_dumps++; + nv_vgpu_info(ctx->vgpu, + "GSPRM: ctrl 0x%x params[0..%u]=%*phN", + req.cmd, req.paramsSize, req.paramsSize, + req_data + GSP_RM_CTRL_HDR_SIZE); + } + out_status = 0; + out_params_size = req.paramsSize; + break; + + case TU104_CTRL_I2C_TRANSACTION: + /* I2C transactions have variable-length FINN-serialized + * params: an all-zero reply fails the driver's DeserializeUp + * and the calling ioctl never completes (nvidia-smi hang). + * Echo the request's serialized params (valid stream) and + * report NOT_SUPPORTED - headless vGPU has no DDC/I2C bus. */ + nv_vgpu_info(ctx->vgpu, "GSPRM: I2C_TRANSACTION -> NOT_SUPPORTED (echo)"); + out_status = 0x56; /* NV_ERR_NOT_SUPPORTED */ + out_params_size = req.paramsSize; + if (req.paramsSize && + req.paramsSize + GSP_RM_CTRL_HDR_SIZE <= GSPRM_MAX_SINGLE_ELEM_DATA) + memcpy(reply + GSP_RM_CTRL_HDR_SIZE, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize); + break; + } + + /* Build the reply header */ + memcpy(reply + 0, &req.hClient, 4); + memcpy(reply + 4, &req.hObject, 4); + memcpy(reply + 8, &req.cmd, 4); + memcpy(reply + 12, &out_status, 4); + memcpy(reply + 16, &out_params_size, 4); + memcpy(reply + 20, &req.flags, 4); + + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GSP_RM_CONTROL, sequence, 0, + reply, GSP_RM_CTRL_HDR_SIZE + out_params_size)) + nv_vgpu_err(ctx->vgpu, "GSPRM: RM_CONTROL reply send failed cmd=0x%x", + req.cmd); + else + rm->rpcs_replied++; + + kfree(reply); +} + +/* + * GSP_RM_FREE (function 10) - NVOS00_PARAMETERS echo with status=NV_OK. + * + * rpc_gsp_free: { hRoot, hObjectParent, hObjectOld, status }. Echo the + * request's params back with status=0 so the driver's free rpc_message is a + * complete well-formed record. (The poll match is function-only; the echo + * keeps the post-poll result struct self-consistent.) + */ +static void gsp_rm_handle_free(struct gsp_emu_context *ctx, const u8 *elem, + u32 sequence) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + const struct gsprm_rpc_header *hdr = + (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE); + const u8 *req = elem + GSPRM_ELEM_HDR_SIZE + sizeof(*hdr); + u8 params[16]; /* NVOS00_PARAMETERS: hRoot, hObjectParent, hObjectOld, status */ + u32 hObjOld = 0; + + if (hdr->length >= sizeof(*hdr) + sizeof(params)) + memcpy(params, req, sizeof(params)); + else + memset(params, 0, sizeof(params)); + + memcpy(&hObjOld, params + 8, 4); + memset(params + 12, 0, 4); /* status = NV_OK */ + + nv_vgpu_info(ctx->vgpu, "GSPRM: FREE hObjOld=0x%x seq=%u cmd_read=%u stat_write=%u", + hObjOld, sequence, rm->cmd_read, rm->stat_write); + + /* Small delay before FREE/UNLOADING replies: mitigates an intermittent + * driver-side reply-matching race in the cleanup path. */ + udelay(2000); + + if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_FREE, sequence, 0, + params, sizeof(params))) + nv_vgpu_err(ctx->vgpu, "GSPRM: FREE reply send failed"); + else + rm->rpcs_replied++; +} + +/* + * Handle one command-queue record (first element). For async init RPCs we + * just consume. Synchronous RPCs get a reply; per-function handlers are + * implemented iteratively (unknown ones get a generic success reply). + */ +static bool gsp_rm_alloc_interesting(u32 class_id) +{ + /* Boring classes already decoded: root/device/subdevice */ + switch (class_id) { + case 0x00: /* NV01_ROOT */ + case 0x80: /* NV01_DEVICE_0 */ + case 0x2080: /* NV20_SUBDEVICE_0 */ + case 0x2081: /* NV2081_BINAPI */ + return false; + default: + return true; + } +} + +static void gsp_rm_handle_element(struct gsp_emu_context *ctx, const u8 *elem) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + const struct gsprm_rpc_header *hdr = + (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE); + u32 function = hdr->function; + u32 sequence = hdr->sequence; + + rm->rpcs_seen++; + + switch (function) { + case GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO: + case GSPRM_RPC_FUNC_SET_REGISTRY: + case GSPRM_RPC_FUNC_CONTINUATION_RECORD: + /* Async init RPCs - fire and forget, just consume. */ + nv_vgpu_info(ctx->vgpu, "GSPRM: async RPC %s(%u) seq=%u len=%u consumed", + gsp_rm_rpc_name(function), function, sequence, + hdr->length); + return; + + case GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO: + gsp_rm_handle_set_guest_system_info(ctx, elem, sequence); + return; + + case GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO: + gsp_rm_handle_get_static_info(ctx, sequence); + return; + + case GSPRM_RPC_FUNC_GSP_RM_CONTROL: + gsp_rm_handle_rm_control(ctx, elem, sequence); + return; + + case GSPRM_RPC_FUNC_FREE: + gsp_rm_handle_free(ctx, elem, sequence); + return; + + default: + /* Synchronous RPC: echo function+sequence with success so the + * driver's rpcRecvPoll completes. */ + rm->unknown_rpcs++; + if (function == 47) { + /* UNLOADING_GUEST_DRIVER: log for unload-cycle debugging */ + nv_vgpu_info(ctx->vgpu, "GSPRM: RPC %s(%u) seq=%u cmd_read=%u stat_write=%u stat_seq=%u", + gsp_rm_rpc_name(function), function, sequence, + rm->cmd_read, rm->stat_write, rm->stat_seq); + } else if (function == GSPRM_RPC_FUNC_GSP_RM_ALLOC) { + /* GSP_RM_ALLOC (103): rpc_gsp_rm_alloc_v03_00 at + * elem+GSPRM_ELEM_HDR_SIZE. NOTE: the RPC header + * 'length' only counts the fixed struct size (64), so + * it must NOT be used to bound the params - read the + * alloc header directly. See handoff PART 6. */ + const u8 *p = elem + GSPRM_ELEM_HDR_SIZE + + sizeof(struct gsprm_rpc_header); + const u8 *params = p + 32; + u32 hc = 0, hp = 0, ho = 0, class_id = 0; + u32 params_size = 0, flags = 0; + + memcpy(&hc, p + 0, 4); + memcpy(&hp, p + 4, 4); + memcpy(&ho, p + 8, 4); + memcpy(&class_id, p + 12, 4); + memcpy(¶ms_size, p + 20, 4); + memcpy(&flags, p + 24, 4); + + nv_vgpu_info(ctx->vgpu, + "GSPRM: GSP_RM_ALLOC class=0x%x paramsSize=%u flags=0x%x hClient=0x%x hParent=0x%x hObject=0x%x", + class_id, params_size, flags, hc, hp, ho); + + /* Full param dump for memory/channel classes and any + * class we have not classified yet (rate-limited). + * %phN is capped at 64 bytes per print, so chunk it. */ + if (gsp_rm_alloc_interesting(class_id) && + rm->alloc_param_dumps < 60) { + u32 shown = min(params_size, (u32)384); + u32 off; + + rm->alloc_param_dumps++; + for (off = 0; off < shown; off += 64) { + u32 n = min(shown - off, (u32)64); + + nv_vgpu_info(ctx->vgpu, + "GSPRM: alloc class=0x%x params[%u..%u]=%*phN", + class_id, off, off + n, (int)n, + params + off); + } + } + } else { + nv_vgpu_info(ctx->vgpu, "GSPRM: RPC %s(%u) seq=%u len=%u -> generic SUCCESS reply (unknown #%llu)", + gsp_rm_rpc_name(function), function, sequence, + hdr->length, rm->unknown_rpcs); + } + /* Small delay before UNLOADING replies: mitigates an intermittent + * driver-side reply-matching race in the cleanup path. */ + if (function == 47) + udelay(2000); + if (gsp_rm_send(ctx, function, sequence, 0, NULL, 0)) + nv_vgpu_err(ctx->vgpu, "GSPRM: reply send failed fn=%u", function); + else + rm->rpcs_replied++; + return; + } +} + +/* + * Poll the command queue TX writePtr; consume and handle new records. + * A record spans elemCount consecutive queue elements. + * + * msgq pointers are SLOT INDICES that wrap at msgCount (msgq.c: + * "writePtr += n; if (>= msgCount) -= msgCount"). Empty when + * writePtr == readPtr. A 62->0 transition is a legitimate wrap, NOT + * queue death. + */ +static void gsp_rm_poll_commands(struct gsp_emu_context *ctx) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + struct gsprm_msgq_tx_header tx; + int ret; + + ret = gsp_rm_read_buf(ctx, rm->cmd_off, &tx, sizeof(tx)); + if (ret) { + /* VM may be gone or device closed; stay quiet, retry later */ + return; + } + + /* A healthy driver ALWAYS has writePtr < msgCount. Anything else + * means the queue header is garbage (buffer freed on unload/crash + * without an observable GSP shutdown) - declare the emulated GSP + * dead so the next probe's WPR2-up check passes and it re-boots. */ + if (tx.writePtr >= rm->cmd_msgcount) { + nv_vgpu_warn(ctx->vgpu, "GSPRM: insane cmd writePtr %u (msgCount=%u), GSP boot declared dead", + tx.writePtr, rm->cmd_msgcount); + gsp_emu_clear_boot_state(ctx, "stale GSP-RM queue header"); + gsp_rm_emu_reset(ctx); + return; + } + + /* Consume records until we catch up with the (wrapped) writePtr */ + while (rm->cmd_read != tx.writePtr) { + u8 *elem; + u64 elem_off = rm->cmd_off + rm->entry_off + + (u64)(rm->cmd_read % rm->cmd_msgcount) * GSPRM_QUEUE_ELEMENT_SIZE; + u32 signature, elem_count; + const struct gsprm_rpc_header *hdr; + + elem = kmalloc(GSPRM_QUEUE_ELEMENT_SIZE, GFP_KERNEL); + if (!elem) + return; + ret = gsp_rm_read_buf(ctx, elem_off, elem, GSPRM_QUEUE_ELEMENT_SIZE); + if (ret) { + kfree(elem); + return; + } + + hdr = (const struct gsprm_rpc_header *)(elem + GSPRM_ELEM_HDR_SIZE); + signature = hdr->signature; + memcpy(&elem_count, elem + 0x28, 4); + + if (signature != GSPRM_RPC_SIGNATURE || elem_count == 0 || + elem_count > rm->cmd_msgcount) { + /* Garbage element: could be teardown (driver stopped + * writing valid elements after UNLOADING_GUEST_DRIVER/ + * FREE sequence) or genuine queue death. Don't declare + * death here - stop polling quietly and let the + * Booter-unload kick (SEC2 CPUCTL) trigger the reset + * via gsp_emu_clear_boot_state. This allows the + * driver's cleanup frees to complete. */ + nv_vgpu_info(ctx->vgpu, "GSPRM: garbage element idx=%u sig=0x%x elemCount=%u fn=%u, stopping poll (teardown?)", + rm->cmd_read, signature, elem_count, + hdr->function); + kfree(elem); + return; + } + + gsp_rm_handle_element(ctx, elem); + kfree(elem); + + /* consume the whole record (all its elements), wrapping + * at msgCount like the driver's own writePtr */ + rm->cmd_read = (rm->cmd_read + elem_count) % rm->cmd_msgcount; + } + + /* Publish our command-queue read position into the status queue RX + * header (MSGQ_FLAGS_SWAP_RX: read pointers live in the peer's queue). */ + gsp_rm_write_buf(ctx, rm->stat_off + 0x20, + &rm->cmd_read, sizeof(rm->cmd_read)); +} + +/* ============================================================ + * Public interface + * ============================================================ */ + +int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx) +{ + struct gsp_rm_emu *rm; + + rm = kzalloc(sizeof(*rm), GFP_KERNEL); + if (!rm) + return -ENOMEM; + + rm->vgpu = vgpu; + ctx->rm_emu = rm; + return 0; +} + +void gsp_rm_emu_fini(struct gsp_emu_context *ctx) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + + if (!rm) + return; + kfree(rm->ptes); + kfree(rm); + ctx->rm_emu = NULL; +} + +/* + * Reset the GSP-RM emulation state on GSP ENGINE reset (guest driver reboots + * the GSP on every probe/unload). The queue buffer GPAs from the previous + * boot are stale (freed by the guest) - drop all parsed state so the next + * Booter completion re-parses the chain. + */ +void gsp_rm_emu_reset(struct gsp_emu_context *ctx) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + + if (!rm || !rm->parsed) + return; + + nv_vgpu_info(ctx->vgpu, "GSPRM: reset (rpcs=%llu replied=%llu unknown=%llu)", + rm->rpcs_seen, rm->rpcs_replied, rm->unknown_rpcs); + + kfree(rm->ptes); + rm->ptes = NULL; + rm->parsed = false; + rm->active = false; + rm->init_done_sent = false; + rm->libos_args_gpa = 0; + rm->gsp_args_gpa = 0; + rm->cmd_read = 0; + rm->stat_write = 0; + rm->stat_seq = 0; + memset(&rm->mq, 0, sizeof(rm->mq)); + rm->cmd_gpa = 0; + rm->stat_gpa = 0; + rm->cmd_size = 0; + rm->stat_size = 0; + rm->cmd_msgcount = 0; + rm->stat_msgcount = 0; +} + +/* + * Called when the guest's Booter Load completes. The GSP MAILBOX0/1 now hold + * the libos init args GPA; parse the queue chain, link the status queue and + * emit GSP_INIT_DONE so the driver's kgspWaitForRmInitDone completes. + */ +void gsp_rm_emu_on_booter_done(struct gsp_emu_context *ctx) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + u32 mb_lo, mb_hi; + u32 b_is_d3hot = 0; + int ret; + + if (!rm || rm->parsed) + return; + + mb_lo = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_MAILBOX0); + mb_hi = gsp_emu_shadow_read32(ctx, GSP_EMU_REG_GSP_MAILBOX0 + 4); + rm->libos_args_gpa = ((u64)mb_hi << 32) | mb_lo; + + nv_vgpu_info(ctx->vgpu, "GSPRM: libos args at gpa=0x%llx (mailbox 0x%x:0x%x)", + rm->libos_args_gpa, mb_hi, mb_lo); + + if (!rm->libos_args_gpa) { + pr_warn_ratelimited("GSPRM: libos args GPA is 0, will retry\n"); + return; + } + + ret = gsp_rm_parse_queue_chain(ctx); + if (ret) { + nv_vgpu_err(ctx->vgpu, "GSPRM: queue chain parse failed: %d", ret); + return; + } + + ret = gsp_rm_link_status_queue(ctx); + if (ret) + return; + + rm->parsed = true; + rm->active = true; + + /* Emit GSP_INIT_DONE (function 0x1001, sequence 0, init_done_v17_00) */ + ret = gsp_rm_send(ctx, GSPRM_RPC_EVENT_GSP_INIT_DONE, 0, 0, + &b_is_d3hot, sizeof(b_is_d3hot)); + if (ret) { + nv_vgpu_err(ctx->vgpu, "GSPRM: GSP_INIT_DONE send failed: %d", ret); + return; + } + rm->init_done_sent = true; + + nv_vgpu_info(ctx->vgpu, "GSPRM: GSP_INIT_DONE emitted, RPC loop active"); +} + +void gsp_rm_emu_poll(struct gsp_emu_context *ctx) +{ + struct gsp_rm_emu *rm = ctx->rm_emu; + + if (!rm) + return; + + /* Retry chain parsing if the first attempt at BOOTER_DONE failed + * (e.g. mailbox not yet visible). Boot stage 2 == BOOTER_DONE. + * Back off hard: at 100us poll cadence an unthrottled retry spams + * dmesg into unusability when the driver stalls mid-bootstrap. */ + if (!rm->parsed) { + if (ctx->boot_stage == 2 && + time_after(jiffies, rm->last_parse_retry + HZ / 4)) { + rm->last_parse_retry = jiffies; + gsp_rm_emu_on_booter_done(ctx); + } + return; + } + + if (!rm->active) + return; + + gsp_rm_poll_commands(ctx); +} diff --git a/kernel/gsp_rm_emulation.h b/kernel/gsp_rm_emulation.h new file mode 100644 index 0000000..a82656f --- /dev/null +++ b/kernel/gsp_rm_emulation.h @@ -0,0 +1,187 @@ +/* + * GSP-RM Emulation (physical-path) for Host-Side vGPU + * + * Emulates the GSP-RM firmware's msgq-based RPC channel for a guest driver + * that believes it is driving a real physical GPU. The guest driver: + * - boots the "GSP" (falcon stages emulated in gsp_emulation.c) + * - programs libos init args address into GSP MAILBOX0/1 + * - expects GSP-RM to msgqInit the status queue in *guest system memory* + * (shared command/status queue buffer described by GSP_ARGUMENTS_CACHED) + * - polls the status queue for NV_VGPU_MSG_EVENT_GSP_INIT_DONE and then + * issues the RM RPC sequence on the command queue. + * + * Since there is no real GSP engine, this module performs the "virtual DMA" + * on behalf of the device using vfio_dma_rw() (CPU copies through the VFIO + * container's IOMMU mappings, IOVA == guest physical address). + */ + +#ifndef _GSP_RM_EMULATION_H_ +#define _GSP_RM_EMULATION_H_ + +#include + +#include "nvidia-vgpu-vfio.h" + +struct gsp_emu_context; + +/* Wire constants (from NVIDIA OpenRM 570.144 msgq/rpc headers - the guest + * driver is the 570.144 stock runfile, whose GSP_MSG_QUEUE_ELEMENT has NO + * MCTP/NVDM transport headers: authTag[16]+aad[16]+checkSum+seqNum+elemCount + * +rpc@0x30). */ +#define GSPRM_MSGQ_VERSION 0 +#define GSPRM_MSGQ_FLAGS_SWAP_RX 1 +#define GSPRM_RPC_SIGNATURE 0x43505256 /* NV_VGPU_MSG_SIGNATURE_VALID */ +#define GSPRM_RPC_HEADER_VERSION 0x03000000 /* MAJOR_TOT(3)<<24|MINOR_TOT(0)<<16 */ + +/* RPC function/event numbers (rpc_global_enums.h) */ +#define GSPRM_RPC_FUNC_SET_GUEST_SYSTEM_INFO 1 +#define GSPRM_RPC_FUNC_ALLOC_ROOT 2 +#define GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO 65 +#define GSPRM_RPC_FUNC_GSP_SET_SYSTEM_INFO 72 +#define GSPRM_RPC_FUNC_SET_REGISTRY 73 +#define GSPRM_RPC_FUNC_CONTINUATION_RECORD 71 +#define GSPRM_RPC_FUNC_GSP_RM_CONTROL 76 +#define GSPRM_RPC_FUNC_GSP_RM_ALLOC 103 +#define GSPRM_RPC_FUNC_FREE 10 +#define GSPRM_RPC_EVENT_GSP_INIT_DONE 0x1001 + +/* rpc_set_guest_system_info_v03_00: version fields the driver reads back + * from the reply to negotiate the RPC interface version (ipVersion). */ +#define GSPRM_SET_GUEST_INFO_SIZE 0x324 +#define GSPRM_VGX_VERSION_MAJOR 0x29 +#define GSPRM_VGX_VERSION_MINOR 0x0C + +/* "RMARGS" packed big-endian into u64 by _kgspGenerateInitArgId() */ +#define GSPRM_LIBOS_ID_RMARGS 0x524D41524753ULL + +#define GSPRM_MAX_LIBOS_REGIONS 32 +#define GSPRM_QUEUE_ELEMENT_SIZE 0x1000 +#define GSPRM_STATUS_QUEUE_SIZE 0x40000 + +/* msgqTxHeader (msgq_priv.h) */ +struct gsprm_msgq_tx_header { + u32 version; + u32 size; + u32 msgSize; + u32 msgCount; + u32 writePtr; + u32 flags; + u32 rxHdrOff; + u32 entryOff; +}; + +/* rpc_message_header_v03_00 (g_rpc-message-header.h) */ +struct gsprm_rpc_header { + u32 header_version; + u32 signature; + u32 length; + u32 function; + u32 rpc_result; + u32 rpc_result_private; + u32 sequence; + u32 spare; /* rpc_message_rpc_union_field_v (spare/cpuRmGfid) */ +}; + +/* LibosMemoryRegionInitArgument (libos_init_args.h) - natural alignment, 32B */ +struct gsprm_libos_region { + u64 id8; + u64 pa; + u64 size; + u8 kind; + u8 loc; + u8 pad[6]; +}; + +/* MESSAGE_QUEUE_INIT_ARGUMENTS (gsp_init_args.h), x86_64 NvLength=u64 */ +struct gsprm_mq_init_args { + u64 sharedMemPhysAddr; + u32 pageTableEntryCount; + u32 pad0; + u64 cmdQueueOffset; + u64 statQueueOffset; + u64 queueElementHdrSize; + u64 queueElementSizeMin; + u64 queueElementSizeMax; + u32 queueHeaderAlign; + u32 queueElementAlign; +}; + +struct gsp_rm_emu { + struct nv_vgpu_device *vgpu; + + bool parsed; /* queue chain parsed + status queue linked */ + bool active; + + unsigned long last_parse_retry; /* jiffies of last on_booter_done retry */ + + u64 libos_args_gpa; /* from GSP MAILBOX0/1 */ + u64 gsp_args_gpa; /* RMARGS region pa */ + + /* queue buffer description (from GSP_ARGUMENTS_CACHED) */ + struct gsprm_mq_init_args mq; + u64 *ptes; /* pageTableEntryCount guest-phys addrs */ + + /* computed queue geometry. cmd_off/stat_off are offsets into the shared + * buffer (linear); GPAs must be computed per-access via the page table + * (buffer is NV_MEMORY_NONCONTIGUOUS). */ + u64 cmd_off; + u64 stat_off; + u64 cmd_gpa; /* GPA of command queue base (diagnostic) */ + u64 stat_gpa; /* GPA of status queue base (diagnostic) */ + u32 cmd_size; /* bytes */ + u32 stat_size; /* bytes */ + u32 cmd_msgcount; + u32 stat_msgcount; + u32 entry_off; /* 0x1000 (msgq entryOff, both queues) */ + + /* GSP-side cursors */ + u32 cmd_read; /* elements consumed from command queue */ + u32 stat_write; /* elements written to status queue */ + u32 stat_seq; /* status record sequence numbers */ + bool init_done_sent; + + /* + * Client page-table geometry, learned from + * NV90F1_CTRL_CMD_VASPACE_COPY_SERVER_RESERVED_PDES (0x90f10106): + * the client hands us the PHYSICAL addresses of its page-table + * levels (root PDE ... leaf PTE) plus the VA range/page shifts. + * Used to translate client GPU VAs (e.g. semaphore/pushbuffer VAs) + * to guest physical addresses. + */ + struct { + u64 va_lo; + u64 va_hi; + u64 page_size; /* coverage of the root level */ + u32 num_levels; + struct { + u64 phys; + u64 size; + u32 aperture; /* 1=sysmem(PA), 2=vidmem(offset) */ + u8 page_shift; + } levels[6]; + } pt; + + /* stats */ + u64 rpcs_seen; + u64 rpcs_replied; + u64 unknown_rpcs; + u32 alloc_dumps; /* raw GSP_RM_ALLOC dumps emitted */ + u32 alloc_param_dumps; /* alloc param dumps emitted */ + u32 ctrl_param_dumps; /* control param dumps emitted */ +}; + +int gsp_rm_emu_init(struct nv_vgpu_device *vgpu, struct gsp_emu_context *ctx); +void gsp_rm_emu_fini(struct gsp_emu_context *ctx); +int gsp_rm_scan_guest_ram(struct nv_vgpu_device *vgpu, u64 needle, + u64 start, u64 end, u32 max_hits); +int gsp_rm_scan_guest_ram_seq(struct nv_vgpu_device *vgpu, const u32 *pat, + u32 npat, u64 start, u64 end, u32 max_hits); +int gsp_rm_scan_guest_ram_pt(struct nv_vgpu_device *vgpu, u64 pa, + u64 start, u64 end, u32 max_hits); +int gsp_rm_scan_guest_ram_masked(struct nv_vgpu_device *vgpu, u64 value, + u64 mask, u64 start, u64 end, u32 max_hits); +void gsp_rm_emu_reset(struct gsp_emu_context *ctx); +void gsp_rm_emu_on_booter_done(struct gsp_emu_context *ctx); +void gsp_rm_emu_poll(struct gsp_emu_context *ctx); + +#endif /* _GSP_RM_EMULATION_H_ */ diff --git a/kernel/gsp_rm_tu104.h b/kernel/gsp_rm_tu104.h new file mode 100644 index 0000000..6a19726 --- /dev/null +++ b/kernel/gsp_rm_tu104.h @@ -0,0 +1,1009 @@ +/* + * TU104 Hardware Tables for GSP-RM Emulation + * + * Static device/engine/interrupt tables for the emulated TU104 GPU. + * Derived from device/engine interface definitions, observed GSP-RM + * behaviour, and hardware reference headers (hwref/turing/tu104). + * + * These tables replace what the real GSP-RM firmware would read from + * PTOP hardware registers during boot. + */ + +#ifndef _GSP_RM_TU104_H_ +#define _GSP_RM_TU104_H_ + +#include + +/* ============================================================ + * Wire structures (from 570.144 reference, ABI-checked) + * ============================================================ */ + +/* NV2080_CTRL_FIFO_DEVICE_ENTRY (ctrl2080fifo.h) - 100 bytes */ +struct tu104_fifo_device_entry { + u32 engineData[16]; /* ENGINE_INFO_TYPE_* indexed */ + u32 pbdmaIds[2]; + u32 pbdmaFaultIds[2]; + u32 numPbdmas; + char engineName[16]; +}; + +/* NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS - 3212 bytes max */ +#define TU104_FIFO_TABLE_MAX_ENTRIES 32 +#define TU104_FIFO_TABLE_MAX_DEVICES 256 + +struct tu104_fifo_get_device_info_params { + u32 baseIndex; + u32 numEntries; + u8 bMore; + u8 _pad[3]; + struct tu104_fifo_device_entry entries[TU104_FIFO_TABLE_MAX_ENTRIES]; +}; + +/* NV2080_CTRL_INTERNAL_DEVICE_INFO (ctrl2080internal.h) - 48 bytes */ +struct tu104_internal_device_info { + u32 faultId; + u32 instanceId; + u32 typeEnum; + u32 resetId; + u32 devicePriBase; + u32 isEngine; + u32 rlEngId; + u32 runlistPriBase; + u32 groupId; + u32 ginTargetId; + u32 deviceBroadcastPriBase; + u32 groupLocalInstanceId; +}; + +/* NV2080_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE_PARAMS - up to 512 entries */ +#define TU104_INTERNAL_TABLE_MAX_ENTRIES 512 + +struct tu104_internal_get_device_info_params { + u32 numEntries; + struct tu104_internal_device_info deviceInfoTable[TU104_INTERNAL_TABLE_MAX_ENTRIES]; +}; + +/* NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS */ +#define TU104_INTR_TABLE_MAX_SIZE 128 + +struct tu104_intr_table_entry { + u16 engineIdx; + u16 _pad0; + u32 pmcIntrMask; + u32 vectorStall; + u32 vectorNonStall; +}; + +/* NV2080_INTR_CATEGORY_SUBTREE_MAP (ctrl2080internal.h) - 2 bytes */ +struct tu104_intr_subtree_map { + u8 subtreeStart; + u8 subtreeEnd; +}; + +/* NV2080_INTR_CATEGORY enum */ +#define TU104_INTR_CATEGORY_DEFAULT 0 +#define TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE 1 +#define TU104_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION 2 +#define TU104_INTR_CATEGORY_RUNLIST 3 +#define TU104_INTR_CATEGORY_RUNLIST_NOTIFICATION 4 +#define TU104_INTR_CATEGORY_UVM_OWNED 5 +#define TU104_INTR_CATEGORY_UVM_SHARED 6 +#define TU104_INTR_CATEGORY_ENUM_COUNT 7 +#define TU104_INTR_INVALID_SUBTREE 0xFF + +struct tu104_intr_get_kernel_table_params { + u32 tableLen; + struct tu104_intr_table_entry table[TU104_INTR_TABLE_MAX_SIZE]; + struct tu104_intr_subtree_map subtreeMap[TU104_INTR_CATEGORY_ENUM_COUNT]; +}; + +/* NV2080_CTRL_GPU_CONSTRUCTED_FALCON_INFO (ctrl2080gpu.h) - 20 bytes */ +struct tu104_constructed_falcon_info { + u32 engDesc; + u32 ctxAttr; + u32 ctxBufferSize; + u32 addrSpaceList; + u32 registerBase; +}; + +#define TU104_MAX_CONSTRUCTED_FALCONS 0x40 + +struct tu104_get_constructed_falcon_info_params { + u32 numConstructedFalcons; + struct tu104_constructed_falcon_info constructedFalconsTable[TU104_MAX_CONSTRUCTED_FALCONS]; +}; + +/* ============================================================ + * ENGINE_INFO_TYPE indices (engine_info.h) - frozen vGPU ABI + * ============================================================ */ +#define EI_ENG_DESC 0 +#define EI_FIFO_TAG 1 +#define EI_RM_ENGINE_TYPE 2 +#define EI_RUNLIST 3 +#define EI_MMU_FAULT_ID 4 +#define EI_RC_MASK 5 +#define EI_RESET 6 +#define EI_INTR 7 +#define EI_MC 8 +#define EI_DEV_TYPE_ENUM 9 +#define EI_INSTANCE_ID 10 +#define EI_RUNLIST_PRI_BASE 11 +#define EI_IS_HOST_DRIVEN_ENGINE 12 +#define EI_RUNLIST_ENGINE_ID 13 +#define EI_CHRAM_PRI_BASE 14 +#define EI_KERNEL_RM_MAX 15 + +/* ============================================================ + * ENG_DESC values (MKENGDESC = classId << 8 | instance) + * ============================================================ */ +#define TU104_ENG_GR0 0xd334df00 /* classId(Graphics)=0xd334df, inst 0 */ +#define TU104_ENG_CE0 0x793ceb00 /* classId(OBJCE)=0x793ceb, inst 0 */ +#define TU104_ENG_CE1 0x793ceb01 +#define TU104_ENG_CE2 0x793ceb02 +#define TU104_ENG_CE3 0x793ceb03 +#define TU104_ENG_CE4 0x793ceb04 +#define TU104_ENG_NVDEC0 0x8f99e100 /* classId(OBJBSP)=0x8f99e1, inst 0 */ +#define TU104_ENG_NVDEC1 0x8f99e101 +#define TU104_ENG_NVENC0 0xe97b6c00 /* classId(OBJMSENC)=0xe97b6c, inst 0 */ +#define TU104_ENG_SEC2 0x28c40800 /* classId(OBJSEC2)=0x28c408, inst 0 */ +#define TU104_ENG_NVJPG 0x2b3a5400 /* classId(OBJNVJPG)=0x2b3a54, inst 0 */ +#define TU104_ENG_SW 0x95a6f500 /* classId(OBJSWENG)=0x95a6f5, inst 0 */ +#define TU104_ENG_FBFLCN 0x8a20bf00 /* classId(OBJFBFLCN)=0x8a20bf, inst 0 */ + +/* ============================================================ + * RM_ENGINE_TYPE values (gpu_engine_type.h) + * ============================================================ */ +#define TU104_RM_ENGINE_GR0 0x01 +#define TU104_RM_ENGINE_CE0 0x09 +#define TU104_RM_ENGINE_CE1 0x0a +#define TU104_RM_ENGINE_CE2 0x0b +#define TU104_RM_ENGINE_CE3 0x0c +#define TU104_RM_ENGINE_CE4 0x0d +#define TU104_RM_ENGINE_NVDEC0 0x1d +#define TU104_RM_ENGINE_NVDEC1 0x1e +#define TU104_RM_ENGINE_NVENC0 0x25 +#define TU104_RM_ENGINE_SW 0x2d +#define TU104_RM_ENGINE_SEC2 0x31 +#define TU104_RM_ENGINE_HOST 0x32 +#define TU104_RM_ENGINE_PMU 0x34 +#define TU104_RM_ENGINE_FBFLCN 0x35 +#define TU104_RM_ENGINE_NVJPG0 0x36 + +/* ============================================================ + * MC_ENGINE_IDX values (engine_idx.h) + * ============================================================ */ +#define TU104_MC_ENGINE_GR 84 +#define TU104_MC_ENGINE_CE0 15 +#define TU104_MC_ENGINE_CE1 16 +#define TU104_MC_ENGINE_CE2 17 +#define TU104_MC_ENGINE_CE3 18 +#define TU104_MC_ENGINE_CE4 19 +#define TU104_MC_ENGINE_NVDEC0 65 +#define TU104_MC_ENGINE_NVDEC1 66 +#define TU104_MC_ENGINE_NVENC0 38 +#define TU104_MC_ENGINE_SEC2 47 +#define TU104_MC_ENGINE_NVJPG 51 + +/* ============================================================ + * NV_PFAULT_MMU_ENG_ID values (tu104/dev_fault.h) + * ============================================================ */ +#define TU104_FAULT_ID_GR 64 +#define TU104_FAULT_ID_CE0 15 +#define TU104_FAULT_ID_CE1 16 +#define TU104_FAULT_ID_CE2 17 +#define TU104_FAULT_ID_CE3 18 +#define TU104_FAULT_ID_CE4 19 +#define TU104_FAULT_ID_NVDEC0 10 +#define TU104_FAULT_ID_NVDEC1 25 +#define TU104_FAULT_ID_NVENC0 11 +#define TU104_FAULT_ID_SEC2 14 +#define TU104_FAULT_ID_NVJPG0 24 +#define TU104_FAULT_ID_GSP 2 +#define TU104_FAULT_ID_PMU 6 + +/* PBDMA fault ID base (NV_PFIFO_CFG0_PBDMA_FAULT_ID_INIT = 32) */ +#define TU104_PBDMA_FAULT_ID_BASE 32 +#define TU104_NUM_PBDMAS 12 + +/* ============================================================ + * ROBUST_CHANNEL values (rc.h) + * ============================================================ */ +#define TU104_RC_GRAPHICS 40 +#define TU104_RC_CE0 22 +#define TU104_RC_CE1 23 +#define TU104_RC_CE2 24 +#define TU104_RC_CE3 25 +#define TU104_RC_CE4 26 +#define TU104_RC_NVDEC0 14 +#define TU104_RC_NVDEC1 15 +#define TU104_RC_NVENC0 11 +#define TU104_RC_SEC2 2 +#define TU104_RC_NVJPG0 32 + +/* ============================================================ + * NV_PTOP_DEVICE_INFO_TYPE_ENUM (tu104/dev_top.h) + * ============================================================ */ +#define TU104_PTOP_TYPE_GRAPHICS 0 +#define TU104_PTOP_TYPE_LCE 19 +#define TU104_PTOP_TYPE_NVDEC 16 +#define TU104_PTOP_TYPE_NVENC 14 +#define TU104_PTOP_TYPE_NVJPG 21 +#define TU104_PTOP_TYPE_SEC 13 +#define TU104_PTOP_TYPE_GSP 20 +#define TU104_PTOP_TYPE_IOCTRL 18 + +/* ============================================================ + * FIFO_TAG values (hardware engine enum, 4 bits) + * ============================================================ */ +#define TU104_TAG_GR 0 +#define TU104_TAG_CE0 1 +#define TU104_TAG_CE1 2 +#define TU104_TAG_CE2 3 +#define TU104_TAG_CE3 4 +#define TU104_TAG_CE4 5 +#define TU104_TAG_NVDEC0 10 +#define TU104_TAG_NVDEC1 11 +#define TU104_TAG_NVENC0 13 +#define TU104_TAG_NVJPG 14 +#define TU104_TAG_SEC2 15 + +/* ============================================================ + * TU104 FIFO Engine Table (11 hardware engines + 1 SW) + * + * Runlist assignment: each engine gets its own runlist. + * Runlist 0=GR, 1-5=CE0-CE4, 6-7=NVDEC0-1, 8=NVENC0, 9=SEC2, 10=NVJPG + * PBDMA assignment: 1:1 with runlists (PBDMA N services runlist N). + * SW engine shares NVJPG's runlist/PBDMA (per fifoConstructSwEngine). + * ============================================================ */ + +static const struct tu104_fifo_device_entry tu104_engine_table[] = { + /* [0] GR0 - Graphics */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_GR0, + [EI_FIFO_TAG] = TU104_TAG_GR, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_GR0, + [EI_RUNLIST] = 0, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_GR, + [EI_RC_MASK] = TU104_RC_GRAPHICS, + [EI_RESET] = 12, /* NV_PMC_ENABLE_PGRAPH bit */ + [EI_INTR] = 0, + [EI_MC] = TU104_MC_ENGINE_GR, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_GRAPHICS, + [EI_INSTANCE_ID] = 0, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {0, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 0, 0}, + .numPbdmas = 1, + .engineName = "GRAPHICS", + }, + /* [1] CE0 */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_CE0, + [EI_FIFO_TAG] = TU104_TAG_CE0, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE0, + [EI_RUNLIST] = 1, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CE0, + [EI_RC_MASK] = TU104_RC_CE0, + [EI_RESET] = 1, + [EI_INTR] = 1, + [EI_MC] = TU104_MC_ENGINE_CE0, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_LCE, + [EI_INSTANCE_ID] = 0, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {1, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 1, 0}, + .numPbdmas = 1, + .engineName = "CE0", + }, + /* [2] CE1 */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_CE1, + [EI_FIFO_TAG] = TU104_TAG_CE1, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE1, + [EI_RUNLIST] = 2, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CE1, + [EI_RC_MASK] = TU104_RC_CE1, + [EI_RESET] = 2, + [EI_INTR] = 2, + [EI_MC] = TU104_MC_ENGINE_CE1, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_LCE, + [EI_INSTANCE_ID] = 1, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {2, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 2, 0}, + .numPbdmas = 1, + .engineName = "CE1", + }, + /* [3] CE2 */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_CE2, + [EI_FIFO_TAG] = TU104_TAG_CE2, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE2, + [EI_RUNLIST] = 3, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CE2, + [EI_RC_MASK] = TU104_RC_CE2, + [EI_RESET] = 3, + [EI_INTR] = 3, + [EI_MC] = TU104_MC_ENGINE_CE2, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_LCE, + [EI_INSTANCE_ID] = 2, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {3, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 3, 0}, + .numPbdmas = 1, + .engineName = "CE2", + }, + /* [4] CE3 */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_CE3, + [EI_FIFO_TAG] = TU104_TAG_CE3, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE3, + [EI_RUNLIST] = 4, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CE3, + [EI_RC_MASK] = TU104_RC_CE3, + [EI_RESET] = 4, + [EI_INTR] = 4, + [EI_MC] = TU104_MC_ENGINE_CE3, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_LCE, + [EI_INSTANCE_ID] = 3, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {4, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 4, 0}, + .numPbdmas = 1, + .engineName = "CE3", + }, + /* [5] CE4 */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_CE4, + [EI_FIFO_TAG] = TU104_TAG_CE4, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_CE4, + [EI_RUNLIST] = 5, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_CE4, + [EI_RC_MASK] = TU104_RC_CE4, + [EI_RESET] = 5, + [EI_INTR] = 5, + [EI_MC] = TU104_MC_ENGINE_CE4, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_LCE, + [EI_INSTANCE_ID] = 4, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {5, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 5, 0}, + .numPbdmas = 1, + .engineName = "CE4", + }, + /* [6] NVDEC0 */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_NVDEC0, + [EI_FIFO_TAG] = TU104_TAG_NVDEC0, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_NVDEC0, + [EI_RUNLIST] = 6, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_NVDEC0, + [EI_RC_MASK] = TU104_RC_NVDEC0, + [EI_RESET] = 14, + [EI_INTR] = 6, + [EI_MC] = TU104_MC_ENGINE_NVDEC0, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_NVDEC, + [EI_INSTANCE_ID] = 0, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {6, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 6, 0}, + .numPbdmas = 1, + .engineName = "NVDEC0", + }, + /* [7] NVDEC1 */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_NVDEC1, + [EI_FIFO_TAG] = TU104_TAG_NVDEC1, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_NVDEC1, + [EI_RUNLIST] = 7, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_NVDEC1, + [EI_RC_MASK] = TU104_RC_NVDEC1, + [EI_RESET] = 15, + [EI_INTR] = 7, + [EI_MC] = TU104_MC_ENGINE_NVDEC1, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_NVDEC, + [EI_INSTANCE_ID] = 1, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {7, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 7, 0}, + .numPbdmas = 1, + .engineName = "NVDEC1", + }, + /* [8] NVENC0 */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_NVENC0, + [EI_FIFO_TAG] = TU104_TAG_NVENC0, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_NVENC0, + [EI_RUNLIST] = 8, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_NVENC0, + [EI_RC_MASK] = TU104_RC_NVENC0, + [EI_RESET] = 17, + [EI_INTR] = 8, + [EI_MC] = TU104_MC_ENGINE_NVENC0, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_NVENC, + [EI_INSTANCE_ID] = 0, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {8, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 8, 0}, + .numPbdmas = 1, + .engineName = "NVENC0", + }, + /* [9] SEC2 */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_SEC2, + [EI_FIFO_TAG] = TU104_TAG_SEC2, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_SEC2, + [EI_RUNLIST] = 9, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_SEC2, + [EI_RC_MASK] = TU104_RC_SEC2, + [EI_RESET] = 19, + [EI_INTR] = 9, + [EI_MC] = TU104_MC_ENGINE_SEC2, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_SEC, + [EI_INSTANCE_ID] = 0, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {9, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 9, 0}, + .numPbdmas = 1, + .engineName = "SEC2", + }, + /* [10] NVJPG */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_NVJPG, + [EI_FIFO_TAG] = TU104_TAG_NVJPG, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_NVJPG0, + [EI_RUNLIST] = 10, + [EI_MMU_FAULT_ID] = TU104_FAULT_ID_NVJPG0, + [EI_RC_MASK] = TU104_RC_NVJPG0, + [EI_RESET] = 20, + [EI_INTR] = 10, + [EI_MC] = TU104_MC_ENGINE_NVJPG, + [EI_DEV_TYPE_ENUM] = TU104_PTOP_TYPE_NVJPG, + [EI_INSTANCE_ID] = 0, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 1, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {10, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 10, 0}, + .numPbdmas = 1, + .engineName = "NVJPG", + }, + /* [11] SW (software engine - shares NVJPG's runlist per fifoConstructSwEngine) */ + { + .engineData = { + [EI_ENG_DESC] = TU104_ENG_SW, + [EI_FIFO_TAG] = 0xffffffff, + [EI_RM_ENGINE_TYPE] = TU104_RM_ENGINE_SW, + [EI_RUNLIST] = 10, /* NVJPG's runlist */ + [EI_MMU_FAULT_ID] = 0xffffffff, + [EI_RC_MASK] = 0, + [EI_RESET] = 0, + [EI_INTR] = 0, + [EI_MC] = 0, + [EI_DEV_TYPE_ENUM] = 0, + [EI_INSTANCE_ID] = 0, + [EI_RUNLIST_PRI_BASE] = 0, + [EI_IS_HOST_DRIVEN_ENGINE] = 0, + [EI_RUNLIST_ENGINE_ID] = 0, + [EI_CHRAM_PRI_BASE] = 0, + [EI_KERNEL_RM_MAX] = 0, + }, + .pbdmaIds = {10, 0}, + .pbdmaFaultIds = {TU104_PBDMA_FAULT_ID_BASE + 10, 0}, + .numPbdmas = 1, + .engineName = "SOFTWARE", + }, +}; + +#define TU104_ENGINE_TABLE_SIZE 12 + +/* ============================================================ + * TU104 Interrupt Table (from intr_phys_tu102.c, kernel table) + * + * These are the CPU-RM (guest) visible interrupts. + * NV_PMC_INTR_INVALID_MASK = 0 (no PMC mask for these). + * NV_INTR_VECTOR_INVALID = 0xFFFFFFFF. + * ============================================================ */ + +#define TU104_NV_INTR_VECTOR_INVALID 0xFFFFFFFF + +static const struct tu104_intr_table_entry tu104_intr_kernel_table[] = { + /* MC_ENGINE_IDX_REPLAYABLE_FAULT = 59 */ + { .engineIdx = 59, .pmcIntrMask = 0, + .vectorStall = 64, /* NV_PFB_PRI_MMU_INT_VECTOR_FAULT_NOTIFY_REPLAYABLE */ + .vectorNonStall = TU104_NV_INTR_VECTOR_INVALID }, + /* MC_ENGINE_IDX_REPLAYABLE_FAULT_ERROR = 62 */ + { .engineIdx = 62, .pmcIntrMask = 0, + .vectorStall = 131, /* NV_PFB_PRI_MMU_INT_VECTOR_FAULT_ERROR_REPLAYABLE */ + .vectorNonStall = TU104_NV_INTR_VECTOR_INVALID }, + /* MC_ENGINE_IDX_ACCESS_CNTR = 60 */ + { .engineIdx = 60, .pmcIntrMask = 0, + .vectorStall = 72, /* HUB_ACCESS_CNTR_INTR_VECTOR */ + .vectorNonStall = TU104_NV_INTR_VECTOR_INVALID }, + /* MC_ENGINE_IDX_CPU_DOORBELL = 73 */ + { .engineIdx = 73, .pmcIntrMask = 0, + .vectorStall = 129, /* NV_CTRL_CPU_DOORBELL_VECTORID_VALUE_CONSTANT */ + .vectorNonStall = TU104_NV_INTR_VECTOR_INVALID }, + /* MC_ENGINE_IDX_PRIV_DOORBELL = 74 */ + { .engineIdx = 74, .pmcIntrMask = 0, + .vectorStall = 130, /* NV_CTRL_VIRTUAL_INTR_VECTORID_VALUE_CONSTANT */ + .vectorNonStall = TU104_NV_INTR_VECTOR_INVALID }, + /* MC_ENGINE_IDX_GR_FECS_LOG = 156 */ + { .engineIdx = 156, .pmcIntrMask = 0, + .vectorStall = TU104_NV_INTR_VECTOR_INVALID, + .vectorNonStall = TU104_NV_INTR_VECTOR_INVALID }, + /* MC_ENGINE_IDX_GSP = 50 */ + { .engineIdx = 50, .pmcIntrMask = 0, + .vectorStall = 155, /* GSP_INTR_0_VECTOR */ + .vectorNonStall = TU104_NV_INTR_VECTOR_INVALID }, + /* MC_ENGINE_IDX_DISP = 2 */ + { .engineIdx = 2, .pmcIntrMask = 0, + .vectorStall = 154, /* PDISP_VECTOR */ + .vectorNonStall = TU104_NV_INTR_VECTOR_INVALID }, +}; + +#define TU104_INTR_KERNEL_TABLE_SIZE 8 + +/* ============================================================ + * TU104 Constructed Falcon Table + * + * Only FBFLCN (framebuffer falcon) is a "constructed falcon" on Turing + * (SEC2, GSP, PMU are skipped as they have their own kernel objects). + * ============================================================ */ + +static const struct tu104_constructed_falcon_info tu104_falcon_table[] = { + { + .engDesc = TU104_ENG_FBFLCN, + .ctxAttr = 0, + .ctxBufferSize = 0x400, /* 1KB context buffer */ + .addrSpaceList = 0, + .registerBase = 0x009a4000, /* NV_PFBFALCON base */ + }, +}; + +#define TU104_FALCON_TABLE_SIZE 1 + +/* ============================================================ + * TU104 Internal Device Info Table + * + * Engines + key non-engine devices. The guest driver queries this + * by typeEnum to find device register bases and fault IDs. + * ============================================================ */ + +static const struct tu104_internal_device_info tu104_internal_device_table[] = { + /* Engines (isEngine=1) */ + { .faultId = TU104_FAULT_ID_GR, .instanceId = 0, .typeEnum = TU104_PTOP_TYPE_GRAPHICS, .resetId = 12, .devicePriBase = 0x00400000, .isEngine = 1, .rlEngId = 0, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 0 }, + { .faultId = TU104_FAULT_ID_CE0, .instanceId = 0, .typeEnum = TU104_PTOP_TYPE_LCE, .resetId = 1, .devicePriBase = 0x00040000, .isEngine = 1, .rlEngId = 1, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 0 }, + { .faultId = TU104_FAULT_ID_CE1, .instanceId = 1, .typeEnum = TU104_PTOP_TYPE_LCE, .resetId = 2, .devicePriBase = 0x00041000, .isEngine = 1, .rlEngId = 2, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 1 }, + { .faultId = TU104_FAULT_ID_CE2, .instanceId = 2, .typeEnum = TU104_PTOP_TYPE_LCE, .resetId = 3, .devicePriBase = 0x00042000, .isEngine = 1, .rlEngId = 3, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 2 }, + { .faultId = TU104_FAULT_ID_CE3, .instanceId = 3, .typeEnum = TU104_PTOP_TYPE_LCE, .resetId = 4, .devicePriBase = 0x00043000, .isEngine = 1, .rlEngId = 4, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 3 }, + { .faultId = TU104_FAULT_ID_CE4, .instanceId = 4, .typeEnum = TU104_PTOP_TYPE_LCE, .resetId = 5, .devicePriBase = 0x00044000, .isEngine = 1, .rlEngId = 5, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 4 }, + { .faultId = TU104_FAULT_ID_NVDEC0, .instanceId = 0, .typeEnum = TU104_PTOP_TYPE_NVDEC, .resetId = 14, .devicePriBase = 0x00060000, .isEngine = 1, .rlEngId = 6, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 0 }, + { .faultId = TU104_FAULT_ID_NVDEC1, .instanceId = 1, .typeEnum = TU104_PTOP_TYPE_NVDEC, .resetId = 15, .devicePriBase = 0x00061000, .isEngine = 1, .rlEngId = 7, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 1 }, + { .faultId = TU104_FAULT_ID_NVENC0, .instanceId = 0, .typeEnum = TU104_PTOP_TYPE_NVENC, .resetId = 17, .devicePriBase = 0x00070000, .isEngine = 1, .rlEngId = 8, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 0 }, + { .faultId = TU104_FAULT_ID_SEC2, .instanceId = 0, .typeEnum = TU104_PTOP_TYPE_SEC, .resetId = 19, .devicePriBase = 0x00840000, .isEngine = 1, .rlEngId = 9, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 0 }, + { .faultId = TU104_FAULT_ID_NVJPG0, .instanceId = 0, .typeEnum = TU104_PTOP_TYPE_NVJPG, .resetId = 20, .devicePriBase = 0x00080000, .isEngine = 1, .rlEngId = 10, .runlistPriBase = 0, .groupId = 0, .ginTargetId = 0, .deviceBroadcastPriBase = 0, .groupLocalInstanceId = 0 }, +}; + +#define TU104_INTERNAL_DEVICE_TABLE_SIZE 11 + +/* ============================================================ + * GspStaticConfigInfo field offsets (ABI-verified, 1656 bytes total) + * ============================================================ */ +#define TU104_GSCI_OFF_grCapsBits 0 /* 23 bytes */ +#define TU104_GSCI_OFF_gidInfo 24 /* 268 bytes */ +#define TU104_GSCI_OFF_SKUInfo 292 /* 48 bytes */ +#define TU104_GSCI_OFF_fbRegionInfoParams 344 /* 776 bytes */ +#define TU104_GSCI_OFF_sriovCaps 1120 /* 80 bytes */ +#define TU104_GSCI_OFF_sriovMaxGfid 1200 /* 4 bytes */ +#define TU104_GSCI_OFF_engineCaps 1204 /* 12 bytes */ +#define TU104_GSCI_OFF_poisonFuseEnabled 1216 /* 1 byte */ +#define TU104_GSCI_OFF_fb_length 1224 /* 8 bytes */ +#define TU104_GSCI_OFF_fbio_mask 1232 /* 8 bytes */ +#define TU104_GSCI_OFF_fb_bus_width 1240 /* 4 bytes */ +#define TU104_GSCI_OFF_fb_ram_type 1244 /* 4 bytes */ +#define TU104_GSCI_OFF_fbp_mask 1248 /* 8 bytes */ +#define TU104_GSCI_OFF_l2_cache_size 1256 /* 4 bytes */ +#define TU104_GSCI_OFF_gpuNameString 1260 /* 64 bytes */ +#define TU104_GSCI_OFF_gpuShortNameString 1324 /* 64 bytes */ +#define TU104_GSCI_OFF_gpuNameString_Unicode 1388 /* 128 bytes */ +#define TU104_GSCI_OFF_bIsMobile 1524 /* 1 byte */ +#define TU104_GSCI_OFF_bar1PdeBase 1536 /* 8 bytes */ +#define TU104_GSCI_OFF_bar2PdeBase 1544 /* 8 bytes */ +#define TU104_GSCI_OFF_bVbiosValid 1552 /* 1 byte */ +#define TU104_GSCI_OFF_vbiosSubVendor 1556 /* 4 bytes */ +#define TU104_GSCI_OFF_vbiosSubDevice 1560 /* 4 bytes */ +#define TU104_GSCI_OFF_hInternalClient 1600 /* 4 bytes */ +#define TU104_GSCI_OFF_hInternalDevice 1604 /* 4 bytes */ +#define TU104_GSCI_OFF_hInternalSubdevice 1608 /* 4 bytes */ +#define TU104_GSCI_OFF_ecidInfo 1616 /* 24 bytes */ +#define TU104_GSCI_OFF_fwWprLayoutOffset 1640 /* 16 bytes */ +#define TU104_GSCI_SIZE 1656 + +/* Internal handle values (g_gpu_nvoc.h, g_resserv_nvoc.h) */ +#define TU104_H_INTERNAL_CLIENT 0xC1E00000 +#define TU104_H_INTERNAL_DEVICE 0xABCD0080 +#define TU104_H_INTERNAL_SUBDEVICE 0xABCD2080 + +/* GSP_RM_CONTROL sub-commands */ +#define TU104_CTRL_INTERNAL_GET_DEVICE_INFO_TABLE 0x20800a40 +#define TU104_CTRL_FIFO_GET_DEVICE_INFO_TABLE 0x20801112 +#define TU104_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE 0x20800a5c +#define TU104_CTRL_GPU_GET_CONSTRUCTED_FALCON_INFO 0x208001b0 +#define TU104_CTRL_INTERNAL_FIFO_GET_NUM_CHANNELS 0x20800a61 +#define TU104_CTRL_BUS_GET_PCI_BAR_INFO 0x20801803 +#define TU104_CTRL_INTERNAL_DISPLAY_GET_STATIC_INFO 0x20800a01 +#define TU104_CTRL_INTERNAL_DISPLAY_GET_IP_VERSION 0x20800a4b +#define TU104_CTRL_I2C_TRANSACTION 0x402c0105 +#define TU104_CTRL_CE_GET_FAULT_METHOD_BUFFER_SIZE 0x20802a08 +#define TU104_CTRL_CE_GET_PHYSICAL_CAPS 0x20802a07 +#define TU104_CTRL_CE_GET_ALL_PHYSICAL_CAPS 0x20802a0b +#define TU104_CTRL_CE_GET_ALL_CAPS 0x20802a0a +#define TU104_CTRL_CE_GET_CE_PCE_MASK 0x20802a02 + +/* CE capability bits (NV2080_CTRL_CE_CAPS_*, ctrl2080ce.h: byte:bit) */ +#define TU104_CE_CAPS0_SYSMEM_READ 0x04 +#define TU104_CE_CAPS0_SYSMEM_WRITE 0x08 +#define TU104_CE_CAPS0_SYSMEM 0x20 +#define TU104_CE_CAPS0_P2P 0x40 +#define TU104_CE_CAPS0_BL_GT_64K 0x80 +#define TU104_CE_CAPS0 (TU104_CE_CAPS0_SYSMEM_READ | \ + TU104_CE_CAPS0_SYSMEM_WRITE | \ + TU104_CE_CAPS0_SYSMEM | \ + TU104_CE_CAPS0_P2P | \ + TU104_CE_CAPS0_BL_GT_64K) +#define TU104_CE_CAPS1 0x00 + +/* ENGINE_TYPE_COPY0 = 0x9, COPY(i) = COPY0 + i (cl2080_notification.h) */ +#define TU104_ENGINE_TYPE_COPY0 0x00000009 +#define TU104_NUM_CES 5 +#define TU104_CTRL_INTERNAL_GMMU_GET_STATIC_INFO 0x20800a59 +#define TU104_CTRL_STATIC_KGR_GET_GLOBAL_SM_ORDER 0x20800a22 +#define TU104_CTRL_STATIC_KGR_GET_INFO 0x20800a2a +#define TU104_CTRL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO 0x20800a32 + +/* NV2080_CTRL_INTERNAL_ENGINE_CONTEXT_BUFFER_INFO (engine context buffer info) + * NV2080_CTRL_INTERNAL_STATIC_GR_CONTEXT_BUFFERS_INFO: engine[ENGINE_ID_COUNT] + * NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS: + * engineContextBuffersInfo[GR_MAX_ENGINES] + * + * The driver's kgraphicsInitializeDeferredStaticData() asks the physical RM + * (GSP-RM) for this table; kgrctxAllocMainCtxBuffer() then sizes the golden + * image ctx buffer from engine[GRAPHICS].size. A zeroed reply produces a + * 0-byte (0-page) allocation -> NV_ERR_NO_MEMORY (mem_desc.c:1353). + * + * The physical RM probes these sizes from the ctxsw ucode at GR init + * (grSetupInitialCtxState -> grDiscoverCtxBufferSize) and reports + * NV_U32_MAX for engine IDs the chip does not support. The values below are + * the emulated TU104 layout (own buffers, no real GR engine behind them): + * sizes are page-aligned scratch regions, NV_U32_MAX = "not supported". */ +#define TU104_CTX_ENGINE_ID_COUNT 26 /* _ENGINE_ID_COUNT (0x1a) */ +#define TU104_CTX_GPU_MAX_ENGINES 8 /* INTERNAL_GR_MAX_ENGINES */ + +/* NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_* */ +#define TU104_CTX_ENG_GRAPHICS 0x00 +#define TU104_CTX_ENG_VLD 0x01 +#define TU104_CTX_ENG_VIDEO 0x02 +#define TU104_CTX_ENG_MPEG 0x03 +#define TU104_CTX_ENG_CAPTURE 0x04 +#define TU104_CTX_ENG_DISPLAY 0x05 +#define TU104_CTX_ENG_ENCRYPTION 0x06 +#define TU104_CTX_ENG_POSTPROCESS 0x07 +#define TU104_CTX_ENG_GRAPHICS_ZCULL 0x08 +#define TU104_CTX_ENG_GRAPHICS_PM 0x09 +#define TU104_CTX_ENG_COMPUTE_PREEMPT 0x0a +#define TU104_CTX_ENG_GRAPHICS_PREEMPT 0x0b +#define TU104_CTX_ENG_GRAPHICS_SPILL 0x0c +#define TU104_CTX_ENG_GRAPHICS_PAGEPOOL 0x0d +#define TU104_CTX_ENG_GRAPHICS_BETACB 0x0e +#define TU104_CTX_ENG_GRAPHICS_RTV 0x0f +#define TU104_CTX_ENG_GRAPHICS_PATCH 0x10 +#define TU104_CTX_ENG_GRAPHICS_BUNDLE_CB 0x11 +#define TU104_CTX_ENG_GRAPHICS_PAGEPOOL_GLOBAL 0x12 +#define TU104_CTX_ENG_GRAPHICS_ATTRIBUTE_CB 0x13 +#define TU104_CTX_ENG_GRAPHICS_RTV_CB_GLOBAL 0x14 +#define TU104_CTX_ENG_GRAPHICS_GFXP_POOL 0x15 +#define TU104_CTX_ENG_GRAPHICS_GFXP_CTRL_BLK 0x16 +#define TU104_CTX_ENG_GRAPHICS_FECS_EVENT 0x17 +#define TU104_CTX_ENG_GRAPHICS_PRIV_ACCESS_MAP 0x18 +#define TU104_CTX_ENG_GRAPHICS_SETUP 0x19 + +/* grabbag: emulated ctx buffer sizes (bytes) */ +#define TU104_CTX_SIZE_MAIN 0x200000 +#define TU104_CTX_SIZE_ZCULL 0x100000 +#define TU104_CTX_SIZE_PM 0x8000 +#define TU104_CTX_SIZE_PREEMPT 0x1000 +#define TU104_CTX_SIZE_SPILL 0x1000 +#define TU104_CTX_SIZE_PAGEPOOL 0x10000 +#define TU104_CTX_SIZE_BETACB 0x1000 +#define TU104_CTX_SIZE_RTV 0x1000 +#define TU104_CTX_SIZE_PATCH 0x2000 +#define TU104_CTX_SIZE_BUNDLE_CB 0x1000 +#define TU104_CTX_SIZE_ATTRIBUTE_CB 0x1000 +#define TU104_CTX_SIZE_RTV_CB_GLOBAL 0x1000 +#define TU104_CTX_SIZE_FECS_EVENT 0x1000 +#define TU104_CTX_SIZE_PRIV_ACCESS_MAP 0x1000 + +struct tu104_ctx_buffer_info { + u32 size; + u32 alignment; +}; + +struct tu104_gr_context_buffers_info { + struct tu104_ctx_buffer_info engine[TU104_CTX_ENGINE_ID_COUNT]; +}; + +struct tu104_gr_get_context_buffers_info_params { + struct tu104_gr_context_buffers_info + engineContextBuffersInfo[TU104_CTX_GPU_MAX_ENGINES]; +}; + +/* FB_GET_INFO_V2 (ctrl2080fb.h): the client sends a list of requested FB info + * indices; the physical RM must echo each index back with its data. + * _fbGetFbInfos() asserts pFbInfos[i].index == returned index, so a zeroed + * reply fires kern_mem_sys_ctrl.c:911 asserts and leaves all FB info zero + * (CUDA/UVMLite read total/usable memory, BAR1 size, ECC/LTC info from here). */ +#define TU104_CTRL_FB_GET_INFO_V2 0x20801303 +#define TU104_FB_INFO_MAX_LIST_SIZE 0x80 + +struct tu104_fb_info_entry { + u32 index; + u32 data; +}; + +struct tu104_fb_get_info_v2_params { + u32 fbInfoListSize; + struct tu104_fb_info_entry fbInfoList[TU104_FB_INFO_MAX_LIST_SIZE]; +}; + +/* NV2080_CTRL_FB_INFO_INDEX_* (subset we populate) */ +#define TU104_FB_INFO_IDX_TILE_REGION_COUNT 0x00 +#define TU104_FB_INFO_IDX_COMPRESSION_SIZE 0x01 +#define TU104_FB_INFO_IDX_PARTITION_COUNT 0x04 +#define TU104_FB_INFO_IDX_BAR1_SIZE 0x05 +#define TU104_FB_INFO_IDX_RAM_SIZE 0x07 +#define TU104_FB_INFO_IDX_TOTAL_RAM_SIZE 0x08 +#define TU104_FB_INFO_IDX_HEAP_SIZE 0x09 +#define TU104_FB_INFO_IDX_MAPPABLE_HEAP_SIZE 0x0a +#define TU104_FB_INFO_IDX_BUS_WIDTH 0x0b +#define TU104_FB_INFO_IDX_RAM_TYPE 0x0d +#define TU104_FB_INFO_IDX_BANK_COUNT 0x0e +#define TU104_FB_INFO_IDX_FB_TAX_SIZE_KB 0x10 +#define TU104_FB_INFO_IDX_LARGEST_FREE_REGION_SIZE_KB 0x12 +#define TU104_FB_INFO_IDX_PARTITION_MASK 0x14 +#define TU104_FB_INFO_IDX_HEAP_FREE 0x16 +#define TU104_FB_INFO_IDX_RAM_LOCATION 0x17 +#define TU104_FB_INFO_IDX_FBP_COUNT 0x19 +#define TU104_FB_INFO_IDX_FBP_MASK 0x1a +#define TU104_FB_INFO_IDX_L2CACHE_SIZE 0x1b +#define TU104_FB_INFO_IDX_MEMORYINFO_VENDOR_ID 0x1c +#define TU104_FB_INFO_IDX_BAR1_AVAIL_SIZE 0x1d +#define TU104_FB_INFO_IDX_BAR1_MAX_CONTIG_AVAIL_SIZE 0x1f +#define TU104_FB_INFO_IDX_USABLE_RAM_SIZE 0x20 +#define TU104_FB_INFO_IDX_LTC_COUNT 0x22 +#define TU104_FB_INFO_IDX_LTS_COUNT 0x23 +#define TU104_FB_INFO_IDX_PSEUDO_CHANNEL_MODE 0x25 +#define TU104_FB_INFO_IDX_ALLOW_PAGE_RETIREMENT 0x2a +#define TU104_FB_INFO_IDX_LTC_MASK 0x2b + +#define TU104_RAM_TYPE_GDDR6 0x11 +#define TU104_RAM_LOCATION_GPU_DEDICATED 0x00 +#define TU104_PSEUDO_CHANNEL_MODE_DISABLED 0x01 +#define TU104_MEMORYINFO_VENDOR_ID_SAMSUNG 0x01 +#define TU104_L2CACHE_SIZE 0x400000 /* TU104: 4MB */ +#define TU104_LTC_COUNT 8 +#define TU104_LTS_COUNT 16 + +/* BUS_GET_INFO_V2 (ctrl2080bus.h) - same flat index/data list pattern as FB. + * UVM's gpuGetSysmemLinkInfo() reads SYSMEM_CONNECTION_TYPE (PCIE) and then + * PCIE_GPU_LINK_CAPS for the link bandwidth; a zeroed reply makes + * getPCIELinkRateMBps() fail with NV_ERR_INVALID_STATE. */ +#define TU104_CTRL_BUS_GET_INFO_V2 0x20801823 +#define TU104_BUS_INFO_MAX_LIST_SIZE 0x34 + +struct tu104_bus_info_entry { + u32 index; + u32 data; +}; + +struct tu104_bus_get_info_v2_params { + u32 busInfoListSize; + struct tu104_bus_info_entry busInfoList[TU104_BUS_INFO_MAX_LIST_SIZE]; +}; + +/* NV2080_CTRL_BUS_INFO_INDEX_* (subset we populate) */ +#define TU104_BUS_INFO_IDX_TYPE 0x00 +#define TU104_BUS_INFO_IDX_INTLINE 0x01 +#define TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CAPS 0x03 +#define TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CAPS 0x04 +#define TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CAPS 0x05 +#define TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CAPS 0x06 +#define TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CTRL_STATUS 0x07 +#define TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CTRL_STATUS 0x08 +#define TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CTRL 0x09 +#define TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CTRL 0x0a +#define TU104_BUS_INFO_IDX_PCIE_GEN_INFO 0x2d +#define TU104_BUS_INFO_IDX_SYSMEM_CONNECTION_TYPE 0x33 + +#define TU104_BUS_INFO_TYPE_PCI_EXPRESS 0x03 +#define TU104_BUS_INFO_SYSMEM_CONN_PCIE 0x00 +/* PCIE link caps / GEN info layout: MAX_SPEED 3:0 (8000MBPS=3), MAX_WIDTH + * 9:4 (lanes), ASPM 11:10, GEN 15:12, CURR_LEVEL 19:16, GPU_GEN 23:20 + * (all GENx = 2 for Gen3), SPEED_CHANGES 24. */ +#define TU104_PCIE_LINK_CAP_GEN3_X16 (0x3u | (16u << 4) | \ + (2u << 12) | (2u << 16) | \ + (2u << 20)) +/* LINK_CTRL_STATUS: ASPM 1:0 (0=disabled), LINK_SPEED 19:16, + * LINK_WIDTH 25:20 (LINK_WIDTH_X16=0x10). */ +#define TU104_PCIE_LINK_CTRL_STATUS_GEN3_X16 ((0x3u << 16) | (0x10u << 20)) + + +/* NV0080_CTRL_GR_INFO entry + infoList indices (ctrl0080gr.h) */ +struct tu104_gr_info_entry { + u32 index; + u32 data; +}; +#define TU104_GR_INFO_MAX_SIZE 58 /* NV0080_CTRL_GR_INFO_MAX_SIZE (0x3a) */ +#define TU104_GR_INFO_MAX_ENGINES 8 + +struct tu104_gr_static_info { + struct tu104_gr_info_entry infoList[TU104_GR_INFO_MAX_SIZE]; +}; + +/* infoList indices we populate (ctrl0080gr.h) */ +#define TU104_GR_INFO_IDX_LITTER_NUM_GPCS 0x14 +#define TU104_GR_INFO_IDX_LITTER_NUM_TPC_PER_GPC 0x17 +#define TU104_GR_INFO_IDX_GPU_CORE_COUNT 0x1d +#define TU104_GR_INFO_IDX_LITTER_NUM_SM_PER_TPC 0x20 +#define TU104_GR_INFO_IDX_FAMILY_MAX_TPC_PER_GPC 0x2a +#define TU104_GR_INFO_IDX_MAX_SUBCONTEXT_COUNT 0x2c +#define TU104_GR_INFO_IDX_MAX_LEGACY_SUBCONTEXT_COUNT 0x2d +#define TU104_GR_INFO_IDX_MAX_PER_ENGINE_SUBCONTEXT_COUNT 0x2e + +/* TU104 max subcontexts (NV_PGRAPH_PRI_FE_CHIP_DEF_INFO_MAX_VEID_COUNT_INIT) */ +#define TU104_MAX_SUBCONTEXT_COUNT 64 + +/* NV2080_CTRL_INTERNAL_STATIC_GR_FLOORSWEEPING_MASKS (ctrl2080internal.h) + * + STATIC_KGR_GET_FLOORSWEEPING_MASKS cmd = 0x20800a26. + * Driver computes numGpcs from floorsweepingMasks.gpcMask - zero kills + * krcWatchdogInit ("numGpcs > 0" assert -> RmInitAdapter bails). */ +#define TU104_CTRL_STATIC_KGR_GET_FLOORSWEEPING_MASKS 0x20800a26 +#define TU104_GR_MAX_GPC 12 +#define TU104_GR_MAX_TPC_PER_GPC_COUNT 10 + +struct tu104_gr_floorsweeping_masks { + u32 gpcMask; + u32 tpcMask[TU104_GR_MAX_GPC]; + u32 tpcCount[TU104_GR_MAX_GPC]; + u32 physGpcMask; + u32 mmuPerGpc[TU104_GR_MAX_GPC]; + u32 tpcToPesMap[TU104_GR_MAX_TPC_PER_GPC_COUNT]; + u32 numPesPerGpc[TU104_GR_MAX_GPC]; + u32 zcullMask[TU104_GR_MAX_GPC]; + u32 physGfxGpcMask; + u32 numGfxTpc; +}; + +/* TU104 SM topology: 6 GPCs x 4 TPCs x 2 SMs = 48 SMs, 24 TPCs (RTX 2080 SUPER) */ +#define TU104_NUM_GPC 6 +#define TU104_TPC_PER_GPC 4 +#define TU104_SM_PER_TPC 2 +#define TU104_NUM_SM (TU104_NUM_GPC * TU104_TPC_PER_GPC * TU104_SM_PER_TPC) +#define TU104_NUM_TPC (TU104_NUM_GPC * TU104_TPC_PER_GPC) + +/* NV2080_CTRL_INTERNAL_STATIC_GR_GLOBAL_SM_ORDER (ctrl2080internal.h) */ +#define TU104_GR_MAX_SM 240 +#define TU104_GR_MAX_ENGINES 8 + +struct tu104_gr_sm_id { + u16 gpcId; + u16 localTpcId; + u16 localSmId; + u16 globalTpcId; + u16 virtualGpcId; + u16 migratableTpcId; +}; + +struct tu104_gr_global_sm_order { + struct tu104_gr_sm_id globalSmId[TU104_GR_MAX_SM]; + u16 numSm; + u16 numTpc; +}; + +/* CE fault method buffer size (ceGetFaultMethodBufferSize_GV100 formula, + * Turing inherits: 135 * (13*totalCes + 2), page-aligned; TU104 has 5 PCEs) */ +#define TU104_CE_FAULT_METHOD_BUFFER_SIZE 12288 + +/* GMMU static info (kgmmuSetAndGetDefaultFaultBufferSize_TU102 semantics): + * the hardware default is reported in NV_VIRTUAL_FUNCTION_PRIV_MMU_FAULT_ + * BUFFER_SIZE_VAL units of NVC369_BUF_SIZE (32B). UVM's faultbufConstruct + * allocates replayableFaultBufferSize and fails with NV_ERR_NO_MEMORY when + * it reads back 0. Turing defaults are 64KB replayable / 4KB legacy; both + * must stay within NV2080_CTRL_INTERNAL_GMMU_FAULT_BUFFER_MAX_PAGES (256). */ +#define TU104_GMMU_REPLAYABLE_FAULT_BUFFER_SIZE 0x10000 +#define TU104_GMMU_NONREPLAYABLE_FAULT_BUFFER_SIZE 0x1000 + +/* Turing display IP version (DISP v04.00 per DispIpHal halspec) */ +#define TU104_DISPLAY_IP_VERSION 0x04000000 + +/* Channels per runlist reported to the guest (consumer Turing convention) */ +#define TU104_FIFO_NUM_CHANNELS_PER_RUNLIST 64 + +/* FB region info params: numFBRegions(4) + 16 × 48B entries = 772 → aligned 776 */ +struct tu104_fb_region { + u64 base; + u64 limit; + u64 reserved; + u32 performance; + u8 supportCompressed; + u8 supportISO; + u8 bProtected; + u8 blackList[17]; + u8 _pad[6]; +}; + +#endif /* _GSP_RM_TU104_H_ */ diff --git a/kernel/gsp_rpc_client.c b/kernel/gsp_rpc_client.c new file mode 100644 index 0000000..eb3d85d --- /dev/null +++ b/kernel/gsp_rpc_client.c @@ -0,0 +1,1174 @@ +/* + * GSP RPC Client for vGPU-IO + * Based on GSP 570.144W firmware decompilation + * Firmware base: 0x04000000 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "memory.h" +#include "gsp_rpc_client.h" + +/* ============================================================ + * FIRMWARE ADDRESSES (from decompilation) + * ============================================================ */ + +#define GSP_FW_BASE 0x04000000UL + +/* Primary RPC Dispatcher */ +#define GSP_RPC_DISPATCH_PRIMARY (GSP_FW_BASE + 0x17A1AC0) /* 0x057a1ac0 */ +#define GSP_RPC_JUMP_TABLE (GSP_FW_BASE + 0x17A3AD8) /* 0x057a3ad8 - 32 entries */ + +/* Secondary RPC Dispatcher */ +#define GSP_RPC_DISPATCH_SECONDARY (GSP_FW_BASE + 0x0ED39DC) /* 0x04ed39dc */ + +/* Specialized Handlers */ +#define GSP_ALLOC_MAIN (GSP_FW_BASE + 0x17E94EC) /* 0x057e94ec */ +#define GSP_SCHEDULE_MAIN (GSP_FW_BASE + 0x11EFBF4) /* 0x051efbf4 */ +#define GSP_BAR_TRAP_HANDLER_JT (GSP_FW_BASE + 0x17675B7) /* 0x057675b7 */ +#define GSP_BAR_TRAP_HANDLER_PRIM (GSP_FW_BASE + 0x0CB8189) /* 0x04cb8189 */ +#define GSP_PIN_HANDLER_SEC (GSP_FW_BASE + 0x0ED34B8) /* 0x04ed34b8 */ + +/* Jump Table Handlers (from 0x059bfeb0) */ +#define GSP_MAP_HANDLER_A (GSP_FW_BASE + 0x1766AB7) /* 0x05766ab7 */ +#define GSP_MAP_HANDLER_B (GSP_FW_BASE + 0x1778A37) /* 0x05778a37 */ +#define GSP_UNMAP_HANDLER_A (GSP_FW_BASE + 0x17669B7) /* 0x057669b7 */ +#define GSP_UNMAP_HANDLER_B (GSP_FW_BASE + 0x1779937) /* 0x05779937 */ +#define GSP_PIN_HANDLER_A (GSP_FW_BASE + 0x17664B7) /* 0x057664b7 */ +#define GSP_PIN_HANDLER_B (GSP_FW_BASE + 0x177A3B7) /* 0x0577a3b7 */ +#define GSP_UNPIN_HANDLER_A (GSP_FW_BASE + 0x17682B7) /* 0x057682b7 */ +#define GSP_UNPIN_HANDLER_B (GSP_FW_BASE + 0x1779FB7) /* 0x05779fb7 */ +#define GSP_SCHEDULE_HANDLER_A (GSP_FW_BASE + 0x1767F37) /* 0x05767f37 */ +#define GSP_SCHEDULE_HANDLER_B (GSP_FW_BASE + 0x175AEB7) /* 0x0575aeb7 */ +#define GSP_YIELD_HANDLER_A (GSP_FW_BASE + 0x175AE37) /* 0x0575ae37 */ +#define GSP_YIELD_HANDLER_B (GSP_FW_BASE + 0x1767337) /* 0x05767337 */ +#define GSP_NOTIFY_HANDLER_A (GSP_FW_BASE + 0x177C8B7) /* 0x0577c8b7 */ +#define GSP_NOTIFY_HANDLER_B (GSP_FW_BASE + 0x1767837) /* 0x05767837 */ + +/* Entry point */ +#define GSP_ENTRY_POINT (GSP_FW_BASE + 0x0009000) /* 0x04009000 */ + +/* ============================================================ + * RPC PROTOCOL DEFINITIONS + * ============================================================ */ + +#define GSP_RPC_ALLOC 0x01 +#define GSP_RPC_FREE 0x02 +#define GSP_RPC_MAP 0x03 +#define GSP_RPC_UNMAP 0x04 +#define GSP_RPC_PIN 0x05 +#define GSP_RPC_UNPIN 0x06 +#define GSP_RPC_SCHEDULE 0x07 +#define GSP_RPC_YIELD 0x08 +#define GSP_RPC_NOTIFY 0x09 +#define GSP_RPC_BAR_TRAP 0x0A +#define GSP_RPC_MIGRATE 0x0B +#define GSP_RPC_DEBUG 0xFF + +/* ALLOC sub-commands (primary dispatcher jump table indices) */ +#define GSP_ALLOC_SUBCMD_DEFAULT 0 +#define GSP_ALLOC_SUBCMD_TYPE1 1 +#define GSP_ALLOC_SUBCMD_TYPE2 3 +#define GSP_ALLOC_SUBCMD_TYPE3 4 +#define GSP_ALLOC_SUBCMD_TYPE4 5 +#define GSP_ALLOC_SUBCMD_TYPE5 8 +#define GSP_ALLOC_SUBCMD_TYPE6 10 +#define GSP_ALLOC_SUBCMD_TYPE7 12 +#define GSP_ALLOC_SUBCMD_TYPE8 17 +#define GSP_ALLOC_SUBCMD_TYPE9 19 +#define GSP_ALLOC_SUBCMD_TYPE10 20 +#define GSP_ALLOC_SUBCMD_TYPE11 21 +#define GSP_ALLOC_SUBCMD_TYPE12 22 +#define GSP_ALLOC_SUBCMD_TYPE13 23 +#define GSP_ALLOC_SUBCMD_TYPE14 24 +#define GSP_ALLOC_SUBCMD_TYPE15 25 +#define GSP_ALLOC_SUBCMD_TYPE16 26 +#define GSP_ALLOC_SUBCMD_TYPE17 27 +#define GSP_ALLOC_SUBCMD_TYPE18 28 +#define GSP_ALLOC_SUBCMD_TYPE19 29 +#define GSP_ALLOC_SUBCMD_TYPE20 30 +#define GSP_ALLOC_SUBCMD_TYPE21 31 + +/* MIGRATE sub-commands */ +#define GSP_MIGRATE_SUBCMD_SAVE 0x04 /* From primary dispatcher branch */ +#define GSP_MIGRATE_SUBCMD_RESTORE 0x05 + +/* PIN sub-commands */ +#define GSP_PIN_SUBCMD_DEFAULT 0x05 /* From secondary dispatcher */ + +/* SCHEDULE sub-commands */ +#define GSP_SCHED_SUBCMD_DEFAULT 0x07 + +/* BAR TRAP sub-commands */ +#define GSP_BAR_TRAP_SUBCMD_DEFAULT 0x0A + +/* ============================================================ + * RPC MESSAGE STRUCTURES + * ============================================================ */ + +/* ============================================================ + * GSP RPC CLIENT CONTEXT + * ============================================================ */ + +struct gsp_client { + struct device *dev; + void __iomem *gsp_fw_base; /* Mapped firmware base */ + void __iomem *rpc_doorbell; /* Doorbell register */ + spinlock_t lock; /* Submission lock */ + struct mutex rpc_mutex; /* RPC serialization */ + atomic_t seq_counter; /* Sequence ID counter */ + struct list_head pending_rpcs; /* Pending RPC list */ + wait_queue_head_t resp_wq; /* Response wait queue */ + struct workqueue_struct *wq; /* Workqueue for async */ + bool initialized; + bool emulation_mode; /* Using already-mapped virtual addresses */ +}; + +struct gsp_pending_rpc { + struct list_head list; + struct gsp_rpc_msg msg; + struct gsp_rpc_response *rsp; + struct completion done; + u32 timeout_ms; +}; + +/* ============================================================ + * LOW-LEVEL RPC SUBMISSION + * ============================================================ */ + +static inline u32 gsp_next_seq(struct gsp_client *client) +{ + return atomic_inc_return(&client->seq_counter); +} + +static int gsp_rpc_submit_raw(struct gsp_client *client, + u32 type, u32 subcmd, + u64 *payload, u32 seq_id, + struct gsp_rpc_response *rsp, + u32 timeout_ms) +{ + struct gsp_pending_rpc *prpc; + u64 *dispatch_ptr; + int ret = 0; + + prpc = kzalloc(sizeof(*prpc), GFP_KERNEL); + if (!prpc) + return -ENOMEM; + + prpc->msg.type = type; + prpc->msg.subcmd = subcmd; + if (payload) + memcpy(prpc->msg.payload, payload, sizeof(prpc->msg.payload)); + prpc->msg.seq_id = seq_id; + prpc->rsp = rsp; + prpc->timeout_ms = timeout_ms; + init_completion(&prpc->done); + + spin_lock(&client->lock); + list_add_tail(&prpc->list, &client->pending_rpcs); + + /* Write RPC message to firmware mailbox */ + dispatch_ptr = (u64 *)(client->gsp_fw_base + 0x1000); /* Mailbox offset */ + for (int i = 0; i < 8; i++) + writeq(((u64 *)&prpc->msg)[i], &dispatch_ptr[i]); + + /* Ring doorbell */ + writel(type, client->rpc_doorbell); + spin_unlock(&client->lock); + + /* Wait for completion */ + if (!wait_for_completion_timeout(&prpc->done, + msecs_to_jiffies(timeout_ms))) { + ret = -ETIMEDOUT; + goto cleanup; + } + + if (rsp) + memcpy(rsp, prpc->rsp, sizeof(*rsp)); + +cleanup: + spin_lock(&client->lock); + list_del(&prpc->list); + spin_unlock(&client->lock); + kfree(prpc); + return ret; +} + +/* ============================================================ + * HIGH-LEVEL RPC API + * ============================================================ */ + +/* Memory Allocation */ +int gsp_rpc_alloc(struct gsp_client *client, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size) +{ + if (gsp_rpc_emulation_is_enabled()) + return gsp_rpc_alloc_emul(client, size, alignment, mem_type, flags, addr_hint, pid, gpu_va, handle, actual_size); + + struct gsp_alloc_req req = {0}; + struct gsp_alloc_rsp rsp = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + req.size = size; + req.alignment = alignment; + req.mem_type = mem_type; + req.flags = flags; + req.addr_hint = addr_hint; + req.pid = pid; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_ALLOC, GSP_ALLOC_SUBCMD_DEFAULT, + (u64 *)&req, seq, &response, 5000); + if (ret) + return ret; + + if (response.status) { + pr_err("GSP ALLOC failed: status=%d\n", response.status); + return -EIO; + } + + memcpy(&rsp, response.result, sizeof(rsp)); + if (gpu_va) *gpu_va = rsp.gpu_va; + if (handle) *handle = rsp.handle; + if (actual_size) *actual_size = rsp.size; + + return 0; +} + +int gsp_rpc_free(struct gsp_client *client, u64 handle) +{ + if (gsp_rpc_emulation_is_enabled()) + return gsp_rpc_free_emul(client, handle); + + struct gsp_rpc_response response = {0}; + u64 payload[6] = { handle, 0, 0, 0, 0, 0 }; + u32 seq = gsp_next_seq(client); + int ret; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_FREE, 0, + payload, seq, &response, 2000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +/* Memory Mapping */ +int gsp_rpc_map(struct gsp_client *client, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va) +{ + if (gsp_rpc_emulation_is_enabled()) + return gsp_rpc_map_emul(client, handle, gpu_va, size, prot, flags, offset, mapped_va); + + struct gsp_map_req req = {0}; + struct gsp_map_rsp rsp = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + req.handle = handle; + req.gpu_va = gpu_va; + req.size = size; + req.prot = prot; + req.flags = flags; + req.offset = offset; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_MAP, 0, + (u64 *)&req, seq, &response, 2000); + if (ret) + return ret; + + if (response.status) + return -EIO; + + memcpy(&rsp, response.result, sizeof(rsp)); + if (mapped_va) *mapped_va = rsp.gpu_va; + + return 0; +} + +int gsp_rpc_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size) +{ + if (gsp_rpc_emulation_is_enabled()) + return gsp_rpc_unmap_emul(client, handle, gpu_va, size); + + struct gsp_rpc_response response = {0}; + u64 payload[6] = { handle, gpu_va, size, 0, 0, 0 }; + u32 seq = gsp_next_seq(client); + int ret; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_UNMAP, 0, + payload, seq, &response, 2000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +/* Page Pinning */ +int gsp_rpc_pin(struct gsp_client *client, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages) +{ + if (gsp_rpc_emulation_is_enabled()) + return gsp_rpc_pin_emul(client, handle, offset, size, flags, dma_addr, num_pages); + + struct gsp_pin_req req = {0}; + struct gsp_pin_rsp rsp = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + req.handle = handle; + req.offset = offset; + req.size = size; + req.flags = flags; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_PIN, GSP_PIN_SUBCMD_DEFAULT, + (u64 *)&req, seq, &response, 5000); + if (ret) + return ret; + + if (response.status) + return -EIO; + + memcpy(&rsp, response.result, sizeof(rsp)); + if (dma_addr) *dma_addr = rsp.dma_addr; + if (num_pages) *num_pages = rsp.num_pages; + + return 0; +} + +int gsp_rpc_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size) +{ + if (gsp_rpc_emulation_is_enabled()) + return gsp_rpc_unpin_emul(client, handle, offset, size); + + struct gsp_rpc_response response = {0}; + u64 payload[6] = { handle, offset, size, 0, 0, 0 }; + u32 seq = gsp_next_seq(client); + int ret; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_UNPIN, 0, + payload, seq, &response, 2000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +/* BAR Trap Emulation */ +int gsp_rpc_bar_trap(struct gsp_client *client, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value) +{ + struct gsp_bar_trap_req req = {0}; + struct gsp_bar_trap_rsp rsp = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + /* In emulation mode there is no real firmware to submit to, and + * submit_raw would ring the very doorbell the emulation thread polls + * (self-triggered RPC livelock + timeout stall). Guest BAR access is + * synchronous MMIO via the shadow mapping and never needs this RPC. + * Real shadow-register handlers are TODO; fail fast and loudly. */ + if (gsp_rpc_emulation_is_enabled()) { + pr_warn_ratelimited("gsp_rpc_bar_trap: not supported in emulation mode (bar=%u addr=0x%llx)\n", + bar_num, bar_addr); + return -ENOTSUPP; + } + + req.bar_addr = bar_addr; + req.access_type = access_type; + req.size = size; + req.value = value; + req.bar_num = bar_num; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_BAR_TRAP, GSP_BAR_TRAP_SUBCMD_DEFAULT, + (u64 *)&req, seq, &response, 2000); + if (ret) + return ret; + + if (response.status) + return -EIO; + + memcpy(&rsp, response.result, sizeof(rsp)); + if (ret_value) *ret_value = rsp.value; + + return rsp.handled ? 0 : -ENODEV; +} + +/* Migration */ +int gsp_rpc_migrate_save(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint) +{ + struct gsp_migrate_req req = {0}; + struct gsp_migrate_rsp rsp = {0}; + struct gsp_rpc_response response = {0}; + u32 seq; + int ret; + + /* Emulation mode: no real firmware; submit_raw would ring the doorbell + * the emulation thread polls (self-triggered livelock + 30s stall). + * Live migration over the emulated vGPU is not yet implemented; + * see kernel/migration.c for the 4-phase host-side protocol work. */ + if (gsp_rpc_emulation_is_enabled()) { + pr_warn_ratelimited("gsp_rpc_migrate_save: not supported in emulation mode\n"); + return -ENOTSUPP; + } + + seq = gsp_next_seq(client); + req.src_gpu_id = src_gpu_id; + req.dst_gpu_id = dst_gpu_id; + req.memory_bitmap = memory_bitmap; + req.page_table_ptr = page_table_ptr; + req.state = 0; /* SAVE */ + req.flags = GSP_MIGRATE_SUBCMD_SAVE; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_MIGRATE, GSP_MIGRATE_SUBCMD_SAVE, + (u64 *)&req, seq, &response, 30000); + if (ret) + return ret; + + if (response.status) + return -EIO; + + memcpy(&rsp, response.result, sizeof(rsp)); + if (checkpoint) *checkpoint = rsp.checkpoint; + + return 0; +} + +int gsp_rpc_migrate_restore(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint) +{ + struct gsp_migrate_req req = {0}; + struct gsp_rpc_response response = {0}; + u32 seq; + int ret; + + /* Emulation mode: no real firmware; see gsp_rpc_migrate_save note. */ + if (gsp_rpc_emulation_is_enabled()) { + pr_warn_ratelimited("gsp_rpc_migrate_restore: not supported in emulation mode\n"); + return -ENOTSUPP; + } + + seq = gsp_next_seq(client); + req.src_gpu_id = src_gpu_id; + req.dst_gpu_id = dst_gpu_id; + req.memory_bitmap = memory_bitmap; + req.page_table_ptr = page_table_ptr; + req.state = 1; /* RESTORE */ + req.flags = GSP_MIGRATE_SUBCMD_RESTORE; + req.context = checkpoint; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_MIGRATE, GSP_MIGRATE_SUBCMD_RESTORE, + (u64 *)&req, seq, &response, 30000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +/* ============================================================ + * EMULATION FALLBACKS + * ============================================================ */ + +static bool gsp_emulation_enabled = false; + +void gsp_rpc_emulation_set(bool enable) +{ + gsp_emulation_enabled = enable; +} + +bool gsp_rpc_emulation_is_enabled(void) +{ + return gsp_emulation_enabled; +} + +/* VRAM allocation tracking for emulation */ +struct gsp_vram_alloc { + struct list_head list; + u64 handle; + u64 gpu_va; + u64 size; + u64 alignment; + u32 mem_type; + u32 flags; + u64 addr_hint; + u32 pid; + bool mapped; + u64 mapped_va; + struct nv_vgpu_simple_mapping *mapping; +}; + +static LIST_HEAD(gsp_vram_allocs); +static DEFINE_SPINLOCK(gsp_vram_lock); +static atomic64_t gsp_handle_counter = ATOMIC64_INIT(1); + +/* Scheduling emulation - local work queue */ +struct gsp_sched_work { + struct list_head list; + struct work_struct work; + u64 work_handle; + u64 callback; + u64 arg; + u32 priority; + u32 flags; + struct completion done; +}; + +static LIST_HEAD(gsp_sched_queue); +static DEFINE_SPINLOCK(gsp_sched_lock); +static struct workqueue_struct *gsp_sched_wq = NULL; + +static void gsp_sched_worker(struct work_struct *work) +{ + struct gsp_sched_work *sched = container_of(work, struct gsp_sched_work, work); + + /* In emulation mode, we just complete successfully */ + complete(&sched->done); +} + +/* Find VRAM allocation by handle */ +static struct gsp_vram_alloc *gsp_find_vram_alloc(u64 handle) +{ + struct gsp_vram_alloc *alloc; + spin_lock(&gsp_vram_lock); + list_for_each_entry(alloc, &gsp_vram_allocs, list) { + if (alloc->handle == handle) { + spin_unlock(&gsp_vram_lock); + return alloc; + } + } + spin_unlock(&gsp_vram_lock); + return NULL; +} + +/* Find VRAM allocation by GPU VA */ +static struct gsp_vram_alloc *gsp_find_vram_by_va(u64 gpu_va) +{ + struct gsp_vram_alloc *alloc; + spin_lock(&gsp_vram_lock); + list_for_each_entry(alloc, &gsp_vram_allocs, list) { + if (alloc->gpu_va <= gpu_va && gpu_va < alloc->gpu_va + alloc->size) { + spin_unlock(&gsp_vram_lock); + return alloc; + } + } + spin_unlock(&gsp_vram_lock); + return NULL; +} + +/* Memory Allocation - Emulation Implementation */ +int gsp_rpc_alloc_emul(struct gsp_client *client, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size) +{ + struct nv_vgpu_device *vgpu = client->dev->driver_data; + struct gsp_vram_alloc *alloc; + u64 va = addr_hint ? addr_hint : 0x100000000ULL; /* Default 4GB */ + dma_addr_t iova; + static u64 next_vidmem_va = 0x100000; /* Start after 1MB (skip WPR2/reserved) */ + u64 fb_size = vgpu->fb_size ? vgpu->fb_size : NV_VGPU_BAR1_SIZE; + + /* For vidmem allocations (golden image, channels, etc.), allocate from + * the emulated GPU's vidmem address range (0 - fb_size). The memmgr + * rejects VAs outside this range. For sysmem allocations, use the + * default 4GB hint. */ + if (!addr_hint) { + if (mem_type == 0 /* vidmem */ || flags & 0x1 /* FBMEM hint */) { + va = ALIGN(next_vidmem_va, alignment); + next_vidmem_va = ALIGN(va + size, PAGE_SIZE); + /* Wrap around if we exceed vidmem (shouldn't happen in practice) */ + if (next_vidmem_va > fb_size - size) + next_vidmem_va = 0x100000; + } else { + va = 0x100000000ULL; + } + } + + alloc = kzalloc(sizeof(*alloc), GFP_KERNEL); + if (!alloc) + return -ENOMEM; + + alloc->handle = atomic64_inc_return(&gsp_handle_counter); + alloc->size = size; + alloc->alignment = alignment; + alloc->mem_type = mem_type; + alloc->flags = flags; + alloc->addr_hint = addr_hint; + alloc->pid = pid; + alloc->mapped = false; + + /* Align VA */ + va = ALIGN(va, alignment); + + /* In emulation, we allocate host memory and map it to GPU VA */ + alloc->mapping = kzalloc(sizeof(*alloc->mapping), GFP_KERNEL); + if (!alloc->mapping) { + kfree(alloc); + return -ENOMEM; + } + + alloc->mapping->gpa = va; + alloc->mapping->size = size; + alloc->mapping->page_count = (size + PAGE_SIZE - 1) / PAGE_SIZE; + alloc->mapping->pages = vzalloc(alloc->mapping->page_count * sizeof(struct page *)); + if (!alloc->mapping->pages) { + kfree(alloc->mapping); + kfree(alloc); + return -ENOMEM; + } + + /* Allocate host pages. For contiguous allocations (golden image, + * large channels), use alloc_pages with a single contiguous block. + * For small allocations, fall back to individual pages. */ + if (size > PAGE_SIZE && size <= (PAGE_SIZE << 9)) { + /* Try contiguous allocation first (order = log2(page_count)) */ + int order = get_order(size); + struct page *contig_page = alloc_pages(GFP_KERNEL | GFP_DMA32, order); + + if (contig_page) { + /* Contiguous allocation succeeded - use it */ + for (int i = 0; i < alloc->mapping->page_count; i++) { + struct page *p = nth_page(contig_page, i); + alloc->mapping->pages[i] = p; + } + } else { + /* Fall back to individual pages (non-contiguous) */ + for (int i = 0; i < alloc->mapping->page_count; i++) { + struct page *page = alloc_page(GFP_KERNEL | GFP_DMA32); + if (!page) { + for (int j = 0; j < i; j++) + __free_page(alloc->mapping->pages[j]); + vfree(alloc->mapping->pages); + kfree(alloc->mapping); + kfree(alloc); + return -ENOMEM; + } + alloc->mapping->pages[i] = page; + } + } + } else { + for (int i = 0; i < alloc->mapping->page_count; i++) { + struct page *page = alloc_page(GFP_KERNEL | GFP_DMA32); + if (!page) { + for (int j = 0; j < i; j++) + __free_page(alloc->mapping->pages[j]); + vfree(alloc->mapping->pages); + kfree(alloc->mapping); + kfree(alloc); + return -ENOMEM; + } + alloc->mapping->pages[i] = page; + } + } + alloc->mapping->pinned = true; + + /* Map to IOVA for DMA (dma_addr_t is 64-bit - do NOT truncate into int; + * with an IOMMU the IOVA may exceed 32 bits) */ + iova = dma_map_page(&vgpu->pdev->dev, alloc->mapping->pages[0], 0, size, DMA_BIDIRECTIONAL); + if (dma_mapping_error(&vgpu->pdev->dev, iova)) { + for (int i = 0; i < alloc->mapping->page_count; i++) + __free_page(alloc->mapping->pages[i]); + vfree(alloc->mapping->pages); + kfree(alloc->mapping); + kfree(alloc); + return -ENOMEM; + } + alloc->mapping->iova = iova; + alloc->mapping->mapped = true; + + alloc->gpu_va = va; + alloc->mapped_va = va; + + spin_lock(&gsp_vram_lock); + list_add_tail(&alloc->list, &gsp_vram_allocs); + spin_unlock(&gsp_vram_lock); + + if (gpu_va) *gpu_va = va; + if (handle) *handle = alloc->handle; + if (actual_size) *actual_size = size; + + nv_vgpu_info(vgpu, "EMUL ALLOC: handle=0x%llx gpu_va=0x%llx size=%llu iova=0x%llx", + alloc->handle, va, size, alloc->mapping->iova); + + return 0; +} + +/* Memory Free - Emulation Implementation */ +int gsp_rpc_free_emul(struct gsp_client *client, u64 handle) +{ + struct nv_vgpu_device *vgpu = client->dev->driver_data; + struct gsp_vram_alloc *alloc; + + alloc = gsp_find_vram_alloc(handle); + if (!alloc) { + nv_vgpu_warn(vgpu, "EMUL FREE: handle 0x%llx not found", handle); + return -ENOENT; + } + + spin_lock(&gsp_vram_lock); + list_del(&alloc->list); + spin_unlock(&gsp_vram_lock); + + if (alloc->mapping) { + if (alloc->mapping->mapped) { + dma_unmap_page(&vgpu->pdev->dev, alloc->mapping->iova, alloc->size, DMA_BIDIRECTIONAL); + } + if (alloc->mapping->pinned) { + for (int i = 0; i < alloc->mapping->page_count; i++) { + if (alloc->mapping->pages[i]) + __free_page(alloc->mapping->pages[i]); + } + } + vfree(alloc->mapping->pages); + kfree(alloc->mapping); + } + + kfree(alloc); + + nv_vgpu_info(vgpu, "EMUL FREE: handle=0x%llx", handle); + return 0; +} + +/* Memory Map - Emulation Implementation */ +int gsp_rpc_map_emul(struct gsp_client *client, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va) +{ + struct nv_vgpu_device *vgpu = client->dev->driver_data; + struct gsp_vram_alloc *alloc = gsp_find_vram_alloc(handle); + + if (!alloc) { + nv_vgpu_warn(vgpu, "EMUL MAP: handle 0x%llx not found", handle); + return -ENOENT; + } + + if (gpu_va == 0) + gpu_va = alloc->gpu_va + offset; + + alloc->mapped_va = gpu_va; + alloc->mapped = true; + + if (mapped_va) + *mapped_va = gpu_va; + + nv_vgpu_info(vgpu, "EMUL MAP: handle=0x%llx gpu_va=0x%llx size=%llu offset=%llu", + handle, gpu_va, size, offset); + + return 0; +} + +/* Memory Unmap - Emulation Implementation */ +int gsp_rpc_unmap_emul(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size) +{ + struct nv_vgpu_device *vgpu = client->dev->driver_data; + struct gsp_vram_alloc *alloc = gsp_find_vram_alloc(handle); + + if (!alloc) { + nv_vgpu_warn(vgpu, "EMUL UNMAP: handle 0x%llx not found", handle); + return -ENOENT; + } + + alloc->mapped = false; + + nv_vgpu_info(vgpu, "EMUL UNMAP: handle=0x%llx gpu_va=0x%llx size=%llu", + handle, gpu_va, size); + + return 0; +} + +/* Page Pin - Emulation Implementation */ +int gsp_rpc_pin_emul(struct gsp_client *client, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages) +{ + struct nv_vgpu_device *vgpu = client->dev->driver_data; + struct gsp_vram_alloc *alloc = gsp_find_vram_alloc(handle); + + if (!alloc || !alloc->mapping) { + nv_vgpu_warn(vgpu, "EMUL PIN: handle 0x%llx not found or no mapping", handle); + return -ENOENT; + } + + if (dma_addr) + *dma_addr = alloc->mapping->iova + offset; + if (num_pages) + *num_pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; + + nv_vgpu_info(vgpu, "EMUL PIN: handle=0x%llx offset=%llu size=%llu dma_addr=0x%llx pages=%u", + handle, offset, size, alloc->mapping->iova + offset, *num_pages); + + return 0; +} + +/* Page Unpin - Emulation Implementation */ +int gsp_rpc_unpin_emul(struct gsp_client *client, u64 handle, u64 offset, u64 size) +{ + struct nv_vgpu_device *vgpu = client->dev->driver_data; + struct gsp_vram_alloc *alloc = gsp_find_vram_alloc(handle); + + if (!alloc) { + nv_vgpu_warn(vgpu, "EMUL UNPIN: handle 0x%llx not found", handle); + return -ENOENT; + } + + nv_vgpu_info(vgpu, "EMUL UNPIN: handle=0x%llx offset=%llu size=%llu", + handle, offset, size); + + return 0; +} + +int gsp_rpc_schedule(struct gsp_client *client, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + struct gsp_sched_req req = {0}; + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + if (gsp_rpc_emulation_is_enabled()) { + struct gsp_sched_work *sched = kzalloc(sizeof(*sched), GFP_KERNEL); + if (!sched) + return -ENOMEM; + + INIT_WORK(&sched->work, gsp_sched_worker); + sched->work_handle = work_handle; + sched->callback = callback; + sched->arg = arg; + sched->priority = priority; + sched->flags = flags; + init_completion(&sched->done); + + spin_lock(&gsp_sched_lock); + list_add_tail(&sched->list, &gsp_sched_queue); + spin_unlock(&gsp_sched_lock); + + if (gsp_sched_wq) + queue_work(gsp_sched_wq, &sched->work); + + wait_for_completion(&sched->done); + + spin_lock(&gsp_sched_lock); + list_del(&sched->list); + spin_unlock(&gsp_sched_lock); + + kfree(sched); + return 0; + } + + req.work_handle = work_handle; + req.callback = callback; + req.arg = arg; + req.priority = priority; + req.flags = flags; + + ret = gsp_rpc_submit_raw(client, GSP_RPC_SCHEDULE, GSP_SCHED_SUBCMD_DEFAULT, + (u64 *)&req, seq, &response, 2000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +int gsp_rpc_yield(struct gsp_client *client) +{ + struct gsp_rpc_response response = {0}; + u32 seq = gsp_next_seq(client); + int ret; + + if (gsp_rpc_emulation_is_enabled()) { + /* In emulation, yield is a no-op success */ + return 0; + } + + ret = gsp_rpc_submit_raw(client, GSP_RPC_YIELD, 0, + NULL, seq, &response, 1000); + if (ret) + return ret; + + return response.status ? -EIO : 0; +} + +/* Notification emulation - async eventfd support */ +struct gsp_notify_cb { + struct list_head list; + u32 type; + void (*callback)(struct gsp_client *, struct gsp_rpc_response *, void *); + void *ctx; +}; + +static LIST_HEAD(gsp_notify_callbacks); +static DEFINE_SPINLOCK(gsp_notify_lock); + +int gsp_rpc_notify_register(struct gsp_client *client, + u32 type, + void (*callback)(struct gsp_client *, + struct gsp_rpc_response *, + void *), + void *ctx) +{ + struct gsp_notify_cb *nc = kzalloc(sizeof(*nc), GFP_KERNEL); + if (!nc) + return -ENOMEM; + + nc->type = type; + nc->callback = callback; + nc->ctx = ctx; + + spin_lock(&gsp_notify_lock); + list_add_tail(&nc->list, &gsp_notify_callbacks); + spin_unlock(&gsp_notify_lock); + + return 0; +} + +void gsp_rpc_notify_unregister(struct gsp_client *client, u32 type) +{ + struct gsp_notify_cb *nc, *tmp; + + spin_lock(&gsp_notify_lock); + list_for_each_entry_safe(nc, tmp, &gsp_notify_callbacks, list) { + if (nc->type == type) { + list_del(&nc->list); + kfree(nc); + } + } + spin_unlock(&gsp_notify_lock); +} + +/* Handle async notifications from GSP */ +void gsp_handle_notification(struct gsp_client *client, struct gsp_rpc_msg *msg) +{ + struct gsp_notify_cb *nc; + + spin_lock(&gsp_notify_lock); + list_for_each_entry(nc, &gsp_notify_callbacks, list) { + if (nc->type == msg->type || nc->type == 0) { + /* Copy message to response format for callback */ + struct gsp_rpc_response rsp = {0}; + rsp.status = 0; + rsp.type = msg->type; + rsp.seq_id = msg->seq_id; + memcpy(rsp.result, msg->payload, sizeof(msg->payload)); + nc->callback(client, &rsp, nc->ctx); + } + } + spin_unlock(&gsp_notify_lock); +} + +/* ============================================================ + * CLIENT INITIALIZATION + * ============================================================ */ + +struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell) +{ + struct gsp_client *client; + resource_size_t fw_size = 0x2000000; /* 32MB */ + + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return ERR_PTR(-ENOMEM); + + client->dev = dev; + spin_lock_init(&client->lock); + mutex_init(&client->rpc_mutex); + INIT_LIST_HEAD(&client->pending_rpcs); + init_waitqueue_head(&client->resp_wq); + atomic_set(&client->seq_counter, 0); + + client->gsp_fw_base = ioremap(fw_base, fw_size); + if (!client->gsp_fw_base) { + kfree(client); + return ERR_PTR(-ENOMEM); + } + + client->rpc_doorbell = ioremap(doorbell, 0x1000); + if (!client->rpc_doorbell) { + iounmap(client->gsp_fw_base); + kfree(client); + return ERR_PTR(-ENOMEM); + } + + client->wq = alloc_workqueue("gsp_rpc", WQ_UNBOUND, 0); + if (!client->wq) { + iounmap(client->rpc_doorbell); + iounmap(client->gsp_fw_base); + kfree(client); + return ERR_PTR(-ENOMEM); + } + + /* Init emulation workqueue for scheduling */ + gsp_sched_wq = alloc_workqueue("gsp_sched", WQ_UNBOUND, 0); + if (!gsp_sched_wq) { + destroy_workqueue(client->wq); + iounmap(client->rpc_doorbell); + iounmap(client->gsp_fw_base); + kfree(client); + return ERR_PTR(-ENOMEM); + } + + client->initialized = true; + pr_info("GSP RPC client initialized: fw_base=%pa, doorbell=%pa\n", &fw_base, &doorbell); + + return client; +} + +/* Emulation variant: uses already-mapped virtual addresses */ +struct gsp_client *gsp_client_init_emulation(struct device *dev, void __iomem *fw_base_virt, void __iomem *doorbell_virt) +{ + struct gsp_client *client; + + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return ERR_PTR(-ENOMEM); + + client->dev = dev; + spin_lock_init(&client->lock); + mutex_init(&client->rpc_mutex); + INIT_LIST_HEAD(&client->pending_rpcs); + init_waitqueue_head(&client->resp_wq); + atomic_set(&client->seq_counter, 0); + + /* Use already-mapped virtual addresses directly */ + client->gsp_fw_base = fw_base_virt; + client->rpc_doorbell = doorbell_virt; + + client->wq = alloc_workqueue("gsp_rpc", WQ_UNBOUND, 0); + if (!client->wq) { + kfree(client); + return ERR_PTR(-ENOMEM); + } + + /* Init emulation workqueue for scheduling */ + gsp_sched_wq = alloc_workqueue("gsp_sched", WQ_UNBOUND, 0); + if (!gsp_sched_wq) { + destroy_workqueue(client->wq); + kfree(client); + return ERR_PTR(-ENOMEM); + } + + client->initialized = true; + client->emulation_mode = true; + pr_info("GSP RPC client initialized (emulation): fw_base=%p, doorbell=%p\n", fw_base_virt, doorbell_virt); + + return client; +} + +void gsp_client_fini(struct gsp_client *client) +{ + if (!client) + return; + + if (client->wq) + destroy_workqueue(client->wq); + + if (gsp_sched_wq) { + destroy_workqueue(gsp_sched_wq); + gsp_sched_wq = NULL; + } + + /* Only iounmap if not in emulation mode (addresses were ioremap'd by us) */ + if (!client->emulation_mode) { + if (client->rpc_doorbell) + iounmap(client->rpc_doorbell); + + if (client->gsp_fw_base) + iounmap(client->gsp_fw_base); + } + + kfree(client); +} + +/* ============================================================ + * WPR MANAGEMENT (from firmware strings) + * ============================================================ */ + +/* WPR mutex operations (from firmware: VPR_WPR_WRITE_MUTEX*) */ +int gsp_wpr_write_mutex_acquire(struct gsp_client *client, u32 wpr_id) +{ + /* Calls firmware function at VPR_WPR_WRITE_MUTEX addresses */ + /* Implementation would use RPC_DEBUG or dedicated WPR RPC */ + (void)client; + (void)wpr_id; + return 0; +} + +int gsp_wpr_write_mutex_release(struct gsp_client *client, u32 wpr_id) +{ + (void)client; + (void)wpr_id; + return 0; +} + +int gsp_wpr_scratch_mutex_acquire(struct gsp_client *client, u32 wpr_id) +{ + /* From firmware: VPR_WPR_SCRATCH_MUTEX */ + (void)client; + (void)wpr_id; + return 0; +} + +int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id) +{ + (void)client; + (void)wpr_id; + return 0; +} + +/* ============================================================ + * MODULE EXPORTS + * ============================================================ */ + +EXPORT_SYMBOL(gsp_client_init); +EXPORT_SYMBOL(gsp_client_init_emulation); +EXPORT_SYMBOL(gsp_client_fini); +EXPORT_SYMBOL(gsp_rpc_alloc); +EXPORT_SYMBOL(gsp_rpc_free); +EXPORT_SYMBOL(gsp_rpc_map); +EXPORT_SYMBOL(gsp_rpc_unmap); +EXPORT_SYMBOL(gsp_rpc_pin); +EXPORT_SYMBOL(gsp_rpc_unpin); +EXPORT_SYMBOL(gsp_rpc_schedule); +EXPORT_SYMBOL(gsp_rpc_yield); +EXPORT_SYMBOL(gsp_rpc_bar_trap); +EXPORT_SYMBOL(gsp_rpc_migrate_save); +EXPORT_SYMBOL(gsp_rpc_migrate_restore); +EXPORT_SYMBOL(gsp_wpr_write_mutex_acquire); +EXPORT_SYMBOL(gsp_wpr_write_mutex_release); +EXPORT_SYMBOL(gsp_wpr_scratch_mutex_acquire); +EXPORT_SYMBOL(gsp_wpr_scratch_mutex_release); +EXPORT_SYMBOL(gsp_rpc_emulation_set); +EXPORT_SYMBOL(gsp_rpc_emulation_is_enabled); +EXPORT_SYMBOL(gsp_rpc_notify_register); +EXPORT_SYMBOL(gsp_rpc_notify_unregister); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("vGPU-IO Team"); +MODULE_DESCRIPTION("GSP 570.144W RPC Client for vGPU-IO"); diff --git a/kernel/gsp_rpc_client.h b/kernel/gsp_rpc_client.h new file mode 100644 index 0000000..4712754 --- /dev/null +++ b/kernel/gsp_rpc_client.h @@ -0,0 +1,355 @@ +/* + * GSP 570.144W RPC Client Header + * Auto-generated from firmware decompilation + */ + +#ifndef _GSP_RPC_CLIENT_H_ +#define _GSP_RPC_CLIENT_H_ + +#include +#include + +/* ============================================================ + * FIRMWARE BASE ADDRESS + * ============================================================ */ +#define GSP_FW_BASE 0x04000000UL + +/* ============================================================ + * RPC DISPATCHER ADDRESSES + * ============================================================ */ + +/* Primary Dispatcher - Jump table switch for 6 RPC types */ +#define GSP_RPC_DISPATCH_PRIMARY (GSP_FW_BASE + 0x17A1AC0) /* 0x057a1ac0 */ +#define GSP_RPC_JUMP_TABLE (GSP_FW_BASE + 0x17A3AD8) /* 0x057a3ad8 - 32 sub-commands */ + +/* Secondary Dispatcher - Full implementation for all 11 RPC types */ +#define GSP_RPC_DISPATCH_SECONDARY (GSP_FW_BASE + 0x0ED39DC) /* 0x04ed39dc */ + +/* Tertiary Dispatcher - Stack=160 bytes */ +#define GSP_RPC_DISPATCH_TERTIARY (GSP_FW_BASE + 0x0CC58F8) /* 0x04cc58f8 */ + +/* Quaternary Dispatcher - Stack=96 bytes */ +#define GSP_RPC_DISPATCH_QUATERNARY (GSP_FW_BASE + 0x1723EF4) /* 0x05723ef4 */ + +/* ============================================================ + * SPECIALIZED HANDLER ADDRESSES + * ============================================================ */ + +#define GSP_ALLOC_MAIN (GSP_FW_BASE + 0x17E94EC) /* 0x057e94ec */ +#define GSP_SCHEDULE_MAIN (GSP_FW_BASE + 0x11EFBF4) /* 0x051efbf4 */ +#define GSP_BAR_TRAP_HANDLER_JT (GSP_FW_BASE + 0x17675B7) /* 0x057675b7 */ +#define GSP_BAR_TRAP_HANDLER_PRIM (GSP_FW_BASE + 0x0CB8189) /* 0x04cb8189 */ +#define GSP_PIN_HANDLER_SEC (GSP_FW_BASE + 0x0ED34B8) /* 0x04ed34b8 */ + +/* Jump Table Handlers (from decoded table at 0x059bfeb0) */ +#define GSP_MAP_HANDLER_A (GSP_FW_BASE + 0x1766AB7) /* 0x05766ab7 */ +#define GSP_MAP_HANDLER_B (GSP_FW_BASE + 0x1778A37) /* 0x05778a37 */ +#define GSP_UNMAP_HANDLER_A (GSP_FW_BASE + 0x17669B7) /* 0x057669b7 */ +#define GSP_UNMAP_HANDLER_B (GSP_FW_BASE + 0x1779937) /* 0x05779937 */ +#define GSP_PIN_HANDLER_A (GSP_FW_BASE + 0x17664B7) /* 0x057664b7 */ +#define GSP_PIN_HANDLER_B (GSP_FW_BASE + 0x177A3B7) /* 0x0577a3b7 */ +#define GSP_UNPIN_HANDLER_A (GSP_FW_BASE + 0x17682B7) /* 0x057682b7 */ +#define GSP_UNPIN_HANDLER_B (GSP_FW_BASE + 0x1779FB7) /* 0x05779fb7 */ +#define GSP_SCHEDULE_HANDLER_A (GSP_FW_BASE + 0x1767F37) /* 0x05767f37 */ +#define GSP_SCHEDULE_HANDLER_B (GSP_FW_BASE + 0x175AEB7) /* 0x0575aeb7 */ +#define GSP_YIELD_HANDLER_A (GSP_FW_BASE + 0x175AE37) /* 0x0575ae37 */ +#define GSP_YIELD_HANDLER_B (GSP_FW_BASE + 0x1767337) /* 0x05767337 */ +#define GSP_NOTIFY_HANDLER_A (GSP_FW_BASE + 0x177C8B7) /* 0x0577c8b7 */ +#define GSP_NOTIFY_HANDLER_B (GSP_FW_BASE + 0x1767837) /* 0x05767837 */ + +/* Entry Point */ +#define GSP_ENTRY_POINT (GSP_FW_BASE + 0x0009000) /* 0x04009000 */ + +/* ============================================================ + * RPC TYPE DEFINITIONS + * ============================================================ */ +enum gsp_rpc_type { + GSP_RPC_ALLOC = 0x01, + GSP_RPC_FREE = 0x02, + GSP_RPC_MAP = 0x03, + GSP_RPC_UNMAP = 0x04, + GSP_RPC_PIN = 0x05, + GSP_RPC_UNPIN = 0x06, + GSP_RPC_SCHEDULE = 0x07, + GSP_RPC_YIELD = 0x08, + GSP_RPC_NOTIFY = 0x09, + GSP_RPC_BAR_TRAP = 0x0A, + GSP_RPC_MIGRATE = 0x0B, + GSP_RPC_DEBUG = 0xFF, + GSP_RPC_MAX_TYPE = 0x10, /* Must be > highest RPC type value */ +}; + +/* ============================================================ + * ALLOC SUB-COMMANDS (Primary Dispatcher Jump Table Indices) + * ============================================================ */ +enum gsp_alloc_subcmd { + GSP_ALLOC_SUBCMD_DEFAULT = 0, + GSP_ALLOC_SUBCMD_TYPE1 = 1, + GSP_ALLOC_SUBCMD_TYPE2 = 3, + GSP_ALLOC_SUBCMD_TYPE3 = 4, + GSP_ALLOC_SUBCMD_TYPE4 = 5, + GSP_ALLOC_SUBCMD_TYPE5 = 8, + GSP_ALLOC_SUBCMD_TYPE6 = 10, + GSP_ALLOC_SUBCMD_TYPE7 = 12, + GSP_ALLOC_SUBCMD_TYPE8 = 17, + GSP_ALLOC_SUBCMD_TYPE9 = 19, + GSP_ALLOC_SUBCMD_TYPE10 = 20, + GSP_ALLOC_SUBCMD_TYPE11 = 21, + GSP_ALLOC_SUBCMD_TYPE12 = 22, + GSP_ALLOC_SUBCMD_TYPE13 = 23, + GSP_ALLOC_SUBCMD_TYPE14 = 24, + GSP_ALLOC_SUBCMD_TYPE15 = 25, + GSP_ALLOC_SUBCMD_TYPE16 = 26, + GSP_ALLOC_SUBCMD_TYPE17 = 27, + GSP_ALLOC_SUBCMD_TYPE18 = 28, + GSP_ALLOC_SUBCMD_TYPE19 = 29, + GSP_ALLOC_SUBCMD_TYPE20 = 30, + GSP_ALLOC_SUBCMD_TYPE21 = 31, +}; + +/* ============================================================ + * MIGRATE SUB-COMMANDS + * ============================================================ */ +enum gsp_migrate_subcmd { + GSP_MIGRATE_SUBCMD_SAVE = 0x04, + GSP_MIGRATE_SUBCMD_RESTORE = 0x05, +}; + +/* ============================================================ + * PIN SUB-COMMANDS + * ============================================================ */ +enum gsp_pin_subcmd { + GSP_PIN_SUBCMD_DEFAULT = 0x05, +}; + +/* ============================================================ + * SCHEDULE SUB-COMMANDS + * ============================================================ */ +enum gsp_sched_subcmd { + GSP_SCHED_SUBCMD_DEFAULT = 0x07, +}; + +/* ============================================================ + * BAR TRAP SUB-COMMANDS + * ============================================================ */ +enum gsp_bar_trap_subcmd { + GSP_BAR_TRAP_SUBCMD_DEFAULT = 0x0A, +}; + +/* ============================================================ + * RPC MESSAGE STRUCTURES + * ============================================================ */ + +struct gsp_rpc_msg { + __le32 type; + __le32 subcmd; + __le64 payload[6]; + __le32 seq_id; + __le32 flags; + __le32 reserved[2]; +} __packed; + +struct gsp_rpc_response { + __le32 status; + __le32 type; + __le64 result[6]; + __le32 seq_id; +} __packed; + +/* ALLOC request/response */ +struct gsp_alloc_req { + __le64 size; + __le64 alignment; + __le32 mem_type; + __le32 flags; + __le64 addr_hint; + __le32 pid; + __le32 reserved; +} __packed; + +struct gsp_alloc_rsp { + __le64 gpu_va; + __le64 handle; + __le64 size; +} __packed; + +/* MAP request/response */ +struct gsp_map_req { + __le64 handle; + __le64 gpu_va; + __le64 size; + __le32 prot; + __le32 flags; + __le64 offset; + __le32 reserved[2]; +} __packed; + +struct gsp_map_rsp { + __le64 gpu_va; +} __packed; + +/* PIN/UNPIN request/response */ +struct gsp_pin_req { + __le64 handle; + __le64 offset; + __le64 size; + __le32 flags; + __le32 reserved[3]; +} __packed; + +struct gsp_pin_rsp { + __le64 dma_addr; + __le32 num_pages; + __le32 reserved; +} __packed; + +/* SCHEDULE request */ +struct gsp_sched_req { + __le64 work_handle; + __le64 callback; + __le64 arg; + __le32 priority; + __le32 flags; + __le32 reserved[2]; +} __packed; + +/* BAR TRAP request/response */ +struct gsp_bar_trap_req { + __le64 bar_addr; + __le32 access_type; + __le32 size; + __le64 value; + __le32 bar_num; + __le32 reserved; +} __packed; + +struct gsp_bar_trap_rsp { + __le64 value; + __le32 handled; + __le32 reserved; +} __packed; + +/* MIGRATE request/response */ +struct gsp_migrate_req { + __le64 src_gpu_id; + __le64 dst_gpu_id; + __le64 memory_bitmap; + __le64 page_table_ptr; + __le32 state; + __le32 flags; + __le64 context; + __le32 reserved; +} __packed; + +struct gsp_migrate_rsp { + __le32 status; + __le32 progress; + __le64 checkpoint; +} __packed; + +/* ============================================================ + * CLIENT OPAQUE STRUCT + * ============================================================ */ +struct gsp_client; + +/* ============================================================ + * PUBLIC API + * ============================================================ */ + + /* Client lifecycle */ + struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell); + struct gsp_client *gsp_client_init_emulation(struct device *dev, void __iomem *fw_base_virt, void __iomem *doorbell_virt); + void gsp_client_fini(struct gsp_client *client); + +/* Memory management */ +int gsp_rpc_alloc(struct gsp_client *client, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size); + +int gsp_rpc_free(struct gsp_client *client, u64 handle); + +int gsp_rpc_map(struct gsp_client *client, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va); + +int gsp_rpc_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size); + +/* Page pinning */ +int gsp_rpc_pin(struct gsp_client *client, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages); + +int gsp_rpc_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size); + +/* Scheduling */ +int gsp_rpc_schedule(struct gsp_client *client, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags); + +int gsp_rpc_yield(struct gsp_client *client); + +/* BAR trap emulation */ +int gsp_rpc_bar_trap(struct gsp_client *client, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value); + +/* Migration */ +int gsp_rpc_migrate_save(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint); + +int gsp_rpc_migrate_restore(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint); + +/* WPR Management */ +int gsp_wpr_write_mutex_acquire(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_write_mutex_release(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_scratch_mutex_acquire(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id); + +/* Emulation control */ +void gsp_rpc_emulation_set(bool enable); +bool gsp_rpc_emulation_is_enabled(void); + +/* Emulation implementations */ +int gsp_rpc_alloc_emul(struct gsp_client *client, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size); +int gsp_rpc_free_emul(struct gsp_client *client, u64 handle); +int gsp_rpc_map_emul(struct gsp_client *client, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va); +int gsp_rpc_unmap_emul(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size); +int gsp_rpc_pin_emul(struct gsp_client *client, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages); +int gsp_rpc_unpin_emul(struct gsp_client *client, u64 handle, u64 offset, u64 size); + +/* Notification */ +int gsp_rpc_notify_register(struct gsp_client *client, + u32 type, + void (*callback)(struct gsp_client *, + struct gsp_rpc_response *, + void *), + void *ctx); +void gsp_rpc_notify_unregister(struct gsp_client *client, u32 type); +void gsp_handle_notification(struct gsp_client *client, struct gsp_rpc_msg *msg); + +/* Client lifecycle */ +struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell); +void gsp_client_fini(struct gsp_client *client); + +/* WPR Management */ +int gsp_wpr_write_mutex_acquire(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_write_mutex_release(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_scratch_mutex_acquire(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id); + +#endif /* _GSP_RPC_CLIENT_H_ */ diff --git a/kernel/gsp_rpc_integration.c b/kernel/gsp_rpc_integration.c new file mode 100644 index 0000000..4d43392 --- /dev/null +++ b/kernel/gsp_rpc_integration.c @@ -0,0 +1,535 @@ +/* + * GSP RPC Integration Layer + * Bridges the firmware-decompiled RPC client with existing gsp_comm.c + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "gsp_comm.h" +#include "gsp_rpc_client.h" +#include "gsp_rpc_integration.h" +#include "bar_emulation.h" +#include "migration.h" + +/* Integration context - wraps the new RPC client */ +struct gsp_rpc_integration { + struct nv_vgpu_device *vgpu; + struct gsp_client *rpc_client; + struct nv_vgpu_gsp_comm_ctx *comm_ctx; + struct mutex lock; + bool initialized; +}; + +/* Global integration instance (per vGPU) */ +static struct gsp_rpc_integration *g_integration = NULL; + +/* ============================================================ + * RPC Message Building Helpers + * ============================================================ */ + +static void gsp_build_alloc_msg(struct gsp_alloc_req *req, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid) +{ + req->size = cpu_to_le64(size); + req->alignment = cpu_to_le64(alignment); + req->mem_type = cpu_to_le32(mem_type); + req->flags = cpu_to_le32(flags); + req->addr_hint = cpu_to_le64(addr_hint); + req->pid = cpu_to_le32(pid); + req->reserved = 0; +} + +static void gsp_build_map_msg(struct gsp_map_req *req, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset) +{ + req->handle = cpu_to_le64(handle); + req->gpu_va = cpu_to_le64(gpu_va); + req->size = cpu_to_le64(size); + req->prot = cpu_to_le32(prot); + req->flags = cpu_to_le32(flags); + req->offset = cpu_to_le64(offset); + req->reserved[0] = 0; + req->reserved[1] = 0; +} + +static void gsp_build_pin_msg(struct gsp_pin_req *req, + u64 handle, u64 offset, u64 size, u32 flags) +{ + req->handle = cpu_to_le64(handle); + req->offset = cpu_to_le64(offset); + req->size = cpu_to_le64(size); + req->flags = cpu_to_le32(flags); + req->reserved[0] = 0; + req->reserved[1] = 0; + req->reserved[2] = 0; +} + +static void gsp_build_sched_msg(struct gsp_sched_req *req, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + req->work_handle = cpu_to_le64(work_handle); + req->callback = cpu_to_le64(callback); + req->arg = cpu_to_le64(arg); + req->priority = cpu_to_le32(priority); + req->flags = cpu_to_le32(flags); + req->reserved[0] = 0; + req->reserved[1] = 0; +} + +static void gsp_build_bar_trap_msg(struct gsp_bar_trap_req *req, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num) +{ + req->bar_addr = cpu_to_le64(bar_addr); + req->access_type = cpu_to_le32(access_type); + req->size = cpu_to_le32(size); + req->value = cpu_to_le64(value); + req->bar_num = cpu_to_le32(bar_num); + req->reserved = 0; +} + +static void gsp_build_migrate_msg(struct gsp_migrate_req *req, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u32 state, u32 flags, u64 context) +{ + req->src_gpu_id = cpu_to_le64(src_gpu_id); + req->dst_gpu_id = cpu_to_le64(dst_gpu_id); + req->memory_bitmap = cpu_to_le64(memory_bitmap); + req->page_table_ptr = cpu_to_le64(page_table_ptr); + req->state = cpu_to_le32(state); + req->flags = cpu_to_le32(flags); + req->context = cpu_to_le64(context); + req->reserved = 0; +} + +/* ============================================================ + * Integration API + * ============================================================ */ + +int nv_vgpu_gsp_rpc_integration_init(struct nv_vgpu_device *vgpu, + struct nv_vgpu_gsp_comm_ctx *comm_ctx) +{ + struct gsp_rpc_integration *integ; + phys_addr_t fw_base, doorbell; + int ret; + + if (!vgpu || !comm_ctx) + return -EINVAL; + + integ = kzalloc(sizeof(*integ), GFP_KERNEL); + if (!integ) + return -ENOMEM; + + integ->vgpu = vgpu; + integ->comm_ctx = comm_ctx; + mutex_init(&integ->lock); + + /* Calculate firmware base and doorbell from BAR0 */ + if (vgpu->bars[0].vaddr) { + fw_base = vgpu->bars[0].gpa; + doorbell = vgpu->bars[0].gpa + NV_VGPU_GSP_DOORBELL_OFFSET; + } else { + /* Default for TU102/TU104 */ + fw_base = GSP_FW_BASE; + doorbell = GSP_FW_BASE + NV_VGPU_GSP_DOORBELL_OFFSET; + } + + /* Initialize the new RPC client */ + integ->rpc_client = gsp_client_init(&vgpu->pdev->dev, fw_base, doorbell); + if (IS_ERR(integ->rpc_client)) { + ret = PTR_ERR(integ->rpc_client); + integ->rpc_client = NULL; + goto err_free; + } + + integ->initialized = true; + g_integration = integ; + + nv_vgpu_info(vgpu, "GSP RPC integration initialized (fw_base=0x%llx, doorbell=0x%llx)", + (unsigned long long)fw_base, (unsigned long long)doorbell); + + return 0; + +err_free: + kfree(integ); + return ret; +} + +void nv_vgpu_gsp_rpc_integration_cleanup(struct nv_vgpu_device *vgpu) +{ + struct gsp_rpc_integration *integ = g_integration; + + if (!integ) + return; + + mutex_lock(&integ->lock); + + if (integ->rpc_client) { + gsp_client_fini(integ->rpc_client); + integ->rpc_client = NULL; + } + + integ->initialized = false; + mutex_unlock(&integ->lock); + + kfree(integ); + g_integration = NULL; + + if (vgpu) + nv_vgpu_info(vgpu, "GSP RPC integration cleaned up"); +} + +/* ============================================================ + * Enhanced RPC Operations (using firmware-decompiled addresses) + * ============================================================ */ + +int nv_vgpu_gsp_rpc_alloc_fw(struct nv_vgpu_device *vgpu, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_alloc_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_alloc_msg(&req, size, alignment, mem_type, flags, addr_hint, pid); + + ret = gsp_rpc_alloc(integ->rpc_client, + size, alignment, mem_type, flags, addr_hint, pid, + gpu_va, handle, actual_size); + + if (ret == 0) { + nv_vgpu_dbg(vgpu, "ALLOC: size=%llu align=%llu -> gpu_va=0x%llx handle=0x%llx", + size, alignment, *gpu_va, *handle); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_free_fw(struct nv_vgpu_device *vgpu, u64 handle) +{ + struct gsp_rpc_integration *integ = g_integration; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_rpc_free(integ->rpc_client, handle); +} + +int nv_vgpu_gsp_rpc_map_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_map_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_map_msg(&req, handle, gpu_va, size, prot, flags, offset); + + ret = gsp_rpc_map(integ->rpc_client, handle, gpu_va, size, + prot, flags, offset, mapped_va); + + if (ret == 0) { + nv_vgpu_dbg(vgpu, "MAP: handle=0x%llx gpu_va=0x%llx size=%llu -> mapped_va=0x%llx", + handle, gpu_va, size, *mapped_va); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_unmap_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size) +{ + struct gsp_rpc_integration *integ = g_integration; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_rpc_unmap(integ->rpc_client, handle, gpu_va, size); +} + +int nv_vgpu_gsp_rpc_pin_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_pin_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_pin_msg(&req, handle, offset, size, flags); + + ret = gsp_rpc_pin(integ->rpc_client, handle, offset, size, + flags, dma_addr, num_pages); + + if (ret == 0) { + nv_vgpu_dbg(vgpu, "PIN: handle=0x%llx offset=%llu size=%llu -> dma_addr=0x%llx pages=%u", + handle, offset, size, *dma_addr, *num_pages); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_unpin_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size) +{ + struct gsp_rpc_integration *integ = g_integration; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_rpc_unpin(integ->rpc_client, handle, offset, size); +} + +int nv_vgpu_gsp_rpc_schedule_fw(struct nv_vgpu_device *vgpu, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_sched_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_sched_msg(&req, work_handle, callback, arg, priority, flags); + + ret = gsp_rpc_schedule(integ->rpc_client, work_handle, callback, arg, + priority, flags); + + if (ret == 0) { + nv_vgpu_dbg(vgpu, "SCHEDULE: work_handle=0x%llx priority=%u", work_handle, priority); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_yield_fw(struct nv_vgpu_device *vgpu) +{ + struct gsp_rpc_integration *integ = g_integration; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_rpc_yield(integ->rpc_client); +} + +int nv_vgpu_gsp_rpc_bar_trap_fw(struct nv_vgpu_device *vgpu, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_bar_trap_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_bar_trap_msg(&req, bar_addr, access_type, size, value, bar_num); + + ret = gsp_rpc_bar_trap(integ->rpc_client, bar_addr, access_type, size, + value, bar_num, ret_value); + + if (ret == 0) { + nv_vgpu_dbg(vgpu, "BAR_TRAP: bar=%u addr=0x%llx type=%u -> value=0x%llx", + bar_num, bar_addr, access_type, *ret_value); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_migrate_save_fw(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_migrate_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_migrate_msg(&req, src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, + GSP_MIGRATE_SUBCMD_SAVE, 0, 0); + + ret = gsp_rpc_migrate_save(integ->rpc_client, + src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, + checkpoint); + + if (ret == 0) { + nv_vgpu_info(vgpu, "MIGRATE SAVE: src=%llu dst=%llu -> checkpoint=0x%llx", + src_gpu_id, dst_gpu_id, *checkpoint); + } + + return ret; +} + +int nv_vgpu_gsp_rpc_migrate_restore_fw(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint) +{ + struct gsp_rpc_integration *integ = g_integration; + struct gsp_migrate_req req; + int ret; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + gsp_build_migrate_msg(&req, src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, + GSP_MIGRATE_SUBCMD_RESTORE, 0, checkpoint); + + ret = gsp_rpc_migrate_restore(integ->rpc_client, + src_gpu_id, dst_gpu_id, + memory_bitmap, page_table_ptr, + checkpoint); + + if (ret == 0) { + nv_vgpu_info(vgpu, "MIGRATE RESTORE: src=%llu dst=%llu checkpoint=0x%llx", + src_gpu_id, dst_gpu_id, checkpoint); + } + + return ret; +} + +/* ============================================================ + * WPR Management (from firmware strings) + * ============================================================ */ + +int nv_vgpu_gsp_wpr_write_mutex_acquire(struct nv_vgpu_device *vgpu, u32 wpr_id) +{ + struct gsp_rpc_integration *integ = g_integration; + + (void)vgpu; /* integ is global, vgpu not directly used */ + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_wpr_write_mutex_acquire(integ->rpc_client, wpr_id); +} + +int nv_vgpu_gsp_wpr_write_mutex_release(struct nv_vgpu_device *vgpu, u32 wpr_id) +{ + struct gsp_rpc_integration *integ = g_integration; + + (void)vgpu; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_wpr_write_mutex_release(integ->rpc_client, wpr_id); +} + +int nv_vgpu_gsp_wpr_scratch_mutex_acquire(struct nv_vgpu_device *vgpu, u32 wpr_id) +{ + struct gsp_rpc_integration *integ = g_integration; + + (void)vgpu; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_wpr_scratch_mutex_acquire(integ->rpc_client, wpr_id); +} + +int nv_vgpu_gsp_wpr_scratch_mutex_release(struct nv_vgpu_device *vgpu, u32 wpr_id) +{ + struct gsp_rpc_integration *integ = g_integration; + + (void)vgpu; + + if (!integ || !integ->initialized || !integ->rpc_client) + return -ENODEV; + + return gsp_wpr_scratch_mutex_release(integ->rpc_client, wpr_id); +} + +/* ============================================================ + * Firmware Address Accessors (for debugging/validation) + * ============================================================ */ + +u64 nv_vgpu_gsp_get_primary_dispatcher(void) +{ + return GSP_RPC_DISPATCH_PRIMARY; +} + +u64 nv_vgpu_gsp_get_secondary_dispatcher(void) +{ + return GSP_RPC_DISPATCH_SECONDARY; +} + +u64 nv_vgpu_gsp_get_alloc_main(void) +{ + return GSP_ALLOC_MAIN; +} + +u64 nv_vgpu_gsp_get_schedule_main(void) +{ + return GSP_SCHEDULE_MAIN; +} + +u64 nv_vgpu_gsp_get_bar_trap_handler_jt(void) +{ + return GSP_BAR_TRAP_HANDLER_JT; +} + +u64 nv_vgpu_gsp_get_bar_trap_handler_primary(void) +{ + return GSP_BAR_TRAP_HANDLER_PRIM; +} + +u64 nv_vgpu_gsp_get_entry_point(void) +{ + return GSP_ENTRY_POINT; +} + +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_integration_init); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_integration_cleanup); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_alloc_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_free_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_map_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_unmap_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_pin_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_unpin_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_schedule_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_yield_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_bar_trap_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_migrate_save_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_rpc_migrate_restore_fw); +EXPORT_SYMBOL(nv_vgpu_gsp_wpr_write_mutex_acquire); +EXPORT_SYMBOL(nv_vgpu_gsp_wpr_write_mutex_release); +EXPORT_SYMBOL(nv_vgpu_gsp_wpr_scratch_mutex_acquire); +EXPORT_SYMBOL(nv_vgpu_gsp_wpr_scratch_mutex_release); +EXPORT_SYMBOL(nv_vgpu_gsp_get_primary_dispatcher); +EXPORT_SYMBOL(nv_vgpu_gsp_get_secondary_dispatcher); +EXPORT_SYMBOL(nv_vgpu_gsp_get_alloc_main); +EXPORT_SYMBOL(nv_vgpu_gsp_get_schedule_main); +EXPORT_SYMBOL(nv_vgpu_gsp_get_bar_trap_handler_jt); +EXPORT_SYMBOL(nv_vgpu_gsp_get_bar_trap_handler_primary); +EXPORT_SYMBOL(nv_vgpu_gsp_get_entry_point); diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c new file mode 100644 index 0000000..ad6c6cb --- /dev/null +++ b/kernel/gsp_sema_service.c @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * gsp_sema_service.c - emulated GPU progress service. + * + * The guest's UVM (and, later, other clients) submits "pushes" into GPU memory + * that end with a copy-engine semaphore release: a method group + * (SET_SEMAPHORE_A/B/PAYLOAD + LAUNCH_DMA) that, on real hardware, makes the + * engine write a payload into a semaphore in system memory. UVM then waits for + * that value. + * + * A real GPU performs those writes. Our emulation must too, or every UVM tracker + * wait times out. This service does it without any cooperation from the guest + * driver: + * + * 1. Discover the GPU-VA -> guest-PA relationship from the guest's own + * bookkeeping. UVM/RM keep structures that record, for each allocation, + * both its GPU VA and its kernel CPU VA (the CPU VA is a direct-map + * address). We first learn the direct-map base from self-referential + * structures in guest memory, then collect {cpu_va, gpu_va} pairs. + * + * 2. Scan the mapped pages for semaphore-release method groups and apply + * them: write the payload to the semaphore's guest-physical slot, in + * order (only when the slot currently holds payload-1). + * + * Everything runs from a kthread: the hypervisor's guest-DMA interface + * (vfio_dma_rw) only works when current->mm is NULL (kthread) or matches the + * mapping's mm (the hypervisor process). + * + * This file is part of the host-side device emulation; see + * open-iov/docs/03-architecture/06-host-module-and-hybrid-model.md. + */ + +#define pr_fmt(fmt) "gsp-resman-vfio: " fmt + +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "gsp_sema_service.h" + +#define SEMA_MAP_MAX 16384 +#define SEMA_PAGE_SIZE 4096 +#define SEMA_SCAN_PAGES_MAX 0x20000 /* 512 MB worth of pages per pass */ + +/* Plausible GPU VA window for the client's system-memory objects (Turing). */ +#define SEMA_VA_LO 0x100000000ULL +#define SEMA_VA_HI 0x160000000ULL + +/* Canonical kernel virtual address range (x86_64). */ +#define KVA_LO 0xffff800000000000ULL +#define KVA_HI 0xffffc00000000000ULL + +/* INC_METHOD header for the CE semaphore group: addr>>2 == 0x240, count == 3 */ +#define CE_SEMA_HDR_ADDR 0x90 +#define CE_SEMA_HDR_COUNT 3 + +struct sema_map_entry { + u64 gpu_page; /* GPU VA, page aligned */ + u64 pa_page; /* guest PA, page aligned */ +}; + +struct sema_svc { + struct task_struct *task; + bool stop; + + u64 direct_base; + u32 map_n; + struct sema_map_entry map[SEMA_MAP_MAX]; + u64 last_build; + + /* stats */ + u64 pairs_seen; + u64 releases_seen; + u64 releases_applied; + u64 builds; +}; + +static struct sema_svc svc; + +/* ------------------------------------------------------------------ */ +/* guest memory helpers */ +/* ------------------------------------------------------------------ */ + +static int svc_read(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) +{ + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, false); +} + +static int svc_write(struct nv_vgpu_device *vgpu, u64 gpa, void *buf, size_t len) +{ + return vfio_dma_rw(&vgpu->vfio_dev, gpa, buf, len, true); +} + +/* ------------------------------------------------------------------ */ +/* discovery */ +/* ------------------------------------------------------------------ */ + +struct base_candidate { + u64 base; + u32 count; +}; + +/* + * Pass 1 of a rebuild: walk guest RAM looking for + * (a) self-referential pointers: value V at offset `off` of page `gpa` + * such that V - (gpa + off) is a 1 GB aligned kernel address. This + * gives the direct-map base (page_offset_base), because such pointers + * are common (list heads, self-references) and consistent. + * (b) {kernel VA, GPU VA} pairs stored adjacently (UVM's rm_mem style: + * vas[CPU] followed by vas[GPU]), collected raw for pass 2. + */ +#define PAIR_MAX 32768 +struct raw_pair { u64 cpu_va; u64 gpu_va; }; + +static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf, + struct base_candidate *cands, u32 *ncand, + struct raw_pair *pairs, u32 *npairs) +{ + u64 gpa; + u32 nonz = 0; + + for (gpa = 0; gpa < (0x140000000ULL); gpa += SEMA_PAGE_SIZE) { + u32 off; + + if (svc_read(vgpu, gpa, buf, SEMA_PAGE_SIZE)) + continue; + + for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 8) { + u64 a, b; + + memcpy(&a, buf + off, 8); + memcpy(&b, buf + off + 8, 8); + + /* (a) self-pointer -> base candidate */ + if (a >= KVA_LO && a < KVA_HI && + (a & 0xfff) == ((gpa + off) & 0xfff)) { + u64 cb = a - gpa - off; + u32 i; + + if ((cb & 0x3fffffffULL) == 0) { + for (i = 0; i < *ncand; i++) { + if (cands[i].base == cb) { + cands[i].count++; + break; + } + } + if (i == *ncand && *ncand < 8) { + cands[*ncand].base = cb; + cands[*ncand].count = 1; + (*ncand)++; + } + } + } + + /* (b) {cpu_va, gpu_va} pair (page aligned: real mappings) */ + if (a >= KVA_LO && a < KVA_HI && + b >= SEMA_VA_LO && b < SEMA_VA_HI && + (a & 0xfff) == 0 && (b & 0xfff) == 0 && + *npairs < PAIR_MAX) { + pairs[*npairs].cpu_va = a; + pairs[*npairs].gpu_va = b; + (*npairs)++; + } + } + nonz++; + } +} + +static void svc_add_map(u64 gpu_va, u64 pa) +{ + u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); + u64 pa_page = pa & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc.map_n; i++) { + if (svc.map[i].gpu_page == gpu_page) { + svc.map[i].pa_page = pa_page; + return; + } + } + if (svc.map_n < SEMA_MAP_MAX) { + svc.map[svc.map_n].gpu_page = gpu_page; + svc.map[svc.map_n].pa_page = pa_page; + svc.map_n++; + } +} + +static int svc_map_lookup(u64 gpu_va, u64 *pa) +{ + u64 gpu_page = gpu_va & ~(u64)(SEMA_PAGE_SIZE - 1); + u32 i; + + for (i = 0; i < svc.map_n; i++) { + if (svc.map[i].gpu_page == gpu_page) { + *pa = svc.map[i].pa_page + (gpu_va & (SEMA_PAGE_SIZE - 1)); + return 0; + } + } + return -ENOENT; +} + +static void svc_rebuild(struct nv_vgpu_device *vgpu) +{ + static u8 *buf; + static struct base_candidate cands[8]; + static struct raw_pair pairs[PAIR_MAX]; + u32 ncand = 0, npairs = 0, i; + u64 best_base = 0; + u32 best_count = 0; + + if (!buf) { + buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + if (!buf) + return; + } + + svc_scan_pass(vgpu, buf, cands, &ncand, pairs, &npairs); + + for (i = 0; i < ncand; i++) { + if (cands[i].count > best_count) { + best_count = cands[i].count; + best_base = cands[i].base; + } + } + + if (best_base && best_count >= 2) { + svc.direct_base = best_base; + for (i = 0; i < npairs; i++) { + u64 cpu = pairs[i].cpu_va; + u64 pa; + + if (cpu < best_base) + continue; + pa = cpu - best_base; + if (pa >= 0x140000000ULL || (pa & 0xfff)) + continue; + if (pairs[i].gpu_va < 0x100000000ULL || + pairs[i].gpu_va >= 0x140000000ULL) + continue; + svc_add_map(pairs[i].gpu_va, pa); + svc.pairs_seen++; + } + } + + svc.builds++; + svc.last_build = jiffies; + + pr_info("sema-svc: rebuild #%llu base=0x%llx (count=%u) pairs=%u map=%u seen=%llu applied=%llu\n", + svc.builds, best_base, best_count, npairs, svc.map_n, + svc.releases_seen, svc.releases_applied); + /* show a few entries so mappings can be verified from dmesg */ + { + u32 shown; + + for (shown = 0; shown < svc.map_n && shown < 8; shown++) + pr_info("sema-svc: map[%u] va=0x%llx -> pa=0x%llx\n", + shown, svc.map[shown].gpu_page, + svc.map[shown].pa_page); + } +} + +/* ------------------------------------------------------------------ */ +/* push application */ +/* ------------------------------------------------------------------ */ + +/* + * Look for CE semaphore-release method groups inside mapped pages: + * + * [INC_METHOD header (addr>>2 == 0x240, count == 3)] + * [upper 32 bits of semaphore VA] + * [lower 32 bits of semaphore VA] + * [payload] + * [... LAUNCH_DMA ...] + * + * Apply by writing payload to the semaphore slot, but only in order + * (current == payload - 1) so that UVM's tracker never sees a jump. + */ +static u32 svc_apply_page(struct nv_vgpu_device *vgpu, u8 *buf, u64 pa_page) +{ + u32 off, applied = 0; + + if (svc_read(vgpu, pa_page, buf, SEMA_PAGE_SIZE)) + return 0; + + for (off = 0; off + 16 <= SEMA_PAGE_SIZE; off += 4) { + u32 hdr, hi, lo, payload; + u64 va, pa, cur; + u32 cur32; + + memcpy(&hdr, buf + off, 4); + if ((hdr & 0xffff) != CE_SEMA_HDR_ADDR) + continue; + if (((hdr >> 16) & 0xff) != CE_SEMA_HDR_COUNT) + continue; + + memcpy(&hi, buf + off + 4, 4); + memcpy(&lo, buf + off + 8, 4); + memcpy(&payload, buf + off + 12, 4); + + if (hi > 0xffff || payload == 0 || payload > 0x10000000) + continue; + + va = ((u64)hi << 32) | lo; + if (va < SEMA_VA_LO || va >= SEMA_VA_HI) + continue; + + svc.releases_seen++; + + if (svc_map_lookup(va, &pa)) + continue; + + cur = 0; + if (svc_read(vgpu, pa, &cur32, sizeof(cur32))) + continue; + cur = cur32; + + /* in-order completion only */ + if (cur + 1 != payload) + continue; + + if (svc_write(vgpu, pa, &payload, sizeof(payload)) == 0) { + svc.releases_applied++; + applied++; + if (svc.releases_applied <= 16) + pr_info("sema-svc: applied va=0x%llx pa=0x%llx payload=%u\n", + va, pa, payload); + } + } + return applied; +} + +static void svc_apply_pushes(struct nv_vgpu_device *vgpu) +{ + static u8 *buf; + u32 i; + + if (!buf) { + buf = kmalloc(SEMA_PAGE_SIZE, GFP_KERNEL); + if (!buf) + return; + } + + for (i = 0; i < svc.map_n; i++) + svc_apply_page(vgpu, buf, svc.map[i].pa_page); +} + +/* ------------------------------------------------------------------ */ +/* thread */ +/* ------------------------------------------------------------------ */ + +static int svc_thread(void *arg) +{ + struct nv_vgpu_device *vgpu; + + pr_info("sema-svc: thread started\n"); + + while (!kthread_should_stop() && !svc.stop) { + vgpu = NULL; + mutex_lock(&nv_vgpu_driver.lock); + { + struct nv_vgpu_device *d; + + list_for_each_entry(d, &nv_vgpu_driver.devices, list) { + if (atomic_read(&d->vfio_dev_open_count) > 0) { + vgpu = d; + break; + } + } + } + mutex_unlock(&nv_vgpu_driver.lock); + + if (!vgpu) { + msleep(500); + continue; + } + + if (!svc.direct_base || + time_after(jiffies, svc.last_build + 10 * HZ)) + svc_rebuild(vgpu); + + if (svc.direct_base && svc.map_n) + svc_apply_pushes(vgpu); + + msleep(100); + } + + pr_info("sema-svc: thread stopping (pairs=%llu releases_seen=%llu applied=%llu map=%u)\n", + svc.pairs_seen, svc.releases_seen, svc.releases_applied, svc.map_n); + return 0; +} + +int gsp_sema_service_start(void) +{ + if (svc.task) + return 0; + svc.stop = false; + svc.task = kthread_run(svc_thread, NULL, "gsp-sema-svc"); + if (IS_ERR(svc.task)) { + int ret = PTR_ERR(svc.task); + + svc.task = NULL; + return ret; + } + return 0; +} + +void gsp_sema_service_stop(void) +{ + if (svc.task) { + svc.stop = true; + kthread_stop(svc.task); + svc.task = NULL; + } +} diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h new file mode 100644 index 0000000..2dfa28d --- /dev/null +++ b/kernel/gsp_sema_service.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _GSP_SEMA_SERVICE_H_ +#define _GSP_SEMA_SERVICE_H_ + +/* + * Emulated-GPU progress service: discovers the guest's GPU-VA -> guest-PA + * mapping and applies the copy-engine semaphore releases found in mapped + * pages, so that the guest's UVM progress waits complete. + * + * See gsp_sema_service.c for the full description. + */ + +int gsp_sema_service_start(void); +void gsp_sema_service_stop(void); + +#endif /* _GSP_SEMA_SERVICE_H_ */ diff --git a/kernel/include/bar_emulation.h b/kernel/include/bar_emulation.h new file mode 100644 index 0000000..326e70e --- /dev/null +++ b/kernel/include/bar_emulation.h @@ -0,0 +1,79 @@ +#ifndef _NV_VGPU_BAR_EMULATION_H_ +#define _NV_VGPU_BAR_EMULATION_H_ + +#include "nvidia-vgpu-vfio.h" + +/* MSI-X Table entry size (16 bytes: addr_lo, addr_hi, data, vector_control) */ +#define NV_VGPU_MSIX_ENTRY_SIZE 16 +#define NV_VGPU_MSIX_TABLE_ENTRIES 32 +#define NV_VGPU_MSIX_TABLE_SIZE (NV_VGPU_MSIX_ENTRY_SIZE * NV_VGPU_MSIX_TABLE_ENTRIES) +#define NV_VGPU_MSIX_PBA_SIZE 8 /* 64 bits for 32 vectors */ + +struct nv_vgpu_msix_entry { + u64 addr; + u32 data; + u32 vector_control; +}; + +struct nv_vgpu_bar_emul_ctx { + struct nv_vgpu_device *vgpu; + void *bar0_shadow; + size_t bar0_shadow_size; + spinlock_t shadow_lock; + + /* MSI-X emulation */ + struct nv_vgpu_msix_entry msix_table[NV_VGPU_MSIX_TABLE_ENTRIES]; + u64 msix_pba; /* Pending Bit Array */ + spinlock_t msix_lock; + bool msix_enabled; + + struct { + u32 offset; + u32 value; + bool is_write; + bool trapped; + } trap_regs[256]; + int trap_count; +}; + +int nv_vgpu_bar_emul_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_bar_emul_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_bar_trap_register(struct nv_vgpu_device *vgpu, int bar_idx, + u32 offset, u32 size); +int nv_vgpu_bar_untrap_register(struct nv_vgpu_device *vgpu, int bar_idx, + u32 offset); + +int nv_vgpu_bar_emul_read(struct nv_vgpu_device *vgpu, int bar_idx, + u64 offset, void *val, size_t size); +int nv_vgpu_bar_emul_write(struct nv_vgpu_device *vgpu, int bar_idx, + u64 offset, void *val, size_t size); + +int nv_vgpu_bar_emul_handle_fault(struct nv_vgpu_device *vgpu, + unsigned long addr, bool write); + +/* MSI-X emulation */ +void nv_vgpu_msix_set_pending(struct nv_vgpu_device *vgpu, int vector); +void nv_vgpu_msix_clear_pending(struct nv_vgpu_device *vgpu, int vector); + +static inline u32 nv_vgpu_read_reg(void __iomem *base, u32 offset) +{ + return readl(base + offset); +} + +static inline void nv_vgpu_write_reg(void __iomem *base, u32 offset, u32 val) +{ + writel(val, base + offset); +} + +static inline u64 nv_vgpu_read_reg64(void __iomem *base, u32 offset) +{ + return readq(base + offset); +} + +static inline void nv_vgpu_write_reg64(void __iomem *base, u32 offset, u64 val) +{ + writeq(val, base + offset); +} + +#endif \ No newline at end of file diff --git a/kernel/include/gsp_comm.h b/kernel/include/gsp_comm.h new file mode 100644 index 0000000..622f298 --- /dev/null +++ b/kernel/include/gsp_comm.h @@ -0,0 +1,155 @@ +#ifndef _NV_VGPU_GSP_COMM_H_ +#define _NV_VGPU_GSP_COMM_H_ + +#include "nvidia-vgpu-vfio.h" + +#define NV_VGPU_GSP_RPC_MAX_SIZE 4096 +#define NV_VGPU_GSP_RPC_TIMEOUT_MS 5000 +#define NV_VGPU_GSP_QUEUE_DEPTH 256 + +#define NV_VGPU_GSP_DOORBELL_OFFSET 0x100000 +#define NV_VGPU_GSP_SHARED_MEM_OFFSET 0x200000 +#define NV_VGPU_GSP_SHARED_MEM_SIZE (64 * 1024) + +#define NV_VGPU_GSP_FIRMWARE_PATH "/lib/firmware/nvidia/tu102/gsp/" + +enum nv_vgpu_gsp_rpc_type { + NV_GSP_RPC_TYPE_ALLOC = 0x01, + NV_GSP_RPC_TYPE_FREE = 0x02, + NV_GSP_RPC_TYPE_MAP = 0x03, + NV_GSP_RPC_TYPE_UNMAP = 0x04, + NV_GSP_RPC_TYPE_PIN = 0x05, + NV_GSP_RPC_TYPE_UNPIN = 0x06, + NV_GSP_RPC_TYPE_SCHEDULE = 0x07, + NV_GSP_RPC_TYPE_YIELD = 0x08, + NV_GSP_RPC_TYPE_NOTIFY = 0x09, + NV_GSP_RPC_TYPE_BAR_TRAP = 0x0A, + NV_GSP_RPC_TYPE_MIGRATE = 0x0B, + NV_GSP_RPC_TYPE_DEBUG = 0xFF, +}; + +struct nv_vgpu_gsp_rpc_header { + u32 type; + u32 sequence; + u32 size; + u32 flags; + u64 timestamp; +}; + +struct nv_vgpu_gsp_rpc_msg { + struct nv_vgpu_gsp_rpc_header hdr; + u8 data[]; +}; + +struct nv_vgpu_gsp_firmware { + void *bootloader; + size_t bootloader_size; + void *booter_load; + size_t booter_load_size; + void *gsp; + size_t gsp_size; +}; + +struct nv_vgpu_gsp_comm_ctx { + struct nv_vgpu_device *vgpu; + + void __iomem *gsp_doorbell; + void __iomem *gsp_shared_mem; + size_t shared_mem_size; + + struct mutex queue_lock; + struct list_head pending_queue; + struct list_head complete_queue; + atomic_t sequence_counter; + + wait_queue_head_t wait_queue; + + struct workqueue_struct *rpc_wq; + struct work_struct rpc_work; + + struct delayed_work poll_work; + bool poll_active; + + spinlock_t irq_lock; + bool irq_enabled; + + struct nv_vgpu_gsp_firmware firmware; + bool firmware_loaded; +}; + +struct nv_vgpu_gsp_rpc_entry { + struct list_head list; + struct nv_vgpu_gsp_rpc_msg *msg; + struct completion done; + int result; +}; + +int nv_vgpu_gsp_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_gsp_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_gsp_send_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size); +int nv_vgpu_gsp_recv_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size, int timeout_ms); + +int nv_vgpu_gsp_alloc(struct nv_vgpu_device *vgpu, u64 size, u32 align, u64 *handle); +int nv_vgpu_gsp_free(struct nv_vgpu_device *vgpu, u64 handle); +int nv_vgpu_gsp_map(struct nv_vgpu_device *vgpu, u64 handle, u64 iova, u64 size, u32 flags); +int nv_vgpu_gsp_unmap(struct nv_vgpu_device *vgpu, u64 handle, u64 iova, u64 size); +int nv_vgpu_gsp_schedule(struct nv_vgpu_device *vgpu, u32 runlist_id, u32 channel_id); +int nv_vgpu_gsp_notify_bar_trap(struct nv_vgpu_device *vgpu, u64 offset, u32 size, void *val); + +int nv_vgpu_gsp_load_firmware(struct nv_vgpu_device *vgpu); +void nv_vgpu_gsp_release_firmware(struct nv_vgpu_device *vgpu); +int nv_vgpu_gsp_boot(struct nv_vgpu_device *vgpu); + +/* ============================================================ + * New Firmware-Decompiled RPC Operations (enhanced APIs) + * ============================================================ */ + +int nv_vgpu_gsp_rpc_alloc(struct nv_vgpu_device *vgpu, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size); + +int nv_vgpu_gsp_rpc_free(struct nv_vgpu_device *vgpu, u64 handle); + +int nv_vgpu_gsp_rpc_map(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va); + +int nv_vgpu_gsp_rpc_unmap(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size); + +int nv_vgpu_gsp_rpc_pin(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages); + +int nv_vgpu_gsp_rpc_unpin(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size); + +int nv_vgpu_gsp_rpc_schedule(struct nv_vgpu_device *vgpu, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags); + +int nv_vgpu_gsp_rpc_yield(struct nv_vgpu_device *vgpu); + +int nv_vgpu_gsp_rpc_bar_trap(struct nv_vgpu_device *vgpu, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value); + +int nv_vgpu_gsp_rpc_migrate_save(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint); + +int nv_vgpu_gsp_rpc_migrate_restore(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint); + +static inline u32 nv_vgpu_gsp_next_sequence(struct nv_vgpu_gsp_comm_ctx *ctx) +{ + return atomic_inc_return(&ctx->sequence_counter); +} + +#endif \ No newline at end of file diff --git a/kernel/include/gsp_rpc_client.h b/kernel/include/gsp_rpc_client.h new file mode 100644 index 0000000..4b48953 --- /dev/null +++ b/kernel/include/gsp_rpc_client.h @@ -0,0 +1,313 @@ +/* + * GSP 570.144W RPC Client Header + * Auto-generated from firmware decompilation + */ + +#ifndef _GSP_RPC_CLIENT_H_ +#define _GSP_RPC_CLIENT_H_ + +#include +#include +#include "nvidia-vgpu-vfio.h" + +/* ============================================================ + * FIRMWARE BASE ADDRESS + * ============================================================ */ +#define GSP_FW_BASE 0x04000000UL + +/* ============================================================ + * RPC DISPATCHER ADDRESSES + * ============================================================ */ + +/* Primary Dispatcher - Jump table switch for 6 RPC types */ +#define GSP_RPC_DISPATCH_PRIMARY (GSP_FW_BASE + 0x17A1AC0) /* 0x057a1ac0 */ +#define GSP_RPC_JUMP_TABLE (GSP_FW_BASE + 0x17A3AD8) /* 0x057a3ad8 - 32 sub-commands */ + +/* Secondary Dispatcher - Full implementation for all 11 RPC types */ +#define GSP_RPC_DISPATCH_SECONDARY (GSP_FW_BASE + 0x0ED39DC) /* 0x04ed39dc */ + +/* Tertiary Dispatcher - Stack=160 bytes */ +#define GSP_RPC_DISPATCH_TERTIARY (GSP_FW_BASE + 0x0CC58F8) /* 0x04cc58f8 */ + +/* Quaternary Dispatcher - Stack=96 bytes */ +#define GSP_RPC_DISPATCH_QUATERNARY (GSP_FW_BASE + 0x1723EF4) /* 0x05723ef4 */ + +/* ============================================================ + * SPECIALIZED HANDLER ADDRESSES + * ============================================================ */ + +#define GSP_ALLOC_MAIN (GSP_FW_BASE + 0x17E94EC) /* 0x057e94ec */ +#define GSP_SCHEDULE_MAIN (GSP_FW_BASE + 0x11EFBF4) /* 0x051efbf4 */ +#define GSP_BAR_TRAP_HANDLER_JT (GSP_FW_BASE + 0x17675B7) /* 0x057675b7 */ +#define GSP_BAR_TRAP_HANDLER_PRIM (GSP_FW_BASE + 0x0CB8189) /* 0x04cb8189 */ +#define GSP_PIN_HANDLER_SEC (GSP_FW_BASE + 0x0ED34B8) /* 0x04ed34b8 */ + +/* Jump Table Handlers (from decoded table at 0x059bfeb0) */ +#define GSP_MAP_HANDLER_A (GSP_FW_BASE + 0x1766AB7) /* 0x05766ab7 */ +#define GSP_MAP_HANDLER_B (GSP_FW_BASE + 0x1778A37) /* 0x05778a37 */ +#define GSP_UNMAP_HANDLER_A (GSP_FW_BASE + 0x17669B7) /* 0x057669b7 */ +#define GSP_UNMAP_HANDLER_B (GSP_FW_BASE + 0x1779937) /* 0x05779937 */ +#define GSP_PIN_HANDLER_A (GSP_FW_BASE + 0x17664B7) /* 0x057664b7 */ +#define GSP_PIN_HANDLER_B (GSP_FW_BASE + 0x177A3B7) /* 0x0577a3b7 */ +#define GSP_UNPIN_HANDLER_A (GSP_FW_BASE + 0x17682B7) /* 0x057682b7 */ +#define GSP_UNPIN_HANDLER_B (GSP_FW_BASE + 0x1779FB7) /* 0x05779fb7 */ +#define GSP_SCHEDULE_HANDLER_A (GSP_FW_BASE + 0x1767F37) /* 0x05767f37 */ +#define GSP_SCHEDULE_HANDLER_B (GSP_FW_BASE + 0x175AEB7) /* 0x0575aeb7 */ +#define GSP_YIELD_HANDLER_A (GSP_FW_BASE + 0x175AE37) /* 0x0575ae37 */ +#define GSP_YIELD_HANDLER_B (GSP_FW_BASE + 0x1767337) /* 0x05767337 */ +#define GSP_NOTIFY_HANDLER_A (GSP_FW_BASE + 0x177C8B7) /* 0x0577c8b7 */ +#define GSP_NOTIFY_HANDLER_B (GSP_FW_BASE + 0x1767837) /* 0x05767837 */ + +/* Entry Point */ +#define GSP_ENTRY_POINT (GSP_FW_BASE + 0x0009000) /* 0x04009000 */ + +/* ============================================================ + * RPC TYPE DEFINITIONS (match existing nv_vgpu_gsp_rpc_type) + * ============================================================ */ +enum gsp_rpc_type { + GSP_RPC_ALLOC = 0x01, + GSP_RPC_FREE = 0x02, + GSP_RPC_MAP = 0x03, + GSP_RPC_UNMAP = 0x04, + GSP_RPC_PIN = 0x05, + GSP_RPC_UNPIN = 0x06, + GSP_RPC_SCHEDULE = 0x07, + GSP_RPC_YIELD = 0x08, + GSP_RPC_NOTIFY = 0x09, + GSP_RPC_BAR_TRAP = 0x0A, + GSP_RPC_MIGRATE = 0x0B, + GSP_RPC_DEBUG = 0xFF, +}; + +/* ============================================================ + * ALLOC SUB-COMMANDS (Primary Dispatcher Jump Table Indices) + * ============================================================ */ +enum gsp_alloc_subcmd { + GSP_ALLOC_SUBCMD_DEFAULT = 0, + GSP_ALLOC_SUBCMD_TYPE1 = 1, + GSP_ALLOC_SUBCMD_TYPE2 = 3, + GSP_ALLOC_SUBCMD_TYPE3 = 4, + GSP_ALLOC_SUBCMD_TYPE4 = 5, + GSP_ALLOC_SUBCMD_TYPE5 = 8, + GSP_ALLOC_SUBCMD_TYPE6 = 10, + GSP_ALLOC_SUBCMD_TYPE7 = 12, + GSP_ALLOC_SUBCMD_TYPE8 = 17, + GSP_ALLOC_SUBCMD_TYPE9 = 19, + GSP_ALLOC_SUBCMD_TYPE10 = 20, + GSP_ALLOC_SUBCMD_TYPE11 = 21, + GSP_ALLOC_SUBCMD_TYPE12 = 22, + GSP_ALLOC_SUBCMD_TYPE13 = 23, + GSP_ALLOC_SUBCMD_TYPE14 = 24, + GSP_ALLOC_SUBCMD_TYPE15 = 25, + GSP_ALLOC_SUBCMD_TYPE16 = 26, + GSP_ALLOC_SUBCMD_TYPE17 = 27, + GSP_ALLOC_SUBCMD_TYPE18 = 28, + GSP_ALLOC_SUBCMD_TYPE19 = 29, + GSP_ALLOC_SUBCMD_TYPE20 = 30, + GSP_ALLOC_SUBCMD_TYPE21 = 31, +}; + +/* ============================================================ + * MIGRATE SUB-COMMANDS + * ============================================================ */ +enum gsp_migrate_subcmd { + GSP_MIGRATE_SUBCMD_SAVE = 0x04, + GSP_MIGRATE_SUBCMD_RESTORE = 0x05, +}; + +/* ============================================================ + * PIN SUB-COMMANDS + * ============================================================ */ +enum gsp_pin_subcmd { + GSP_PIN_SUBCMD_DEFAULT = 0x05, +}; + +/* ============================================================ + * SCHEDULE SUB-COMMANDS + * ============================================================ */ +enum gsp_sched_subcmd { + GSP_SCHED_SUBCMD_DEFAULT = 0x07, +}; + +/* ============================================================ + * BAR TRAP SUB-COMMANDS + * ============================================================ */ +enum gsp_bar_trap_subcmd { + GSP_BAR_TRAP_SUBCMD_DEFAULT = 0x0A, +}; + +/* ============================================================ + * RPC MESSAGE STRUCTURES + * ============================================================ */ + +struct gsp_rpc_msg { + __le32 type; + __le32 subcmd; + __le64 payload[6]; + __le32 seq_id; + __le32 flags; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +struct gsp_rpc_response { + __le32 status; + __le32 type; + __le64 result[6]; + __le32 seq_id; +} __attribute__((packed, aligned(8))); + +/* ALLOC request/response */ +struct gsp_alloc_req { + __le64 size; + __le64 alignment; + __le32 mem_type; + __le32 flags; + __le64 addr_hint; + __le32 pid; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +struct gsp_alloc_rsp { + __le64 gpu_va; + __le64 handle; + __le64 size; +} __attribute__((packed, aligned(8))); + +/* MAP request/response */ +struct gsp_map_req { + __le64 handle; + __le64 gpu_va; + __le64 size; + __le32 prot; + __le32 flags; + __le64 offset; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +struct gsp_map_rsp { + __le64 gpu_va; +} __attribute__((packed, aligned(8))); + +/* PIN/UNPIN request/response */ +struct gsp_pin_req { + __le64 handle; + __le64 offset; + __le64 size; + __le32 flags; + __le32 reserved[2]; +} __attribute__((packed, aligned(8))); + +struct gsp_pin_rsp { + __le64 dma_addr; + __le32 num_pages; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +/* SCHEDULE request */ +struct gsp_sched_req { + __le64 work_handle; + __le64 callback; + __le64 arg; + __le32 priority; + __le32 flags; + __le32 reserved[2]; +} __attribute__((packed, aligned(8))); + +/* BAR TRAP request/response */ +struct gsp_bar_trap_req { + __le64 bar_addr; + __le32 access_type; + __le32 size; + __le64 value; + __le32 bar_num; +} __attribute__((packed, aligned(8))); + +struct gsp_bar_trap_rsp { + __le64 value; + __le32 handled; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +/* MIGRATE request/response */ +struct gsp_migrate_req { + __le64 src_gpu_id; + __le64 dst_gpu_id; + __le64 memory_bitmap; + __le64 page_table_ptr; + __le32 state; + __le32 flags; + __le64 context; + __le32 reserved; +} __attribute__((packed, aligned(8))); + +struct gsp_migrate_rsp { + __le32 status; + __le32 progress; + __le64 checkpoint; +} __attribute__((packed, aligned(8))); + +/* ============================================================ + * CLIENT OPAQUE STRUCT + * ============================================================ */ +struct gsp_client; + +/* ============================================================ + * PUBLIC API + * ============================================================ */ + +/* Client lifecycle */ +struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell); +void gsp_client_fini(struct gsp_client *client); + +/* Memory management */ +int gsp_rpc_alloc(struct gsp_client *client, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size); + +int gsp_rpc_free(struct gsp_client *client, u64 handle); + +int gsp_rpc_map(struct gsp_client *client, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va); + +int gsp_rpc_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size); + +/* Page pinning */ +int gsp_rpc_pin(struct gsp_client *client, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages); + +int gsp_rpc_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size); + +/* Scheduling */ +int gsp_rpc_schedule(struct gsp_client *client, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags); + +int gsp_rpc_yield(struct gsp_client *client); + +/* BAR trap emulation */ +int gsp_rpc_bar_trap(struct gsp_client *client, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value); + +/* Migration */ +int gsp_rpc_migrate_save(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint); + +int gsp_rpc_migrate_restore(struct gsp_client *client, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint); + +/* WPR Management */ +int gsp_wpr_write_mutex_acquire(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_write_mutex_release(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_scratch_mutex_acquire(struct gsp_client *client, u32 wpr_id); +int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id); + +#endif /* _GSP_RPC_CLIENT_H_ */ diff --git a/kernel/include/gsp_rpc_integration.h b/kernel/include/gsp_rpc_integration.h new file mode 100644 index 0000000..eea3bf3 --- /dev/null +++ b/kernel/include/gsp_rpc_integration.h @@ -0,0 +1,97 @@ +/* + * GSP RPC Integration Layer Header + * Bridges the firmware-decompiled RPC client with existing gsp_comm.c + */ + +#ifndef _NV_VGPU_GSP_RPC_INTEGRATION_H_ +#define _NV_VGPU_GSP_RPC_INTEGRATION_H_ + +#include "nvidia-vgpu-vfio.h" +#include "gsp_comm.h" + +/* Initialize the RPC integration layer */ +int nv_vgpu_gsp_rpc_integration_init(struct nv_vgpu_device *vgpu, + struct nv_vgpu_gsp_comm_ctx *comm_ctx); + +/* Cleanup the RPC integration layer */ +void nv_vgpu_gsp_rpc_integration_cleanup(struct nv_vgpu_device *vgpu); + +/* ============================================================ + * Enhanced RPC Operations (using firmware-decompiled addresses) + * ============================================================ */ + +/* Memory allocation with full firmware parameter support */ +int nv_vgpu_gsp_rpc_alloc_fw(struct nv_vgpu_device *vgpu, + u64 size, u64 alignment, u32 mem_type, + u32 flags, u64 addr_hint, u32 pid, + u64 *gpu_va, u64 *handle, u64 *actual_size); + +/* Memory free */ +int nv_vgpu_gsp_rpc_free_fw(struct nv_vgpu_device *vgpu, u64 handle); + +/* Memory mapping with full firmware parameter support */ +int nv_vgpu_gsp_rpc_map_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size, + u32 prot, u32 flags, u64 offset, + u64 *mapped_va); + +/* Memory unmapping */ +int nv_vgpu_gsp_rpc_unmap_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 gpu_va, u64 size); + +/* Page pinning for DMA */ +int nv_vgpu_gsp_rpc_pin_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size, + u32 flags, u64 *dma_addr, u32 *num_pages); + +/* Page unpinning */ +int nv_vgpu_gsp_rpc_unpin_fw(struct nv_vgpu_device *vgpu, + u64 handle, u64 offset, u64 size); + +/* Work scheduling */ +int nv_vgpu_gsp_rpc_schedule_fw(struct nv_vgpu_device *vgpu, + u64 work_handle, u64 callback, u64 arg, + u32 priority, u32 flags); + +/* Yield current work */ +int nv_vgpu_gsp_rpc_yield_fw(struct nv_vgpu_device *vgpu); + +/* BAR trap emulation */ +int nv_vgpu_gsp_rpc_bar_trap_fw(struct nv_vgpu_device *vgpu, + u64 bar_addr, u32 access_type, u32 size, + u64 value, u32 bar_num, u64 *ret_value); + +/* Migration - save state */ +int nv_vgpu_gsp_rpc_migrate_save_fw(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 *checkpoint); + +/* Migration - restore state */ +int nv_vgpu_gsp_rpc_migrate_restore_fw(struct nv_vgpu_device *vgpu, + u64 src_gpu_id, u64 dst_gpu_id, + u64 memory_bitmap, u64 page_table_ptr, + u64 checkpoint); + +/* ============================================================ + * WPR Management (from firmware strings: VPR_WPR_WRITE_MUTEX*) + * ============================================================ */ + +int nv_vgpu_gsp_wpr_write_mutex_acquire(struct nv_vgpu_device *vgpu, u32 wpr_id); +int nv_vgpu_gsp_wpr_write_mutex_release(struct nv_vgpu_device *vgpu, u32 wpr_id); +int nv_vgpu_gsp_wpr_scratch_mutex_acquire(struct nv_vgpu_device *vgpu, u32 wpr_id); +int nv_vgpu_gsp_wpr_scratch_mutex_release(struct nv_vgpu_device *vgpu, u32 wpr_id); + +/* ============================================================ + * Firmware Address Accessors (for debugging/validation) + * ============================================================ */ + +u64 nv_vgpu_gsp_get_primary_dispatcher(void); /* 0x057a1ac0 */ +u64 nv_vgpu_gsp_get_secondary_dispatcher(void); /* 0x04ed39dc */ +u64 nv_vgpu_gsp_get_alloc_main(void); /* 0x057e94ec */ +u64 nv_vgpu_gsp_get_schedule_main(void); /* 0x051efbf4 */ +u64 nv_vgpu_gsp_get_bar_trap_handler_jt(void); /* 0x057675b7 */ +u64 nv_vgpu_gsp_get_bar_trap_handler_primary(void); /* 0x04cb8189 */ +u64 nv_vgpu_gsp_get_entry_point(void); /* 0x04009000 */ + +#endif /* _NV_VGPU_GSP_RPC_INTEGRATION_H_ */ diff --git a/kernel/include/ioctl.h b/kernel/include/ioctl.h new file mode 100644 index 0000000..36d28ac --- /dev/null +++ b/kernel/include/ioctl.h @@ -0,0 +1,126 @@ +#ifndef _NV_VGPU_IOCTL_H_ +#define _NV_VGPU_IOCTL_H_ + +#include "nvidia-vgpu-vfio.h" + +#define NV_VGPU_IOCTL_MAGIC 'N' + +#define NV_VGPU_IOCTL_GET_INFO \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x01, struct nv_vgpu_ioctl_get_info) + +#define NV_VGPU_IOCTL_MAP_BAR \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x02, struct nv_vgpu_ioctl_map_bar) + +#define NV_VGPU_IOCTL_UNMAP_BAR \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x03, struct nv_vgpu_ioctl_unmap_bar) + +#define NV_VGPU_IOCTL_ENABLE_IRQ \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x04, struct nv_vgpu_ioctl_irq) + +#define NV_VGPU_IOCTL_DISABLE_IRQ \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x05, struct nv_vgpu_ioctl_irq) + +#define NV_VGPU_IOCTL_GSP_RPC \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x06, struct nv_vgpu_ioctl_gsp_rpc) + +#define NV_VGPU_IOCTL_MIGRATE_SAVE \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x07, struct nv_vgpu_ioctl_migrate) + +#define NV_VGPU_IOCTL_MIGRATE_RESTORE \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x08, struct nv_vgpu_ioctl_migrate) + +#define NV_VGPU_IOCTL_SET_GSP_FIRMWARE \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x09, struct nv_vgpu_ioctl_firmware) + +#define NV_VGPU_IOCTL_REGISTER_VGPU_TYPE \ + _IOWR(NV_VGPU_IOCTL_MAGIC, 0x0A, struct nv_vgpu_ioctl_register_vgpu_type) + +/* vGPU type info structure (shared between main.c and ioctl.c) */ +struct nv_vgpu_type_info { + char name[64]; + char description[128]; + u64 vdev_id; + u64 pdev_id; + u32 max_instances; +}; + +struct nv_vgpu_ioctl_get_info { + u32 vgpu_type; + u32 vgpu_id; + u32 gpu_instance_id; + u32 compute_instance_id; + u32 state; + u64 fb_base; + u64 fb_size; + u64 bar0_config_base; + u32 num_bars; + struct { + u32 index; + u64 gpa; + u64 size; + u64 iova; + bool mapped; + bool trapped; + } bars[NV_VGPU_MAX_REGIONS]; +}; + +struct nv_vgpu_ioctl_map_bar { + u32 bar_index; + u64 gpa; + u64 size; + u64 iova; + u32 flags; +}; + +struct nv_vgpu_ioctl_unmap_bar { + u32 bar_index; + u64 iova; + u64 size; +}; + +struct nv_vgpu_ioctl_irq { + u32 irq_index; + int fd; +}; + +struct nv_vgpu_ioctl_gsp_rpc { + u32 type; + u32 flags; + u64 data_ptr; + u32 data_size; + u32 result; +}; + +struct nv_vgpu_ioctl_migrate { + u64 data_ptr; + u32 data_size; + u32 flags; + int result; +}; + +struct nv_vgpu_ioctl_firmware { + u32 chip_id; + u64 firmware_ptr; + u32 firmware_size; + u32 firmware_type; +}; + +struct nv_vgpu_ioctl_register_vgpu_type { + char name[64]; + char description[128]; + u64 vdev_id; + u64 pdev_id; + u32 max_instances; +}; + +int nv_vgpu_ioctl_get_info(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_map_bar(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_unmap_bar(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_enable_irq(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_disable_irq(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_gsp_rpc(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_migrate_save(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_migrate_restore(struct nv_vgpu_device *vgpu, void __user *arg); +int nv_vgpu_ioctl_register_vgpu_type(void __user *arg); + +#endif \ No newline at end of file diff --git a/kernel/include/irq.h b/kernel/include/irq.h new file mode 100644 index 0000000..d06ac5b --- /dev/null +++ b/kernel/include/irq.h @@ -0,0 +1,45 @@ +#ifndef _NV_VGPU_IRQ_H_ +#define _NV_VGPU_IRQ_H_ + +#include "nvidia-vgpu-vfio.h" + +#define NV_VGPU_MAX_MSI_VECTORS 32 + +struct nv_vgpu_msi_irq { + int virq; + u32 vector; + bool enabled; + struct eventfd_ctx *trigger; + struct irq_domain *domain; + struct msi_msg msg; +}; + +struct nv_vgpu_irq_ctx { + struct nv_vgpu_device *vgpu; + struct mutex lock; + struct nv_vgpu_msi_irq msis[NV_VGPU_MAX_MSI_VECTORS]; + int msi_count; + bool msi_enabled; + bool msix_enabled; +}; + +int nv_vgpu_irq_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_irq_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_irq_enable(struct nv_vgpu_device *vgpu, int irq_idx); +int nv_vgpu_irq_disable(struct nv_vgpu_device *vgpu, int irq_idx); + +int nv_vgpu_irq_setup_msi(struct nv_vgpu_device *vgpu, int num_vectors); +int nv_vgpu_irq_setup_msix(struct nv_vgpu_device *vgpu, int num_vectors); +void nv_vgpu_irq_teardown_msi(struct nv_vgpu_device *vgpu); +void nv_vgpu_irq_teardown_msix(struct nv_vgpu_device *vgpu); + +int nv_vgpu_irq_set_trigger(struct nv_vgpu_device *vgpu, int irq_idx, int fd); +int nv_vgpu_irq_get_virq(struct nv_vgpu_device *vgpu, int irq_idx); + +static inline struct nv_vgpu_irq_ctx *vgpu_to_irq_ctx(struct nv_vgpu_device *vgpu) +{ + return (struct nv_vgpu_irq_ctx *)vgpu->irq_ctx; +} + +#endif \ No newline at end of file diff --git a/kernel/include/memory.h b/kernel/include/memory.h new file mode 100644 index 0000000..07e57f2 --- /dev/null +++ b/kernel/include/memory.h @@ -0,0 +1,56 @@ +#ifndef _NV_VGPU_MEMORY_H_ +#define _NV_VGPU_MEMORY_H_ + +#include "nvidia-vgpu-vfio.h" + +#define NV_VGPU_MAX_MAPPINGS 1024 + +struct nv_vgpu_mapping { + u64 gpa; + u64 iova; + u64 size; + struct page **pages; + int page_count; + bool pinned; + bool mapped; + struct list_head list; +}; + +struct nv_vgpu_simple_mapping { + u64 gpa; + u64 size; + u64 iova; + struct page **pages; + int page_count; + bool pinned; + bool mapped; + struct list_head list; +}; + +struct nv_vgpu_mm_ctx { + struct nv_vgpu_device *vgpu; + struct mutex lock; + struct list_head mappings; + u32 mapping_count; + struct iommu_domain *iommu_domain; + struct dma_iommu_mapping *dma_mapping; + bool use_simple; +}; + +int nv_vgpu_memory_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_memory_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_memory_map(struct nv_vgpu_device *vgpu, u64 gpa, u64 size, u64 *iova); +int nv_vgpu_memory_unmap(struct nv_vgpu_device *vgpu, u64 iova, u64 size); + +int nv_vgpu_memory_pin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size); +int nv_vgpu_memory_unpin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size); + +int nv_vgpu_memory_mmap(struct nv_vgpu_device *vgpu, struct vm_area_struct *vma); + +static inline struct nv_vgpu_mm_ctx *vgpu_to_mm_ctx(struct nv_vgpu_device *vgpu) +{ + return vgpu->mm_ctx; +} + +#endif \ No newline at end of file diff --git a/kernel/include/migration.h b/kernel/include/migration.h new file mode 100644 index 0000000..17cf8db --- /dev/null +++ b/kernel/include/migration.h @@ -0,0 +1,75 @@ +#ifndef _NV_VGPU_MIGRATION_H_ +#define _NV_VGPU_MIGRATION_H_ + +#include "nvidia-vgpu-vfio.h" + +#define NV_VGPU_MIGRATION_MAGIC 0x4E564750 +#define NV_VGPU_MIGRATION_VERSION 1 + +struct nv_vgpu_migration_ctx { + struct nv_vgpu_device *vgpu; + struct mutex lock; + struct nv_vgpu_migration_state state; + struct task_struct *save_thread; + struct task_struct *restore_thread; + bool in_progress; +}; + +struct nv_vgpu_migration_header { + u32 magic; + u32 version; + u32 flags; + u32 num_sections; + u64 total_size; + u64 timestamp; +}; + +enum nv_vgpu_migration_section_type { + NV_VGPU_MIG_SECTION_DEVICE_STATE = 0x01, + NV_VGPU_MIG_SECTION_BAR_STATE = 0x02, + NV_VGPU_MIG_SECTION_MEMORY_STATE = 0x03, + NV_VGPU_MIG_SECTION_IRQ_STATE = 0x04, + NV_VGPU_MIG_SECTION_GSP_STATE = 0x05, + NV_VGPU_MIG_SECTION_ENGINE_STATE = 0x06, + NV_VGPU_MIG_SECTION_CHANNEL_STATE = 0x07, + NV_VGPU_MIG_SECTION_CONTEXT_STATE = 0x08, +}; + +struct nv_vgpu_migration_section { + u32 type; + u32 flags; + u64 offset; + u64 size; + u32 checksum; +}; + +struct nv_vgpu_migration_device_state { + u32 vgpu_type; + u32 vgpu_id; + u32 gpu_instance_id; + u32 compute_instance_id; + u32 state; + u64 fb_base; + u64 fb_size; + u64 bar0_config_base; +}; + +struct nv_vgpu_migration_bar_state { + u32 bar_index; + u64 gpa; + u64 size; + u64 iova; + u8 data[]; +}; + +int nv_vgpu_migration_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_migration_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_migration_save_state(struct nv_vgpu_device *vgpu, u8 **data, size_t *size); +int nv_vgpu_migration_restore_state(struct nv_vgpu_device *vgpu, u8 *data, size_t size); + +int nv_vgpu_migration_prepare(struct nv_vgpu_device *vgpu); +int nv_vgpu_migration_complete(struct nv_vgpu_device *vgpu); +int nv_vgpu_migration_cancel(struct nv_vgpu_device *vgpu); + +#endif \ No newline at end of file diff --git a/kernel/include/nv-hypervisor.h b/kernel/include/nv-hypervisor.h new file mode 100644 index 0000000..a2b2649 --- /dev/null +++ b/kernel/include/nv-hypervisor.h @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1999-2018 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _NV_HYPERVISOR_H_ +#define _NV_HYPERVISOR_H_ + +#include + +// Enums for supported hypervisor types. +// New hypervisor type should be added before OS_HYPERVISOR_UNKNOWN +typedef enum _HYPERVISOR_TYPE +{ + OS_HYPERVISOR_XEN = 0, + OS_HYPERVISOR_VMWARE, + OS_HYPERVISOR_HYPERV, + OS_HYPERVISOR_KVM, + OS_HYPERVISOR_UNKNOWN +} HYPERVISOR_TYPE; + +#define CMD_VFIO_WAKE_REMOVE_GPU 1 +#define CMD_VGPU_VFIO_PRESENT 2 +#define CMD_VFIO_PCI_CORE_PRESENT 3 + +#define MAX_VF_COUNT_PER_GPU 64 + +typedef enum _VGPU_TYPE_INFO +{ + VGPU_TYPE_NAME = 0, + VGPU_TYPE_DESCRIPTION, + VGPU_TYPE_INSTANCES, +} VGPU_TYPE_INFO; + +typedef struct +{ + void *nv; + NvU32 domain; + NvU32 bus; + NvU32 device; + NvU32 return_status; +} vgpu_vfio_info; + +typedef struct +{ + NvU32 domain; + NvU8 bus; + NvU8 slot; + NvU8 function; + NvBool isNvidiaAttached; + NvBool isMdevAttached; +} vgpu_vf_pci_info; + +typedef enum VGPU_CMD_PROCESS_VF_INFO_E +{ + NV_VGPU_SAVE_VF_INFO = 0, + NV_VGPU_REMOVE_VF_PCI_INFO = 1, + NV_VGPU_REMOVE_VF_MDEV_INFO = 2, + NV_VGPU_GET_VF_INFO = 3 +} VGPU_CMD_PROCESS_VF_INFO; + +typedef enum VGPU_DEVICE_STATE_E +{ + NV_VGPU_DEV_UNUSED = 0, + NV_VGPU_DEV_OPENED = 1, + NV_VGPU_DEV_IN_USE = 2 +} VGPU_DEVICE_STATE; + +/* + * Function prototypes + */ + +HYPERVISOR_TYPE NV_API_CALL nv_get_hypervisor_type(void); + +#endif // _NV_HYPERVISOR_H_ diff --git a/kernel/include/nv-vgpu-vfio-interface.h b/kernel/include/nv-vgpu-vfio-interface.h new file mode 100644 index 0000000..2553722 --- /dev/null +++ b/kernel/include/nv-vgpu-vfio-interface.h @@ -0,0 +1,83 @@ +/* SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + */ + +#ifndef _NV_VGPU_VFIO_INTERFACE_H_ +#define _NV_VGPU_VFIO_INTERFACE_H_ + +#include +#include +#include +#include + +/* Use standard kernel types */ +typedef int NV_STATUS; +typedef unsigned char NvU8; +typedef unsigned int NvU32; +typedef unsigned short NvU16; +typedef unsigned long long NvU64; +typedef int NvS32; +typedef unsigned char NvBool; + +#define NV_OK 0 +#define NV_ERR_NOT_SUPPORTED -1 +#define NV_ERR_GENERIC -2 +#define NV_ERR_INVALID_ARGUMENT -3 +#define NV_ERR_NO_MEMORY -4 + +typedef enum { + VGPU_DEVICE_STATE_UNINITIALIZED = 0, + VGPU_DEVICE_STATE_INITIALIZING, + VGPU_DEVICE_STATE_READY, + VGPU_DEVICE_STATE_RUNNING, + VGPU_DEVICE_STATE_SUSPENDED, + VGPU_DEVICE_STATE_ERROR, + VGPU_DEVICE_STATE_MIGRATING, +} VGPU_DEVICE_STATE; + +typedef NV_STATUS (*vgpu_vfio_probe_t)(struct pci_dev *, NvU32, NvU32 *); +typedef void (*vgpu_vfio_remove_t)(struct pci_dev *); +typedef NV_STATUS (*vgpu_vfio_inject_interrupt_t)(void *); + +typedef struct +{ + vgpu_vfio_probe_t probe; + vgpu_vfio_remove_t remove; + vgpu_vfio_inject_interrupt_t inject_interrupt; +} vgpu_vfio_ops_t; + +typedef NV_STATUS (*vgpu_vfio_vgpu_create_t) (struct pci_dev *dev, const NvU8 *uuid, + NvU32 vgpu_type_id, NvU16 *vgpu_id, + NvU32 *gpu_pci_id, NvU32 gpu_pci_bdf); +typedef NV_STATUS (*vgpu_vfio_vgpu_destroy_t) (const NvU8 *uuid, NvU16 vgpu_id); +typedef NV_STATUS (*vgpu_vfio_start_t) (const NvU8 *uuid, void *wait_queue, NvS32 *return_status, + NvU8 *vm_name, NvU32 qemu_pid); +typedef NV_STATUS (*vgpu_vfio_bar_info_t) (struct pci_dev *dev, const NvU8 *, NvU64 *, + NvU32 bar_index, void *); +typedef NV_STATUS (*vgpu_vfio_get_description) (struct pci_dev *, NvU32, char *); +typedef NV_STATUS (*vgpu_vfio_get_name) (struct pci_dev *, NvU32, char *); +typedef NV_STATUS (*vgpu_vfio_get_instance) (struct pci_dev *, NvU32, char *); +typedef NV_STATUS (*vgpu_vfio_sparse_mmap_t) (struct pci_dev *, const NvU8 *, NvU64 **, NvU64 **, NvU32 *); +typedef void (*vgpu_vfio_update_request_t) (const NvU8 *, NvU64 *, NvU64 *, VGPU_DEVICE_STATE, const char *); + +typedef struct +{ + const char *version_string; + vgpu_vfio_vgpu_create_t vgpu_create; + vgpu_vfio_vgpu_destroy_t vgpu_delete; + vgpu_vfio_bar_info_t vgpu_bar_info; + vgpu_vfio_start_t vgpu_start; + vgpu_vfio_get_description get_description; + vgpu_vfio_get_instance get_instances; + vgpu_vfio_get_name get_name; + vgpu_vfio_sparse_mmap_t get_sparse_mmap; + vgpu_vfio_update_request_t update_request; +} rm_vgpu_vfio_ops_t; + +extern NV_STATUS nvidia_vgpu_vfio_set_ops(vgpu_vfio_ops_t *vgpu_vfio_ops); +extern NV_STATUS nvidia_vgpu_vfio_get_ops(rm_vgpu_vfio_ops_t *ops); + +extern NV_STATUS nvidia_vgpu_vfio_probe(struct pci_dev *); +extern void nvidia_vgpu_vfio_remove(struct pci_dev *, NvBool); + +#endif /* _NV_VGPU_VFIO_INTERFACE_H_ */ \ No newline at end of file diff --git a/kernel/include/nvidia-vgpu-vfio.h b/kernel/include/nvidia-vgpu-vfio.h new file mode 100644 index 0000000..961bba2 --- /dev/null +++ b/kernel/include/nvidia-vgpu-vfio.h @@ -0,0 +1,232 @@ +#ifndef _NVIDIA_VGPU_VFIO_H_ +#define _NVIDIA_VGPU_VFIO_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define OPEN_IOV_VFIO_VERSION "0.1.0" +#define OPEN_IOV_VFIO_NAME "open-iov-vfio" + +#define NVIDIA_VGPU_VFIO_VERSION OPEN_IOV_VFIO_VERSION +#define NVIDIA_VGPU_VFIO_NAME OPEN_IOV_VFIO_NAME + +#define NVIDIA_VGPU_MAX_VF_COUNT 32 +#define NVIDIA_VGPU_MAX_REGIONS 8 +#define NVIDIA_VGPU_MAX_IRQS 16 + +#define NV_VGPU_MAX_REGIONS NVIDIA_VGPU_MAX_REGIONS +#define NV_VGPU_MAX_IRQS NVIDIA_VGPU_MAX_IRQS + +#define NV_VGPU_BAR0_SIZE (16 * 1024 * 1024) +#define NV_VGPU_BAR1_SIZE (256 * 1024 * 1024) +#define NV_VGPU_BAR2_SIZE (32 * 1024 * 1024) +#define NV_VGPU_BAR3_SIZE (32 * 1024 * 1024) + +enum nv_vgpu_type { + NV_VGPU_TYPE_UNKNOWN = 0, + NV_VGPU_TYPE_AMPERE, + NV_VGPU_TYPE_HOPPER, + NV_VGPU_TYPE_TURING, + NV_VGPU_TYPE_PASCAL, + NV_VGPU_TYPE_MAXWELL, + NV_VGPU_TYPE_KEPLER, +}; + +enum nv_vgpu_state { + NV_VGPU_STATE_UNINITIALIZED = 0, + NV_VGPU_STATE_INITIALIZING, + NV_VGPU_STATE_READY, + NV_VGPU_STATE_RUNNING, + NV_VGPU_STATE_SUSPENDED, + NV_VGPU_STATE_ERROR, + NV_VGPU_STATE_MIGRATING, +}; + +struct nv_vgpu_bar { + u64 gpa; + u64 size; + u64 iova; + void *vaddr; + bool mapped; + bool trapped; + struct mutex lock; +}; + +struct nv_vgpu_irq { + int virq; + int type; + bool masked; + struct eventfd_ctx *trigger; + struct irq_domain *domain; +}; + +struct nv_vgpu_region { + u64 offset; + u64 size; + u64 iova; + int fd; + void *vaddr; + bool readonly; +}; + +struct nv_vgpu_migration_ctx; + +struct nv_vgpu_migration_state { + u32 version; + u32 flags; + u64 timestamp; + u8 *data; + size_t data_size; + struct completion done; + int result; + struct nv_vgpu_migration_ctx *ctx; +}; + +struct nv_vgpu_device { + /* VFIO device for VFIO group - must be first for vfio_alloc_device */ + struct vfio_device vfio_dev; + + struct mdev_device *mdev; + struct pci_dev *pdev; + struct device dev; + struct mutex lock; + enum nv_vgpu_type type; + enum nv_vgpu_state state; + u32 vgpu_id; + u32 gpu_instance_id; + u32 compute_instance_id; + + struct nv_vgpu_bar bars[NV_VGPU_MAX_REGIONS]; + struct nv_vgpu_irq irqs[NV_VGPU_MAX_IRQS]; + struct nv_vgpu_region regions[NV_VGPU_MAX_REGIONS]; + + struct nv_vgpu_migration_state migration; + + struct work_struct init_work; + struct work_struct cleanup_work; + + void *gsp_comm_ctx; + void *bar_emul_ctx; + void *mm_ctx; + void *irq_ctx; + + u64 fb_base; + u64 fb_size; + u64 bar0_config_base; + + struct list_head list; + + bool dry_run; + + /* PCI Config Space for VFIO mdev */ + u8 pci_config[4096]; + bool config_space_dirty; + + /* VFIO device open/close tracking */ + atomic_t vfio_dev_open_count; + wait_queue_head_t vfio_dev_close_wait; + + /* VFIO device registration state */ + bool vfio_dev_registered; + }; + +typedef struct nv_vgpu_device nv_vgpu_device; + +struct nv_vgpu_driver_data { + struct mutex lock; + struct list_head devices; + u32 device_count; + struct class *class; + struct bus_type *bus; +}; + +extern struct nv_vgpu_driver_data nv_vgpu_driver; + +int nv_vgpu_device_create(struct mdev_device *mdev); +void nv_vgpu_device_destroy(struct mdev_device *mdev); +int nv_vgpu_device_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_device_cleanup(struct nv_vgpu_device *vgpu); + +int nv_vgpu_bar_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_bar_cleanup(struct nv_vgpu_device *vgpu); +int nv_vgpu_bar_trap(struct nv_vgpu_device *vgpu, int bar_idx); +void nv_vgpu_bar_untrap(struct nv_vgpu_device *vgpu, int bar_idx); + +int nv_vgpu_irq_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_irq_cleanup(struct nv_vgpu_device *vgpu); +int nv_vgpu_irq_enable(struct nv_vgpu_device *vgpu, int irq_idx); +int nv_vgpu_irq_disable(struct nv_vgpu_device *vgpu, int irq_idx); + +int nv_vgpu_memory_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_memory_cleanup(struct nv_vgpu_device *vgpu); +int nv_vgpu_memory_map(struct nv_vgpu_device *vgpu, u64 gpa, u64 size, u64 *iova); +int nv_vgpu_memory_unmap(struct nv_vgpu_device *vgpu, u64 iova, u64 size); + +int nv_vgpu_gsp_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_gsp_cleanup(struct nv_vgpu_device *vgpu); +int nv_vgpu_gsp_send_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size); +int nv_vgpu_gsp_recv_rpc(struct nv_vgpu_device *vgpu, void *msg, size_t size, int timeout_ms); +int nv_vgpu_gsp_load_firmware(struct nv_vgpu_device *vgpu); +void nv_vgpu_gsp_release_firmware(struct nv_vgpu_device *vgpu); +int nv_vgpu_gsp_boot(struct nv_vgpu_device *vgpu); + +/* mdev driver/parent registration (main.c) */ +int nv_vgpu_create_mdev_types(void); +void nv_vgpu_free_type_info(void); +int nv_vgpu_mdev_driver_register(void); +void nv_vgpu_mdev_driver_unregister(void); +int nv_vgpu_mdev_register_parent(struct pci_dev *pdev); +void nv_vgpu_mdev_unregister_parent(struct pci_dev *pdev); + +int nv_vgpu_migration_init(struct nv_vgpu_device *vgpu); +void nv_vgpu_migration_cleanup(struct nv_vgpu_device *vgpu); +int nv_vgpu_migration_save_state(struct nv_vgpu_device *vgpu, u8 **data, size_t *size); +int nv_vgpu_migration_restore_state(struct nv_vgpu_device *vgpu, u8 *data, size_t size); +int nv_vgpu_migration_prepare(struct nv_vgpu_device *vgpu); +int nv_vgpu_migration_complete(struct nv_vgpu_device *vgpu); +int nv_vgpu_migration_cancel(struct nv_vgpu_device *vgpu); +u8 *nv_vgpu_migration_get_data(struct nv_vgpu_device *vgpu, size_t *size); + +static inline struct nv_vgpu_device *mdev_to_vgpu(struct mdev_device *mdev) +{ + return dev_get_drvdata(&mdev->dev); +} + +/* NULL-tolerant logging: emulation paths may run with a NULL vgpu + * (e.g. unit tests, early init) - fall back to plain pr_*. */ +#define nv_vgpu_dbg(vgpu, fmt, ...) do { \ + if (vgpu) \ + dev_dbg(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__); \ + else \ + pr_debug("%s: " fmt, __func__, ##__VA_ARGS__); \ +} while (0) + +#define nv_vgpu_info(vgpu, fmt, ...) do { \ + if (vgpu) \ + dev_info(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__); \ + else \ + pr_info("%s: " fmt, __func__, ##__VA_ARGS__); \ +} while (0) + +#define nv_vgpu_warn(vgpu, fmt, ...) do { \ + if (vgpu) \ + dev_warn(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__); \ + else \ + pr_warn("%s: " fmt, __func__, ##__VA_ARGS__); \ +} while (0) + +#define nv_vgpu_err(vgpu, fmt, ...) do { \ + if (vgpu) \ + dev_err(&(vgpu)->dev, "%s: " fmt, __func__, ##__VA_ARGS__); \ + else \ + pr_err("%s: " fmt, __func__, ##__VA_ARGS__); \ +} while (0) + +#endif \ No newline at end of file diff --git a/kernel/include/nvstatus.h b/kernel/include/nvstatus.h new file mode 100644 index 0000000..4f5284d --- /dev/null +++ b/kernel/include/nvstatus.h @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2014-2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef SDK_NVSTATUS_H +#define SDK_NVSTATUS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "nvtypes.h" + +typedef NvU32 NV_STATUS; + +#define NV_STATUS_CODE( name, code, string ) name = (code), + +enum +{ + #include "nvstatuscodes.h" +}; + +#undef NV_STATUS_CODE + +/*! + * @def NV_STATUS_LEVEL_OK + * @see NV_STATUS_LEVEL + * @brief Success: No error or special condition + */ +#define NV_STATUS_LEVEL_OK 0 + +/*! + * @def NV_STATUS_LEVEL_WARN + * @see NV_STATUS_LEVEL + * @brief Success, but there is an special condition + * + * @details In general, NV_STATUS_LEVEL_WARN status codes are handled the + * same as NV_STATUS_LEVEL_OK, but are usefil to indicate that + * there is a condition that may be specially handled. + * + * Therefore, in most cases, client function should test for + * status <= NV_STATUS_LEVEL_WARN or status > NV_STATUS_LEVEL_WARN + * to determine success v. failure of a call. + */ +#define NV_STATUS_LEVEL_WARN 1 + +/*! + * @def NV_STATUS_LEVEL_ERR + * @see NV_STATUS_LEVEL + * @brief Unrecoverable error condition + */ +#define NV_STATUS_LEVEL_ERR 3 + +/*! + * @def NV_STATUS_LEVEL + * @see NV_STATUS_LEVEL_OK + * @see NV_STATUS_LEVEL_WARN + * @see NV_STATUS_LEVEL_ERR + * @brief Level of the status code + * + * @warning IMPORTANT: When comparing NV_STATUS_LEVEL(_S) against one of + * these constants, it is important to use '<=' or '>' (rather + * than '<' or '>='). + * + * For example. do: + * if (NV_STATUS_LEVEL(status) <= NV_STATUS_LEVEL_WARN) + * rather than: + * if (NV_STATUS_LEVEL(status) < NV_STATUS_LEVEL_ERR) + * + * By being consistent in this manner, it is easier to systematically + * add additional level constants. New levels are likely to lower + * (rather than raise) the severity of _ERR codes. For example, + * if we were to add NV_STATUS_LEVEL_RETRY to indicate hardware + * failures that may be recoverable (e.g. RM_ERR_TIMEOUT_RETRY + * or RM_ERR_BUSY_RETRY), it would be less severe than + * NV_STATUS_LEVEL_ERR the level to which these status codes now + * belong. Using '<=' and '>' ensures your code is not broken in + * cases like this. + */ +#define NV_STATUS_LEVEL(_S) \ + ((_S) == NV_OK? NV_STATUS_LEVEL_OK: \ + ((_S) != NV_ERR_GENERIC && (_S) & 0x00010000? NV_STATUS_LEVEL_WARN: \ + NV_STATUS_LEVEL_ERR)) + +/*! + * @def NV_STATUS_LEVEL + * @see NV_STATUS_LEVEL_OK + * @see NV_STATUS_LEVEL_WARN + * @see NV_STATUS_LEVEL_ERR + * @brief Character representing status code level + */ +#define NV_STATUS_LEVEL_CHAR(_S) \ + ((_S) == NV_OK? '0': \ + ((_S) != NV_ERR_GENERIC && (_S) & 0x00010000? 'W': \ + 'E')) + +// Function definitions +const char *nvstatusToString(NV_STATUS nvStatusIn); + +#ifdef __cplusplus +} +#endif + +#endif /* SDK_NVSTATUS_H */ diff --git a/kernel/include/nvtypes.h b/kernel/include/nvtypes.h new file mode 100644 index 0000000..7810d34 --- /dev/null +++ b/kernel/include/nvtypes.h @@ -0,0 +1,830 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef NVTYPES_INCLUDED +#define NVTYPES_INCLUDED + +#include "cpuopsys.h" + +#ifndef NVTYPES_USE_STDINT +#define NVTYPES_USE_STDINT 0 +#endif + +#if NVTYPES_USE_STDINT +#ifdef __cplusplus +#include +#include +#else +#include +#include +#endif // __cplusplus +#endif // NVTYPES_USE_STDINT + +#ifndef __cplusplus +// Header includes to make sure wchar_t is defined for C-file compilation +// (C++ is not affected as it is a fundamental type there) +// _MSC_VER is a hack to avoid failures for old setup of UEFI builds which are +// currently set to msvc100 but do not properly set the include paths +#if defined(NV_WINDOWS) && (!defined(_MSC_VER) || (_MSC_VER > 1600)) +#include +#define NV_HAS_WCHAR_T_TYPEDEF 1 +#endif +#endif // __cplusplus + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(MAKE_NV64TYPES_8BYTES_ALIGNED) && defined(__i386__) +// ensure or force 8-bytes alignment of NV 64-bit types +#define OPTIONAL_ALIGN8_ATTR __attribute__((aligned(8))) +#else +// nothing needed +#define OPTIONAL_ALIGN8_ATTR +#endif // MAKE_NV64TYPES_8BYTES_ALIGNED && i386 + + /***************************************************************************\ +|* Typedefs *| + \***************************************************************************/ + +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +//Typedefs for MISRA COMPLIANCE +typedef unsigned long long UInt64; +typedef signed long long Int64; +typedef unsigned int UInt32; +typedef signed int Int32; +typedef unsigned short UInt16; +typedef signed short Int16; +typedef unsigned char UInt8 ; +typedef signed char Int8 ; + +typedef void Void; +typedef float float32_t; +typedef double float64_t; +#endif + + +// Floating point types +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef float32_t NvF32; /* IEEE Single Precision (S1E8M23) */ +typedef float64_t NvF64 OPTIONAL_ALIGN8_ATTR; /* IEEE Double Precision (S1E11M52) */ +#else +typedef float NvF32; /* IEEE Single Precision (S1E8M23) */ +typedef double NvF64 OPTIONAL_ALIGN8_ATTR; /* IEEE Double Precision (S1E11M52) */ +#endif + + +// 8-bit: 'char' is the only 8-bit in the C89 standard and after. +#if NVTYPES_USE_STDINT +typedef uint8_t NvV8; /* "void": enumerated or multiple fields */ +typedef uint8_t NvU8; /* 0 to 255 */ +typedef int8_t NvS8; /* -128 to 127 */ +#else +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef UInt8 NvV8; /* "void": enumerated or multiple fields */ +typedef UInt8 NvU8; /* 0 to 255 */ +typedef Int8 NvS8; /* -128 to 127 */ +#else +typedef unsigned char NvV8; /* "void": enumerated or multiple fields */ +typedef unsigned char NvU8; /* 0 to 255 */ +typedef signed char NvS8; /* -128 to 127 */ +#endif +#endif // NVTYPES_USE_STDINT + + +#if NVTYPES_USE_STDINT +typedef uint16_t NvV16; /* "void": enumerated or multiple fields */ +typedef uint16_t NvU16; /* 0 to 65535 */ +typedef int16_t NvS16; /* -32768 to 32767 */ +#else +// 16-bit: If the compiler tells us what we can use, then use it. +#ifdef __INT16_TYPE__ +typedef unsigned __INT16_TYPE__ NvV16; /* "void": enumerated or multiple fields */ +typedef unsigned __INT16_TYPE__ NvU16; /* 0 to 65535 */ +typedef signed __INT16_TYPE__ NvS16; /* -32768 to 32767 */ + +// The minimal standard for C89 and after +#else // __INT16_TYPE__ +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef UInt16 NvV16; /* "void": enumerated or multiple fields */ +typedef UInt16 NvU16; /* 0 to 65535 */ +typedef Int16 NvS16; /* -32768 to 32767 */ +#else +typedef unsigned short NvV16; /* "void": enumerated or multiple fields */ +typedef unsigned short NvU16; /* 0 to 65535 */ +typedef signed short NvS16; /* -32768 to 32767 */ +#endif +#endif // __INT16_TYPE__ +#endif // NVTYPES_USE_STDINT + +// wchar type (fixed size types consistent across Linux/Windows boundaries) +#if defined(NV_HAS_WCHAR_T_TYPEDEF) + typedef wchar_t NvWchar; +#else + typedef NvV16 NvWchar; +#endif + +// Macro to build an NvU32 from four bytes, listed from msb to lsb +#define NvU32_BUILD(a, b, c, d) \ + ((NvU32)( \ + (((NvU32)(a) & 0xff) << 24) | \ + (((NvU32)(b) & 0xff) << 16) | \ + (((NvU32)(c) & 0xff) << 8) | \ + (((NvU32)(d) & 0xff)))) + +// Macro to build an NvU64 from two DWORDS, listed from msb to lsb +#define NvU64_BUILD(a, b) \ + ((NvU64)( \ + (((NvU64)(a) & ~0U) << 32) | \ + (((NvU64)(b) & ~0U)))) + +#if NVTYPES_USE_STDINT +typedef uint32_t NvV32; /* "void": enumerated or multiple fields */ +typedef uint32_t NvU32; /* 0 to 4294967295 */ +typedef int32_t NvS32; /* -2147483648 to 2147483647 */ +#else +// 32-bit: If the compiler tells us what we can use, then use it. +#ifdef __INT32_TYPE__ +typedef unsigned __INT32_TYPE__ NvV32; /* "void": enumerated or multiple fields */ +typedef unsigned __INT32_TYPE__ NvU32; /* 0 to 4294967295 */ +typedef signed __INT32_TYPE__ NvS32; /* -2147483648 to 2147483647 */ + +// Older compilers +#else // __INT32_TYPE__ + +// For historical reasons, NvU32/NvV32 are defined to different base intrinsic +// types than NvS32 on some platforms. +// Mainly for 64-bit linux, where long is 64 bits and win9x, where int is 16 bit. +#if (defined(NV_UNIX) || defined(vxworks) || defined(NV_WINDOWS_CE) || \ + defined(__arm) || defined(__IAR_SYSTEMS_ICC__) || defined(NV_QNX) || \ + defined(NV_INTEGRITY) || defined(NV_MODS) || \ + defined(__GNUC__) || defined(__clang__) || defined(NV_MACINTOSH_64)) && \ + (!defined(NV_MACINTOSH) || defined(NV_MACINTOSH_64)) +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef UInt32 NvV32; /* "void": enumerated or multiple fields */ +typedef UInt32 NvU32; /* 0 to 4294967295 */ +#else +typedef unsigned int NvV32; /* "void": enumerated or multiple fields */ +typedef unsigned int NvU32; /* 0 to 4294967295 */ +#endif + +// The minimal standard for C89 and after +#else // (defined(NV_UNIX) || defined(vxworks) || ... +typedef unsigned long NvV32; /* "void": enumerated or multiple fields */ +typedef unsigned long NvU32; /* 0 to 4294967295 */ +#endif // (defined(NV_UNIX) || defined(vxworks) || ... + +// Mac OS 32-bit still needs this +#if defined(NV_MACINTOSH) && !defined(NV_MACINTOSH_64) +typedef signed long NvS32; /* -2147483648 to 2147483647 */ +#else +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef Int32 NvS32; /* -2147483648 to 2147483647 */ +#else +typedef signed int NvS32; /* -2147483648 to 2147483647 */ +#endif +#endif // defined(NV_MACINTOSH) && !defined(NV_MACINTOSH_64) +#endif // __INT32_TYPE__ +#endif // NVTYPES_USE_STDINT + + + +#if NVTYPES_USE_STDINT +typedef uint64_t NvU64 OPTIONAL_ALIGN8_ATTR; /* 0 to 18446744073709551615 */ +typedef int64_t NvS64 OPTIONAL_ALIGN8_ATTR; /* -9223372036854775808 to 9223372036854775807 */ + +#define NvU64_fmtX PRIX64 +#define NvU64_fmtx PRIx64 +#define NvU64_fmtu PRIu64 +#define NvU64_fmto PRIo64 +#define NvS64_fmtd PRId64 +#define NvS64_fmti PRIi64 +#else +// 64-bit types for compilers that support them, plus some obsolete variants +#if defined(__GNUC__) || defined(__clang__) || defined(__arm) || \ + defined(__IAR_SYSTEMS_ICC__) || defined(__ghs__) || defined(_WIN64) || \ + defined(__SUNPRO_C) || defined(__SUNPRO_CC) || defined (__xlC__) +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +typedef UInt64 NvU64 OPTIONAL_ALIGN8_ATTR; /* 0 to 18446744073709551615 */ +typedef Int64 NvS64 OPTIONAL_ALIGN8_ATTR; /* -9223372036854775808 to 9223372036854775807 */ +#else +typedef unsigned long long NvU64 OPTIONAL_ALIGN8_ATTR; /* 0 to 18446744073709551615 */ +typedef long long NvS64 OPTIONAL_ALIGN8_ATTR; /* -9223372036854775808 to 9223372036854775807 */ +#endif + +#define NvU64_fmtX "llX" +#define NvU64_fmtx "llx" +#define NvU64_fmtu "llu" +#define NvU64_fmto "llo" +#define NvS64_fmtd "lld" +#define NvS64_fmti "lli" + +// Microsoft since 2003 -- https://msdn.microsoft.com/en-us/library/29dh1w7z.aspx +#else +typedef unsigned __int64 NvU64 OPTIONAL_ALIGN8_ATTR; /* 0 to 18446744073709551615 */ +typedef __int64 NvS64 OPTIONAL_ALIGN8_ATTR; /* -9223372036854775808 to 9223372036854775807 */ + +#define NvU64_fmtX "I64X" +#define NvU64_fmtx "I64x" +#define NvU64_fmtu "I64u" +#define NvU64_fmto "I64o" +#define NvS64_fmtd "I64d" +#define NvS64_fmti "I64i" + +#endif +#endif // NVTYPES_USE_STDINT + +#ifdef NV_TYPESAFE_HANDLES +/* + * Can't use opaque pointer as clients might be compiled with mismatched + * pointer sizes. TYPESAFE check will eventually be removed once all clients + * have transistioned safely to NvHandle. + * The plan is to then eventually scale up the handle to be 64-bits. + */ +typedef struct +{ + NvU32 val; +} NvHandle; +#else +/* + * For compatibility with modules that haven't moved typesafe handles. + */ +typedef NvU32 NvHandle; +#endif // NV_TYPESAFE_HANDLES + +/* Boolean type */ +typedef NvU8 NvBool; +#define NV_TRUE ((NvBool)(0 == 0)) +#define NV_FALSE ((NvBool)(0 != 0)) + +/* Tristate type: NV_TRISTATE_FALSE, NV_TRISTATE_TRUE, NV_TRISTATE_INDETERMINATE */ +typedef NvU8 NvTristate; +#define NV_TRISTATE_FALSE ((NvTristate) 0) +#define NV_TRISTATE_TRUE ((NvTristate) 1) +#define NV_TRISTATE_INDETERMINATE ((NvTristate) 2) + +/* Macros to extract the low and high parts of a 64-bit unsigned integer */ +/* Also designed to work if someone happens to pass in a 32-bit integer */ +#ifdef NV_MISRA_COMPLIANCE_REQUIRED +#define NvU64_HI32(n) ((NvU32)((((NvU64)(n)) >> 32) & 0xffffffffU)) +#define NvU64_LO32(n) ((NvU32)(( (NvU64)(n)) & 0xffffffffU)) +#else +#define NvU64_HI32(n) ((NvU32)((((NvU64)(n)) >> 32) & 0xffffffff)) +#define NvU64_LO32(n) ((NvU32)(( (NvU64)(n)) & 0xffffffff)) +#endif +#define NvU40_HI32(n) ((NvU32)((((NvU64)(n)) >> 8) & 0xffffffffU)) +#define NvU40_HI24of32(n) ((NvU32)( (NvU64)(n) & 0xffffff00U)) + +/* Macros to get the MSB and LSB of a 32 bit unsigned number */ +#define NvU32_HI16(n) ((NvU16)((((NvU32)(n)) >> 16) & 0xffffU)) +#define NvU32_LO16(n) ((NvU16)(( (NvU32)(n)) & 0xffffU)) + + /***************************************************************************\ +|* *| +|* 64 bit type definitions for use in interface structures. *| +|* *| + \***************************************************************************/ + +#if defined(NV_64_BITS) + +typedef void* NvP64; /* 64 bit void pointer */ +typedef NvU64 NvUPtr; /* pointer sized unsigned int */ +typedef NvS64 NvSPtr; /* pointer sized signed int */ +typedef NvU64 NvLength; /* length to agree with sizeof */ + +#define NvP64_VALUE(n) (n) +#define NvP64_fmt "%p" + +#define KERNEL_POINTER_FROM_NvP64(p,v) ((p)(v)) +#define NvP64_PLUS_OFFSET(p,o) (NvP64)((NvU64)(p) + (NvU64)(o)) + +#define NvUPtr_fmtX NvU64_fmtX +#define NvUPtr_fmtx NvU64_fmtx +#define NvUPtr_fmtu NvU64_fmtu +#define NvUPtr_fmto NvU64_fmto +#define NvSPtr_fmtd NvS64_fmtd +#define NvSPtr_fmti NvS64_fmti + +#else + +typedef NvU64 NvP64; /* 64 bit void pointer */ +typedef NvU32 NvUPtr; /* pointer sized unsigned int */ +typedef NvS32 NvSPtr; /* pointer sized signed int */ +typedef NvU32 NvLength; /* length to agree with sizeof */ + +#define NvP64_VALUE(n) ((void *)(NvUPtr)(n)) +#define NvP64_fmt "0x%llx" + +#define KERNEL_POINTER_FROM_NvP64(p,v) ((p)(NvUPtr)(v)) +#define NvP64_PLUS_OFFSET(p,o) ((p) + (NvU64)(o)) + +#define NvUPtr_fmtX "X" +#define NvUPtr_fmtx "x" +#define NvUPtr_fmtu "u" +#define NvUPtr_fmto "o" +#define NvSPtr_fmtd "d" +#define NvSPtr_fmti "i" + +#endif + +#define NvP64_NULL (NvP64)0 + +/*! + * Helper macro to pack an @ref NvU64_ALIGN32 structure from a @ref NvU64. + * + * @param[out] pDst Pointer to NvU64_ALIGN32 structure to pack + * @param[in] pSrc Pointer to NvU64 with which to pack + */ +#define NvU64_ALIGN32_PACK(pDst, pSrc) \ +do { \ + (pDst)->lo = NvU64_LO32(*(pSrc)); \ + (pDst)->hi = NvU64_HI32(*(pSrc)); \ +} while (NV_FALSE) + +/*! + * Helper macro to unpack a @ref NvU64_ALIGN32 structure into a @ref NvU64. + * + * @param[out] pDst Pointer to NvU64 in which to unpack + * @param[in] pSrc Pointer to NvU64_ALIGN32 structure from which to unpack + */ +#define NvU64_ALIGN32_UNPACK(pDst, pSrc) \ +do { \ + (*(pDst)) = NvU64_ALIGN32_VAL(pSrc); \ +} while (NV_FALSE) + +/*! + * Helper macro to unpack a @ref NvU64_ALIGN32 structure as a @ref NvU64. + * + * @param[in] pSrc Pointer to NvU64_ALIGN32 structure to unpack + */ +#define NvU64_ALIGN32_VAL(pSrc) \ + ((NvU64) ((NvU64)((pSrc)->lo) | (((NvU64)(pSrc)->hi) << 32U))) + +/*! + * Helper macro to check whether the 32 bit aligned 64 bit number is zero. + * + * @param[in] _pU64 Pointer to NvU64_ALIGN32 structure. + * + * @return + * NV_TRUE _pU64 is zero. + * NV_FALSE otherwise. + */ +#define NvU64_ALIGN32_IS_ZERO(_pU64) \ + (((_pU64)->lo == 0U) && ((_pU64)->hi == 0U)) + +/*! + * Helper macro to sub two 32 aligned 64 bit numbers on 64 bit processor. + * + * @param[in] pSrc1 Pointer to NvU64_ALIGN32 source 1 structure. + * @param[in] pSrc2 Pointer to NvU64_ALIGN32 source 2 structure. + * @param[in/out] pDst Pointer to NvU64_ALIGN32 dest. structure. + */ +#define NvU64_ALIGN32_ADD(pDst, pSrc1, pSrc2) \ +do { \ + NvU64 __dst, __src1, __scr2; \ + \ + NvU64_ALIGN32_UNPACK(&__src1, (pSrc1)); \ + NvU64_ALIGN32_UNPACK(&__scr2, (pSrc2)); \ + __dst = __src1 + __scr2; \ + NvU64_ALIGN32_PACK((pDst), &__dst); \ +} while (NV_FALSE) + +/*! + * Helper macro to sub two 32 aligned 64 bit numbers on 64 bit processor. + * + * @param[in] pSrc1 Pointer to NvU64_ALIGN32 source 1 structure. + * @param[in] pSrc2 Pointer to NvU64_ALIGN32 source 2 structure. + * @param[in/out] pDst Pointer to NvU64_ALIGN32 dest. structure. + */ +#define NvU64_ALIGN32_SUB(pDst, pSrc1, pSrc2) \ +do { \ + NvU64 __dst, __src1, __scr2; \ + \ + NvU64_ALIGN32_UNPACK(&__src1, (pSrc1)); \ + NvU64_ALIGN32_UNPACK(&__scr2, (pSrc2)); \ + __dst = __src1 - __scr2; \ + NvU64_ALIGN32_PACK((pDst), &__dst); \ +} while (NV_FALSE) + +/*! + * Structure for representing 32 bit aligned NvU64 (64-bit unsigned integer) + * structures. This structure must be used because the 32 bit processor and + * 64 bit processor compilers will pack/align NvU64 differently. + * + * One use case is RM being 64 bit proc whereas PMU being 32 bit proc, this + * alignment difference will result in corrupted transactions between the RM + * and PMU. + * + * See the @ref NvU64_ALIGN32_PACK and @ref NvU64_ALIGN32_UNPACK macros for + * packing and unpacking these structures. + * + * @note The intention of this structure is to provide a datatype which will + * packed/aligned consistently and efficiently across all platforms. + * We don't want to use "NV_DECLARE_ALIGNED(NvU64, 8)" because that + * leads to memory waste on our 32-bit uprocessors (e.g. FALCONs) where + * DMEM efficiency is vital. + */ +typedef struct +{ + /*! + * Low 32 bits. + */ + NvU32 lo; + /*! + * High 32 bits. + */ + NvU32 hi; +} NvU64_ALIGN32; + +/* Useful macro to hide required double cast */ +#define NV_PTR_TO_NvP64(n) (NvP64)(NvUPtr)(n) +#define NV_SIGN_EXT_PTR_TO_NvP64(p) ((NvP64)(NvS64)(NvSPtr)(p)) +#define KERNEL_POINTER_TO_NvP64(p) ((NvP64)(uintptr_t)(p)) + + /***************************************************************************\ +|* *| +|* Limits for common types. *| +|* *| + \***************************************************************************/ + +/* Explanation of the current form of these limits: + * + * - Decimal is used, as hex values are by default positive. + * - Casts are not used, as usage in the preprocessor itself (#if) ends poorly. + * - The subtraction of 1 for some MIN values is used to get around the fact + * that the C syntax actually treats -x as NEGATE(x) instead of a distinct + * number. Since 214748648 isn't a valid positive 32-bit signed value, we + * take the largest valid positive signed number, negate it, and subtract 1. + */ +#define NV_S8_MIN (-128) +#define NV_S8_MAX (+127) +#define NV_U8_MIN (0U) +#define NV_U8_MAX (+255U) +#define NV_S16_MIN (-32768) +#define NV_S16_MAX (+32767) +#define NV_U16_MIN (0U) +#define NV_U16_MAX (+65535U) +#define NV_S32_MIN (-2147483647 - 1) +#define NV_S32_MAX (+2147483647) +#define NV_U32_MIN (0U) +#define NV_U32_MAX (+4294967295U) +#define NV_S64_MIN (-9223372036854775807LL - 1LL) +#define NV_S64_MAX (+9223372036854775807LL) +#define NV_U64_MIN (0ULL) +#define NV_U64_MAX (+18446744073709551615ULL) + +/* Aligns fields in structs so they match up between 32 and 64 bit builds */ +#if defined(__GNUC__) || defined(__clang__) || defined(NV_QNX) +#define NV_ALIGN_BYTES(size) __attribute__ ((aligned (size))) +#elif defined(__arm) +#define NV_ALIGN_BYTES(size) __align(ALIGN) +#else +// XXX This is dangerously nonportable! We really shouldn't provide a default +// version of this that doesn't do anything. +#define NV_ALIGN_BYTES(size) +#endif + +// NV_DECLARE_ALIGNED() can be used on all platforms. +// This macro form accounts for the fact that __declspec on Windows is required +// before the variable type, +// and NV_ALIGN_BYTES is required after the variable name. +#if defined(__GNUC__) || defined(__clang__) || defined(NV_QNX) +#define NV_DECLARE_ALIGNED(TYPE_VAR, ALIGN) TYPE_VAR __attribute__ ((aligned (ALIGN))) +#elif defined(_MSC_VER) +#define NV_DECLARE_ALIGNED(TYPE_VAR, ALIGN) __declspec(align(ALIGN)) TYPE_VAR +#elif defined(__arm) +#define NV_DECLARE_ALIGNED(TYPE_VAR, ALIGN) __align(ALIGN) TYPE_VAR +#endif + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +#define NV_ALIGNOF(type) _Alignof(type) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) +#define NV_ALIGNOF(type) alignof(type) +#elif defined(__GNUC__) || defined(__clang__) +#define NV_ALIGNOF(type) __alignof__(type) +#elif defined(_MSC_VER) +#define NV_ALIGNOF(type) __alignof(type) +#endif + + /***************************************************************************\ +|* Function Declaration Types *| + \***************************************************************************/ + +// stretching the meaning of "nvtypes", but this seems to least offensive +// place to re-locate these from nvos.h which cannot be included by a number +// of builds that need them + +#if defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER) + #define NV_ATTRIBUTE_UNUSED __attribute__((__unused__)) +#else + #define NV_ATTRIBUTE_UNUSED +#endif + +#if defined(_MSC_VER) + + #if _MSC_VER >= 1310 + #define NV_NOINLINE __declspec(noinline) + #else + #define NV_NOINLINE + #endif + + #define NV_INLINE __inline + + #if _MSC_VER >= 1200 + #define NV_FORCEINLINE __forceinline + #else + #define NV_FORCEINLINE __inline + #endif + + #define NV_APIENTRY __stdcall + #define NV_FASTCALL __fastcall + #define NV_CDECLCALL __cdecl + #define NV_STDCALL __stdcall + + #define NV_FORCERESULTCHECK + + #define NV_FORMAT_PRINTF(_f, _a) + +#else // ! defined(_MSC_VER) + + #if defined(__GNUC__) + #if (__GNUC__ > 3) || \ + ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) && (__GNUC_PATCHLEVEL__ >= 1)) + #define NV_NOINLINE __attribute__((__noinline__)) + #endif + #elif defined(__clang__) + #if __has_attribute(noinline) + #define NV_NOINLINE __attribute__((__noinline__)) + #endif + #elif defined(__arm) && (__ARMCC_VERSION >= 300000) + #define NV_NOINLINE __attribute__((__noinline__)) + #elif (defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590)) ||\ + (defined(__SUNPRO_CC) && (__SUNPRO_CC >= 0x590)) + #define NV_NOINLINE __attribute__((__noinline__)) + #elif defined (__INTEL_COMPILER) + #define NV_NOINLINE __attribute__((__noinline__)) + #endif + + #if !defined(NV_NOINLINE) + #define NV_NOINLINE + #endif + + /* GreenHills compiler defines __GNUC__, but doesn't support + * __inline__ keyword. */ + #if defined(__ghs__) + #define NV_INLINE inline + #elif defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER) + #define NV_INLINE __inline__ + #elif defined (macintosh) || defined(__SUNPRO_C) || defined(__SUNPRO_CC) + #define NV_INLINE inline + #elif defined(__arm) + #define NV_INLINE __inline + #else + #define NV_INLINE + #endif + + /* Don't force inline on DEBUG builds -- it's annoying for debuggers. */ + #if !defined(DEBUG) + /* GreenHills compiler defines __GNUC__, but doesn't support + * __attribute__ or __inline__ keyword. */ + #if defined(__ghs__) + #define NV_FORCEINLINE inline + #elif defined(__GNUC__) + // GCC 3.1 and beyond support the always_inline function attribute. + #if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) + #define NV_FORCEINLINE __attribute__((__always_inline__)) __inline__ + #else + #define NV_FORCEINLINE __inline__ + #endif + #elif defined(__clang__) + #if __has_attribute(always_inline) + #define NV_FORCEINLINE __attribute__((__always_inline__)) __inline__ + #else + #define NV_FORCEINLINE __inline__ + #endif + #elif defined(__arm) && (__ARMCC_VERSION >= 220000) + // RVDS 2.2 also supports forceinline, but ADS 1.2 does not + #define NV_FORCEINLINE __forceinline + #else /* defined(__GNUC__) */ + #define NV_FORCEINLINE NV_INLINE + #endif + #else + #define NV_FORCEINLINE NV_INLINE + #endif + + #define NV_APIENTRY + #define NV_FASTCALL + #define NV_CDECLCALL + #define NV_STDCALL + + /* + * The 'warn_unused_result' function attribute prompts GCC to issue a + * warning if the result of a function tagged with this attribute + * is ignored by a caller. In combination with '-Werror', it can be + * used to enforce result checking in RM code; at this point, this + * is only done on UNIX. + */ + #if defined(__GNUC__) && defined(NV_UNIX) + #if (__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4)) + #define NV_FORCERESULTCHECK __attribute__((__warn_unused_result__)) + #else + #define NV_FORCERESULTCHECK + #endif + #elif defined(__clang__) + #if __has_attribute(warn_unused_result) + #define NV_FORCERESULTCHECK __attribute__((__warn_unused_result__)) + #else + #define NV_FORCERESULTCHECK + #endif + #else /* defined(__GNUC__) */ + #define NV_FORCERESULTCHECK + #endif + + /* + * Functions decorated with NV_FORMAT_PRINTF(f, a) have a format string at + * parameter number 'f' and variadic arguments start at parameter number 'a'. + * (Note that for C++ methods, there is an implicit 'this' parameter so + * explicit parameters are numbered from 2.) + */ + #if defined(__GNUC__) + #define NV_FORMAT_PRINTF(_f, _a) __attribute__((format(printf, _f, _a))) + #else + #define NV_FORMAT_PRINTF(_f, _a) + #endif + +#endif // defined(_MSC_VER) + + /***************************************************************************\ +|* *| +|* Definitions of binary/bytes representations of floating point values *| +|* *| +|* Intended for use when passing values across kernel contexts in which *| +|* floating point use is forbidden *| +|* *| + \***************************************************************************/ +// +// Determine whether we have to turn FP32 support off based on compiler flags. +// For now, this is only done when we can detect that we're compiling for x86_64 +// with GCC/clang and SSE is disabled. +// +// This is also currently only used to control NvF32Bytes definitions. +// +// In the future, it could be possible to: +// 1.) Define this more generally +// and +// 2.) Use it in more places +// +// This would allow us to fully prevent usage of NvF32 in codebases where it is +// not supported. +// +#if !defined(NVTYPES_NV_F32_SUPPORTED) && NVCPU_IS_X86_64 && (defined(__GNUC__) || defined(__clang__)) && !defined(__SSE__) +#define NVTYPES_NV_F32_SUPPORTED 0 +#else +#define NVTYPES_NV_F32_SUPPORTED 1 +#endif + +/*! + * A struct containing the 32-bit representation of an (IEEE-754 binary32) + * floating point value, i.e., the raw underlying bytes of an @ref NvF32 + * + * This struct must be used in places where a value may be passed across a + * kernel boundary, as use of floating point within some kernels is, in general, + * banned. + */ +typedef struct +{ + NvU32 value; +} NvF32Bytes; + +#if !defined(NVDEC_1_0) +/*! + * @brief Maps the bytes of a given @ref NvF32 to an @ref NvF32Bytes + * + * @param[in] pfValue + * Pointer to value to convert + * + * @return @ref NvF32Bytes byte-mapped from @p *pfValue + */ +static NV_FORCEINLINE NvF32Bytes +nvF32BytesMapFromNvF32 +( + const NvF32 *pfValue +) +{ + union + { + NvF32 f; + NvF32Bytes b; + } value; + + value.f = *pfValue; + + return value.b; +} + +#if NVTYPES_NV_F32_SUPPORTED +/*! + * @brief Maps the bytes of a given @ref NvF32Bytes to an @ref NvF32 + * + * @param[in] pfValue + * Pointer to value to convert + * + * @return @ref NvF32 byte-mapped from @p *pfValue + */ +static NV_FORCEINLINE NvF32 +nvF32BytesMapToNvF32 +( + const NvF32Bytes *pfValue +) +{ + union + { + NvF32Bytes b; + NvF32 f; + } value; + + value.b = *pfValue; + + return value.f; +} +#endif // NVTYPES_NV_F32_SUPPORTED + +/*! + * @brief Maps the bytes of a given @ref NvU32 to an @ref NvF32Bytes + * + * @details This function is intended for first "moving" raw byte values into + * the "floating point domain." + * + * @param[in] pfValue + * Pointer to value to convert + * + * @return @ref NvF32Bytes byte-mapped from @p *pfValue + */ +static NV_FORCEINLINE NvF32Bytes +nvF32BytesMapFromNvU32 +( + const NvU32 *pfValue +) +{ + union + { + NvU32 u; + NvF32Bytes b; + } value; + + value.u = *pfValue; + + return value.b; +} + +/*! + * @brief Maps the bytes of a given @ref NvF32Bytes to an @ref NvU32 + * + * @param[in] pfValue + * Pointer to value to convert + * + * @return @ref NvU32 byte-mapped from @p *pfValue + */ +static NV_FORCEINLINE NvU32 +nvF32BytesMapToNvU32 +( + const NvF32Bytes *pfValue +) +{ + union + { + NvF32Bytes b; + NvU32 u; + } value; + + value.b = *pfValue; + + return value.u; +} +#endif // !defined(NVDEC_1_0) + +#ifdef __cplusplus +} +#endif + +#endif /* NVTYPES_INCLUDED */ diff --git a/kernel/ioctl.c b/kernel/ioctl.c new file mode 100644 index 0000000..387a249 --- /dev/null +++ b/kernel/ioctl.c @@ -0,0 +1,307 @@ +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "ioctl.h" +#include "bar_emulation.h" +#include "gsp_comm.h" +#include "memory.h" +#include "irq.h" +#include "migration.h" + +/* Forward declarations from main.c */ +extern int nv_vgpu_register_type_info(struct nv_vgpu_type_info *info); +extern int nv_vgpu_create_mdev_types(void); +extern void nv_vgpu_free_type_info(void); +extern struct mdev_driver nv_vgpu_mdev_driver; + +int nv_vgpu_ioctl_get_info(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_get_info info; + int i; + + memset(&info, 0, sizeof(info)); + + mutex_lock(&vgpu->lock); + info.vgpu_type = vgpu->type; + info.vgpu_id = vgpu->vgpu_id; + info.gpu_instance_id = vgpu->gpu_instance_id; + info.compute_instance_id = vgpu->compute_instance_id; + info.state = vgpu->state; + info.fb_base = vgpu->fb_base; + info.fb_size = vgpu->fb_size; + info.bar0_config_base = vgpu->bar0_config_base; + info.num_bars = 0; + + for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) { + if (vgpu->bars[i].size > 0) { + info.bars[info.num_bars].index = i; + info.bars[info.num_bars].gpa = vgpu->bars[i].gpa; + info.bars[info.num_bars].size = vgpu->bars[i].size; + info.bars[info.num_bars].iova = vgpu->bars[i].iova; + info.bars[info.num_bars].mapped = vgpu->bars[i].mapped; + info.bars[info.num_bars].trapped = vgpu->bars[i].trapped; + info.num_bars++; + } + } + mutex_unlock(&vgpu->lock); + + if (copy_to_user(arg, &info, sizeof(info))) + return -EFAULT; + + return 0; +} + +int nv_vgpu_ioctl_map_bar(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_map_bar params; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (params.bar_index >= NV_VGPU_MAX_REGIONS) + return -EINVAL; + + mutex_lock(&vgpu->lock); + + ret = nv_vgpu_memory_pin(vgpu, params.gpa, params.size); + if (ret) { + mutex_unlock(&vgpu->lock); + return ret; + } + + ret = nv_vgpu_memory_map(vgpu, params.gpa, params.size, ¶ms.iova); + if (ret) { + nv_vgpu_memory_unpin(vgpu, params.gpa, params.size); + mutex_unlock(&vgpu->lock); + return ret; + } + + vgpu->bars[params.bar_index].gpa = params.gpa; + vgpu->bars[params.bar_index].size = params.size; + vgpu->bars[params.bar_index].iova = params.iova; + vgpu->bars[params.bar_index].mapped = true; + + if (params.flags & 0x1) { + ret = nv_vgpu_bar_trap(vgpu, params.bar_index); + if (ret) + nv_vgpu_warn(vgpu, "Failed to trap BAR %d: %d", params.bar_index, ret); + else + vgpu->bars[params.bar_index].trapped = true; + } + + mutex_unlock(&vgpu->lock); + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return 0; +} + +int nv_vgpu_ioctl_unmap_bar(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_unmap_bar params; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (params.bar_index >= NV_VGPU_MAX_REGIONS) + return -EINVAL; + + mutex_lock(&vgpu->lock); + + if (vgpu->bars[params.bar_index].trapped) { + nv_vgpu_bar_untrap(vgpu, params.bar_index); + vgpu->bars[params.bar_index].trapped = false; + } + + ret = nv_vgpu_memory_unmap(vgpu, params.iova, params.size); + if (ret) { + mutex_unlock(&vgpu->lock); + return ret; + } + + nv_vgpu_memory_unpin(vgpu, vgpu->bars[params.bar_index].gpa, params.size); + + vgpu->bars[params.bar_index].mapped = false; + vgpu->bars[params.bar_index].iova = 0; + + mutex_unlock(&vgpu->lock); + return 0; +} + +int nv_vgpu_ioctl_enable_irq(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_irq params; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + ret = nv_vgpu_irq_enable(vgpu, params.irq_index); + if (ret) + return ret; + + if (params.fd >= 0) { + ret = nv_vgpu_irq_set_trigger(vgpu, params.irq_index, params.fd); + if (ret) + nv_vgpu_irq_disable(vgpu, params.irq_index); + } + + return ret; +} + +int nv_vgpu_ioctl_disable_irq(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_irq params; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + ret = nv_vgpu_irq_set_trigger(vgpu, params.irq_index, -1); + if (ret) + return ret; + + return nv_vgpu_irq_disable(vgpu, params.irq_index); +} + +int nv_vgpu_ioctl_gsp_rpc(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_gsp_rpc params; + void *data = NULL; + int ret = 0; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (params.data_size > 0 && params.data_size <= NV_VGPU_GSP_RPC_MAX_SIZE) { + data = kmalloc(params.data_size, GFP_KERNEL); + if (!data) + return -ENOMEM; + + if (copy_from_user(data, (void __user *)params.data_ptr, params.data_size)) { + kfree(data); + return -EFAULT; + } + } + + ret = nv_vgpu_gsp_send_rpc(vgpu, data, params.data_size); + params.result = ret; + + if (data) + kfree(data); + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return 0; +} + +int nv_vgpu_ioctl_migrate_save(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_migrate params; + u8 *data = NULL; + size_t size = 0; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + ret = nv_vgpu_migration_prepare(vgpu); + if (ret) + return ret; + + ret = nv_vgpu_migration_complete(vgpu); + if (ret) + return ret; + + data = nv_vgpu_migration_get_data(vgpu, &size); + + if (params.data_ptr && params.data_size >= size) { + if (copy_to_user((void __user *)params.data_ptr, data, size)) { + ret = -EFAULT; + } + } + + params.data_size = size; + params.result = ret; + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return ret; +} + +int nv_vgpu_ioctl_migrate_restore(struct nv_vgpu_device *vgpu, void __user *arg) +{ + struct nv_vgpu_ioctl_migrate params; + u8 *data = NULL; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + if (!params.data_ptr || params.data_size == 0) + return -EINVAL; + + data = kmalloc(params.data_size, GFP_KERNEL); + if (!data) + return -ENOMEM; + + if (copy_from_user(data, (void __user *)params.data_ptr, params.data_size)) { + kfree(data); + return -EFAULT; + } + + ret = nv_vgpu_migration_restore_state(vgpu, data, params.data_size); + params.result = ret; + + kfree(data); + + if (copy_to_user(arg, ¶ms, sizeof(params))) + return -EFAULT; + + return ret; +} + +int nv_vgpu_ioctl_register_vgpu_type(void __user *arg) +{ + struct nv_vgpu_ioctl_register_vgpu_type params; + struct nv_vgpu_type_info info; + int ret; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + memset(&info, 0, sizeof(info)); + strncpy(info.name, params.name, sizeof(info.name) - 1); + strncpy(info.description, params.description, sizeof(info.description) - 1); + info.vdev_id = params.vdev_id; + info.pdev_id = params.pdev_id; + info.max_instances = params.max_instances; + + ret = nv_vgpu_register_type_info(&info); + if (ret) + return ret; + + /* Re-create mdev types with updated info */ + nv_vgpu_free_type_info(); + ret = nv_vgpu_create_mdev_types(); + if (ret) + return ret; + + /* Re-register mdev driver with new types */ + mdev_unregister_driver(&nv_vgpu_mdev_driver); + ret = mdev_register_driver(&nv_vgpu_mdev_driver); + if (ret) { + pr_err("Failed to re-register mdev driver: %d\n", ret); + return ret; + } + + pr_info("vGPU type '%s' registered via ioctl\n", info.name); + return 0; +} \ No newline at end of file diff --git a/kernel/irq.c b/kernel/irq.c new file mode 100644 index 0000000..f4b77e2 --- /dev/null +++ b/kernel/irq.c @@ -0,0 +1,246 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "irq.h" + +static irqreturn_t nv_vgpu_msi_handler(int irq, void *data) +{ + struct nv_vgpu_msi_irq *msi = data; + + if (msi->trigger) + eventfd_signal(msi->trigger); + + return IRQ_HANDLED; +} + +int nv_vgpu_irq_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_irq_ctx *ctx; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: IRQ init validation"); + /* Check MSI/MSI-X capability */ + if (pci_find_capability(vgpu->pdev, PCI_CAP_ID_MSIX)) { + nv_vgpu_info(vgpu, "DRY-RUN: MSI-X capability present"); + } else if (pci_find_capability(vgpu->pdev, PCI_CAP_ID_MSI)) { + nv_vgpu_info(vgpu, "DRY-RUN: MSI capability present"); + } else { + nv_vgpu_warn(vgpu, "DRY-RUN: No MSI/MSI-X capability found"); + } + return 0; + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + mutex_init(&ctx->lock); + ctx->msi_count = 0; + ctx->msi_enabled = false; + ctx->msix_enabled = false; + + vgpu->irq_ctx = ctx; + return 0; +} + +void nv_vgpu_irq_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + + if (!ctx) + return; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: IRQ cleanup (no hardware resources to release)"); + kfree(ctx); + vgpu->irq_ctx = NULL; + return; + } + + nv_vgpu_irq_teardown_msix(vgpu); + nv_vgpu_irq_teardown_msi(vgpu); + + kfree(ctx); + vgpu->irq_ctx = NULL; +} + +int nv_vgpu_irq_setup_msi(struct nv_vgpu_device *vgpu, int num_vectors) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + struct pci_dev *pdev = vgpu->pdev; + int ret, i; + + if (!ctx || num_vectors > NV_VGPU_MAX_MSI_VECTORS) + return -EINVAL; + + ret = pci_alloc_irq_vectors(pdev, 1, num_vectors, PCI_IRQ_MSI); + if (ret < 0) + return ret; + + ctx->msi_count = ret; + ctx->msi_enabled = true; + + for (i = 0; i < ctx->msi_count; i++) { + ctx->msis[i].virq = pci_irq_vector(pdev, i); + ctx->msis[i].vector = i; + ctx->msis[i].enabled = false; + ctx->msis[i].trigger = NULL; + + ret = request_irq(ctx->msis[i].virq, nv_vgpu_msi_handler, + 0, "nv-vgpu-msi", &ctx->msis[i]); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to request MSI %d: %d", i, ret); + ctx->msis[i].virq = 0; + } + } + + return 0; +} + +void nv_vgpu_irq_teardown_msi(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + int i; + + if (!ctx || !ctx->msi_enabled) + return; + + for (i = 0; i < ctx->msi_count; i++) { + if (ctx->msis[i].virq) + free_irq(ctx->msis[i].virq, &ctx->msis[i]); + if (ctx->msis[i].trigger) + eventfd_ctx_put(ctx->msis[i].trigger); + } + + pci_free_irq_vectors(vgpu->pdev); + ctx->msi_enabled = false; + ctx->msi_count = 0; +} + +int nv_vgpu_irq_setup_msix(struct nv_vgpu_device *vgpu, int num_vectors) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + struct pci_dev *pdev = vgpu->pdev; + int ret, i; + + if (!ctx || num_vectors > NV_VGPU_MAX_MSI_VECTORS) + return -EINVAL; + + ret = pci_alloc_irq_vectors(pdev, 1, num_vectors, PCI_IRQ_MSIX); + if (ret < 0) + return ret; + + ctx->msi_count = ret; + ctx->msix_enabled = true; + + for (i = 0; i < ctx->msi_count; i++) { + ctx->msis[i].virq = pci_irq_vector(pdev, i); + ctx->msis[i].vector = i; + ctx->msis[i].enabled = false; + ctx->msis[i].trigger = NULL; + + ret = request_irq(ctx->msis[i].virq, nv_vgpu_msi_handler, + 0, "nv-vgpu-msix", &ctx->msis[i]); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to request MSIX %d: %d", i, ret); + ctx->msis[i].virq = 0; + } + } + + return 0; +} + +void nv_vgpu_irq_teardown_msix(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + int i; + + if (!ctx || !ctx->msix_enabled) + return; + + for (i = 0; i < ctx->msi_count; i++) { + if (ctx->msis[i].virq) + free_irq(ctx->msis[i].virq, &ctx->msis[i]); + if (ctx->msis[i].trigger) + eventfd_ctx_put(ctx->msis[i].trigger); + } + + pci_free_irq_vectors(vgpu->pdev); + ctx->msix_enabled = false; + ctx->msi_count = 0; +} + +int nv_vgpu_irq_enable(struct nv_vgpu_device *vgpu, int irq_idx) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + + if (!ctx || irq_idx < 0 || irq_idx >= ctx->msi_count) + return -EINVAL; + + if (!ctx->msis[irq_idx].virq) + return -ENODEV; + + ctx->msis[irq_idx].enabled = true; + enable_irq(ctx->msis[irq_idx].virq); + return 0; +} + +int nv_vgpu_irq_disable(struct nv_vgpu_device *vgpu, int irq_idx) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + + if (!ctx || irq_idx < 0 || irq_idx >= ctx->msi_count) + return -EINVAL; + + if (!ctx->msis[irq_idx].virq) + return -ENODEV; + + disable_irq(ctx->msis[irq_idx].virq); + ctx->msis[irq_idx].enabled = false; + return 0; +} + +int nv_vgpu_irq_set_trigger(struct nv_vgpu_device *vgpu, int irq_idx, int fd) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + struct eventfd_ctx *trigger; + + if (!ctx || irq_idx < 0 || irq_idx >= ctx->msi_count) + return -EINVAL; + + if (fd < 0) { + if (ctx->msis[irq_idx].trigger) { + eventfd_ctx_put(ctx->msis[irq_idx].trigger); + ctx->msis[irq_idx].trigger = NULL; + } + return 0; + } + + trigger = eventfd_ctx_fdget(fd); + if (IS_ERR(trigger)) + return PTR_ERR(trigger); + + if (ctx->msis[irq_idx].trigger) + eventfd_ctx_put(ctx->msis[irq_idx].trigger); + + ctx->msis[irq_idx].trigger = trigger; + return 0; +} + +int nv_vgpu_irq_get_virq(struct nv_vgpu_device *vgpu, int irq_idx) +{ + struct nv_vgpu_irq_ctx *ctx = vgpu->irq_ctx; + + if (!ctx || irq_idx < 0 || irq_idx >= ctx->msi_count) + return -EINVAL; + + return ctx->msis[irq_idx].virq; +} \ No newline at end of file diff --git a/kernel/main.c b/kernel/main.c new file mode 100644 index 0000000..177c729 --- /dev/null +++ b/kernel/main.c @@ -0,0 +1,2532 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "bar_emulation.h" +#include "gsp_comm.h" +#include "gsp_emulation.h" +#include "gsp_rm_emulation.h" +#include "gsp_sema_service.h" +#include "memory.h" +#include "irq.h" +#include "migration.h" +#include "ioctl.h" + +#include "nv-vgpu-vfio-interface.h" + +#define NV_VERSION_STRING "550.163.01" + +/* ============================================================ + * Forward declarations + * ============================================================ */ + +static int nv_vgpu_mdev_create(struct mdev_device *mdev); +static void nv_vgpu_mdev_remove(struct mdev_device *mdev); +static void nv_vgpu_device_init_work(struct work_struct *work); +static void nv_vgpu_device_cleanup_work(struct work_struct *work); + +/* VFIO device operations */ +static int nv_vgpu_vfio_dev_match(struct vfio_device *vdev, char *buf); +static int nv_vgpu_vfio_dev_init(struct vfio_device *vdev); +static void nv_vgpu_vfio_dev_release(struct vfio_device *vdev); +static int nv_vgpu_vfio_dev_open(struct vfio_device *vdev); +static void nv_vgpu_vfio_dev_close(struct vfio_device *vdev); +static ssize_t nv_vgpu_vfio_dev_read(struct vfio_device *vdev, char __user *buf, size_t count, loff_t *ppos); +static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __user *buf, size_t count, loff_t *ppos); +static long nv_vgpu_vfio_dev_ioctl(struct vfio_device *vdev, unsigned int cmd, unsigned long arg); +static int nv_vgpu_vfio_dev_mmap(struct vfio_device *vdev, struct vm_area_struct *vma); +static void nv_vgpu_vfio_dev_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length); + +/* PCI Config Space handlers for VFIO mdev */ +static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev *pdev); +static ssize_t nv_vgpu_config_read(struct nv_vgpu_device *vgpu, char __user *buf, size_t count, loff_t *ppos); +static ssize_t nv_vgpu_config_write(struct nv_vgpu_device *vgpu, const char __user *buf, size_t count, loff_t *ppos); + +/* vGPU type management */ +int nv_vgpu_register_type_info(struct nv_vgpu_type_info *info); +struct nv_vgpu_type_info *nv_vgpu_find_type_info(const char *name); + +struct mdev_type *nv_vgpu_mdev_types[16]; +struct nv_vgpu_type_info g_vgpu_type_infos[16]; +int g_num_vgpu_types = 0; +static DEFINE_MUTEX(g_vgpu_type_lock); + +/* + * get_available - report available instances for an mdev type. + * + * The mdev core's shared parent->available_instances counter can drift + * (e.g. if a remove path ever fails to complete). Compute availability + * from our own authoritative device list instead. + */ +static unsigned int nv_vgpu_get_available(struct mdev_type *mtype) +{ + struct nv_vgpu_type_info *info; + struct nv_vgpu_device *vgpu; + const char *name = mtype->sysfs_name; + unsigned int active = 0, max = 1; + + /* Type dirs may carry an extra "nvidia-" prefix vs our type names */ + if (str_has_prefix(name, "nvidia-")) + name += strlen("nvidia-"); + info = nv_vgpu_find_type_info(name); + if (info && info->max_instances) + max = info->max_instances; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + if (vgpu->mdev && vgpu->mdev->type == mtype) + active++; + } + mutex_unlock(&nv_vgpu_driver.lock); + + return active >= max ? 0 : max - active; +} + +/* + * show_description - mdev type description for management tools. + * + * / parses "framebuffer=M" from this string for EVERY + * supported type before creating an mdev, so the field is mandatory + * (missing description file -> readFile failure -> no vGPU). + * 256M matches the vGPU BAR1 (VRAM aperture) size we expose to guests. + */ +static ssize_t nv_vgpu_show_description(struct mdev_type *mtype, char *buf) +{ + return sysfs_emit(buf, "num_heads=1, frl_config=60, framebuffer=256M, " + "max_resolution=5120x2880, max_instance=1\n"); +} + +struct mdev_driver nv_vgpu_mdev_driver = { + .device_api = VFIO_DEVICE_API_PCI_STRING, + .max_instances = 8, + .probe = nv_vgpu_mdev_create, + .remove = nv_vgpu_mdev_remove, + .get_available = nv_vgpu_get_available, + .show_description = nv_vgpu_show_description, + .driver = { + .name = "nvidia-vgpu-vfio-mdev", + }, +}; + +/* VFIO device operations for vGPU */ +static const struct vfio_device_ops nv_vgpu_vfio_dev_ops = { + .name = "nvidia-vgpu-vfio", + .init = nv_vgpu_vfio_dev_init, + .release = nv_vgpu_vfio_dev_release, + .open_device = nv_vgpu_vfio_dev_open, + .close_device = nv_vgpu_vfio_dev_close, + .read = nv_vgpu_vfio_dev_read, + .write = nv_vgpu_vfio_dev_write, + .ioctl = nv_vgpu_vfio_dev_ioctl, + .mmap = nv_vgpu_vfio_dev_mmap, + .dma_unmap = nv_vgpu_vfio_dev_dma_unmap, + .match = nv_vgpu_vfio_dev_match, + .bind_iommufd = vfio_iommufd_emulated_bind, + .unbind_iommufd = vfio_iommufd_emulated_unbind, + .attach_ioas = vfio_iommufd_emulated_attach_ioas, + .detach_ioas = vfio_iommufd_emulated_detach_ioas, +}; + +/* ============================================================ + * Symbols we provide to nvidia.ko + * ============================================================ */ + +static vgpu_vfio_ops_t *g_vgpu_vfio_ops = NULL; +static rm_vgpu_vfio_ops_t g_rm_vgpu_vfio_ops; + +/* mdev parent structure for our physical device */ +static struct mdev_parent *g_mdev_parent = NULL; + +/* VFIO ops implementation */ +static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, NvU32 *vgpu_id); +static void nv_vgpu_vfio_remove(struct pci_dev *pdev); +static NV_STATUS nv_vgpu_vfio_inject_interrupt(void *vgpu_dev); + +static vgpu_vfio_ops_t nv_vgpu_vfio_ops = { + .probe = nv_vgpu_vfio_probe, + .remove = nv_vgpu_vfio_remove, + .inject_interrupt = nv_vgpu_vfio_inject_interrupt, +}; + +/* RM ops implementation - callbacks from nvidia.ko */ +static NV_STATUS nv_vgpu_create_callback(struct pci_dev *pdev, const NvU8 *uuid, + NvU32 type_id, NvU16 *vgpu_id, + NvU32 *gpu_pci_id, NvU32 gpu_pci_bdf) +{ + pr_info("gsp-resman-vfio: vgpu_create callback: type_id=%u\n", type_id); + if (vgpu_id) + *vgpu_id = 1; + if (gpu_pci_id) + *gpu_pci_id = 0x10de1ed3; + return NV_OK; +} + +static NV_STATUS nv_vgpu_delete_callback(const NvU8 *uuid, NvU16 vgpu_id) +{ + pr_info("gsp-resman-vfio: vgpu_delete callback: vgpu_id=%u\n", vgpu_id); + return NV_OK; +} + +static NV_STATUS nv_vgpu_bar_info_callback(struct pci_dev *pdev, const NvU8 *uuid, + NvU64 *size, NvU32 bar_index, void *vgpu_dev) +{ + pr_info("gsp-resman-vfio: vgpu_bar_info callback: bar_index=%u\n", bar_index); + if (size) { + switch (bar_index) { + case 0: + *size = NV_VGPU_BAR0_SIZE; /* 16MB */ + break; + case 1: + *size = NV_VGPU_BAR1_SIZE; /* 256MB */ + break; + case 2: + *size = NV_VGPU_BAR2_SIZE; /* 32MB */ + break; + case 3: + *size = NV_VGPU_BAR3_SIZE; /* 32MB */ + break; + default: + *size = 0; + break; + } + } + return NV_OK; +} + +static NV_STATUS nv_vgpu_get_name_callback(struct pci_dev *pdev, NvU32 type_id, char *buf) +{ + pr_info("gsp-resman-vfio: get_name callback: type_id=%u\n", type_id); + if (buf) { + switch (type_id) { + case 0: /* Ampere */ + strcpy(buf, "NVIDIA A100"); + break; + case 1: /* Hopper */ + strcpy(buf, "NVIDIA H100"); + break; + case 2: /* Turing */ + strcpy(buf, "NVIDIA T4"); + break; + case 3: /* Pascal */ + strcpy(buf, "NVIDIA P4"); + break; + default: + strcpy(buf, "NVIDIA vGPU"); + break; + } + } + return NV_OK; +} + +static NV_STATUS nv_vgpu_get_description_callback(struct pci_dev *pdev, NvU32 type_id, char *buf) +{ + pr_info("gsp-resman-vfio: get_description callback: type_id=%u\n", type_id); + if (buf) { + switch (type_id) { + case 0: + strcpy(buf, "NVIDIA Ampere Architecture vGPU"); + break; + case 1: + strcpy(buf, "NVIDIA Hopper Architecture vGPU"); + break; + case 2: + strcpy(buf, "NVIDIA Turing Architecture vGPU"); + break; + case 3: + strcpy(buf, "NVIDIA Pascal Architecture vGPU"); + break; + default: + strcpy(buf, "NVIDIA vGPU Description"); + break; + } + } + return NV_OK; +} + +static NV_STATUS nv_vgpu_get_instances_callback(struct pci_dev *pdev, NvU32 type_id, char *buf) +{ + pr_info("gsp-resman-vfio: get_instances callback: type_id=%u\n", type_id); + if (buf) + strcpy(buf, "1"); + return NV_OK; +} + +static void nv_vgpu_update_request_callback(const NvU8 *uuid, NvU64 *vgpu_dev, + NvU64 *vgpu_vm, VGPU_DEVICE_STATE device_state, + const char *params) +{ + pr_info("gsp-resman-vfio: update_request callback: state=%u\n", device_state); +} + +static NV_STATUS nv_vgpu_start_callback(const NvU8 *uuid, void *wait_queue, NvS32 *return_status, + NvU8 *vm_name, NvU32 qemu_pid) +{ + pr_info("gsp-resman-vfio: vgpu_start callback: qemu_pid=%u\n", qemu_pid); + if (return_status) + *return_status = 0; + return NV_OK; +} + +static NV_STATUS nv_vgpu_sparse_mmap_callback(struct pci_dev *pdev, const NvU8 *uuid, + NvU64 **pages, NvU64 **offsets, NvU32 *count) +{ + pr_info("gsp-resman-vfio: sparse_mmap callback\n"); + if (count) + *count = 0; + return NV_OK; +} + +/* Build RM ops structure */ +static void populate_rm_ops(void) +{ + memset(&g_rm_vgpu_vfio_ops, 0, sizeof(g_rm_vgpu_vfio_ops)); + g_rm_vgpu_vfio_ops.version_string = NV_VERSION_STRING; + g_rm_vgpu_vfio_ops.vgpu_create = nv_vgpu_create_callback; + g_rm_vgpu_vfio_ops.vgpu_delete = nv_vgpu_delete_callback; + g_rm_vgpu_vfio_ops.vgpu_bar_info = nv_vgpu_bar_info_callback; + g_rm_vgpu_vfio_ops.vgpu_start = nv_vgpu_start_callback; + g_rm_vgpu_vfio_ops.get_name = nv_vgpu_get_name_callback; + g_rm_vgpu_vfio_ops.get_description = nv_vgpu_get_description_callback; + g_rm_vgpu_vfio_ops.get_instances = nv_vgpu_get_instances_callback; + g_rm_vgpu_vfio_ops.get_sparse_mmap = nv_vgpu_sparse_mmap_callback; + g_rm_vgpu_vfio_ops.update_request = nv_vgpu_update_request_callback; +} + +/* ============================================================ + * vGPU Type Management (userspace provides info via ioctl) + * ============================================================ */ + +int nv_vgpu_register_type_info(struct nv_vgpu_type_info *info) +{ + int i; + + mutex_lock(&g_vgpu_type_lock); + + /* Check if already registered */ + for (i = 0; i < g_num_vgpu_types; i++) { + if (strcmp(g_vgpu_type_infos[i].name, info->name) == 0) { + /* Update existing */ + g_vgpu_type_infos[i] = *info; + mutex_unlock(&g_vgpu_type_lock); + return 0; + } + } + + /* Add new */ + if (g_num_vgpu_types >= 16) { + mutex_unlock(&g_vgpu_type_lock); + return -ENOSPC; + } + + g_vgpu_type_infos[g_num_vgpu_types] = *info; + g_num_vgpu_types++; + + mutex_unlock(&g_vgpu_type_lock); + + pr_info("Registered vGPU type: name='%s', vdev=0x%llx, pdev=0x%llx, max_inst=%u\n", + info->name, info->vdev_id, info->pdev_id, info->max_instances); + return 0; +} +EXPORT_SYMBOL(nv_vgpu_register_type_info); + +struct nv_vgpu_type_info *nv_vgpu_find_type_info(const char *name) +{ + int i; + + mutex_lock(&g_vgpu_type_lock); + for (i = 0; i < g_num_vgpu_types; i++) { + if (strcmp(g_vgpu_type_infos[i].name, name) == 0) { + mutex_unlock(&g_vgpu_type_lock); + return &g_vgpu_type_infos[i]; + } + } + mutex_unlock(&g_vgpu_type_lock); + return NULL; +} +EXPORT_SYMBOL(nv_vgpu_find_type_info); + +static void nv_vgpu_create_static_types(void) +{ + int i; + struct nv_vgpu_type_info static_types[4] = { + { .name = "nvidia-vgpu-ampere", .description = "NVIDIA Ampere vGPU", .vdev_id = 0x10de2235, .pdev_id = 0x10de2235, .max_instances = 1 }, + { .name = "nvidia-vgpu-hopper", .description = "NVIDIA Hopper vGPU", .vdev_id = 0x10de2331, .pdev_id = 0x10de2331, .max_instances = 1 }, + { .name = "nvidia-vgpu-turing", .description = "NVIDIA Turing vGPU", .vdev_id = 0x10de1ed3, .pdev_id = 0x10de1ed3, .max_instances = 1 }, + { .name = "nvidia-vgpu-pascal", .description = "NVIDIA Pascal vGPU", .vdev_id = 0x10de1db1, .pdev_id = 0x10de1db1, .max_instances = 1 }, + }; + + for (i = 0; i < 4; i++) { + nv_vgpu_register_type_info(&static_types[i]); + } +} + +int nv_vgpu_create_mdev_types(void) +{ + int i, ret; + + for (i = 0; i < g_num_vgpu_types; i++) { + nv_vgpu_mdev_types[i] = kzalloc(sizeof(struct mdev_type), GFP_KERNEL); + if (!nv_vgpu_mdev_types[i]) { + ret = -ENOMEM; + goto err_types; + } + nv_vgpu_mdev_types[i]->sysfs_name = kstrdup(g_vgpu_type_infos[i].name, GFP_KERNEL); + nv_vgpu_mdev_types[i]->pretty_name = kstrdup(g_vgpu_type_infos[i].description[0] ? g_vgpu_type_infos[i].description : g_vgpu_type_infos[i].name, GFP_KERNEL); + if (!nv_vgpu_mdev_types[i]->sysfs_name || !nv_vgpu_mdev_types[i]->pretty_name) { + ret = -ENOMEM; + goto err_types; + } + } + + return 0; + +err_types: + for (i = 0; i < g_num_vgpu_types; i++) { + kfree(nv_vgpu_mdev_types[i]->sysfs_name); + kfree(nv_vgpu_mdev_types[i]->pretty_name); + kfree(nv_vgpu_mdev_types[i]); + nv_vgpu_mdev_types[i] = NULL; + } + return ret; +} +EXPORT_SYMBOL(nv_vgpu_create_mdev_types); + +void nv_vgpu_free_type_info(void) +{ + int i; + for (i = 0; i < g_num_vgpu_types; i++) { + kfree(nv_vgpu_mdev_types[i]->sysfs_name); + kfree(nv_vgpu_mdev_types[i]->pretty_name); + kfree(nv_vgpu_mdev_types[i]); + nv_vgpu_mdev_types[i] = NULL; + } + g_num_vgpu_types = 0; +} +EXPORT_SYMBOL(nv_vgpu_free_type_info); + +/* get_ops: called by nvidia-vgpu-vfio to get RM ops */ +NV_STATUS nvidia_vgpu_vfio_get_ops(rm_vgpu_vfio_ops_t *ops) +{ + if (!ops) + return NV_ERR_INVALID_ARGUMENT; + + populate_rm_ops(); + memcpy(ops, &g_rm_vgpu_vfio_ops, sizeof(rm_vgpu_vfio_ops_t)); + pr_info("gsp-resman-vfio: get_ops provided RM ops\n"); + return NV_OK; +} +EXPORT_SYMBOL(nvidia_vgpu_vfio_get_ops); + +/* set_ops: called by nvidia-vgpu-vfio to provide VFIO ops */ +NV_STATUS nvidia_vgpu_vfio_set_ops(vgpu_vfio_ops_t *ops) +{ + if (!ops) + return NV_ERR_INVALID_ARGUMENT; + + g_vgpu_vfio_ops = ops; + pr_info("gsp-resman-vfio: set_ops received VFIO ops\n"); + return NV_OK; +} +EXPORT_SYMBOL(nvidia_vgpu_vfio_set_ops); + +/* mdev driver registration - called by nvidia.ko */ +int nv_vgpu_mdev_driver_register(void) +{ + pr_info("gsp-resman-vfio: mdev driver register\n"); + return mdev_register_driver(&nv_vgpu_mdev_driver); +} +EXPORT_SYMBOL(nv_vgpu_mdev_driver_register); + +void nv_vgpu_mdev_driver_unregister(void) +{ + pr_info("gsp-resman-vfio: mdev driver unregister\n"); + mdev_unregister_driver(&nv_vgpu_mdev_driver); +} +EXPORT_SYMBOL(nv_vgpu_mdev_driver_unregister); + +/* mdev parent device registration - called by nvidia.ko */ + +int nv_vgpu_mdev_register_parent(struct pci_dev *pdev) +{ + struct mdev_parent *parent; + int ret; + + pr_info("gsp-resman-vfio: mdev parent register for %s\n", pci_name(pdev)); + + if (g_mdev_parent) { + pr_warn("mdev parent already registered\n"); + return 0; + } + + parent = kzalloc(sizeof(*parent), GFP_KERNEL); + if (!parent) + return -ENOMEM; + + parent->dev = &pdev->dev; + + ret = mdev_register_parent(parent, &pdev->dev, &nv_vgpu_mdev_driver, + nv_vgpu_mdev_types, g_num_vgpu_types); + if (ret) { + pr_err("Failed to register mdev parent: %d\n", ret); + kfree(parent); + return ret; + } + + g_mdev_parent = parent; + pr_info("mdev parent registered successfully with %d types\n", g_num_vgpu_types); + return 0; +} +EXPORT_SYMBOL(nv_vgpu_mdev_register_parent); + +void nv_vgpu_mdev_unregister_parent(struct pci_dev *pdev) +{ + pr_info("gsp-resman-vfio: mdev parent unregister for %s\n", pci_name(pdev)); + + if (g_mdev_parent) { + mdev_unregister_parent(g_mdev_parent); + kfree(g_mdev_parent); + g_mdev_parent = NULL; + } +} +EXPORT_SYMBOL(nv_vgpu_mdev_unregister_parent); + +/* NOTE: No MODULE_SOFTDEP on nvidia - this module is fully standalone. + * It exports the nvidia_vgpu_vfio_* interface (nvidia.ko imports it when + * present) and registers the mdev parent on the physical GPU directly. + * Load order for the matched stack: open-iov-vfio.ko BEFORE nvidia.ko. */ + +MODULE_AUTHOR("Open-IOV Project"); +MODULE_DESCRIPTION("Open-IOV vGPU VFIO Mediated Device Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(OPEN_IOV_VFIO_VERSION); + +struct nv_vgpu_driver_data nv_vgpu_driver = { + .lock = __MUTEX_INITIALIZER(nv_vgpu_driver.lock), + .devices = LIST_HEAD_INIT(nv_vgpu_driver.devices), + .device_count = 0, +}; + +static int nv_vgpu_mdev_create(struct mdev_device *mdev) +{ + struct nv_vgpu_device *vgpu; + struct pci_dev *pdev; + struct mdev_parent *parent; + int ret; + + vgpu = vfio_alloc_device(nv_vgpu_device, vfio_dev, &mdev->dev, + &nv_vgpu_vfio_dev_ops); + if (IS_ERR(vgpu)) + return PTR_ERR(vgpu); + + vgpu->mdev = mdev; + vgpu->state = NV_VGPU_STATE_UNINITIALIZED; + mutex_init(&vgpu->lock); + INIT_WORK(&vgpu->init_work, nv_vgpu_device_init_work); + INIT_WORK(&vgpu->cleanup_work, nv_vgpu_device_cleanup_work); + + /* Initialize VFIO device registration state */ + vgpu->vfio_dev_registered = false; + + /* Initialize VFIO device open/close tracking */ + atomic_set(&vgpu->vfio_dev_open_count, 0); + init_waitqueue_head(&vgpu->vfio_dev_close_wait); + +/* Set drvdata so we can retrieve it in remove */ + dev_set_drvdata(&mdev->dev, vgpu); + + parent = mdev->type->parent; + if (!parent || !parent->dev) { + ret = -ENODEV; + goto err_put_vdev; + } + pdev = to_pci_dev(parent->dev); + if (!pdev) { + ret = -ENODEV; + goto err_put_vdev; + } +vgpu->pdev = pci_dev_get(pdev); + + /* Initialize PCI config space for VFIO mdev */ + ret = nv_vgpu_config_space_init(vgpu, pdev); + if (ret) { + nv_vgpu_err(vgpu, "PCI config space init failed: %d", ret); + goto err_put; + } + + mutex_lock(&nv_vgpu_driver.lock); + list_add_tail(&vgpu->list, &nv_vgpu_driver.devices); + nv_vgpu_driver.device_count++; + mutex_unlock(&nv_vgpu_driver.lock); + + /* Register VFIO device (required for VFIO_DEVICE_API_PCI_STRING). + * This creates a new IOMMU group for the emulated device. + * Do NOT add mdev to parent's IOMMU group first - vfio_register_emulated_iommu_dev + * creates its own IOMMU group for the emulated device. */ + ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_dev); + if (ret) { + nv_vgpu_err(vgpu, "Failed to register VFIO device: %d", ret); + goto err_list_del; + } + vgpu->vfio_dev_registered = true; + + schedule_work(&vgpu->init_work); + + return 0; + + err_list_del: + mutex_lock(&nv_vgpu_driver.lock); + list_del(&vgpu->list); + nv_vgpu_driver.device_count--; + mutex_unlock(&nv_vgpu_driver.lock); + err_put: + pci_dev_put(vgpu->pdev); + err_put_vdev: + vfio_put_device(&vgpu->vfio_dev); + return ret; +} + +static void nv_vgpu_mdev_remove(struct mdev_device *mdev) +{ + struct nv_vgpu_device *vgpu = dev_get_drvdata(&mdev->dev); + long timeout; + + if (!vgpu) { + printk(KERN_ERR "gsp-resman-vfio: No vGPU device data found for mdev %s\n", + dev_name(&mdev->dev)); + return; + } + + pr_info("gsp-resman-vfio: mdev remove started for %s\n", dev_name(&mdev->dev)); + + /* Cancel pending/running init work first */ + cancel_work_sync(&vgpu->init_work); + + /* Wait for any open file descriptors to close before proceeding. + * The VFIO device must not be unregistered while userspace has it open, + * otherwise vfio_device_group_unregister can crash with NULL pointer deref. + * vfio_unregister_group_dev() itself blocks until fds drain, so this + * pre-wait exists to surface a clear warning if userspace is stuck. */ + pr_info("gsp-resman-vfio: waiting for vfio_dev_open_count to reach 0 (current: %d)\n", + atomic_read(&vgpu->vfio_dev_open_count)); + + timeout = wait_event_timeout(vgpu->vfio_dev_close_wait, + atomic_read(&vgpu->vfio_dev_open_count) == 0, + msecs_to_jiffies(5000)); + if (timeout == 0) { + pr_warn("gsp-resman-vfio: timeout waiting for vfio device close (open_count=%d), proceeding anyway\n", + atomic_read(&vgpu->vfio_dev_open_count)); + } else { + pr_info("gsp-resman-vfio: all vfio device file descriptors closed\n"); + } + + /* Unregister the VFIO device. This removes it from its IOMMU group, + * deletes the cdev, and waits for any remaining userspace fds to + * close. After this returns, no userspace can access the device. */ + if (vgpu->vfio_dev_registered) { + vfio_unregister_group_dev(&vgpu->vfio_dev); + vgpu->vfio_dev_registered = false; + } + + /* Now that no userspace can touch the device, release all per-device + * resources (GSP emulation thread, memory, IRQs, BARs, migration). + * cleanup_work is never scheduled elsewhere, so invoke it directly. */ + nv_vgpu_device_cleanup_work(&vgpu->cleanup_work); + + /* Remove from driver device list (get_available accounts per-type) */ + mutex_lock(&nv_vgpu_driver.lock); + list_del(&vgpu->list); + nv_vgpu_driver.device_count--; + mutex_unlock(&nv_vgpu_driver.lock); + + pci_dev_put(vgpu->pdev); + vgpu->pdev = NULL; + + /* Balance vfio_alloc_device() in create; frees the vgpu struct once + * the vfio_device refcount drops to zero (ops->release fires first). */ + vfio_put_device(&vgpu->vfio_dev); + + pr_info("gsp-resman-vfio: mdev remove callback completed for %s\n", dev_name(&mdev->dev)); +} + +/* VFIO device operations implementations */ +static int nv_vgpu_vfio_dev_init(struct vfio_device *vdev) +{ + pr_info("gsp-resman-vfio: VFIO device init\n"); + return 0; +} + +static void nv_vgpu_vfio_dev_release(struct vfio_device *vdev) +{ + pr_info("gsp-resman-vfio: VFIO device release\n"); +} + +static int nv_vgpu_vfio_dev_open(struct vfio_device *vdev) +{ + struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev); + + pr_info("gsp-resman-vfio: VFIO device open\n"); + atomic_inc(&vgpu->vfio_dev_open_count); + return 0; +} + +static void nv_vgpu_vfio_dev_close(struct vfio_device *vdev) +{ + struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev); + int i; + + pr_info("gsp-resman-vfio: VFIO device close\n"); + + /* Release eventfd trigger references wired via SET_IRQS; the next + * opener re-wires its own. Prevents fd/ctx leaks across VM restarts. */ + for (i = 0; i < NV_VGPU_MAX_IRQS; i++) { + if (vgpu->irqs[i].trigger) { + eventfd_ctx_put(vgpu->irqs[i].trigger); + vgpu->irqs[i].trigger = NULL; + } + } + + atomic_dec(&vgpu->vfio_dev_open_count); + wake_up(&vgpu->vfio_dev_close_wait); +} + +static ssize_t nv_vgpu_vfio_dev_read(struct vfio_device *vdev, char __user *buf, + size_t count, loff_t *ppos) +{ + struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev); + loff_t pos = *ppos; + + pr_info("gsp-resman-vfio: VFIO device read count=%zu pos=%lld (0x%llx)\n", count, pos, pos); + + /* Config space region: VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_CONFIG_REGION_INDEX) = 0x70000000000 */ + if (pos >= 0x70000000000ULL && pos < 0x70000000000ULL + 4096) { + loff_t cfg_pos = pos - 0x70000000000ULL; + size_t to_read = min(count, (size_t)(4096 - cfg_pos)); + if (copy_to_user(buf, vgpu->pci_config + cfg_pos, to_read)) + return -EFAULT; + *ppos += to_read; + return to_read; + } + + /* BAR0 region: VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX) = 0x0 */ + if (pos >= 0x0 && pos < 0x0 + (256 * 1024 * 1024)) { + /* This would be handled by mmap, not read */ + return -EINVAL; + } + + return 0; +} + +static ssize_t nv_vgpu_vfio_dev_write(struct vfio_device *vdev, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev); + loff_t pos = *ppos; + + pr_info("gsp-resman-vfio: VFIO device write count=%zu pos=%lld (0x%llx)\n", count, pos, pos); + + /* Config space region - route through the per-register filter so the + * guest can only modify writable registers (not vendor/device IDs, + * BAR addresses, capabilities structures, etc.) */ + if (pos >= 0x70000000000ULL && pos < 0x70000000000ULL + 4096) { + loff_t cfg_pos = pos - 0x70000000000ULL; + ssize_t ret; + + ret = nv_vgpu_config_write(vgpu, buf, count, &cfg_pos); + if (ret > 0) + *ppos += ret; + return ret; + } + + return 0; +} + +static long nv_vgpu_vfio_dev_ioctl(struct vfio_device *vdev, unsigned int cmd, + unsigned long arg) +{ + struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev); + unsigned long minsz; + int ret; + + pr_info("gsp-resman-vfio: VFIO device ioctl cmd=0x%x\n", cmd); + + if (cmd == VFIO_DEVICE_GET_INFO) { + struct vfio_device_info info; + + minsz = offsetofend(struct vfio_device_info, num_irqs); + + pr_info("gsp-resman-vfio: VFIO_DEVICE_GET_INFO minsz=%lu argsz=%u\n", minsz, info.argsz); + + if (copy_from_user(&info, (void __user *)arg, minsz)) { + pr_err("gsp-resman-vfio: VFIO_DEVICE_GET_INFO copy_from_user failed\n"); + return -EFAULT; + } + + pr_info("gsp-resman-vfio: VFIO_DEVICE_GET_INFO argsz=%u minsz=%lu\n", info.argsz, minsz); + + if (info.argsz < minsz) { + pr_err("gsp-resman-vfio: VFIO_DEVICE_GET_INFO argsz too small\n"); + return -EINVAL; + } + + info.flags = VFIO_DEVICE_FLAGS_PCI | VFIO_DEVICE_FLAGS_RESET; + info.num_regions = VFIO_PCI_NUM_REGIONS; + info.num_irqs = VFIO_PCI_NUM_IRQS; + + pr_info("gsp-resman-vfio: VFIO_DEVICE_GET_INFO returning flags=0x%x num_regions=%u num_irqs=%u\n", + info.flags, info.num_regions, info.num_irqs); + + ret = copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + pr_info("gsp-resman-vfio: VFIO_DEVICE_GET_INFO copy_to_user ret=%d\n", ret); + return ret; + } + + if (cmd == VFIO_DEVICE_GET_REGION_INFO) { + struct vfio_region_info info; + + minsz = offsetofend(struct vfio_region_info, offset); + + if (copy_from_user(&info, (void __user *)arg, minsz)) + return -EFAULT; + + if (info.argsz < minsz) + return -EINVAL; + + /* PCI config space region */ + if (info.index == VFIO_PCI_CONFIG_REGION_INDEX) { + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = 4096; /* 4KB config space */ + info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE; + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* BAR0 - GPU registers */ + if (info.index == VFIO_PCI_BAR0_REGION_INDEX) { + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = vgpu->bars[0].size ? vgpu->bars[0].size : (256 * 1024 * 1024); /* 256MB default */ + info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP; + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* BAR1 - VRAM (if present) */ + if (info.index == VFIO_PCI_BAR1_REGION_INDEX) { + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = vgpu->bars[1].size ? vgpu->bars[1].size : 0; + if (info.size) { + info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP; + } else { + info.flags = 0; + } + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* BAR2 - Additional BAR (if present) */ + if (info.index == VFIO_PCI_BAR2_REGION_INDEX) { + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = vgpu->bars[2].size ? vgpu->bars[2].size : 0; + if (info.size) { + info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP; + } else { + info.flags = 0; + } + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* BAR3 - Additional BAR (if present) */ + if (info.index == VFIO_PCI_BAR3_REGION_INDEX) { + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = vgpu->bars[3].size ? vgpu->bars[3].size : 0; + if (info.size) { + info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP; + } else { + info.flags = 0; + } + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* BAR4 - Additional BAR (if present) */ + if (info.index == VFIO_PCI_BAR4_REGION_INDEX) { + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = vgpu->bars[4].size ? vgpu->bars[4].size : 0; + if (info.size) { + info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP; + } else { + info.flags = 0; + } + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* BAR5 - Additional BAR (if present) */ + if (info.index == VFIO_PCI_BAR5_REGION_INDEX) { + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = vgpu->bars[5].size ? vgpu->bars[5].size : 0; + if (info.size) { + info.flags = VFIO_REGION_INFO_FLAG_READ | VFIO_REGION_INFO_FLAG_WRITE | VFIO_REGION_INFO_FLAG_MMAP; + } else { + info.flags = 0; + } + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* ROM region */ + if (info.index == VFIO_PCI_ROM_REGION_INDEX) { + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = 0; + info.flags = 0; + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* VGA region */ + if (info.index == VFIO_PCI_VGA_REGION_INDEX) { + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = 0; + info.flags = 0; + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + return -EINVAL; + } + + if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { + struct vfio_irq_info info; + + minsz = offsetofend(struct vfio_irq_info, count); + + if (copy_from_user(&info, (void __user *)arg, minsz)) + return -EFAULT; + + if (info.argsz < minsz) + return -EINVAL; + + /* INTx */ + if (info.index == VFIO_PCI_INTX_IRQ_INDEX) { + info.flags = VFIO_IRQ_INFO_EVENTFD; + info.count = 1; + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* MSI */ + if (info.index == VFIO_PCI_MSI_IRQ_INDEX) { + info.flags = VFIO_IRQ_INFO_EVENTFD; + info.count = 1; + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* MSI-X */ + if (info.index == VFIO_PCI_MSIX_IRQ_INDEX) { + info.flags = VFIO_IRQ_INFO_EVENTFD | VFIO_IRQ_INFO_AUTOMASKED; + info.count = 32; /* 32 MSI-X vectors */ + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + /* ERR IRQ */ + if (info.index == VFIO_PCI_ERR_IRQ_INDEX) { + info.flags = VFIO_IRQ_INFO_EVENTFD; + info.count = 1; + return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; + } + + return -EINVAL; + } + + if (cmd == VFIO_DEVICE_SET_IRQS) { + struct vfio_irq_set *irq_set_ptr; + struct vfio_irq_set irq_set_hdr; + unsigned long minsz = offsetof(struct vfio_irq_set, data); + int *fds = NULL; + + pr_info("gsp-resman-vfio: VFIO_DEVICE_SET_IRQS\n"); + + /* First copy the fixed part to get argsz */ + if (copy_from_user(&irq_set_hdr, (void __user *)arg, minsz)) { + pr_err("gsp-resman-vfio: SET_IRQS copy_from_user hdr failed\n"); + return -EFAULT; + } + + pr_info("gsp-resman-vfio: SET_IRQS hdr: argsz=%u flags=0x%x index=%u start=%u count=%u\n", + irq_set_hdr.argsz, irq_set_hdr.flags, irq_set_hdr.index, irq_set_hdr.start, irq_set_hdr.count); + + if (irq_set_hdr.argsz < minsz) { + pr_err("gsp-resman-vfio: SET_IRQS argsz too small: %u < %lu\n", irq_set_hdr.argsz, minsz); + return -EINVAL; + } + + /* Allocate buffer for full struct including data array */ + irq_set_ptr = kmalloc(irq_set_hdr.argsz, GFP_KERNEL); + if (!irq_set_ptr) { + pr_err("gsp-resman-vfio: SET_IRQS kmalloc failed\n"); + return -ENOMEM; + } + + if (copy_from_user(irq_set_ptr, (void __user *)arg, irq_set_hdr.argsz)) { + pr_err("gsp-resman-vfio: SET_IRQS copy_from_user full failed\n"); + kfree(irq_set_ptr); + return -EFAULT; + } + + pr_info("gsp-resman-vfio: SET_IRQS full: argsz=%u flags=0x%x index=%u start=%u count=%u\n", + irq_set_ptr->argsz, irq_set_ptr->flags, irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count); + + /* Validate data type */ + if ((irq_set_ptr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) == 0 || + (irq_set_ptr->flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) == 0) { + pr_err("gsp-resman-vfio: SET_IRQS invalid flags\n"); + kfree(irq_set_ptr); + return -EINVAL; + } + + /* Handle DATA_EVENTFD */ + if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_EVENTFD) { + int i, idx, wired = 0; + + if (irq_set_ptr->count == 0) { + kfree(irq_set_ptr); + return -EINVAL; + } + + if (irq_set_ptr->index > VFIO_PCI_REQ_IRQ_INDEX || + irq_set_ptr->start + irq_set_ptr->count > NV_VGPU_MAX_IRQS) { + pr_err("gsp-resman-vfio: SET_IRQS out of range: index=%u start=%u count=%u\n", + irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count); + kfree(irq_set_ptr); + return -EINVAL; + } + + fds = kcalloc(irq_set_ptr->count, sizeof(int), GFP_KERNEL); + if (!fds) { + kfree(irq_set_ptr); + return -ENOMEM; + } + + /* Copy eventfds directly from userspace (after the fixed header) */ + if (copy_from_user(fds, (void __user *)arg + minsz, irq_set_ptr->count * sizeof(int))) { + pr_err("gsp-resman-vfio: SET_IRQS copy_from_user data failed\n"); + kfree(fds); + kfree(irq_set_ptr); + return -EFAULT; + } + + /* Wire each eventfd to its interrupt vector trigger. + * fd >= 0: take a reference and store; fd == -1: clear. + * The GSP emulation engine fires vector 0 via + * gsp_emu_inject_completion_interrupt(). */ + for (i = 0; i < irq_set_ptr->count; i++) { + idx = irq_set_ptr->start + i; + + if (fds[i] >= 0) { + struct eventfd_ctx *trigger; + + trigger = eventfd_ctx_fdget(fds[i]); + if (IS_ERR(trigger)) { + pr_err("gsp-resman-vfio: SET_IRQS bad eventfd %d for vector %d\n", + fds[i], idx); + continue; + } + if (vgpu->irqs[idx].trigger) + eventfd_ctx_put(vgpu->irqs[idx].trigger); + vgpu->irqs[idx].trigger = trigger; + vgpu->irqs[idx].type = irq_set_ptr->index; + wired++; + } else if (fds[i] == -1) { + if (vgpu->irqs[idx].trigger) { + eventfd_ctx_put(vgpu->irqs[idx].trigger); + vgpu->irqs[idx].trigger = NULL; + } + } + } + + pr_info("gsp-resman-vfio: SET_IRQS wired %d/%d eventfds (index=%d start=%d)\n", + wired, irq_set_ptr->count, irq_set_ptr->index, irq_set_ptr->start); + + kfree(fds); + kfree(irq_set_ptr); + return 0; + } + + /* Handle DATA_NONE (mask/unmask) */ + if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_NONE) { + pr_info("gsp-resman-vfio: SET_IRQS DATA_NONE index=%d start=%d count=%d action=%x\n", + irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count, irq_set_ptr->flags & VFIO_IRQ_SET_ACTION_TYPE_MASK); + kfree(irq_set_ptr); + return 0; + } + + /* Handle DATA_BOOL */ + if (irq_set_ptr->flags & VFIO_IRQ_SET_DATA_BOOL) { + pr_info("gsp-resman-vfio: SET_IRQS DATA_BOOL index=%d start=%d count=%d\n", + irq_set_ptr->index, irq_set_ptr->start, irq_set_ptr->count); + kfree(irq_set_ptr); + return 0; + } + + kfree(irq_set_ptr); + return -EINVAL; + } + + return -ENOTTY; +} + +/* Convert any kernel virtual address (lowmem or vmalloc) to a PFN for mmap. + * virt_to_phys() is only valid for direct-mapped lowmem; vmalloc/vmap + * addresses (vzalloc, ioremap, kvzalloc fallback) must go through + * vmalloc_to_pfn(). Using virt_to_phys() on a vmalloc address produces a + * garbage PFN, which KVM then refuses to map (EPT reserved-bit WARN) and + * the guest faults/hangs touching the BAR. */ +static unsigned long nv_vgpu_mmap_virt_to_pfn(const void *vaddr) +{ + if (is_vmalloc_addr(vaddr)) + return vmalloc_to_pfn(vaddr); + return virt_to_phys((void *)vaddr) >> PAGE_SHIFT; +} + +static int nv_vgpu_vfio_dev_mmap(struct vfio_device *vdev, struct vm_area_struct *vma) +{ + struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev); + unsigned long offset = vma->vm_pgoff << PAGE_SHIFT; + unsigned long size = vma->vm_end - vma->vm_start; + unsigned long pfn; + void *vaddr; + struct nv_vgpu_bar_emul_ctx *bar_ctx = (struct nv_vgpu_bar_emul_ctx *)vgpu->bar_emul_ctx; + unsigned long region_base; + unsigned long region_size; + + pr_info("gsp-resman-vfio: VFIO device mmap offset=0x%lx size=0x%lx\n", offset, size); + + /* Config space region */ + region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_CONFIG_REGION_INDEX); + region_size = 4096; + if (offset >= region_base && offset < region_base + region_size) { + if (offset + size > region_base + region_size) + return -EINVAL; + vaddr = vgpu->pci_config + (offset - region_base); + pfn = nv_vgpu_mmap_virt_to_pfn(vaddr); + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot); + } + + /* BAR0 region - map to BAR0 emulation shadow */ + region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR0_REGION_INDEX); + region_size = bar_ctx ? bar_ctx->bar0_shadow_size : (256 * 1024 * 1024); + if (offset >= region_base && offset < region_base + region_size) { + unsigned long shadow_off = offset - region_base; + unsigned long pos; + + if (!bar_ctx || !bar_ctx->bar0_shadow) + return -ENODEV; + if (offset + size > region_base + region_size) + return -EINVAL; + /* bar0_shadow is vzalloc'd: physically non-contiguous pages, so + * map one page at a time via vmalloc_to_pfn(). */ + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + for (pos = 0; pos < size; pos += PAGE_SIZE) { + pfn = vmalloc_to_pfn(bar_ctx->bar0_shadow + shadow_off + pos); + if (remap_pfn_range(vma, vma->vm_start + pos, pfn, PAGE_SIZE, + vma->vm_page_prot)) + return -EAGAIN; + } + /* Debug: log PRAMIN window mapping for BAR2 verify debugging */ + if (shadow_off <= 0x70e000 && shadow_off + size > 0x70e000) { + pr_info("gsp-resman-vfio: BAR0 mmap covers PRAMIN[0xe000] shadow_off=0x%lx size=0x%lx\n", + shadow_off, size); + } + return 0; + } + + /* BAR1 region - VRAM (emulated vidmem buffer, per-page vmalloc_to_pfn) */ + region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR1_REGION_INDEX); + region_size = vgpu->bars[1].size; + if (offset >= region_base && offset < region_base + region_size) { + unsigned long vram_off = offset - region_base; + unsigned long pos; + + if (!vgpu->bars[1].vaddr || region_size == 0) + return -ENODEV; + if (offset + size > region_base + region_size) + return -EINVAL; + /* vidmem buffer is vzalloc'd: physically non-contiguous pages, so + * map one page at a time via vmalloc_to_pfn() (like BAR0 shadow). + * Guest VRAM accesses hit our clean host-RAM backing, not the + * host GPU's real BAR1. */ + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + for (pos = 0; pos < size; pos += PAGE_SIZE) { + pfn = vmalloc_to_pfn(vgpu->bars[1].vaddr + vram_off + pos); + if (remap_pfn_range(vma, vma->vm_start + pos, pfn, PAGE_SIZE, + vma->vm_page_prot)) + return -EAGAIN; + } + return 0; + } + + /* BAR2 region - ioremap'd host BAR; map backing physical address */ + region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR2_REGION_INDEX); + region_size = vgpu->bars[2].size; + if (offset >= region_base && offset < region_base + region_size) { + if (!vgpu->bars[2].vaddr || region_size == 0) + return -ENODEV; + if (offset + size > region_base + region_size) + return -EINVAL; + pfn = (vgpu->bars[2].gpa + (offset - region_base)) >> PAGE_SHIFT; + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot); + } + + /* BAR3 region - emulated shadow buffer (not physical GPU BAR) */ + region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR3_REGION_INDEX); + region_size = vgpu->bars[3].size; + if (offset >= region_base && offset < region_base + region_size) { + unsigned long bar3_off = offset - region_base; + unsigned long pos; + + if (!vgpu->bars[3].vaddr || region_size == 0) + return -ENODEV; + if (offset + size > region_base + region_size) + return -EINVAL; + /* BAR3 shadow is vzalloc'd: per-page vmalloc_to_pfn like BAR0 shadow. + * Guest BAR3 accesses hit our clean host-RAM backing, not the + * host GPU's real instance memory aperture. */ + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); + for (pos = 0; pos < size; pos += PAGE_SIZE) { + pfn = vmalloc_to_pfn(vgpu->bars[3].vaddr + bar3_off + pos); + if (remap_pfn_range(vma, vma->vm_start + pos, pfn, PAGE_SIZE, + vma->vm_page_prot)) + return -EAGAIN; + } + return 0; + } + + /* BAR4 region - map backing physical address if present */ + region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR4_REGION_INDEX); + region_size = vgpu->bars[4].size; + if (offset >= region_base && offset < region_base + region_size) { + if (!vgpu->bars[4].vaddr || region_size == 0) + return -ENODEV; + if (offset + size > region_base + region_size) + return -EINVAL; + pfn = (vgpu->bars[4].gpa + (offset - region_base)) >> PAGE_SHIFT; + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot); + } + + /* BAR5 region - map backing physical address if present */ + region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_BAR5_REGION_INDEX); + region_size = vgpu->bars[5].size; + if (offset >= region_base && offset < region_base + region_size) { + if (!vgpu->bars[5].vaddr || region_size == 0) + return -ENODEV; + if (offset + size > region_base + region_size) + return -EINVAL; + pfn = (vgpu->bars[5].gpa + (offset - region_base)) >> PAGE_SHIFT; + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + return remap_pfn_range(vma, vma->vm_start, pfn, size, vma->vm_page_prot); + } + + /* ROM region */ + region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_ROM_REGION_INDEX); + region_size = 0; + if (offset >= region_base && offset < region_base + region_size) { + return -EINVAL; + } + + /* VGA region */ + region_base = VFIO_PCI_INDEX_TO_OFFSET(VFIO_PCI_VGA_REGION_INDEX); + region_size = 0; + if (offset >= region_base && offset < region_base + region_size) { + return -EINVAL; + } + + pr_err("gsp-resman-vfio: mmap failed for unknown offset 0x%lx\n", offset); + return -EINVAL; +} + +static void nv_vgpu_vfio_dev_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) +{ + pr_info("gsp-resman-vfio: VFIO device dma_unmap iova=0x%llx len=0x%llx\n", iova, length); +} + +static int nv_vgpu_vfio_dev_match(struct vfio_device *vdev, char *buf) +{ + struct nv_vgpu_device *vgpu = container_of(vdev, struct nv_vgpu_device, vfio_dev); + + /* Match by mdev UUID */ + if (vgpu->mdev && !strcmp(dev_name(&vgpu->mdev->dev), buf)) { + return 1; + } + + /* Also match by vfio device name (vfio0, vfio1, etc.) */ + if (!strcmp(dev_name(&vdev->device), buf)) { + return 1; + } + + return 0; +} + +static int __init nv_vgpu_init(void) +{ + int ret; + struct pci_dev *pdev = NULL; + + pr_info("gsp-resman-vfio: init started\n"); + + nv_vgpu_driver.class = class_create("nvidia-vgpu"); + if (IS_ERR(nv_vgpu_driver.class)) { + ret = PTR_ERR(nv_vgpu_driver.class); + pr_err("Failed to create nvidia-vgpu class: %d\n", ret); + return ret; + } + pr_info("gsp-resman-vfio: class created\n"); + + /* Create static vGPU types as fallback (userspace can override via ioctl) */ + nv_vgpu_create_static_types(); + pr_info("gsp-resman-vfio: static types created\n"); + + /* Create mdev types from registered vGPU types */ + ret = nv_vgpu_create_mdev_types(); + if (ret) { + pr_err("Failed to create mdev types: %d\n", ret); + goto err_class; + } + pr_info("gsp-resman-vfio: mdev types created\n"); + + /* Find the parent PCI device to register mdev parent */ + pdev = pci_get_device(0x10de, 0x1ed3, NULL); + if (!pdev) { + pr_err("Failed to find NVIDIA GPU (10de:1ed3) for mdev parent registration\n"); + ret = -ENODEV; + goto err_types; + } + pr_info("gsp-resman-vfio: found GPU %s\n", pci_name(pdev)); + + /* Register mdev parent via nvidia.ko */ + ret = nv_vgpu_mdev_register_parent(pdev); + if (ret) { + pr_err("Failed to register mdev parent: %d\n", ret); + goto err_put_pdev; + } + pr_info("gsp-resman-vfio: mdev parent registered\n"); + + ret = mdev_register_driver(&nv_vgpu_mdev_driver); + if (ret) { + pr_err("Failed to register mdev driver: %d\n", ret); + goto err_types; + } + pr_info("gsp-resman-vfio: mdev driver registered\n"); + + pci_dev_put(pdev); + pdev = NULL; + + /* Provide VFIO ops to host driver (nvidia.ko) */ + ret = nvidia_vgpu_vfio_set_ops(&nv_vgpu_vfio_ops); + if (ret) { + pr_err("Failed to set VFIO ops: %d\n", ret); + goto err_driver; + } + pr_info("gsp-resman-vfio: VFIO ops set\n"); + + pr_info("NVIDIA vGPU VFIO driver loaded (version %s)\n", NVIDIA_VGPU_VFIO_VERSION); + /* Start the emulated-GPU progress service: it discovers the guest's + * GPU-VA -> guest-PA map and applies copy-engine semaphore releases so + * the guest's UVM waits complete. Idle until a VM opens a device. */ + ret = gsp_sema_service_start(); + if (ret) + pr_warn("gsp-resman-vfio: sema service start failed: %d\n", ret); + + pr_info("gsp-resman-vfio: init complete\n"); + return 0; + +err_driver: + mdev_unregister_driver(&nv_vgpu_mdev_driver); +err_types: + nv_vgpu_free_type_info(); +err_put_pdev: + if (pdev) + pci_dev_put(pdev); +err_class: + class_destroy(nv_vgpu_driver.class); + return ret; +} + +static void __exit nv_vgpu_exit(void) +{ + struct pci_dev *pdev = NULL; + + pr_info("gsp-resman-vfio: nv_vgpu_exit: starting cleanup\n"); + + /* Break circular dependency with nvidia.ko FIRST + * by clearing the VFIO ops pointer. This signals nvidia.ko + * to release its references to our module. */ + pr_info("gsp-resman-vfio: nv_vgpu_exit: clearing VFIO ops to break circular dependency\n"); + g_vgpu_vfio_ops = NULL; + + if (nv_vgpu_driver.device_count > 0) { + struct nv_vgpu_device *vgpu; + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + if (vgpu->pdev) { + pdev = vgpu->pdev; + break; + } + } + mutex_unlock(&nv_vgpu_driver.lock); + } + + if (!pdev) { + pdev = pci_get_device(0x10de, 0x1ed3, NULL); + } + + gsp_sema_service_stop(); + + pr_info("gsp-resman-vfio: nv_vgpu_exit: unregistering mdev driver\n"); + /* Unregister mdev driver FIRST, then mdev parent */ + mdev_unregister_driver(&nv_vgpu_mdev_driver); + pr_info("gsp-resman-vfio: nv_vgpu_exit: mdev driver unregistered\n"); + + /* Unregister mdev parent via nvidia.ko */ + if (pdev) { + pr_info("gsp-resman-vfio: nv_vgpu_exit: unregistering mdev parent\n"); + nv_vgpu_mdev_unregister_parent(pdev); + pr_info("gsp-resman-vfio: mdev parent unregistered for %s\n", dev_name(&pdev->dev)); + pci_dev_put(pdev); + } + + pr_info("gsp-resman-vfio: nv_vgpu_exit: destroying class\n"); + class_destroy(nv_vgpu_driver.class); + + pr_info("gsp-resman-vfio: nv_vgpu_exit: freeing type info\n"); + nv_vgpu_free_type_info(); + + pr_info("NVIDIA vGPU VFIO driver unloaded\n"); +} + +static void nv_vgpu_device_init_work(struct work_struct *work) +{ + struct nv_vgpu_device *vgpu = container_of(work, struct nv_vgpu_device, init_work); + int ret; + + mutex_lock(&vgpu->lock); + vgpu->state = NV_VGPU_STATE_INITIALIZING; + mutex_unlock(&vgpu->lock); + + ret = nv_vgpu_bar_init(vgpu); + if (ret) { + nv_vgpu_err(vgpu, "BAR init failed: %d", ret); + goto err_state; + } + + ret = nv_vgpu_irq_init(vgpu); + if (ret) { + nv_vgpu_err(vgpu, "IRQ init failed: %d", ret); + goto err_bar; + } + + ret = nv_vgpu_memory_init(vgpu); + if (ret) { + nv_vgpu_err(vgpu, "Memory init failed: %d", ret); + goto err_irq; + } + + /* Physical GSP is managed by host nvidia.ko - skip nv_vgpu_gsp_init to avoid conflict */ + nv_vgpu_info(vgpu, "Skipping physical GSP init (host nvidia.ko manages physical GSP)"); + + /* Initialize GSP emulation engine (host-side) for guest GSP RPC handling */ + ret = gsp_emu_init(vgpu); + if (ret) { + nv_vgpu_err(vgpu, "GSP emulation init failed: %d", ret); + goto err_memory; + } + + ret = nv_vgpu_migration_init(vgpu); + if (ret) { + nv_vgpu_warn(vgpu, "Migration init failed: %d", ret); + } + + mutex_lock(&vgpu->lock); + vgpu->state = NV_VGPU_STATE_READY; + mutex_unlock(&vgpu->lock); + + nv_vgpu_info(vgpu, "vGPU device initialized successfully"); + return; + +err_memory: + nv_vgpu_memory_cleanup(vgpu); +err_irq: + nv_vgpu_irq_cleanup(vgpu); +err_bar: + nv_vgpu_bar_cleanup(vgpu); +err_state: + mutex_lock(&vgpu->lock); + vgpu->state = NV_VGPU_STATE_ERROR; + mutex_unlock(&vgpu->lock); +} + +static void nv_vgpu_device_cleanup_work(struct work_struct *work) +{ + struct nv_vgpu_device *vgpu = container_of(work, struct nv_vgpu_device, cleanup_work); + + nv_vgpu_migration_cleanup(vgpu); + gsp_emu_cleanup(vgpu); + nv_vgpu_memory_cleanup(vgpu); + nv_vgpu_irq_cleanup(vgpu); + nv_vgpu_bar_cleanup(vgpu); + + mutex_lock(&vgpu->lock); + vgpu->state = NV_VGPU_STATE_UNINITIALIZED; + mutex_unlock(&vgpu->lock); + + nv_vgpu_info(vgpu, "vGPU device cleaned up"); +} + +static NV_STATUS nv_vgpu_vfio_probe(struct pci_dev *pdev, NvU32 vgpu_type_id, NvU32 *vgpu_id) +{ + pr_info("gsp-resman-vfio: VFIO probe called for vGPU type %u\n", vgpu_type_id); + return NV_OK; +} + +static void nv_vgpu_vfio_remove(struct pci_dev *pdev) +{ + pr_info("gsp-resman-vfio: VFIO remove called\n"); + /* Clear VFIO ops to break circular dependency with nvidia.ko */ + g_vgpu_vfio_ops = NULL; +} + +static NV_STATUS nv_vgpu_vfio_inject_interrupt(void *vgpu_dev) +{ + pr_info("gsp-resman-vfio: VFIO inject interrupt called\n"); + return NV_OK; +} + +/* ============================================================ + * PCI Config Space for VFIO mdev + * ============================================================ */ + +static int nv_vgpu_config_space_init(struct nv_vgpu_device *vgpu, struct pci_dev *pdev) +{ + u8 *cfg = vgpu->pci_config; + u16 vendor_id = 0x10de; /* NVIDIA */ + u16 device_id = 0x1ed3; /* TU104 vGPU (default) */ + u16 subsys_vendor = 0x10de; + u16 subsys_device = 0x12b4; + u8 revision = 0xa1; + u8 prog_if = 0x00; + u8 subclass = 0x00; + u8 class_code = 0x03; /* Display controller */ + u8 cache_line = 0x10; + u8 latency = 0x00; + u8 header_type = 0x00; /* Normal device */ + u8 bist = 0x00; + int i; + + memset(cfg, 0, 4096); + + /* Look up vGPU type info from mdev type name */ + if (vgpu->mdev && vgpu->mdev->type && vgpu->mdev->type->sysfs_name) { + for (i = 0; i < g_num_vgpu_types; i++) { + if (strcmp(vgpu->mdev->type->sysfs_name, g_vgpu_type_infos[i].name) == 0) { + vendor_id = (u16)((g_vgpu_type_infos[i].vdev_id >> 16) & 0xFFFF); + device_id = (u16)(g_vgpu_type_infos[i].vdev_id & 0xFFFF); + subsys_vendor = (u16)((g_vgpu_type_infos[i].pdev_id >> 16) & 0xFFFF); + subsys_device = (u16)(g_vgpu_type_infos[i].pdev_id & 0xFFFF); + nv_vgpu_info(vgpu, "Using vGPU type: name='%s', vendor=0x%04x, device=0x%04x, subsys_vendor=0x%04x, subsys_device=0x%04x", + g_vgpu_type_infos[i].name, vendor_id, device_id, subsys_vendor, subsys_device); + break; + } + } + } + + /* PCI Header (offset 0x00-0x3F) */ + cfg[0x00] = vendor_id & 0xff; + cfg[0x01] = (vendor_id >> 8) & 0xff; + cfg[0x02] = device_id & 0xff; + cfg[0x03] = (device_id >> 8) & 0xff; + + /* Command register (0x04) - Memory + IO + BusMaster */ + cfg[0x04] = 0x07; + cfg[0x05] = 0x00; + + /* Status register (0x06) */ + cfg[0x06] = 0x10; + cfg[0x07] = 0x00; + + /* Revision ID (0x08) */ + cfg[0x08] = revision; + + /* Class code (0x09-0x0B) */ + cfg[0x09] = prog_if; + cfg[0x0a] = subclass; + cfg[0x0b] = class_code; + + /* Cache line size (0x0C) */ + cfg[0x0c] = cache_line; + + /* Latency timer (0x0D) */ + cfg[0x0d] = latency; + + /* Header type (0x0E) */ + cfg[0x0e] = header_type; + + /* BIST (0x0F) */ + cfg[0x0f] = bist; + + /* BAR0 - 16MB Memory space (32-bit, non-prefetchable) - matches TU104 */ + cfg[0x10] = 0x00; /* 32-bit, non-prefetchable */ + cfg[0x11] = 0x00; + cfg[0x12] = 0x00; + cfg[0x13] = 0x00; + + /* BAR1 - 256MB Memory space (64-bit, prefetchable) - VRAM */ + cfg[0x14] = 0x0c; /* 64-bit, prefetchable */ + cfg[0x15] = 0x00; + cfg[0x16] = 0x00; + cfg[0x17] = 0x00; + cfg[0x18] = 0x00; /* Upper 32 bits */ + cfg[0x19] = 0x00; + cfg[0x1a] = 0x00; + cfg[0x1b] = 0x00; + + /* BAR2 - Not present (consumed by BAR1 upper half) */ + cfg[0x1c] = 0x00; + cfg[0x1d] = 0x00; + cfg[0x1e] = 0x00; + cfg[0x1f] = 0x00; + + /* BAR3 - 32MB Memory space (64-bit, prefetchable) */ + cfg[0x20] = 0x0c; /* 64-bit, prefetchable */ + cfg[0x21] = 0x00; + cfg[0x22] = 0x00; + cfg[0x23] = 0x00; + cfg[0x24] = 0x00; /* Upper 32 bits */ + cfg[0x25] = 0x00; + cfg[0x26] = 0x00; + cfg[0x27] = 0x00; + + /* BAR4 - Not present (consumed by BAR3 upper half) */ + cfg[0x28] = 0x00; + cfg[0x29] = 0x00; + cfg[0x2a] = 0x00; + cfg[0x2b] = 0x00; + + /* CardBus CIS Pointer (0x2C) - Not used */ + cfg[0x2c] = 0x00; + cfg[0x2d] = 0x00; + cfg[0x2e] = 0x00; + cfg[0x2f] = 0x00; + + /* Subsystem Vendor ID (0x2E) and Subsystem ID (0x2F) - wait, correct offsets: */ + /* Subsystem Vendor ID at 0x2C, Subsystem ID at 0x2E */ + cfg[0x2c] = subsys_vendor & 0xff; + cfg[0x2d] = (subsys_vendor >> 8) & 0xff; + cfg[0x2e] = subsys_device & 0xff; + cfg[0x2f] = (subsys_device >> 8) & 0xff; + + /* Expansion ROM Base Address (0x30-0x33) - Not used */ + cfg[0x30] = 0x00; + cfg[0x31] = 0x00; + cfg[0x32] = 0x00; + cfg[0x33] = 0x00; + + /* Capabilities Pointer (0x34) - Point to Power Management at 0x40 */ + cfg[0x34] = 0x40; + cfg[0x35] = 0x00; + cfg[0x36] = 0x00; + cfg[0x37] = 0x00; + + /* Interrupt Line (0x3C) - Will be set by guest */ + cfg[0x3c] = 0x00; + /* Interrupt Pin (0x3D) - INTA */ + cfg[0x3d] = 0x01; + /* Min_Gnt (0x3E) */ + cfg[0x3e] = 0x00; + /* Max_Lat (0x3F) */ + cfg[0x3f] = 0x00; + + /* Subsystem Vendor ID (0x3C) */ + cfg[0x3c] = subsys_vendor & 0xff; + cfg[0x3d] = (subsys_vendor >> 8) & 0xff; + + /* Subsystem ID (0x3E) */ + cfg[0x3e] = subsys_device & 0xff; + cfg[0x3f] = (subsys_device >> 8) & 0xff; + + /* Expansion ROM Base Address (0x40-0x43) - Not used */ + + /* Capabilities Pointer (0x34) - Point to Power Management at 0x40 */ + cfg[0x34] = 0x40; + cfg[0x35] = 0x00; + cfg[0x36] = 0x00; + cfg[0x37] = 0x00; + + /* Interrupt Line (0x3C) - Will be set by guest */ + cfg[0x3c] = 0x00; + /* Interrupt Pin (0x3D) - INTA */ + cfg[0x3d] = 0x01; + /* Min_Gnt (0x3E) */ + cfg[0x3e] = 0x00; + /* Max_Lat (0x3F) */ + cfg[0x3f] = 0x00; + + /* Power Management Capability (0x40) */ + cfg[0x40] = 0x01; /* Capability ID: Power Management */ + cfg[0x41] = 0x48; /* Next capability: MSI at 0x48 */ + cfg[0x42] = 0x03; /* PME support from D0, D3hot, D3cold */ + cfg[0x43] = 0x00; + cfg[0x44] = 0x00; /* PM Control/Status */ + cfg[0x45] = 0x00; + cfg[0x46] = 0x00; + cfg[0x47] = 0x00; + + /* MSI Capability (0x48) */ + cfg[0x48] = 0x05; /* Capability ID: MSI */ + cfg[0x49] = 0x68; /* Next capability: MSI-X at 0x68 */ + cfg[0x4a] = 0x80; /* 64-bit capable, 1 vector */ + cfg[0x4b] = 0x00; + cfg[0x4c] = 0x00; /* Message Address (low) - set by guest */ + cfg[0x4d] = 0x00; + cfg[0x4e] = 0x00; + cfg[0x4f] = 0x00; + cfg[0x50] = 0x00; /* Message Address (high) - 64-bit */ + cfg[0x51] = 0x00; + cfg[0x52] = 0x00; + cfg[0x53] = 0x00; + cfg[0x54] = 0x01; /* Message Data - set by guest */ + cfg[0x55] = 0x00; + + /* MSI-X Capability (0x68) - real TU104 puts the PCIe capability at + * 0x78, so the MSI-X cap lives at 0x68 (0x68-0x73). This matters: + * the driver's NV_PCFG mirror (BAR0+0x88000, mirror size 0x1000) + * is intercepted by vgpuDevReadReg032, which forwards a BAR0 read + * at 0x88000+off to PCI config space offset `off` + * (kbifGetPciConfigSpacePriMirror_GM107 = NV_PCFG = 0x88000). The + * driver reads NV_XVE_LINK_CAPABILITIES = 0x84 expecting the PCIe + * Link Capabilities register there, which is only true when the + * PCIe capability base is 0x78 (cap+0x0C = 0x84). */ + cfg[0x68] = 0x11; /* Capability ID: MSI-X */ + cfg[0x69] = 0x78; /* Next capability: PCI Express at 0x78 */ + cfg[0x6a] = 0x1f; /* Table size = 31 (32 vectors), 64-bit */ + cfg[0x6b] = 0x00; + /* Table Offset/BIR (0x6c) - Table at BAR0 offset 0x2000 */ + cfg[0x6c] = 0x00; + cfg[0x6d] = 0x20; + cfg[0x6e] = 0x00; + cfg[0x6f] = 0x00; + /* PBA Offset/BIR (0x70) - PBA at BAR0 offset 0x3000 */ + cfg[0x70] = 0x00; + cfg[0x71] = 0x30; + cfg[0x72] = 0x00; + cfg[0x73] = 0x00; + + /* PCI Express Capability (0x78) - matches the real TU104 layout: + * cap+0x00 0x78 cap header, cap+0x04 0x7C Device Capabilities, + * cap+0x08 0x80 Device Control/Status, + * cap+0x0C 0x84 Link Capabilities <- NV_XVE_LINK_CAPABILITIES + * cap+0x10 0x88 Link Control/Status <- NV_XVE_LINK_CONTROL_STATUS + * Gen3 x16 (MAX_SPEED 3 = 8 GT/s, MAX_WIDTH 0x10 = x16), so UVM's + * getPCIELinkRateMBps -> calculatePCIELinkRateMBps succeeds. */ + cfg[0x78] = 0x10; /* Capability ID: PCI Express */ + cfg[0x79] = 0x00; /* Next capability: None (end of list) */ + cfg[0x7a] = 0x02; /* Device type: PCIe Endpoint, version 2 */ + cfg[0x7b] = 0x00; + cfg[0x7c] = 0x00; /* Device Capabilities */ + cfg[0x7d] = 0x00; + cfg[0x7e] = 0x00; + cfg[0x7f] = 0x00; + cfg[0x80] = 0x00; /* Device Control */ + cfg[0x81] = 0x00; + cfg[0x82] = 0x10; /* Device Status */ + cfg[0x83] = 0x00; + cfg[0x84] = 0x03; /* Link Capabilities: MAX_SPEED 3:0 = 3 (8 GT/s) */ + cfg[0x85] = 0x01; /* ... MAX_WIDTH 9:4 = 0x10 (x16) -> 0x103 */ + cfg[0x86] = 0x00; + cfg[0x87] = 0x00; + cfg[0x88] = 0x00; /* Link Control */ + cfg[0x89] = 0x00; + cfg[0x8a] = 0x03; /* Link Status: speed 3:0 = 3 (Gen3) */ + cfg[0x8b] = 0x01; /* ... width 9:4 = 0x10 (x16) */ + cfg[0x8c] = 0x00; /* Slot Capabilities */ + cfg[0x8d] = 0x00; + cfg[0x8e] = 0x00; + cfg[0x8f] = 0x00; + cfg[0x90] = 0x00; /* Slot Control */ + cfg[0x91] = 0x00; + cfg[0x92] = 0x00; /* Slot Status */ + cfg[0x93] = 0x00; + cfg[0x94] = 0x00; /* Root Capabilities */ + cfg[0x95] = 0x00; + cfg[0x96] = 0x00; /* Root Control */ + cfg[0x97] = 0x00; + cfg[0x98] = 0x00; /* Root Status */ + cfg[0x99] = 0x00; + cfg[0x9a] = 0x00; + cfg[0x9b] = 0x00; + cfg[0x9c] = 0x00; /* Device Capabilities 2 */ + cfg[0x9d] = 0x00; + cfg[0x9e] = 0x00; + cfg[0x9f] = 0x00; + cfg[0xa0] = 0x00; /* Device Control 2 */ + cfg[0xa1] = 0x00; + cfg[0xa2] = 0x00; + cfg[0xa3] = 0x00; + cfg[0xa4] = 0x00; /* Link Control 2 */ + cfg[0xa5] = 0x00; + cfg[0xa6] = 0x00; /* Link Status 2 */ + cfg[0xa7] = 0x00; + + vgpu->config_space_dirty = false; + nv_vgpu_info(vgpu, "PCI config space initialized (vendor=0x%04x, device=0x%04x, subsys_vendor=0x%04x, subsys_device=0x%04x)", vendor_id, device_id, subsys_vendor, subsys_device); + nv_vgpu_info(vgpu, "PCIe cap: base=0x%02x hdr=%02x%02x%02x%02x devcap=%02x%02x%02x%02x devctl=%02x%02x linkcap=%02x%02x%02x%02x linkctl=%02x%02x linksta=%02x%02x", + cfg[0x78], cfg[0x78], cfg[0x79], cfg[0x7a], cfg[0x7b], + cfg[0x7c], cfg[0x7d], cfg[0x7e], cfg[0x7f], + cfg[0x80], cfg[0x81], + cfg[0x84], cfg[0x85], cfg[0x86], cfg[0x87], + cfg[0x88], cfg[0x89], cfg[0x8a], cfg[0x8b]); + return 0; +} + +static ssize_t nv_vgpu_config_read(struct nv_vgpu_device *vgpu, char __user *buf, size_t count, loff_t *ppos) +{ + loff_t pos = *ppos; + + if (pos >= 256) + return 0; + + if (pos + count > 256) + count = 256 - pos; + + if (copy_to_user(buf, vgpu->pci_config + pos, count)) + return -EFAULT; + + *ppos += count; + return count; +} + +static ssize_t nv_vgpu_config_write(struct nv_vgpu_device *vgpu, const char __user *buf, size_t count, loff_t *ppos) +{ + loff_t pos = *ppos; + size_t written = 0; + u8 val; + + if (pos >= 256) + return -EINVAL; + + if (pos + count > 256) + count = 256 - pos; + + /* Handle specific writable registers */ + while (count--) { + if (copy_from_user(&val, buf++, 1)) + return -EFAULT; + + switch (pos) { + case 0x04: /* Command register */ + /* Allow Memory, IO, BusMaster */ + vgpu->pci_config[pos] = val & 0x07; + break; + case 0x0c: /* Cache line size */ + vgpu->pci_config[pos] = val; + break; + case 0x0d: /* Latency timer */ + vgpu->pci_config[pos] = val; + break; + case 0x3c: /* Interrupt line */ + vgpu->pci_config[pos] = val; + break; + case 0x4c: case 0x4d: case 0x4e: case 0x4f: /* MSI Message Address Low */ + case 0x50: case 0x51: case 0x52: case 0x53: /* MSI Message Address High */ + case 0x54: case 0x55: /* MSI Message Data */ + vgpu->pci_config[pos] = val; + break; + case 0x6a: case 0x6b: /* MSI-X Message Control */ + vgpu->pci_config[pos] = val; + break; + case 0x6c: case 0x6d: case 0x6e: case 0x6f: /* MSI-X Table Offset/BIR */ + vgpu->pci_config[pos] = val; + break; + case 0x70: case 0x71: case 0x72: case 0x73: /* MSI-X PBA Offset/BIR */ + vgpu->pci_config[pos] = val; + break; + case 0x80: case 0x81: /* Device Control */ + case 0x88: case 0x89: /* Link Control */ + case 0x90: case 0x91: /* Slot Control */ + case 0x96: case 0x97: /* Root Control */ + case 0xa0: case 0xa1: /* Device Control 2 */ + case 0xa4: case 0xa5: /* Link Control 2 */ + vgpu->pci_config[pos] = val; + break; + default: + /* Most config space is read-only for guest */ + break; + } + pos++; + written++; + } + + vgpu->config_space_dirty = true; + *ppos = pos; + return written; +} + +/* ============================================================ + * Exported symbols for nvidia.ko (vGPU VFIO interface) + * ============================================================ */ + +NV_STATUS nvidia_vgpu_vfio_probe(struct pci_dev *pdev) +{ + pr_info("gsp-resman-vfio: nvidia_vgpu_vfio_probe called for %s\n", pci_name(pdev)); + return NV_OK; +} +EXPORT_SYMBOL(nvidia_vgpu_vfio_probe); + +void nvidia_vgpu_vfio_remove(struct pci_dev *pdev, NvBool remove_device) +{ + pr_info("gsp-resman-vfio: nvidia_vgpu_vfio_remove called for %s, remove_device=%d\n", + pci_name(pdev), remove_device); +} +EXPORT_SYMBOL(nvidia_vgpu_vfio_remove); + +/* + * Debug: scan the guest's RAM for a 64-bit value to locate guest structures + * without knowing the guest's GPU page tables (see handoff PART 6). + * echo 0x121000000 > /sys/module/open_iov_vfio/parameters/scan_guest_ram + * Scans 0..8GB of guest physical address space, 4KB at a time. + * + * The scan MUST run in a kthread: vfio_iommu_type1_dma_rw only works when + * current->mm is NULL (kthread) or matches the mapping's mm (QEMU). From a + * user process it fails with -EFAULT for every page. + */ +struct nv_vgpu_scan_req { + u64 needle; + struct completion done; +}; + +static int nv_vgpu_scan_thread(void *arg) +{ + struct nv_vgpu_scan_req *req = arg; + struct nv_vgpu_device *vgpu; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + gsp_rm_scan_guest_ram(vgpu, req->needle, 0, 8ULL << 30, 32); + } + mutex_unlock(&nv_vgpu_driver.lock); + + complete(&req->done); + return 0; +} + +static int nv_vgpu_scan_set(const char *val, const struct kernel_param *kp) +{ + static struct nv_vgpu_scan_req req; + struct task_struct *task; + u64 needle = 0; + int ret; + + ret = kstrtoull(val, 0, &needle); + if (ret) + return ret; + + req.needle = needle; + init_completion(&req.done); + + task = kthread_run(nv_vgpu_scan_thread, &req, "vgpu-scan"); + if (IS_ERR(task)) + return PTR_ERR(task); + + wait_for_completion(&req.done); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_scan_ops = { + .set = nv_vgpu_scan_set, +}; + +module_param_cb(scan_guest_ram, &nv_vgpu_scan_ops, NULL, 0200); +MODULE_PARM_DESC(scan_guest_ram, "Scan guest RAM for a 64-bit value (hex)"); + +/* + * Debug: scan the emulated vidmem (BAR1 shadow, 256MB host buffer) for a + * 32/64-bit value. Used to find guest page tables / channel structures that + * the guest writes into its vidmem aperture. + * echo 0x21000000 > /sys/module/open_iov_vfio/parameters/scan_vidmem + */ +static int nv_vgpu_scan_vidmem_set(const char *val, const struct kernel_param *kp) +{ + struct nv_vgpu_device *vgpu; + u64 needle = 0; + int ret; + + ret = kstrtoull(val, 0, &needle); + if (ret) + return ret; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + size_t off; + u32 hits = 0; + + if (!base || !size) + continue; + + for (off = 0; off + 8 <= size && hits < 16; off += 4) { + u64 v64; + u32 v32; + + memcpy(&v64, base + off, 8); + memcpy(&v32, base + off, 4); + if (v64 == needle) { + pr_info("gsp-resman-vfio: vidmem scan: u64 0x%llx at vidmem off 0x%zx\n", + needle, off); + hits++; + } + if (v32 == (u32)needle) { + pr_info("gsp-resman-vfio: vidmem scan: u32 0x%x at vidmem off 0x%zx\n", + (u32)needle, off); + hits++; + } + } + pr_info("gsp-resman-vfio: vidmem scan: needle=0x%llx size=0x%zx hits=%u\n", + needle, size, hits); + } + mutex_unlock(&nv_vgpu_driver.lock); + + return 0; +} + +static const struct kernel_param_ops nv_vgpu_scan_vidmem_ops = { + .set = nv_vgpu_scan_vidmem_set, +}; + +module_param_cb(scan_vidmem, &nv_vgpu_scan_vidmem_ops, NULL, 0200); +MODULE_PARM_DESC(scan_vidmem, "Scan emulated vidmem for a 32/64-bit value (hex)"); + +/* + * Debug: scan guest RAM and emulated vidmem for a sequence of up to 4 u32 + * values (comma-separated hex), e.g. a CE semaphore-release method sequence: + * echo 0x240,0x1,0x244,0x21000000 > .../parameters/scan_seq + */ +struct nv_vgpu_seq_req { + u32 pat[4]; + u32 npat; + struct completion done; +}; + +static int nv_vgpu_seq_thread(void *arg) +{ + struct nv_vgpu_seq_req *req = arg; + struct nv_vgpu_device *vgpu; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + gsp_rm_scan_guest_ram_seq(vgpu, req->pat, req->npat, + 0, 8ULL << 30, 8); + } + mutex_unlock(&nv_vgpu_driver.lock); + complete(&req->done); + return 0; +} + +static int nv_vgpu_scan_seq_set(const char *val, const struct kernel_param *kp) +{ + static struct nv_vgpu_seq_req req; + struct task_struct *task; + char buf[128]; + char *tok, *cur; + + memset(&req, 0, sizeof(req)); + strscpy(buf, val, sizeof(buf)); + cur = buf; + while (req.npat < 4 && (tok = strsep(&cur, ",")) != NULL) { + u32 v; + + if (kstrtou32(tok, 0, &v)) + break; + req.pat[req.npat++] = v; + } + if (!req.npat) + return -EINVAL; + + init_completion(&req.done); + task = kthread_run(nv_vgpu_seq_thread, &req, "vgpu-seq"); + if (IS_ERR(task)) + return PTR_ERR(task); + wait_for_completion(&req.done); + + /* Also scan the emulated vidmem (plain memory, no kthread needed) */ + mutex_lock(&nv_vgpu_driver.lock); + { + struct nv_vgpu_device *vgpu; + + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + size_t off; + u32 hits = 0; + + if (!base || !size) + continue; + for (off = 0; off + req.npat * 4 <= size; off += 4) { + u32 i, ok = 1; + + for (i = 0; i < req.npat; i++) { + u32 v; + + memcpy(&v, base + off + i * 4, 4); + if (v != req.pat[i]) { + ok = 0; + break; + } + } + if (ok && hits < 8) { + pr_info("gsp-resman-vfio: seq scan: hit in vidmem at off 0x%zx\n", + off); + hits++; + } + } + pr_info("gsp-resman-vfio: seq scan: vidmem hits=%u\n", hits); + } + } + mutex_unlock(&nv_vgpu_driver.lock); + + return 0; +} + +static const struct kernel_param_ops nv_vgpu_scan_seq_ops = { + .set = nv_vgpu_scan_seq_set, +}; + +module_param_cb(scan_seq, &nv_vgpu_scan_seq_ops, NULL, 0200); +MODULE_PARM_DESC(scan_seq, "Scan vidmem for a u32 sequence (hex,comma separated)"); + +/* + * Debug: find page-table entries pointing at a physical address, in guest RAM + * and emulated vidmem: + * echo 0x12442b000 > .../parameters/scan_pt + */ +struct nv_vgpu_pt_req { + u64 pa; + struct completion done; +}; + +static int nv_vgpu_pt_thread(void *arg) +{ + struct nv_vgpu_pt_req *req = arg; + struct nv_vgpu_device *vgpu; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) + gsp_rm_scan_guest_ram_pt(vgpu, req->pa, 0, 8ULL << 30, 16); + mutex_unlock(&nv_vgpu_driver.lock); + complete(&req->done); + return 0; +} + +static int nv_vgpu_scan_pt_set(const char *val, const struct kernel_param *kp) +{ + static struct nv_vgpu_pt_req req; + struct task_struct *task; + u64 pa = 0; + int ret; + + ret = kstrtoull(val, 0, &pa); + if (ret) + return ret; + + req.pa = pa; + init_completion(&req.done); + task = kthread_run(nv_vgpu_pt_thread, &req, "vgpu-pt"); + if (IS_ERR(task)) + return PTR_ERR(task); + wait_for_completion(&req.done); + + /* vidmem shadow scan (plain memory) */ + mutex_lock(&nv_vgpu_driver.lock); + { + struct nv_vgpu_device *vgpu; + + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + size_t off; + u32 hits = 0; + u64 pa_page = pa & ~0xfffULL; + + if (!base || !size) + continue; + for (off = 0; off + 8 <= size; off += 4) { + u64 v64; + u32 v32; + + memcpy(&v64, base + off, 8); + memcpy(&v32, base + off, 4); + if ((v64 & 0xfffffffffffff000ULL) == pa_page) { + pr_info("gsp-resman-vfio: pt scan: vidmem u64 pte 0x%llx at off 0x%zx\n", + v64, off); + if (++hits >= 16) + break; + } + if ((v32 & 0xfffff000U) == (u32)pa_page) { + pr_info("gsp-resman-vfio: pt scan: vidmem u32 pte 0x%x at off 0x%zx\n", + v32, off); + if (++hits >= 16) + break; + } + } + pr_info("gsp-resman-vfio: pt scan: vidmem pa=0x%llx hits=%u\n", + pa, hits); + } + } + mutex_unlock(&nv_vgpu_driver.lock); + + return 0; +} + +static const struct kernel_param_ops nv_vgpu_scan_pt_ops = { + .set = nv_vgpu_scan_pt_set, +}; + +module_param_cb(scan_pt, &nv_vgpu_scan_pt_ops, NULL, 0200); +MODULE_PARM_DESC(scan_pt, "Find PTEs pointing at a physical address (hex)"); + +/* + * Debug: scan guest RAM for (v & mask) == value (kthread context): + * echo 0x121000000,0xfffff000 > .../parameters/scan_masked + * Used to find stored page-base VAs (base = VA - index*size). + */ +struct nv_vgpu_masked_req { + u64 value; + u64 mask; + struct completion done; +}; + +static int nv_vgpu_masked_thread(void *arg) +{ + struct nv_vgpu_masked_req *req = arg; + struct nv_vgpu_device *vgpu; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) + gsp_rm_scan_guest_ram_masked(vgpu, req->value, req->mask, + 0, 8ULL << 30, 12); + mutex_unlock(&nv_vgpu_driver.lock); + complete(&req->done); + return 0; +} + +static int nv_vgpu_scan_masked_set(const char *val, const struct kernel_param *kp) +{ + static struct nv_vgpu_masked_req req; + struct task_struct *task; + char buf[96]; + char *comma; + + memset(&req, 0, sizeof(req)); + strscpy(buf, val, sizeof(buf)); + comma = strchr(buf, ','); + if (!comma) + return -EINVAL; + *comma = '\0'; + if (kstrtoull(buf, 0, &req.value) || kstrtoull(comma + 1, 0, &req.mask)) + return -EINVAL; + + init_completion(&req.done); + task = kthread_run(nv_vgpu_masked_thread, &req, "vgpu-mask"); + if (IS_ERR(task)) + return PTR_ERR(task); + wait_for_completion(&req.done); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_scan_masked_ops = { + .set = nv_vgpu_scan_masked_set, +}; + +module_param_cb(scan_masked, &nv_vgpu_scan_masked_ops, NULL, 0200); +MODULE_PARM_DESC(scan_masked, "Scan guest RAM for (v & mask) == value (hex,hex)"); + +/* + * Debug: fill the entire emulated vidmem (BAR1 shadow) with a u32 value: + * echo 0x1 > .../parameters/fill_vidmem + * Used to test whether a guest-visible structure (e.g. a UVM semaphore) + * lives in the vidmem aperture. + */ +static int nv_vgpu_fill_vidmem_set(const char *val, const struct kernel_param *kp) +{ + struct nv_vgpu_device *vgpu; + u32 value = 0; + int ret; + + ret = kstrtou32(val, 0, &value); + if (ret) + return ret; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + size_t o; + + if (!base || !size) + continue; + for (o = 0; o + 4 <= size; o += 4) + memcpy(base + o, &value, 4); + pr_info("gsp-resman-vfio: fill_vidmem value=0x%x size=0x%zx done\n", + value, size); + } + mutex_unlock(&nv_vgpu_driver.lock); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_fill_vidmem_ops = { + .set = nv_vgpu_fill_vidmem_set, +}; + +module_param_cb(fill_vidmem, &nv_vgpu_fill_vidmem_ops, NULL, 0200); +MODULE_PARM_DESC(fill_vidmem, "Fill emulated vidmem with a u32 value (hex)"); + +/* + * Debug: write a u32 to a guest physical address (kthread context, via + * vfio_dma_rw). Used to complete an emulated GPU semaphore release: + * echo 0x121000000,0x1 > .../parameters/poke_guest + */ +struct nv_vgpu_poke_req { + u64 gpa; + u32 value; + struct completion done; + int rc; +}; + +static int nv_vgpu_poke_thread(void *arg) +{ + struct nv_vgpu_poke_req *req = arg; + struct nv_vgpu_device *vgpu; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) { + req->rc = -ENODEV; + continue; + } + req->rc = vfio_dma_rw(&vgpu->vfio_dev, req->gpa, &req->value, + sizeof(req->value), true); + pr_info("gsp-resman-vfio: poke gpa=0x%llx value=0x%x rc=%d\n", + req->gpa, req->value, req->rc); + } + mutex_unlock(&nv_vgpu_driver.lock); + complete(&req->done); + return 0; +} + +static int nv_vgpu_poke_set(const char *val, const struct kernel_param *kp) +{ + static struct nv_vgpu_poke_req req; + struct task_struct *task; + char buf[64]; + char *comma; + + memset(&req, 0, sizeof(req)); + strscpy(buf, val, sizeof(buf)); + comma = strchr(buf, ','); + if (!comma) + return -EINVAL; + *comma = '\0'; + if (kstrtoull(buf, 0, &req.gpa) || kstrtou32(comma + 1, 0, &req.value)) + return -EINVAL; + + init_completion(&req.done); + task = kthread_run(nv_vgpu_poke_thread, &req, "vgpu-poke"); + if (IS_ERR(task)) + return PTR_ERR(task); + wait_for_completion(&req.done); + + return req.rc; +} + +static const struct kernel_param_ops nv_vgpu_poke_ops = { + .set = nv_vgpu_poke_set, +}; + +module_param_cb(poke_guest, &nv_vgpu_poke_ops, NULL, 0200); +MODULE_PARM_DESC(poke_guest, "Write u32 to guest PA (hex gpa,hex value)"); + +/* + * Debug: fill a guest physical range with a u32 value (kthread context): + * echo 0x121000000,0x1,1024 > .../parameters/poke_fill + * (gpa, value, count of dwords) + */ +static u32 *nv_vgpu_fill_buf; +static u32 nv_vgpu_fill_count; + +static int nv_vgpu_poke_fill_thread(void *arg) +{ + struct nv_vgpu_poke_req *req = arg; + struct nv_vgpu_device *vgpu; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + if (atomic_read(&vgpu->vfio_dev_open_count) <= 0) { + req->rc = -ENODEV; + continue; + } + req->rc = vfio_dma_rw(&vgpu->vfio_dev, req->gpa, + nv_vgpu_fill_buf, + nv_vgpu_fill_count * sizeof(u32), true); + pr_info("gsp-resman-vfio: poke_fill gpa=0x%llx count=%u rc=%d\n", + req->gpa, nv_vgpu_fill_count, req->rc); + } + mutex_unlock(&nv_vgpu_driver.lock); + complete(&req->done); + return 0; +} + +static int nv_vgpu_poke_fill_set(const char *val, const struct kernel_param *kp) +{ + u64 gpa = 0; + u32 value = 0, count = 0; + char buf[96]; + char *tok, *cur; + struct task_struct *task; + static struct nv_vgpu_poke_req req; + int rc; + + strscpy(buf, val, sizeof(buf)); + cur = buf; + tok = strsep(&cur, ","); + if (!tok || kstrtoull(tok, 0, &gpa)) + return -EINVAL; + tok = strsep(&cur, ","); + if (!tok || kstrtou32(tok, 0, &value)) + return -EINVAL; + tok = strsep(&cur, ","); + if (!tok || kstrtou32(tok, 0, &count) || !count || count > 4096) + return -EINVAL; + + nv_vgpu_fill_buf = kmalloc_array(count, sizeof(u32), GFP_KERNEL); + if (!nv_vgpu_fill_buf) + return -ENOMEM; + for (u32 i = 0; i < count; i++) + nv_vgpu_fill_buf[i] = value; + nv_vgpu_fill_count = count; + + memset(&req, 0, sizeof(req)); + req.gpa = gpa; + init_completion(&req.done); + + task = kthread_run(nv_vgpu_poke_fill_thread, &req, "vgpu-fill"); + if (IS_ERR(task)) { + kfree(nv_vgpu_fill_buf); + nv_vgpu_fill_buf = NULL; + return PTR_ERR(task); + } + wait_for_completion(&req.done); + rc = req.rc; + kfree(nv_vgpu_fill_buf); + nv_vgpu_fill_buf = NULL; + return rc; +} + +static const struct kernel_param_ops nv_vgpu_poke_fill_ops = { + .set = nv_vgpu_poke_fill_set, +}; + +module_param_cb(poke_fill, &nv_vgpu_poke_fill_ops, NULL, 0200); +MODULE_PARM_DESC(poke_fill, "Fill guest PA range with u32 (gpa,value,count)"); + +/* + * Debug: dump a range of the emulated vidmem (BAR1 shadow): + * echo 0x20000,0x200 > .../parameters/dump_vidmem + * echo summary > .../parameters/dump_vidmem (find nonzero areas) + */ +static int nv_vgpu_dump_vidmem_set(const char *val, const struct kernel_param *kp) +{ + struct nv_vgpu_device *vgpu; + unsigned long off = 0, len = 0; + char buf[64]; + char *comma; + + strscpy(buf, val, sizeof(buf)); + if (!strncmp(buf, "summary", 7)) { + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + size_t o; + u32 nz = 0; + + if (!base || !size) + continue; + for (o = 0; o + 4 <= size; o += 4) { + u32 v; + + memcpy(&v, base + o, 4); + if (v) { + if (nz < 32) + pr_info("gsp-resman-vfio: vidmem nz[0x%zx]=0x%x\n", + o, v); + nz++; + } + } + pr_info("gsp-resman-vfio: vidmem summary: size=0x%zx nonzero_dwords=%u\n", + size, nz); + } + mutex_unlock(&nv_vgpu_driver.lock); + return 0; + } + + comma = strchr(buf, ','); + if (comma) + *comma = '\0'; + if (kstrtoul(buf, 0, &off)) + return -EINVAL; + if (comma && kstrtoul(comma + 1, 0, &len)) + return -EINVAL; + if (!len) + len = 0x100; + if (len > 0x1000) + len = 0x1000; + + mutex_lock(&nv_vgpu_driver.lock); + list_for_each_entry(vgpu, &nv_vgpu_driver.devices, list) { + u8 *base = vgpu->bars[1].vaddr; + size_t size = vgpu->bars[1].size; + unsigned long o; + + if (!base || !size || off + len > size) + continue; + for (o = 0; o < len; o += 64) { + u32 n = min_t(unsigned long, len - o, 64); + + pr_info("gsp-resman-vfio: vidmem[0x%lx..0x%lx]=%*phN\n", + off + o, off + o + n, (int)n, base + off + o); + } + } + mutex_unlock(&nv_vgpu_driver.lock); + return 0; +} + +static const struct kernel_param_ops nv_vgpu_dump_vidmem_ops = { + .set = nv_vgpu_dump_vidmem_set, +}; + +module_param_cb(dump_vidmem, &nv_vgpu_dump_vidmem_ops, NULL, 0200); +MODULE_PARM_DESC(dump_vidmem, "Dump emulated vidmem range (hex offset,hex len)"); + +module_init(nv_vgpu_init); +module_exit(nv_vgpu_exit); diff --git a/kernel/memory.c b/kernel/memory.c new file mode 100644 index 0000000..cd81496 --- /dev/null +++ b/kernel/memory.c @@ -0,0 +1,274 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "memory.h" + +static LIST_HEAD(simple_mappings); +static DEFINE_MUTEX(simple_lock); + +int nv_vgpu_memory_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_mm_ctx *ctx; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: Memory init validation (simple fallback)"); + return 0; + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + mutex_init(&ctx->lock); + INIT_LIST_HEAD(&ctx->mappings); + ctx->mapping_count = 0; + ctx->iommu_domain = NULL; + ctx->use_simple = true; + + vgpu->mm_ctx = ctx; + nv_vgpu_info(vgpu, "Memory init: using simple fallback (no IOMMU)"); + return 0; +} + +void nv_vgpu_memory_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping, *tmp; + + if (!ctx) + return; + + if (ctx->use_simple) { + mutex_lock(&simple_lock); + list_for_each_entry_safe(mapping, tmp, &simple_mappings, list) { + if (mapping->mapped) { + dma_unmap_page(&vgpu->pdev->dev, mapping->iova, mapping->size, DMA_BIDIRECTIONAL); + mapping->mapped = false; + } + if (mapping->pinned) { + for (int i = 0; i < mapping->page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + mapping->pinned = false; + } + list_del(&mapping->list); + vfree(mapping->pages); + kfree(mapping); + } + mutex_unlock(&simple_lock); + } else if (ctx->iommu_domain) { + iommu_detach_device(ctx->iommu_domain, &vgpu->pdev->dev); + iommu_domain_free(ctx->iommu_domain); + } + + kfree(ctx); + vgpu->mm_ctx = NULL; +} + +int nv_vgpu_memory_pin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping; + u64 start_pfn = gpa >> PAGE_SHIFT; + u64 end_pfn = (gpa + size + PAGE_SIZE - 1) >> PAGE_SHIFT; + int page_count = end_pfn - start_pfn; + int i, ret = 0; + + if (!ctx) + return -ENODEV; + + if (!ctx->use_simple) + return -ENODEV; + + mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) + return -ENOMEM; + + mapping->gpa = gpa; + mapping->size = size; + mapping->page_count = page_count; + mapping->pages = vzalloc(page_count * sizeof(struct page *)); + if (!mapping->pages) { + kfree(mapping); + return -ENOMEM; + } + + for (i = 0; i < page_count; i++) { + u64 pfn = start_pfn + i; + struct page *page = pfn_to_page(pfn); + if (!page) { + ret = -EFAULT; + goto err_unlock; + } + get_page(page); + mapping->pages[i] = page; + } + + mapping->pinned = true; + mutex_lock(&simple_lock); + list_add_tail(&mapping->list, &simple_mappings); + mutex_unlock(&simple_lock); + + return 0; + +err_unlock: + for (i = 0; i < page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + vfree(mapping->pages); + kfree(mapping); + return ret; +} + +int nv_vgpu_memory_unpin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping; + + if (!ctx || !ctx->use_simple) + return -ENODEV; + + mutex_lock(&simple_lock); + list_for_each_entry(mapping, &simple_mappings, list) { + if (mapping->gpa == gpa && mapping->size == size) { + if (mapping->mapped) { + mutex_unlock(&simple_lock); + return -EBUSY; + } else if (mapping->pinned) { + for (int i = 0; i < mapping->page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + mapping->pinned = false; + } + list_del(&mapping->list); + vfree(mapping->pages); + kfree(mapping); + break; + } + } + mutex_unlock(&simple_lock); + + return 0; +} + +int nv_vgpu_memory_map(struct nv_vgpu_device *vgpu, u64 gpa, u64 size, u64 *iova) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping; + u64 map_iova; + int ret = 0; + + if (!ctx || !ctx->use_simple) + return -ENODEV; + + mutex_lock(&simple_lock); + list_for_each_entry(mapping, &simple_mappings, list) { + if (mapping->gpa == gpa && mapping->size == size) + break; + } + if (&mapping->list == &simple_mappings) { + mutex_unlock(&simple_lock); + return -ENOENT; + } + + if (!mapping->pinned) { + mutex_unlock(&simple_lock); + return -EINVAL; + } + + map_iova = dma_map_page(&vgpu->pdev->dev, mapping->pages[0], 0, size, DMA_BIDIRECTIONAL); + if (dma_mapping_error(&vgpu->pdev->dev, map_iova)) { + mutex_unlock(&simple_lock); + return -ENOMEM; + } + + mapping->iova = map_iova; + mapping->mapped = true; + *iova = map_iova; + mutex_unlock(&simple_lock); + + return ret; +} + +int nv_vgpu_memory_unmap(struct nv_vgpu_device *vgpu, u64 iova, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping; + int ret = 0; + + if (!ctx || !ctx->use_simple) + return -ENODEV; + + mutex_lock(&simple_lock); + list_for_each_entry(mapping, &simple_mappings, list) { + if (mapping->iova == iova && mapping->size == size) { + if (!mapping->mapped) { + mutex_unlock(&simple_lock); + return -ENOENT; + } + dma_unmap_page(&vgpu->pdev->dev, iova, size, DMA_BIDIRECTIONAL); + mapping->mapped = false; + mapping->iova = 0; + break; + } + } + mutex_unlock(&simple_lock); + + return ret; +} + +static vm_fault_t nv_vgpu_vm_fault(struct vm_fault *vmf) +{ + struct nv_vgpu_device *vgpu = vmf->vma->vm_private_data; + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_simple_mapping *mapping; + u64 offset = vmf->address - vmf->vma->vm_start + vmf->vma->vm_pgoff * PAGE_SIZE; + u64 pfn; + vm_fault_t ret = VM_FAULT_SIGBUS; + + if (!ctx) + return VM_FAULT_SIGBUS; + + if (!ctx->use_simple) + return VM_FAULT_SIGBUS; + + mutex_lock(&simple_lock); + list_for_each_entry(mapping, &simple_mappings, list) { + if (mapping->mapped && offset >= mapping->iova && + offset < mapping->iova + mapping->size) { + pfn = (mapping->gpa + offset - mapping->iova) >> PAGE_SHIFT; + vmf->page = pfn_to_page(pfn); + get_page(vmf->page); + ret = VM_FAULT_NOPAGE; + break; + } + } + mutex_unlock(&simple_lock); + + return ret; +} + +static const struct vm_operations_struct nv_vgpu_vm_ops = { + .fault = nv_vgpu_vm_fault, +}; + +int nv_vgpu_memory_mmap(struct nv_vgpu_device *vgpu, struct vm_area_struct *vma) +{ + vma->vm_ops = &nv_vgpu_vm_ops; + vma->vm_private_data = vgpu; + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); + return 0; +} diff --git a/kernel/memory.c.iommu b/kernel/memory.c.iommu new file mode 100644 index 0000000..335446b --- /dev/null +++ b/kernel/memory.c.iommu @@ -0,0 +1,335 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "memory.h" + +int nv_vgpu_memory_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_mm_ctx *ctx; + int ret; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: Memory init validation"); + /* Check IOMMU support */ + if (iommu_present(&pci_bus_type)) { + nv_vgpu_info(vgpu, "DRY-RUN: IOMMU present"); + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + ctx->iommu_domain = iommu_domain_alloc(&pci_bus_type); + if (ctx->iommu_domain) { + nv_vgpu_info(vgpu, "DRY-RUN: IOMMU domain allocatable"); + iommu_domain_free(ctx->iommu_domain); + } else { + nv_vgpu_warn(vgpu, "DRY-RUN: IOMMU domain allocation failed"); + } + kfree(ctx); + } else { + nv_vgpu_warn(vgpu, "DRY-RUN: No IOMMU present"); + } + return 0; + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + mutex_init(&ctx->lock); + INIT_LIST_HEAD(&ctx->mappings); + ctx->mapping_count = 0; + + ctx->iommu_domain = iommu_domain_alloc(&pci_bus_type); + if (!ctx->iommu_domain) { + ret = -ENODEV; + goto err_free; + } + + ret = iommu_attach_device(ctx->iommu_domain, &vgpu->pdev->dev); + if (ret) { + nv_vgpu_warn(vgpu, "Failed to attach IOMMU domain: %d", ret); + } + + vgpu->mm_ctx = ctx; + return 0; + +err_free: + kfree(ctx); + return ret; +} + +void nv_vgpu_memory_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping, *tmp; + + if (!ctx) + return; + + if (vgpu->dry_run) { + nv_vgpu_info(vgpu, "DRY-RUN: Memory cleanup (no hardware resources to release)"); + if (ctx->iommu_domain) + iommu_domain_free(ctx->iommu_domain); + kfree(ctx); + vgpu->mm_ctx = NULL; + return; + } + + mutex_lock(&ctx->lock); + list_for_each_entry_safe(mapping, tmp, &ctx->mappings, list) { + if (mapping->mapped) { + iommu_unmap(ctx->iommu_domain, mapping->iova, mapping->size); + mapping->mapped = false; + } + if (mapping->pinned) { + int i; + for (i = 0; i < mapping->page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + mapping->pinned = false; + } + list_del(&mapping->list); + vfree(mapping->pages); + kfree(mapping); + ctx->mapping_count--; + } + mutex_unlock(&ctx->lock); + + if (ctx->iommu_domain) { + iommu_detach_device(ctx->iommu_domain, &vgpu->pdev->dev); + iommu_domain_free(ctx->iommu_domain); + } + + kfree(ctx); + vgpu->mm_ctx = NULL; +} + +static struct nv_vgpu_mapping *nv_vgpu_find_mapping(struct nv_vgpu_mm_ctx *ctx, u64 iova, u64 size) +{ + struct nv_vgpu_mapping *mapping; + + list_for_each_entry(mapping, &ctx->mappings, list) { + if (mapping->iova == iova && mapping->size == size) + return mapping; + } + return NULL; +} + +int nv_vgpu_memory_pin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping; + u64 start_pfn = gpa >> PAGE_SHIFT; + u64 end_pfn = (gpa + size + PAGE_SIZE - 1) >> PAGE_SHIFT; + int page_count = end_pfn - start_pfn; + int i, ret = 0; + + if (!ctx) + return -ENODEV; + + mapping = kzalloc(sizeof(*mapping), GFP_KERNEL); + if (!mapping) + return -ENOMEM; + + mapping->gpa = gpa; + mapping->size = size; + mapping->page_count = page_count; + mapping->pages = vzalloc(page_count * sizeof(struct page *)); + if (!mapping->pages) { + kfree(mapping); + return -ENOMEM; + } + + mutex_lock(&ctx->lock); + + for (i = 0; i < page_count; i++) { + u64 pfn = start_pfn + i; + struct page *page = pfn_to_page(pfn); + if (!page) { + ret = -EFAULT; + goto err_unlock; + } + get_page(page); + mapping->pages[i] = page; + } + + mapping->pinned = true; + list_add_tail(&mapping->list, &ctx->mappings); + ctx->mapping_count++; + + mutex_unlock(&ctx->lock); + return 0; + +err_unlock: + for (i = 0; i < page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + vfree(mapping->pages); + kfree(mapping); + mutex_unlock(&ctx->lock); + return ret; +} + +int nv_vgpu_memory_unpin(struct nv_vgpu_device *vgpu, u64 gpa, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping; + u64 iova = 0; + int ret = 0; + + if (!ctx) + return -ENODEV; + + mutex_lock(&ctx->lock); + list_for_each_entry(mapping, &ctx->mappings, list) { + if (mapping->gpa == gpa && mapping->size == size) { + if (mapping->mapped) { + iova = mapping->iova; + ret = -EBUSY; + } else if (mapping->pinned) { + int i; + for (i = 0; i < mapping->page_count; i++) { + if (mapping->pages[i]) + put_page(mapping->pages[i]); + } + mapping->pinned = false; + } + list_del(&mapping->list); + ctx->mapping_count--; + vfree(mapping->pages); + kfree(mapping); + break; + } + } + mutex_unlock(&ctx->lock); + + return ret; +} + +int nv_vgpu_memory_map(struct nv_vgpu_device *vgpu, u64 gpa, u64 size, u64 *iova) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping; + u64 map_iova; + int ret; + + if (!ctx || !ctx->iommu_domain) + return -ENODEV; + + mutex_lock(&ctx->lock); + mapping = nv_vgpu_find_mapping(ctx, 0, 0); + if (!mapping || mapping->gpa != gpa || mapping->size != size) { + list_for_each_entry(mapping, &ctx->mappings, list) { + if (mapping->gpa == gpa && mapping->size == size) + break; + } + if (&mapping->list == &ctx->mappings) { + mutex_unlock(&ctx->lock); + return -ENOENT; + } + } + + if (!mapping->pinned) { + mutex_unlock(&ctx->lock); + return -EINVAL; + } + + map_iova = dma_map_page(&vgpu->pdev->dev, mapping->pages[0], 0, size, DMA_BIDIRECTIONAL); + if (dma_mapping_error(&vgpu->pdev->dev, map_iova)) { + mutex_unlock(&ctx->lock); + return -ENOMEM; + } + + mapping->iova = map_iova; + mapping->mapped = true; + *iova = map_iova; + + ret = iommu_map(ctx->iommu_domain, map_iova, gpa, size, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE, GFP_KERNEL); + if (ret) { + dma_unmap_page(&vgpu->pdev->dev, map_iova, size, DMA_BIDIRECTIONAL); + mapping->mapped = false; + mapping->iova = 0; + } + + mutex_unlock(&ctx->lock); + return ret; +} + +int nv_vgpu_memory_unmap(struct nv_vgpu_device *vgpu, u64 iova, u64 size) +{ + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping; + int ret = 0; + + if (!ctx || !ctx->iommu_domain) + return -ENODEV; + + mutex_lock(&ctx->lock); + mapping = nv_vgpu_find_mapping(ctx, iova, size); + if (!mapping || !mapping->mapped) { + mutex_unlock(&ctx->lock); + return -ENOENT; + } + + ret = iommu_unmap(ctx->iommu_domain, iova, size); + if (ret == 0) { + dma_unmap_page(&vgpu->pdev->dev, iova, size, DMA_BIDIRECTIONAL); + mapping->mapped = false; + mapping->iova = 0; + } + + mutex_unlock(&ctx->lock); + return ret; +} + +static vm_fault_t nv_vgpu_vm_fault(struct vm_fault *vmf) +{ + struct nv_vgpu_device *vgpu = vmf->vma->vm_private_data; + struct nv_vgpu_mm_ctx *ctx = vgpu->mm_ctx; + struct nv_vgpu_mapping *mapping; + u64 offset = vmf->address - vmf->vma->vm_start + vmf->vma->vm_pgoff * PAGE_SIZE; + u64 pfn; + vm_fault_t ret = VM_FAULT_SIGBUS; + + if (!ctx) + return VM_FAULT_SIGBUS; + + mutex_lock(&ctx->lock); + list_for_each_entry(mapping, &ctx->mappings, list) { + if (mapping->mapped && offset >= mapping->iova && + offset < mapping->iova + mapping->size) { + pfn = (mapping->gpa + offset - mapping->iova) >> PAGE_SHIFT; + vmf->page = pfn_to_page(pfn); + get_page(vmf->page); + ret = VM_FAULT_NOPAGE; + break; + } + } + mutex_unlock(&ctx->lock); + + return ret; +} + +static const struct vm_operations_struct nv_vgpu_vm_ops = { + .fault = nv_vgpu_vm_fault, +}; + +int nv_vgpu_memory_mmap(struct nv_vgpu_device *vgpu, struct vm_area_struct *vma) +{ + vma->vm_ops = &nv_vgpu_vm_ops; + vma->vm_private_data = vgpu; + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); + return 0; +} \ No newline at end of file diff --git a/kernel/migration.c b/kernel/migration.c new file mode 100644 index 0000000..c8814a2 --- /dev/null +++ b/kernel/migration.c @@ -0,0 +1,324 @@ +#include +#include +#include +#include +#include + +#include "nvidia-vgpu-vfio.h" +#include "migration.h" + +static int nv_vgpu_migration_save_thread(void *data) +{ + struct nv_vgpu_device *vgpu = data; + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + u8 *buffer; + size_t size; + int ret; + + ret = nv_vgpu_migration_save_state(vgpu, &buffer, &size); + if (ret) { + ctx->state.result = ret; + } else { + ctx->state.data = buffer; + ctx->state.data_size = size; + ctx->state.result = 0; + } + + complete(&ctx->state.done); + return 0; +} + +static int nv_vgpu_migration_restore_thread(void *data) +{ + struct nv_vgpu_device *vgpu = data; + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + int ret; + + ret = nv_vgpu_migration_restore_state(vgpu, ctx->state.data, ctx->state.data_size); + ctx->state.result = ret; + complete(&ctx->state.done); + return 0; +} + +int nv_vgpu_migration_init(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_migration_ctx *ctx; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->vgpu = vgpu; + mutex_init(&ctx->lock); + init_completion(&ctx->state.done); + ctx->in_progress = false; + + vgpu->migration.ctx = ctx; + return 0; +} + +void nv_vgpu_migration_cleanup(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + + if (!ctx) + return; + + if (ctx->save_thread) { + kthread_stop(ctx->save_thread); + ctx->save_thread = NULL; + } + if (ctx->restore_thread) { + kthread_stop(ctx->restore_thread); + ctx->restore_thread = NULL; + } + + if (ctx->state.data) { + kfree(ctx->state.data); + ctx->state.data = NULL; + } + + kfree(ctx); + vgpu->migration.ctx = NULL; +} + +int nv_vgpu_migration_save_state(struct nv_vgpu_device *vgpu, u8 **data, size_t *size) +{ + struct nv_vgpu_migration_header *hdr; + struct nv_vgpu_migration_section *sections; + struct nv_vgpu_migration_device_state *dev_state; + struct nv_vgpu_migration_bar_state *bar_state; + u8 *buffer; + size_t total_size = 0; + int i, num_sections = 0; + + mutex_lock(&vgpu->lock); + + total_size = sizeof(struct nv_vgpu_migration_header) + + (8 * sizeof(struct nv_vgpu_migration_section)); + + for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) { + if (vgpu->bars[i].mapped) { + total_size += sizeof(struct nv_vgpu_migration_bar_state) + vgpu->bars[i].size; + num_sections++; + } + } + + buffer = kmalloc(total_size, GFP_KERNEL); + if (!buffer) { + mutex_unlock(&vgpu->lock); + return -ENOMEM; + } + + hdr = (struct nv_vgpu_migration_header *)buffer; + hdr->magic = NV_VGPU_MIGRATION_MAGIC; + hdr->version = NV_VGPU_MIGRATION_VERSION; + hdr->flags = 0; + hdr->num_sections = num_sections + 1; + hdr->total_size = total_size; + hdr->timestamp = ktime_get_ns(); + + sections = (struct nv_vgpu_migration_section *)(buffer + sizeof(*hdr)); + + dev_state = (struct nv_vgpu_migration_device_state *)(buffer + sizeof(*hdr) + + (hdr->num_sections * sizeof(*sections))); + + dev_state->vgpu_type = vgpu->type; + dev_state->vgpu_id = vgpu->vgpu_id; + dev_state->gpu_instance_id = vgpu->gpu_instance_id; + dev_state->compute_instance_id = vgpu->compute_instance_id; + dev_state->state = vgpu->state; + dev_state->fb_base = vgpu->fb_base; + dev_state->fb_size = vgpu->fb_size; + dev_state->bar0_config_base = vgpu->bar0_config_base; + + sections[0].type = NV_VGPU_MIG_SECTION_DEVICE_STATE; + sections[0].flags = 0; + sections[0].offset = (u8 *)dev_state - buffer; + sections[0].size = sizeof(*dev_state); + sections[0].checksum = crc32(0, dev_state, sizeof(*dev_state)); + + u8 *data_ptr = (u8 *)dev_state + sizeof(*dev_state); + for (i = 0; i < NV_VGPU_MAX_REGIONS; i++) { + if (!vgpu->bars[i].mapped) + continue; + + bar_state = (struct nv_vgpu_migration_bar_state *)data_ptr; + bar_state->bar_index = i; + bar_state->gpa = vgpu->bars[i].gpa; + bar_state->size = vgpu->bars[i].size; + bar_state->iova = vgpu->bars[i].iova; + + if (vgpu->bars[i].vaddr && vgpu->bars[i].size > 0) { + memcpy(bar_state->data, vgpu->bars[i].vaddr, vgpu->bars[i].size); + } + + sections[num_sections].type = NV_VGPU_MIG_SECTION_BAR_STATE; + sections[num_sections].flags = 0; + sections[num_sections].offset = data_ptr - buffer; + sections[num_sections].size = sizeof(*bar_state) + vgpu->bars[i].size; + sections[num_sections].checksum = crc32(0, bar_state, sections[num_sections].size); + + data_ptr += sections[num_sections].size; + num_sections++; + } + + hdr->num_sections = num_sections; + hdr->total_size = data_ptr - buffer; + + *data = buffer; + *size = hdr->total_size; + + mutex_unlock(&vgpu->lock); + return 0; +} + +int nv_vgpu_migration_restore_state(struct nv_vgpu_device *vgpu, u8 *data, size_t size) +{ + struct nv_vgpu_migration_header *hdr = (struct nv_vgpu_migration_header *)data; + struct nv_vgpu_migration_section *sections; + struct nv_vgpu_migration_device_state *dev_state; + struct nv_vgpu_migration_bar_state *bar_state; + int i; + + if (!data || size < sizeof(*hdr)) + return -EINVAL; + + if (hdr->magic != NV_VGPU_MIGRATION_MAGIC) + return -EINVAL; + + if (hdr->version > NV_VGPU_MIGRATION_VERSION) + return -EINVAL; + + sections = (struct nv_vgpu_migration_section *)(data + sizeof(*hdr)); + + mutex_lock(&vgpu->lock); + + for (i = 0; i < hdr->num_sections; i++) { + u32 checksum; + u8 *section_data = data + sections[i].offset; + + checksum = crc32(0, section_data, sections[i].size); + if (checksum != sections[i].checksum) { + nv_vgpu_err(vgpu, "Migration checksum mismatch section %d", i); + mutex_unlock(&vgpu->lock); + return -EIO; + } + + switch (sections[i].type) { + case NV_VGPU_MIG_SECTION_DEVICE_STATE: + dev_state = (struct nv_vgpu_migration_device_state *)section_data; + vgpu->type = dev_state->vgpu_type; + vgpu->vgpu_id = dev_state->vgpu_id; + vgpu->gpu_instance_id = dev_state->gpu_instance_id; + vgpu->compute_instance_id = dev_state->compute_instance_id; + vgpu->state = dev_state->state; + vgpu->fb_base = dev_state->fb_base; + vgpu->fb_size = dev_state->fb_size; + vgpu->bar0_config_base = dev_state->bar0_config_base; + break; + + case NV_VGPU_MIG_SECTION_BAR_STATE: + bar_state = (struct nv_vgpu_migration_bar_state *)section_data; + if (bar_state->bar_index < NV_VGPU_MAX_REGIONS) { + vgpu->bars[bar_state->bar_index].gpa = bar_state->gpa; + vgpu->bars[bar_state->bar_index].size = bar_state->size; + vgpu->bars[bar_state->bar_index].iova = bar_state->iova; + if (vgpu->bars[bar_state->bar_index].vaddr && bar_state->size > 0) { + memcpy(vgpu->bars[bar_state->bar_index].vaddr, bar_state->data, bar_state->size); + } + } + break; + + default: + nv_vgpu_warn(vgpu, "Unknown migration section type: %d", sections[i].type); + break; + } + } + + mutex_unlock(&vgpu->lock); + return 0; +} + +int nv_vgpu_migration_prepare(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + + if (!ctx) + return -ENODEV; + + mutex_lock(&ctx->lock); + if (ctx->in_progress) { + mutex_unlock(&ctx->lock); + return -EBUSY; + } + ctx->in_progress = true; + ctx->state.version = NV_VGPU_MIGRATION_VERSION; + ctx->state.flags = 0; + ctx->state.timestamp = ktime_get_ns(); + reinit_completion(&ctx->state.done); + mutex_unlock(&ctx->lock); + + ctx->save_thread = kthread_run(nv_vgpu_migration_save_thread, vgpu, "nv-vgpu-mig-save"); + if (IS_ERR(ctx->save_thread)) { + ctx->in_progress = false; + return PTR_ERR(ctx->save_thread); + } + + return 0; +} + +int nv_vgpu_migration_complete(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + int ret; + + if (!ctx) + return -ENODEV; + + wait_for_completion(&ctx->state.done); + ret = ctx->state.result; + + mutex_lock(&ctx->lock); + ctx->in_progress = false; + ctx->save_thread = NULL; + mutex_unlock(&ctx->lock); + + return ret; +} + +int nv_vgpu_migration_cancel(struct nv_vgpu_device *vgpu) +{ + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + + if (!ctx) + return -ENODEV; + + if (ctx->save_thread) { + kthread_stop(ctx->save_thread); + ctx->save_thread = NULL; + } + if (ctx->restore_thread) { + kthread_stop(ctx->restore_thread); + ctx->restore_thread = NULL; + } + + mutex_lock(&ctx->lock); + ctx->in_progress = false; + mutex_unlock(&ctx->lock); + + return 0; +} + +u8 *nv_vgpu_migration_get_data(struct nv_vgpu_device *vgpu, size_t *size) +{ + struct nv_vgpu_migration_ctx *ctx = vgpu->migration.ctx; + + if (!ctx || !ctx->state.data) + return NULL; + + if (size) + *size = ctx->state.data_size; + + return ctx->state.data; +} \ No newline at end of file diff --git a/kernel/tests/Makefile b/kernel/tests/Makefile new file mode 100644 index 0000000..56c819a --- /dev/null +++ b/kernel/tests/Makefile @@ -0,0 +1,24 @@ +obj-m += gsp_emulation_test.o + +ccflags-y := -I$(PWD)/../ -I$(PWD)/../include + +# Resolve open-iov-vfio.ko exports (gsp_emu_process_rpc, +# gsp_client_init_emulation, gsp_rpc_emulation_set, ...). +# Requires 'make kernel' (or 'make -C kernel') to have run first. +KBUILD_EXTRA_SYMBOLS := $(abspath $(PWD)/../Module.symvers) + +KDIR := /lib/modules/$(shell uname -r)/build +PWD := $(shell pwd) + +all: + $(MAKE) -C $(KDIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(KDIR) M=$(PWD) clean + +# Requires open-iov-vfio.ko loaded first (symbol dependency) +install: + sudo insmod gsp_emulation_test.ko + +uninstall: + sudo rmmod gsp_emulation_test diff --git a/kernel/tests/gsp_emulation_test.c b/kernel/tests/gsp_emulation_test.c new file mode 100644 index 0000000..d63fcdb --- /dev/null +++ b/kernel/tests/gsp_emulation_test.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * GSP Emulation Unit Tests + * + * Drives gsp_emu_process_rpc() end-to-end with crafted shared-memory + * messages and validates dispatch, response layout, and protocol handling. + * + * Standalone-testable RPC paths (no physical GPU dependency): + * SCHEDULE (emulated workqueue), YIELD (no-op), NOTIFY (log-only), + * DEBUG (log-only), protocol validation (seq_id, unknown type). + * + * NOT covered here: ALLOC/FREE/MAP/UNMAP/PIN/UNPIN emulation is coupled + * to a physical device (client->dev->driver_data -> vgpu -> pdev + + * dma_map_page), and BAR_TRAP/MIGRATE submit raw RPCs to real firmware. + * Those paths are exercised by the live QEMU guest integration test + * (scripts/test_vgpu_stack.sh). + * + * Requires open-iov-vfio.ko loaded first (links against its exports). + */ + +#include +#include +#include +#include + +#include "gsp_emulation.h" +#include "gsp_rpc_client.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Open-IOV"); +MODULE_DESCRIPTION("GSP Emulation Unit Tests"); +MODULE_SOFTDEP("pre: open_iov_vfio"); + +#define SHMEM_SIZE 4096 +#define RSP_OFFSET 0x80 +#define UNKNOWN_TYPE 0x77 /* not a valid GSP_RPC_* type */ + +static struct gsp_emu_context *test_ctx; +static struct gsp_client *test_client; +static void *test_shmem; +static u32 test_seq; + +static int test_failures; + +#define T_CHECK(cond, name, ...) do { \ + if (!(cond)) { \ + pr_err("GSP_EMU_TEST: FAIL %s: " #name ": " __VA_ARGS__); \ + test_failures++; \ + return -1; \ + } \ +} while (0) + +/* + * run_rpc - place a request in fake shared memory, run one dispatch, + * and copy back the response. Returns the dispatch return code. + */ +static int run_rpc(u32 type, u32 subcmd, const void *payload, size_t plen, + struct gsp_rpc_response *rsp) +{ + struct gsp_rpc_msg *msg = test_shmem; + struct gsp_rpc_response *r = test_shmem + RSP_OFFSET; + int ret; + + memset(test_shmem, 0, SHMEM_SIZE); + msg->type = type; + msg->subcmd = subcmd; + if (payload && plen) + memcpy(msg->payload, payload, min(plen, sizeof(msg->payload))); + msg->seq_id = ++test_seq; + + ret = gsp_emu_process_rpc(test_ctx); + + memcpy(rsp, r, sizeof(*rsp)); + if (rsp->seq_id != msg->seq_id) { + pr_err("GSP_EMU_TEST: response seq mismatch (got %u, want %u)\n", + rsp->seq_id, msg->seq_id); + return -EBADMSG; + } + return ret; +} + +/* seq_id == 0 must be rejected by gsp_emu_process_rpc itself */ +static int test_seq_id_zero(void) +{ + struct gsp_rpc_msg *msg = test_shmem; + int ret; + + pr_info("GSP_EMU_TEST: Testing seq_id=0 rejection...\n"); + + memset(test_shmem, 0, SHMEM_SIZE); + msg->type = GSP_RPC_YIELD; + msg->seq_id = 0; + + ret = gsp_emu_process_rpc(test_ctx); + T_CHECK(ret == -EINVAL, seq_zero, "expected -EINVAL, got %d\n", ret); + + pr_info("GSP_EMU_TEST: seq_id=0 correctly rejected\n"); + return 0; +} + +/* uninitialized context must be rejected */ +static int test_uninitialized_ctx(void) +{ + struct gsp_emu_context *bad; + int ret; + + pr_info("GSP_EMU_TEST: Testing uninitialized ctx rejection...\n"); + + bad = kzalloc(sizeof(*bad), GFP_KERNEL); + T_CHECK(bad != NULL, alloc, "out of memory\n"); + bad->initialized = false; + + ret = gsp_emu_process_rpc(bad); + kfree(bad); + T_CHECK(ret == -ENODEV, uninit, "expected -ENODEV, got %d\n", ret); + + pr_info("GSP_EMU_TEST: uninitialized ctx correctly rejected\n"); + return 0; +} + +/* unknown RPC type must be rejected with -ENOTSUPP in ret and status */ +static int test_unknown_type(void) +{ + struct gsp_rpc_response rsp; + int ret; + + pr_info("GSP_EMU_TEST: Testing unknown type rejection...\n"); + + ret = run_rpc(UNKNOWN_TYPE, 0, NULL, 0, &rsp); + T_CHECK(ret == -ENOTSUPP, unkret, "expected -ENOTSUPP, got %d\n", ret); + T_CHECK((s32)rsp.status == -ENOTSUPP, unkstatus, + "expected status -ENOTSUPP, got %d\n", (s32)rsp.status); + T_CHECK(rsp.type == UNKNOWN_TYPE, unktype, + "expected type echo 0x%x, got 0x%x\n", UNKNOWN_TYPE, rsp.type); + + pr_info("GSP_EMU_TEST: unknown type correctly rejected\n"); + return 0; +} + +static int test_schedule(void) +{ + struct gsp_sched_req sreq = {0}; + struct gsp_rpc_response rsp; + int ret; + + pr_info("GSP_EMU_TEST: Testing SCHEDULE...\n"); + + sreq.work_handle = 0x1234; + sreq.callback = 0; /* emulation worker never invokes callback */ + sreq.arg = 0; + sreq.priority = 10; + sreq.flags = 0; + + ret = run_rpc(GSP_RPC_SCHEDULE, 0, &sreq, sizeof(sreq), &rsp); + T_CHECK(ret == 0, schedret, "dispatch returned %d\n", ret); + T_CHECK((s32)rsp.status == 0, schedstatus, + "status %d\n", (s32)rsp.status); + T_CHECK(rsp.type == GSP_RPC_SCHEDULE, schedtype, + "type echo 0x%x\n", rsp.type); + + pr_info("GSP_EMU_TEST: SCHEDULE succeeded\n"); + return 0; +} + +static int test_yield(void) +{ + struct gsp_rpc_response rsp; + int ret; + + pr_info("GSP_EMU_TEST: Testing YIELD...\n"); + + ret = run_rpc(GSP_RPC_YIELD, 0, NULL, 0, &rsp); + T_CHECK(ret == 0, yieldret, "dispatch returned %d\n", ret); + T_CHECK((s32)rsp.status == 0, yieldstatus, + "status %d\n", (s32)rsp.status); + + pr_info("GSP_EMU_TEST: YIELD succeeded\n"); + return 0; +} + +static int test_notify(void) +{ + struct gsp_rpc_response rsp; + int ret; + + pr_info("GSP_EMU_TEST: Testing NOTIFY...\n"); + + /* subcmd carries the event type; dispatch is log-only, must succeed + * even with a NULL vgpu (exercises NULL-safe logging macros). */ + ret = run_rpc(GSP_RPC_NOTIFY, 1, NULL, 0, &rsp); + T_CHECK(ret == 0, notifyret, "dispatch returned %d\n", ret); + T_CHECK((s32)rsp.status == 0, notifystatus, + "status %d\n", (s32)rsp.status); + + pr_info("GSP_EMU_TEST: NOTIFY succeeded\n"); + return 0; +} + +static int test_debug(void) +{ + struct gsp_rpc_response rsp; + int ret; + + pr_info("GSP_EMU_TEST: Testing DEBUG...\n"); + + ret = run_rpc(GSP_RPC_DEBUG, 0, NULL, 0, &rsp); + T_CHECK(ret == 0, dbgret, "dispatch returned %d\n", ret); + T_CHECK((s32)rsp.status == 0, dbgstatus, + "status %d\n", (s32)rsp.status); + + pr_info("GSP_EMU_TEST: DEBUG succeeded\n"); + return 0; +} + +/* hammer the dispatch loop with mixed SCHEDULE/YIELD traffic */ +static int test_dispatch_stress(void) +{ + struct gsp_sched_req sreq = {0}; + struct gsp_rpc_response rsp; + int i, ret; + + pr_info("GSP_EMU_TEST: Testing dispatch stress (100 iterations)...\n"); + + sreq.work_handle = 0xBEEF; + sreq.priority = 5; + + for (i = 0; i < 100; i++) { + u32 type = (i & 1) ? GSP_RPC_YIELD : GSP_RPC_SCHEDULE; + + ret = run_rpc(type, 0, &sreq, + (type == GSP_RPC_SCHEDULE) ? sizeof(sreq) : 0, + &rsp); + T_CHECK(ret == 0, stress, "iteration %d returned %d\n", i, ret); + T_CHECK((s32)rsp.status == 0, stressstatus, + "iteration %d status %d\n", i, (s32)rsp.status); + } + + pr_info("GSP_EMU_TEST: dispatch stress passed (100/100)\n"); + return 0; +} + +static int init_test_context(void) +{ + int ret; + + /* Route all RPC paths through their emulation implementations */ + gsp_rpc_emulation_set(true); + + test_shmem = vzalloc(SHMEM_SIZE); + if (!test_shmem) + return -ENOMEM; + + /* NULL fw/doorbell virt addrs: emulation paths never touch them */ + test_client = gsp_client_init_emulation(NULL, NULL, NULL); + if (IS_ERR(test_client)) { + ret = PTR_ERR(test_client); + test_client = NULL; + goto err_shmem; + } + + test_ctx = kzalloc(sizeof(*test_ctx), GFP_KERNEL); + if (!test_ctx) { + ret = -ENOMEM; + goto err_client; + } + + test_ctx->vgpu = NULL; /* NULL-safe logging macros handle this */ + test_ctx->bar0_vaddr = NULL; /* PBA write is guarded */ + test_ctx->shared_mem_virt = test_shmem; + test_ctx->shared_mem_size = SHMEM_SIZE; + test_ctx->rpc_client = test_client; + spin_lock_init(&test_ctx->lock); + test_ctx->running = true; + test_ctx->initialized = true; + /* msix_trigger[] zeroed by kzalloc; eventfd_signal is guarded */ + + pr_info("GSP_EMU_TEST: Test context initialized\n"); + return 0; + +err_client: + gsp_client_fini(test_client); + test_client = NULL; +err_shmem: + vfree(test_shmem); + test_shmem = NULL; + gsp_rpc_emulation_set(false); + return ret; +} + +static void cleanup_test_context(void) +{ + if (test_ctx) { + test_ctx->running = false; + kfree(test_ctx); + test_ctx = NULL; + } + if (test_client) { + gsp_client_fini(test_client); + test_client = NULL; + } + if (test_shmem) { + vfree(test_shmem); + test_shmem = NULL; + } + gsp_rpc_emulation_set(false); + pr_info("GSP_EMU_TEST: Test context cleaned up\n"); +} + +static int run_all_tests(void) +{ + test_failures = 0; + + if (init_test_context()) { + pr_err("GSP_EMU_TEST: Failed to initialize test context\n"); + return -1; + } + + test_seq_id_zero(); + test_uninitialized_ctx(); + test_unknown_type(); + test_schedule(); + test_yield(); + test_notify(); + test_debug(); + test_dispatch_stress(); + + cleanup_test_context(); + + if (test_failures == 0) { + pr_info("GSP_EMU_TEST: ALL TESTS PASSED\n"); + return 0; + } + + pr_err("GSP_EMU_TEST: %d TEST(S) FAILED\n", test_failures); + return -1; +} + +static int __init gsp_emulation_test_init(void) +{ + pr_info("GSP_EMU_TEST: Starting GSP emulation unit tests...\n"); + return run_all_tests(); +} + +static void __exit gsp_emulation_test_exit(void) +{ + pr_info("GSP_EMU_TEST: Module unloaded\n"); +} + +module_init(gsp_emulation_test_init); +module_exit(gsp_emulation_test_exit); diff --git a/kernel/tests/gsp_rpc_test.c b/kernel/tests/gsp_rpc_test.c new file mode 100644 index 0000000..f609066 --- /dev/null +++ b/kernel/tests/gsp_rpc_test.c @@ -0,0 +1,425 @@ +/* + * GSP RPC Client Test/Validation Harness + * Tests message encoding/decoding, RPC structure validation + */ + +#include +#include +#include +#include +#include + +#include "../include/gsp_rpc_client.h" +#include "../include/nvidia-vgpu-vfio.h" + +#define TEST_PASS 0 +#define TEST_FAIL -1 + +static int test_count = 0; +static int pass_count = 0; +static int fail_count = 0; + +#define TEST_ASSERT(cond, msg) \ + do { \ + test_count++; \ + if (cond) { \ + pass_count++; \ + pr_info("PASS: %s\n", msg); \ + } else { \ + fail_count++; \ + pr_err("FAIL: %s\n", msg); \ + } \ + } while (0) + +#define TEST_ASSERT_EQ(a, b, msg) \ + TEST_ASSERT((a) == (b), msg " (expected " #b ", got " #a ")") + +/* ============================================================ + * Structure Size/Alignment Tests + * ============================================================ */ + +static int test_structure_sizes(void) +{ + pr_info("=== Testing Structure Sizes ===\n"); + + TEST_ASSERT_EQ(sizeof(struct gsp_rpc_msg), 72, "gsp_rpc_msg size"); + TEST_ASSERT_EQ(sizeof(struct gsp_rpc_response), 64, "gsp_rpc_response size"); + TEST_ASSERT_EQ(sizeof(struct gsp_alloc_req), 40, "gsp_alloc_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_alloc_rsp), 24, "gsp_alloc_rsp size"); + TEST_ASSERT_EQ(sizeof(struct gsp_map_req), 48, "gsp_map_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_map_rsp), 8, "gsp_map_rsp size"); + TEST_ASSERT_EQ(sizeof(struct gsp_pin_req), 40, "gsp_pin_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_pin_rsp), 16, "gsp_pin_rsp size"); + TEST_ASSERT_EQ(sizeof(struct gsp_sched_req), 40, "gsp_sched_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_bar_trap_req), 32, "gsp_bar_trap_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_bar_trap_rsp), 16, "gsp_bar_trap_rsp size"); + TEST_ASSERT_EQ(sizeof(struct gsp_migrate_req), 56, "gsp_migrate_req size"); + TEST_ASSERT_EQ(sizeof(struct gsp_migrate_rsp), 16, "gsp_migrate_rsp size"); + + return TEST_PASS; +} + +static int test_structure_alignment(void) +{ + pr_info("=== Testing Structure Alignment ===\n"); + + TEST_ASSERT_EQ(__alignof__(struct gsp_rpc_msg), 8, "gsp_rpc_msg alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_rpc_response), 8, "gsp_rpc_response alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_alloc_req), 8, "gsp_alloc_req alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_map_req), 8, "gsp_map_req alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_pin_req), 8, "gsp_pin_req alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_sched_req), 8, "gsp_sched_req alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_bar_trap_req), 8, "gsp_bar_trap_req alignment"); + TEST_ASSERT_EQ(__alignof__(struct gsp_migrate_req), 8, "gsp_migrate_req alignment"); + + return TEST_PASS; +} + +/* ============================================================ + * Message Encoding/Decoding Tests + * ============================================================ */ + +static int test_rpc_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_alloc_req *alloc_req = (struct gsp_alloc_req *)msg.payload; + + pr_info("=== Testing RPC Message Encoding ===\n"); + + /* Test ALLOC message encoding */ + msg.type = cpu_to_le32(GSP_RPC_ALLOC); + msg.subcmd = cpu_to_le32(GSP_ALLOC_SUBCMD_DEFAULT); + msg.seq_id = cpu_to_le32(0x12345678); + msg.flags = cpu_to_le32(0x00000001); + + alloc_req->size = cpu_to_le64(0x100000); + alloc_req->alignment = cpu_to_le64(0x1000); + alloc_req->mem_type = cpu_to_le32(0x2); + alloc_req->flags = cpu_to_le32(0x3); + alloc_req->addr_hint = cpu_to_le64(0x0); + alloc_req->pid = cpu_to_le32(0x456); + alloc_req->reserved = 0; + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_ALLOC, "RPC type field"); + TEST_ASSERT_EQ(le32_to_cpu(msg.subcmd), GSP_ALLOC_SUBCMD_DEFAULT, "RPC subcmd field"); + TEST_ASSERT_EQ(le32_to_cpu(msg.seq_id), 0x12345678, "RPC seq_id field"); + TEST_ASSERT_EQ(le32_to_cpu(msg.flags), 0x00000001, "RPC flags field"); + TEST_ASSERT_EQ(le64_to_cpu(alloc_req->size), 0x100000, "ALLOC size field"); + TEST_ASSERT_EQ(le64_to_cpu(alloc_req->alignment), 0x1000, "ALLOC alignment field"); + TEST_ASSERT_EQ(le32_to_cpu(alloc_req->mem_type), 0x2, "ALLOC mem_type field"); + TEST_ASSERT_EQ(le32_to_cpu(alloc_req->flags), 0x3, "ALLOC flags field"); + TEST_ASSERT_EQ(le64_to_cpu(alloc_req->addr_hint), 0x0, "ALLOC addr_hint field"); + TEST_ASSERT_EQ(le32_to_cpu(alloc_req->pid), 0x456, "ALLOC pid field"); + + return TEST_PASS; +} + +static int test_map_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_map_req *map_req = (struct gsp_map_req *)msg.payload; + + pr_info("=== Testing MAP Message Encoding ===\n"); + + msg.type = cpu_to_le32(GSP_RPC_MAP); + msg.subcmd = cpu_to_le32(0); + msg.seq_id = cpu_to_le32(0xABCDEF00); + + map_req->handle = cpu_to_le64(0xDEADBEEF); + map_req->gpu_va = cpu_to_le64(0x80000000); + map_req->size = cpu_to_le64(0x200000); + map_req->prot = cpu_to_le32(0x7); + map_req->flags = cpu_to_le32(0x1); + map_req->offset = cpu_to_le64(0x0); + map_req->reserved = 0; + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_MAP, "MAP RPC type"); + TEST_ASSERT_EQ(le64_to_cpu(map_req->handle), 0xDEADBEEF, "MAP handle"); + TEST_ASSERT_EQ(le64_to_cpu(map_req->gpu_va), 0x80000000, "MAP gpu_va"); + TEST_ASSERT_EQ(le64_to_cpu(map_req->size), 0x200000, "MAP size"); + TEST_ASSERT_EQ(le32_to_cpu(map_req->prot), 0x7, "MAP prot"); + TEST_ASSERT_EQ(le32_to_cpu(map_req->flags), 0x1, "MAP flags"); + TEST_ASSERT_EQ(le64_to_cpu(map_req->offset), 0x0, "MAP offset"); + + return TEST_PASS; +} + +static int test_pin_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_pin_req *pin_req = (struct gsp_pin_req *)msg.payload; + + pr_info("=== Testing PIN Message Encoding ===\n"); + + msg.type = cpu_to_le32(GSP_RPC_PIN); + msg.subcmd = cpu_to_le32(GSP_PIN_SUBCMD_DEFAULT); + + pin_req->handle = cpu_to_le64(0xCAFEBABE); + pin_req->offset = cpu_to_le64(0x1000); + pin_req->size = cpu_to_le64(0x10000); + pin_req->flags = cpu_to_le32(0x3); + pin_req->reserved[0] = 0; + pin_req->reserved[1] = 0; + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_PIN, "PIN RPC type"); + TEST_ASSERT_EQ(le32_to_cpu(msg.subcmd), GSP_PIN_SUBCMD_DEFAULT, "PIN subcmd"); + TEST_ASSERT_EQ(le64_to_cpu(pin_req->handle), 0xCAFEBABE, "PIN handle"); + TEST_ASSERT_EQ(le64_to_cpu(pin_req->offset), 0x1000, "PIN offset"); + TEST_ASSERT_EQ(le64_to_cpu(pin_req->size), 0x10000, "PIN size"); + TEST_ASSERT_EQ(le32_to_cpu(pin_req->flags), 0x3, "PIN flags"); + + return TEST_PASS; +} + +static int test_sched_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_sched_req *sched_req = (struct gsp_sched_req *)msg.payload; + + pr_info("=== Testing SCHEDULE Message Encoding ===\n"); + + msg.type = cpu_to_le32(GSP_RPC_SCHEDULE); + msg.subcmd = cpu_to_le32(GSP_SCHED_SUBCMD_DEFAULT); + + sched_req->work_handle = cpu_to_le64(0x11111111); + sched_req->callback = cpu_to_le64(0x22222222); + sched_req->arg = cpu_to_le64(0x33333333); + sched_req->priority = cpu_to_le32(0x5); + sched_req->flags = cpu_to_le32(0x0); + sched_req->reserved[0] = 0; + sched_req->reserved[1] = 0; + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_SCHEDULE, "SCHED RPC type"); + TEST_ASSERT_EQ(le64_to_cpu(sched_req->work_handle), 0x11111111, "SCHED work_handle"); + TEST_ASSERT_EQ(le64_to_cpu(sched_req->callback), 0x22222222, "SCHED callback"); + TEST_ASSERT_EQ(le64_to_cpu(sched_req->arg), 0x33333333, "SCHED arg"); + TEST_ASSERT_EQ(le32_to_cpu(sched_req->priority), 0x5, "SCHED priority"); + + return TEST_PASS; +} + +static int test_bar_trap_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_bar_trap_req *bar_req = (struct gsp_bar_trap_req *)msg.payload; + + pr_info("=== Testing BAR_TRAP Message Encoding ===\n"); + + msg.type = cpu_to_le32(GSP_RPC_BAR_TRAP); + msg.subcmd = cpu_to_le32(GSP_BAR_TRAP_SUBCMD_DEFAULT); + + bar_req->bar_addr = cpu_to_le64(0x00100000); + bar_req->access_type = cpu_to_le32(0x1); + bar_req->size = cpu_to_le32(0x4); + bar_req->value = cpu_to_le64(0xDEADBEEF); + bar_req->bar_num = cpu_to_le32(0x1); + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_BAR_TRAP, "BAR_TRAP RPC type"); + TEST_ASSERT_EQ(le64_to_cpu(bar_req->bar_addr), 0x00100000, "BAR_TRAP bar_addr"); + TEST_ASSERT_EQ(le32_to_cpu(bar_req->access_type), 0x1, "BAR_TRAP access_type"); + TEST_ASSERT_EQ(le32_to_cpu(bar_req->size), 0x4, "BAR_TRAP size"); + TEST_ASSERT_EQ(le64_to_cpu(bar_req->value), 0xDEADBEEF, "BAR_TRAP value"); + TEST_ASSERT_EQ(le32_to_cpu(bar_req->bar_num), 0x1, "BAR_TRAP bar_num"); + + return TEST_PASS; +} + +static int test_migrate_msg_encoding(void) +{ + struct gsp_rpc_msg msg = {0}; + struct gsp_migrate_req *mig_req = (struct gsp_migrate_req *)msg.payload; + + pr_info("=== Testing MIGRATE Message Encoding ===\n"); + + msg.type = cpu_to_le32(GSP_RPC_MIGRATE); + msg.subcmd = cpu_to_le32(GSP_MIGRATE_SUBCMD_SAVE); + + mig_req->src_gpu_id = cpu_to_le64(0x1); + mig_req->dst_gpu_id = cpu_to_le64(0x2); + mig_req->memory_bitmap = cpu_to_le64(0xFFFFFFFFFFFFFFFF); + mig_req->page_table_ptr = cpu_to_le64(0x80000000); + mig_req->state = cpu_to_le32(GSP_MIGRATE_SUBCMD_SAVE); + mig_req->flags = cpu_to_le32(0); + mig_req->context = cpu_to_le64(0); + mig_req->reserved = 0; + + TEST_ASSERT_EQ(le32_to_cpu(msg.type), GSP_RPC_MIGRATE, "MIGRATE RPC type"); + TEST_ASSERT_EQ(le32_to_cpu(msg.subcmd), GSP_MIGRATE_SUBCMD_SAVE, "MIGRATE subcmd"); + TEST_ASSERT_EQ(le64_to_cpu(mig_req->src_gpu_id), 0x1, "MIGRATE src_gpu_id"); + TEST_ASSERT_EQ(le64_to_cpu(mig_req->dst_gpu_id), 0x2, "MIGRATE dst_gpu_id"); + TEST_ASSERT_EQ(le64_to_cpu(mig_req->memory_bitmap), 0xFFFFFFFFFFFFFFFF, "MIGRATE memory_bitmap"); + TEST_ASSERT_EQ(le64_to_cpu(mig_req->page_table_ptr), 0x80000000, "MIGRATE page_table_ptr"); + + /* Test RESTORE subcmd */ + msg.subcmd = cpu_to_le32(GSP_MIGRATE_SUBCMD_RESTORE); + mig_req->state = cpu_to_le32(GSP_MIGRATE_SUBCMD_RESTORE); + mig_req->flags = cpu_to_le32(GSP_MIGRATE_SUBCMD_RESTORE); + mig_req->context = cpu_to_le64(0xDEADBEEF); + + TEST_ASSERT_EQ(le32_to_cpu(msg.subcmd), GSP_MIGRATE_SUBCMD_RESTORE, "MIGRATE restore subcmd"); + TEST_ASSERT_EQ(le64_to_cpu(mig_req->context), 0xDEADBEEF, "MIGRATE checkpoint context"); + + return TEST_PASS; +} + +/* ============================================================ + * Firmware Address Validation Tests + * ============================================================ */ + +static int test_firmware_addresses(void) +{ + pr_info("=== Testing Firmware Address Constants ===\n"); + + TEST_ASSERT_EQ(GSP_FW_BASE, 0x04000000UL, "GSP_FW_BASE"); + TEST_ASSERT_EQ(GSP_RPC_DISPATCH_PRIMARY, 0x057a1ac0UL, "Primary Dispatcher"); + TEST_ASSERT_EQ(GSP_RPC_JUMP_TABLE, 0x057a3ad8UL, "Jump Table"); + TEST_ASSERT_EQ(GSP_RPC_DISPATCH_SECONDARY, 0x04ed39dcUL, "Secondary Dispatcher"); + TEST_ASSERT_EQ(GSP_RPC_DISPATCH_TERTIARY, 0x04cc58f8UL, "Tertiary Dispatcher"); + TEST_ASSERT_EQ(GSP_RPC_DISPATCH_QUATERNARY, 0x05723ef4UL, "Quaternary Dispatcher"); + TEST_ASSERT_EQ(GSP_ALLOC_MAIN, 0x057e94ecUL, "ALLOC Main"); + TEST_ASSERT_EQ(GSP_SCHEDULE_MAIN, 0x051efbf4UL, "SCHEDULE Main"); + TEST_ASSERT_EQ(GSP_BAR_TRAP_HANDLER_JT, 0x057675b7UL, "BAR_TRAP Handler (JT)"); + TEST_ASSERT_EQ(GSP_BAR_TRAP_HANDLER_PRIM, 0x04cb8189UL, "BAR_TRAP Handler (Primary)"); + TEST_ASSERT_EQ(GSP_PIN_HANDLER_SEC, 0x04ed34b8UL, "PIN Handler (Secondary)"); + TEST_ASSERT_EQ(GSP_MAP_HANDLER_A, 0x05766ab7UL, "MAP Handler A"); + TEST_ASSERT_EQ(GSP_MAP_HANDLER_B, 0x05778a37UL, "MAP Handler B"); + TEST_ASSERT_EQ(GSP_UNMAP_HANDLER_A, 0x057669b7UL, "UNMAP Handler A"); + TEST_ASSERT_EQ(GSP_UNMAP_HANDLER_B, 0x05779937UL, "UNMAP Handler B"); + TEST_ASSERT_EQ(GSP_PIN_HANDLER_A, 0x057664b7UL, "PIN Handler A"); + TEST_ASSERT_EQ(GSP_PIN_HANDLER_B, 0x0577a3b7UL, "PIN Handler B"); + TEST_ASSERT_EQ(GSP_UNPIN_HANDLER_A, 0x057682b7UL, "UNPIN Handler A"); + TEST_ASSERT_EQ(GSP_UNPIN_HANDLER_B, 0x05779fb7UL, "UNPIN Handler B"); + TEST_ASSERT_EQ(GSP_SCHEDULE_HANDLER_A, 0x05767f37UL, "SCHEDULE Handler A"); + TEST_ASSERT_EQ(GSP_SCHEDULE_HANDLER_B, 0x0575aeb7UL, "SCHEDULE Handler B"); + TEST_ASSERT_EQ(GSP_YIELD_HANDLER_A, 0x0575ae37UL, "YIELD Handler A"); + TEST_ASSERT_EQ(GSP_YIELD_HANDLER_B, 0x05767337UL, "YIELD Handler B"); + TEST_ASSERT_EQ(GSP_NOTIFY_HANDLER_A, 0x0577c8b7UL, "NOTIFY Handler A"); + TEST_ASSERT_EQ(GSP_NOTIFY_HANDLER_B, 0x05767837UL, "NOTIFY Handler B"); + TEST_ASSERT_EQ(GSP_ENTRY_POINT, 0x04009000UL, "Entry Point"); + + return TEST_PASS; +} + +/* ============================================================ + * RPC Type/Subcommand Validation Tests + * ============================================================ */ + +static int test_rpc_types(void) +{ + pr_info("=== Testing RPC Type Constants ===\n"); + + TEST_ASSERT_EQ(GSP_RPC_ALLOC, 0x01, "RPC_ALLOC"); + TEST_ASSERT_EQ(GSP_RPC_FREE, 0x02, "RPC_FREE"); + TEST_ASSERT_EQ(GSP_RPC_MAP, 0x03, "RPC_MAP"); + TEST_ASSERT_EQ(GSP_RPC_UNMAP, 0x04, "RPC_UNMAP"); + TEST_ASSERT_EQ(GSP_RPC_PIN, 0x05, "RPC_PIN"); + TEST_ASSERT_EQ(GSP_RPC_UNPIN, 0x06, "RPC_UNPIN"); + TEST_ASSERT_EQ(GSP_RPC_SCHEDULE, 0x07, "RPC_SCHEDULE"); + TEST_ASSERT_EQ(GSP_RPC_YIELD, 0x08, "RPC_YIELD"); + TEST_ASSERT_EQ(GSP_RPC_NOTIFY, 0x09, "RPC_NOTIFY"); + TEST_ASSERT_EQ(GSP_RPC_BAR_TRAP, 0x0A, "RPC_BAR_TRAP"); + TEST_ASSERT_EQ(GSP_RPC_MIGRATE, 0x0B, "RPC_MIGRATE"); + TEST_ASSERT_EQ(GSP_RPC_DEBUG, 0xFF, "RPC_DEBUG"); + + return TEST_PASS; +} + +static int test_alloc_subcmds(void) +{ + pr_info("=== Testing ALLOC Sub-commands ===\n"); + + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_DEFAULT, 0, "ALLOC_SUBCMD_DEFAULT"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE1, 1, "ALLOC_SUBCMD_TYPE1"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE2, 3, "ALLOC_SUBCMD_TYPE2"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE3, 4, "ALLOC_SUBCMD_TYPE3"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE4, 5, "ALLOC_SUBCMD_TYPE4"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE5, 8, "ALLOC_SUBCMD_TYPE5"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE6, 10, "ALLOC_SUBCMD_TYPE6"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE7, 12, "ALLOC_SUBCMD_TYPE7"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE8, 17, "ALLOC_SUBCMD_TYPE8"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE9, 19, "ALLOC_SUBCMD_TYPE9"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE10, 20, "ALLOC_SUBCMD_TYPE10"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE11, 21, "ALLOC_SUBCMD_TYPE11"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE12, 22, "ALLOC_SUBCMD_TYPE12"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE13, 23, "ALLOC_SUBCMD_TYPE13"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE14, 24, "ALLOC_SUBCMD_TYPE14"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE15, 25, "ALLOC_SUBCMD_TYPE15"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE16, 26, "ALLOC_SUBCMD_TYPE16"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE17, 27, "ALLOC_SUBCMD_TYPE17"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE18, 28, "ALLOC_SUBCMD_TYPE18"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE19, 29, "ALLOC_SUBCMD_TYPE19"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE20, 30, "ALLOC_SUBCMD_TYPE20"); + TEST_ASSERT_EQ(GSP_ALLOC_SUBCMD_TYPE21, 31, "ALLOC_SUBCMD_TYPE21"); + + return TEST_PASS; +} + +/* ============================================================ + * Payload Size Validation + * ============================================================ */ + +static int test_payload_capacity(void) +{ + pr_info("=== Testing Payload Capacity ===\n"); + + /* gsp_rpc_msg has 6 x u64 payload = 48 bytes */ + TEST_ASSERT(sizeof(struct gsp_alloc_req) <= 48, "ALLOC req fits in payload"); + TEST_ASSERT(sizeof(struct gsp_map_req) <= 48, "MAP req fits in payload"); + TEST_ASSERT(sizeof(struct gsp_pin_req) <= 48, "PIN req fits in payload"); + TEST_ASSERT(sizeof(struct gsp_sched_req) <= 48, "SCHED req fits in payload"); + TEST_ASSERT(sizeof(struct gsp_bar_trap_req) <= 48, "BAR_TRAP req fits in payload"); + /* MIGRATE req is 56 bytes - uses extended payload format */ + TEST_ASSERT(sizeof(struct gsp_migrate_req) == 56, "MIGRATE req size is 56 bytes"); + + return TEST_PASS; +} + +/* ============================================================ + * Module Init/Exit + * ============================================================ */ + +static int __init gsp_rpc_test_init(void) +{ + pr_info("========================================\n"); + pr_info("GSP RPC Client Test Suite Starting\n"); + pr_info("========================================\n"); + + test_structure_sizes(); + test_structure_alignment(); + test_rpc_msg_encoding(); + test_map_msg_encoding(); + test_pin_msg_encoding(); + test_sched_msg_encoding(); + test_bar_trap_msg_encoding(); + test_migrate_msg_encoding(); + test_firmware_addresses(); + test_rpc_types(); + test_alloc_subcmds(); + test_payload_capacity(); + + pr_info("========================================\n"); + pr_info("Test Results: %d total, %d passed, %d failed\n", + test_count, pass_count, fail_count); + pr_info("========================================\n"); + + if (fail_count > 0) { + pr_err("TEST SUITE FAILED\n"); + return -EINVAL; + } + + pr_info("TEST SUITE PASSED\n"); + return 0; +} + +static void __exit gsp_rpc_test_exit(void) +{ + pr_info("GSP RPC Client Test Suite Exited\n"); +} + +module_init(gsp_rpc_test_init); +module_exit(gsp_rpc_test_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Open-IOV Project"); +MODULE_DESCRIPTION("GSP RPC Client Test/Validation Harness"); +MODULE_VERSION("1.0"); diff --git a/kernel/version/version.c b/kernel/version/version.c new file mode 100644 index 0000000..2564a15 --- /dev/null +++ b/kernel/version/version.c @@ -0,0 +1,129 @@ +/* + * Driver Version Adaptation Layer + * Handles interface differences between NVIDIA driver versions + */ + +#include +#include +#include +#include +#include + +#include "version.h" + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Open-IOV"); +MODULE_DESCRIPTION("Driver Version Adaptation Layer"); + +/* Version 550 interface */ +static struct driver_interface interface_550 = { + .version = "550", + .major = 550, + .minor = 0, + .has_vgpu_vfio = true, + .has_migration = true, + .has_scheduling = true, + .has_notifications = true, + .has_wpr = true, +}; + +/* Version 560 interface */ +static struct driver_interface interface_560 = { + .version = "560", + .major = 560, + .minor = 0, + .has_vgpu_vfio = true, + .has_migration = true, + .has_scheduling = true, + .has_notifications = true, + .has_wpr = true, +}; + +/* Version 570 interface */ +static struct driver_interface interface_570 = { + .version = "570", + .major = 570, + .minor = 0, + .has_vgpu_vfio = true, + .has_migration = true, + .has_scheduling = true, + .has_notifications = true, + .has_wpr = true, +}; + +/* Version 580 interface */ +static struct driver_interface interface_580 = { + .version = "580", + .major = 580, + .minor = 0, + .has_vgpu_vfio = true, + .has_migration = true, + .has_scheduling = true, + .has_notifications = true, + .has_wpr = true, +}; + +static struct driver_interface *interfaces[] = { + &interface_550, + &interface_560, + &interface_570, + &interface_580, + NULL +}; + +struct driver_interface *version_get_interface(const char *version_str) +{ + int major = 0, minor = 0; + + if (version_str) { + sscanf(version_str, "%d.%d", &major, &minor); + } + + for (int i = 0; interfaces[i]; i++) { + if (interfaces[i]->major == major) + return interfaces[i]; + } + + /* Default to latest known */ + return &interface_570; +} + +struct driver_interface *version_get_by_kernel_version(void) +{ + /* Try to detect from kernel version or module version */ + return &interface_570; +} + +bool version_has_feature(struct driver_interface *iface, enum driver_feature feature) +{ + if (!iface) + return false; + + switch (feature) { + case DRV_FEAT_VGPU_VFIO: + return iface->has_vgpu_vfio; + case DRV_FEAT_MIGRATION: + return iface->has_migration; + case DRV_FEAT_SCHEDULING: + return iface->has_scheduling; + case DRV_FEAT_NOTIFICATIONS: + return iface->has_notifications; + case DRV_FEAT_WPR: + return iface->has_wpr; + default: + return false; + } +} + +const char *version_get_string(struct driver_interface *iface) +{ + return iface ? iface->version : "unknown"; +} + +EXPORT_SYMBOL_GPL(version_get_interface); +EXPORT_SYMBOL_GPL(version_get_by_kernel_version); +EXPORT_SYMBOL_GPL(version_has_feature); +EXPORT_SYMBOL_GPL(version_get_string); + +MODULE_DESCRIPTION("Driver Version Adaptation Layer"); +MODULE_LICENSE("GPL"); \ No newline at end of file diff --git a/kernel/version/version.h b/kernel/version/version.h new file mode 100644 index 0000000..818d939 --- /dev/null +++ b/kernel/version/version.h @@ -0,0 +1,35 @@ +#ifndef _VERSION_H_ +#define _VERSION_H_ + +#include + +/* Driver features */ +enum driver_feature { + DRV_FEAT_VGPU_VFIO, + DRV_FEAT_MIGRATION, + DRV_FEAT_SCHEDULING, + DRV_FEAT_NOTIFICATIONS, + DRV_FEAT_WPR, +}; + +/* Driver interface definition */ +struct driver_interface { + const char *version; + int major; + int minor; + bool has_vgpu_vfio; + bool has_migration; + bool has_scheduling; + bool has_notifications; + bool has_wpr; +}; + +/* Version registry */ +struct driver_interface *version_get_interface(const char *version_str); +struct driver_interface *version_get_by_kernel_version(void); + +/* Feature queries */ +bool version_has_feature(struct driver_interface *iface, enum driver_feature feature); +const char *version_get_string(struct driver_interface *iface); + +#endif /* _VERSION_H_ */ \ No newline at end of file -- cgit v1.2.3