summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_rpc_client.c393
-rw-r--r--kernel/gsp_rpc_client.h36
-rw-r--r--kernel/include/memory.h11
-rw-r--r--kernel/memory.c11
4 files changed, 333 insertions, 118 deletions
diff --git a/kernel/gsp_rpc_client.c b/kernel/gsp_rpc_client.c
index a1579f7..46d0d50 100644
--- a/kernel/gsp_rpc_client.c
+++ b/kernel/gsp_rpc_client.c
@@ -16,6 +16,10 @@
#include <linux/delay.h>
#include <asm/io.h>
+#include "nvidia-vgpu-vfio.h"
+#include "memory.h"
+#include "gsp_rpc_client.h"
+
/* ============================================================
* FIRMWARE ADDRESSES (from decompilation)
* ============================================================ */
@@ -113,113 +117,6 @@
* RPC MESSAGE STRUCTURES
* ============================================================ */
-struct gsp_rpc_msg {
- u32 type; /* RPC type (0x01-0x0B, 0xFF) */
- u32 subcmd; /* Sub-command for jump table dispatch */
- u64 payload[6]; /* Variable payload */
- u32 seq_id; /* Sequence ID for async ops */
- u32 flags; /* Flags */
- u32 reserved[2]; /* Padding to 64 bytes */
-} __packed;
-
-struct gsp_rpc_response {
- u32 status; /* 0 = success, negative = error */
- u32 type; /* Echo of request type */
- u64 result[6]; /* Return values */
- u32 seq_id; /* Echo of request seq_id */
-} __packed;
-
-/* ALLOC payload */
-struct gsp_alloc_req {
- u64 size; /* Allocation size in bytes */
- u64 alignment; /* Alignment requirement */
- u32 mem_type; /* Memory type (0=VRAM, 1=SYSMEM, 2=COHERENT) */
- u32 flags; /* Allocation flags */
- u64 addr_hint; /* Hint address (optional) */
- u32 pid; /* Process ID */
- u32 reserved;
-};
-
-struct gsp_alloc_rsp {
- u64 gpu_va; /* GPU virtual address */
- u64 handle; /* Allocation handle */
- u64 size; /* Actual allocated size */
-};
-
-/* MAP payload */
-struct gsp_map_req {
- u64 handle; /* Allocation handle from ALLOC */
- u64 gpu_va; /* GPU virtual address to map at */
- u64 size; /* Size to map */
- u32 prot; /* Protection (R/W/X) */
- u32 flags; /* Map flags */
- u64 offset; /* Offset into allocation */
- u32 reserved[2];
-};
-
-struct gsp_map_rsp {
- u64 gpu_va; /* Mapped GPU VA */
-};
-
-/* PIN/UNPIN payload */
-struct gsp_pin_req {
- u64 handle; /* Allocation handle */
- u64 offset; /* Offset into allocation */
- u64 size; /* Size to pin */
- u32 flags; /* Pin flags (0=read, 1=write, 2=read+write) */
- u32 reserved[3];
-};
-
-struct gsp_pin_rsp {
- u64 dma_addr; /* DMA address for device */
- u32 num_pages; /* Number of pages pinned */
- u32 reserved;
-};
-
-/* SCHEDULE payload */
-struct gsp_sched_req {
- u64 work_handle; /* Work descriptor handle */
- u64 callback; /* Callback function pointer */
- u64 arg; /* Callback argument */
- u32 priority; /* Priority */
- u32 flags; /* Schedule flags */
- u32 reserved[2];
-};
-
-/* BAR TRAP payload */
-struct gsp_bar_trap_req {
- u64 bar_addr; /* BAR register address */
- u32 access_type; /* 0=read, 1=write */
- u32 size; /* Access size (1/2/4/8) */
- u64 value; /* Value (for write) */
- u32 bar_num; /* BAR number (0-5) */
- u32 reserved;
-};
-
-struct gsp_bar_trap_rsp {
- u64 value; /* Value (for read) */
- u32 handled; /* 1 if handled */
- u32 reserved;
-};
-
-/* MIGRATE payload */
-struct gsp_migrate_req {
- u64 src_gpu_id; /* Source GPU ID */
- u64 dst_gpu_id; /* Destination GPU ID */
- u64 memory_bitmap; /* Memory bitmap pointer */
- u64 page_table_ptr; /* Page table pointer */
- u32 state; /* Migration state */
- u32 flags; /* Migration flags */
- u64 context; /* Migration context */
- u32 reserved;
-};
-
-struct gsp_migrate_rsp {
- u32 status; /* Migration status */
- u32 progress; /* Progress percentage */
- u64 checkpoint; /* Checkpoint handle */
-};
-
/* ============================================================
* GSP RPC CLIENT CONTEXT
* ============================================================ */
@@ -317,6 +214,9 @@ int gsp_rpc_alloc(struct gsp_client *client,
u32 flags, u64 addr_hint, u32 pid,
u64 *gpu_va, u64 *handle, u64 *actual_size)
{
+ if (gsp_rpc_emulation_is_enabled())
+ return gsp_rpc_alloc_emul(client, size, alignment, mem_type, flags, addr_hint, pid, gpu_va, handle, actual_size);
+
struct gsp_alloc_req req = {0};
struct gsp_alloc_rsp rsp = {0};
struct gsp_rpc_response response = {0};
@@ -350,6 +250,9 @@ int gsp_rpc_alloc(struct gsp_client *client,
int gsp_rpc_free(struct gsp_client *client, u64 handle)
{
+ if (gsp_rpc_emulation_is_enabled())
+ return gsp_rpc_free_emul(client, handle);
+
struct gsp_rpc_response response = {0};
u64 payload[6] = { handle, 0, 0, 0, 0, 0 };
u32 seq = gsp_next_seq(client);
@@ -369,6 +272,9 @@ int gsp_rpc_map(struct gsp_client *client,
u32 prot, u32 flags, u64 offset,
u64 *mapped_va)
{
+ if (gsp_rpc_emulation_is_enabled())
+ return gsp_rpc_map_emul(client, handle, gpu_va, size, prot, flags, offset, mapped_va);
+
struct gsp_map_req req = {0};
struct gsp_map_rsp rsp = {0};
struct gsp_rpc_response response = {0};
@@ -398,6 +304,9 @@ int gsp_rpc_map(struct gsp_client *client,
int gsp_rpc_unmap(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size)
{
+ if (gsp_rpc_emulation_is_enabled())
+ return gsp_rpc_unmap_emul(client, handle, gpu_va, size);
+
struct gsp_rpc_response response = {0};
u64 payload[6] = { handle, gpu_va, size, 0, 0, 0 };
u32 seq = gsp_next_seq(client);
@@ -416,6 +325,9 @@ int gsp_rpc_pin(struct gsp_client *client,
u64 handle, u64 offset, u64 size,
u32 flags, u64 *dma_addr, u32 *num_pages)
{
+ if (gsp_rpc_emulation_is_enabled())
+ return gsp_rpc_pin_emul(client, handle, offset, size, flags, dma_addr, num_pages);
+
struct gsp_pin_req req = {0};
struct gsp_pin_rsp rsp = {0};
struct gsp_rpc_response response = {0};
@@ -444,6 +356,9 @@ int gsp_rpc_pin(struct gsp_client *client,
int gsp_rpc_unpin(struct gsp_client *client, u64 handle, u64 offset, u64 size)
{
+ if (gsp_rpc_emulation_is_enabled())
+ return gsp_rpc_unpin_emul(client, handle, offset, size);
+
struct gsp_rpc_response response = {0};
u64 payload[6] = { handle, offset, size, 0, 0, 0 };
u32 seq = gsp_next_seq(client);
@@ -563,6 +478,26 @@ bool gsp_rpc_emulation_is_enabled(void)
return gsp_emulation_enabled;
}
+/* VRAM allocation tracking for emulation */
+struct gsp_vram_alloc {
+ struct list_head list;
+ u64 handle;
+ u64 gpu_va;
+ u64 size;
+ u64 alignment;
+ u32 mem_type;
+ u32 flags;
+ u64 addr_hint;
+ u32 pid;
+ bool mapped;
+ u64 mapped_va;
+ struct nv_vgpu_simple_mapping *mapping;
+};
+
+static LIST_HEAD(gsp_vram_allocs);
+static DEFINE_SPINLOCK(gsp_vram_lock);
+static atomic64_t gsp_handle_counter = ATOMIC64_INIT(1);
+
/* Scheduling emulation - local work queue */
struct gsp_sched_work {
struct list_head list;
@@ -587,6 +522,250 @@ static void gsp_sched_worker(struct work_struct *work)
complete(&sched->done);
}
+/* Find VRAM allocation by handle */
+static struct gsp_vram_alloc *gsp_find_vram_alloc(u64 handle)
+{
+ struct gsp_vram_alloc *alloc;
+ spin_lock(&gsp_vram_lock);
+ list_for_each_entry(alloc, &gsp_vram_allocs, list) {
+ if (alloc->handle == handle) {
+ spin_unlock(&gsp_vram_lock);
+ return alloc;
+ }
+ }
+ spin_unlock(&gsp_vram_lock);
+ return NULL;
+}
+
+/* Find VRAM allocation by GPU VA */
+static struct gsp_vram_alloc *gsp_find_vram_by_va(u64 gpu_va)
+{
+ struct gsp_vram_alloc *alloc;
+ spin_lock(&gsp_vram_lock);
+ list_for_each_entry(alloc, &gsp_vram_allocs, list) {
+ if (alloc->gpu_va <= gpu_va && gpu_va < alloc->gpu_va + alloc->size) {
+ spin_unlock(&gsp_vram_lock);
+ return alloc;
+ }
+ }
+ spin_unlock(&gsp_vram_lock);
+ return NULL;
+}
+
+/* Memory Allocation - Emulation Implementation */
+int gsp_rpc_alloc_emul(struct gsp_client *client,
+ u64 size, u64 alignment, u32 mem_type,
+ u32 flags, u64 addr_hint, u32 pid,
+ u64 *gpu_va, u64 *handle, u64 *actual_size)
+{
+ struct nv_vgpu_device *vgpu = client->dev->driver_data;
+ struct gsp_vram_alloc *alloc;
+ u64 va = addr_hint ? addr_hint : 0x100000000ULL; /* Default 4GB */
+ int ret;
+
+ alloc = kzalloc(sizeof(*alloc), GFP_KERNEL);
+ if (!alloc)
+ return -ENOMEM;
+
+ alloc->handle = atomic64_inc_return(&gsp_handle_counter);
+ alloc->size = size;
+ alloc->alignment = alignment;
+ alloc->mem_type = mem_type;
+ alloc->flags = flags;
+ alloc->addr_hint = addr_hint;
+ alloc->pid = pid;
+ alloc->mapped = false;
+
+ /* Align VA */
+ va = ALIGN(va, alignment);
+
+ /* In emulation, we allocate host memory and map it to GPU VA */
+ alloc->mapping = kzalloc(sizeof(*alloc->mapping), GFP_KERNEL);
+ if (!alloc->mapping) {
+ kfree(alloc);
+ return -ENOMEM;
+ }
+
+ alloc->mapping->gpa = va;
+ alloc->mapping->size = size;
+ alloc->mapping->page_count = (size + PAGE_SIZE - 1) / PAGE_SIZE;
+ alloc->mapping->pages = vzalloc(alloc->mapping->page_count * sizeof(struct page *));
+ if (!alloc->mapping->pages) {
+ kfree(alloc->mapping);
+ kfree(alloc);
+ return -ENOMEM;
+ }
+
+ /* Allocate host pages */
+ for (int i = 0; i < alloc->mapping->page_count; i++) {
+ struct page *page = alloc_page(GFP_KERNEL | GFP_DMA32);
+ if (!page) {
+ for (int j = 0; j < i; j++)
+ __free_page(alloc->mapping->pages[j]);
+ vfree(alloc->mapping->pages);
+ kfree(alloc->mapping);
+ kfree(alloc);
+ return -ENOMEM;
+ }
+ alloc->mapping->pages[i] = page;
+ }
+ alloc->mapping->pinned = true;
+
+ /* Map to IOVA for DMA */
+ ret = dma_map_page(&vgpu->pdev->dev, alloc->mapping->pages[0], 0, size, DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(&vgpu->pdev->dev, ret)) {
+ for (int i = 0; i < alloc->mapping->page_count; i++)
+ __free_page(alloc->mapping->pages[i]);
+ vfree(alloc->mapping->pages);
+ kfree(alloc->mapping);
+ kfree(alloc);
+ return -ENOMEM;
+ }
+ alloc->mapping->iova = ret;
+ alloc->mapping->mapped = true;
+
+ alloc->gpu_va = va;
+ alloc->mapped_va = va;
+
+ spin_lock(&gsp_vram_lock);
+ list_add_tail(&alloc->list, &gsp_vram_allocs);
+ spin_unlock(&gsp_vram_lock);
+
+ if (gpu_va) *gpu_va = va;
+ if (handle) *handle = alloc->handle;
+ if (actual_size) *actual_size = size;
+
+ nv_vgpu_info(vgpu, "EMUL ALLOC: handle=0x%llx gpu_va=0x%llx size=%llu iova=0x%llx",
+ alloc->handle, va, size, alloc->mapping->iova);
+
+ return 0;
+}
+
+/* Memory Free - Emulation Implementation */
+int gsp_rpc_free_emul(struct gsp_client *client, u64 handle)
+{
+ struct nv_vgpu_device *vgpu = client->dev->driver_data;
+ struct gsp_vram_alloc *alloc;
+
+ alloc = gsp_find_vram_alloc(handle);
+ if (!alloc) {
+ nv_vgpu_warn(vgpu, "EMUL FREE: handle 0x%llx not found", handle);
+ return -ENOENT;
+ }
+
+ spin_lock(&gsp_vram_lock);
+ list_del(&alloc->list);
+ spin_unlock(&gsp_vram_lock);
+
+ if (alloc->mapping) {
+ if (alloc->mapping->mapped) {
+ dma_unmap_page(&vgpu->pdev->dev, alloc->mapping->iova, alloc->size, DMA_BIDIRECTIONAL);
+ }
+ if (alloc->mapping->pinned) {
+ for (int i = 0; i < alloc->mapping->page_count; i++) {
+ if (alloc->mapping->pages[i])
+ __free_page(alloc->mapping->pages[i]);
+ }
+ }
+ vfree(alloc->mapping->pages);
+ kfree(alloc->mapping);
+ }
+
+ kfree(alloc);
+
+ nv_vgpu_info(vgpu, "EMUL FREE: handle=0x%llx", handle);
+ return 0;
+}
+
+/* Memory Map - Emulation Implementation */
+int gsp_rpc_map_emul(struct gsp_client *client,
+ u64 handle, u64 gpu_va, u64 size,
+ u32 prot, u32 flags, u64 offset,
+ u64 *mapped_va)
+{
+ struct nv_vgpu_device *vgpu = client->dev->driver_data;
+ struct gsp_vram_alloc *alloc = gsp_find_vram_alloc(handle);
+
+ if (!alloc) {
+ nv_vgpu_warn(vgpu, "EMUL MAP: handle 0x%llx not found", handle);
+ return -ENOENT;
+ }
+
+ if (gpu_va == 0)
+ gpu_va = alloc->gpu_va + offset;
+
+ alloc->mapped_va = gpu_va;
+ alloc->mapped = true;
+
+ if (mapped_va)
+ *mapped_va = gpu_va;
+
+ nv_vgpu_info(vgpu, "EMUL MAP: handle=0x%llx gpu_va=0x%llx size=%llu offset=%llu",
+ handle, gpu_va, size, offset);
+
+ return 0;
+}
+
+/* Memory Unmap - Emulation Implementation */
+int gsp_rpc_unmap_emul(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size)
+{
+ struct nv_vgpu_device *vgpu = client->dev->driver_data;
+ struct gsp_vram_alloc *alloc = gsp_find_vram_alloc(handle);
+
+ if (!alloc) {
+ nv_vgpu_warn(vgpu, "EMUL UNMAP: handle 0x%llx not found", handle);
+ return -ENOENT;
+ }
+
+ alloc->mapped = false;
+
+ nv_vgpu_info(vgpu, "EMUL UNMAP: handle=0x%llx gpu_va=0x%llx size=%llu",
+ handle, gpu_va, size);
+
+ return 0;
+}
+
+/* Page Pin - Emulation Implementation */
+int gsp_rpc_pin_emul(struct gsp_client *client,
+ u64 handle, u64 offset, u64 size,
+ u32 flags, u64 *dma_addr, u32 *num_pages)
+{
+ struct nv_vgpu_device *vgpu = client->dev->driver_data;
+ struct gsp_vram_alloc *alloc = gsp_find_vram_alloc(handle);
+
+ if (!alloc || !alloc->mapping) {
+ nv_vgpu_warn(vgpu, "EMUL PIN: handle 0x%llx not found or no mapping", handle);
+ return -ENOENT;
+ }
+
+ if (dma_addr)
+ *dma_addr = alloc->mapping->iova + offset;
+ if (num_pages)
+ *num_pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
+
+ nv_vgpu_info(vgpu, "EMUL PIN: handle=0x%llx offset=%llu size=%llu dma_addr=0x%llx pages=%u",
+ handle, offset, size, alloc->mapping->iova + offset, *num_pages);
+
+ return 0;
+}
+
+/* Page Unpin - Emulation Implementation */
+int gsp_rpc_unpin_emul(struct gsp_client *client, u64 handle, u64 offset, u64 size)
+{
+ struct nv_vgpu_device *vgpu = client->dev->driver_data;
+ struct gsp_vram_alloc *alloc = gsp_find_vram_alloc(handle);
+
+ if (!alloc) {
+ nv_vgpu_warn(vgpu, "EMUL UNPIN: handle 0x%llx not found", handle);
+ return -ENOENT;
+ }
+
+ nv_vgpu_info(vgpu, "EMUL UNPIN: handle=0x%llx offset=%llu size=%llu",
+ handle, offset, size);
+
+ return 0;
+}
+
int gsp_rpc_schedule(struct gsp_client *client,
u64 work_handle, u64 callback, u64 arg,
u32 priority, u32 flags)
diff --git a/kernel/gsp_rpc_client.h b/kernel/gsp_rpc_client.h
index 64bfe5b..54c216e 100644
--- a/kernel/gsp_rpc_client.h
+++ b/kernel/gsp_rpc_client.h
@@ -314,4 +314,40 @@ int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id);
void gsp_rpc_emulation_set(bool enable);
bool gsp_rpc_emulation_is_enabled(void);
+/* Emulation implementations */
+int gsp_rpc_alloc_emul(struct gsp_client *client,
+ u64 size, u64 alignment, u32 mem_type,
+ u32 flags, u64 addr_hint, u32 pid,
+ u64 *gpu_va, u64 *handle, u64 *actual_size);
+int gsp_rpc_free_emul(struct gsp_client *client, u64 handle);
+int gsp_rpc_map_emul(struct gsp_client *client,
+ u64 handle, u64 gpu_va, u64 size,
+ u32 prot, u32 flags, u64 offset,
+ u64 *mapped_va);
+int gsp_rpc_unmap_emul(struct gsp_client *client, u64 handle, u64 gpu_va, u64 size);
+int gsp_rpc_pin_emul(struct gsp_client *client,
+ u64 handle, u64 offset, u64 size,
+ u32 flags, u64 *dma_addr, u32 *num_pages);
+int gsp_rpc_unpin_emul(struct gsp_client *client, u64 handle, u64 offset, u64 size);
+
+/* Notification */
+int gsp_rpc_notify_register(struct gsp_client *client,
+ u32 type,
+ void (*callback)(struct gsp_client *,
+ struct gsp_rpc_response *,
+ void *),
+ void *ctx);
+void gsp_rpc_notify_unregister(struct gsp_client *client, u32 type);
+void gsp_handle_notification(struct gsp_client *client, struct gsp_rpc_msg *msg);
+
+/* Client lifecycle */
+struct gsp_client *gsp_client_init(struct device *dev, phys_addr_t fw_base, phys_addr_t doorbell);
+void gsp_client_fini(struct gsp_client *client);
+
+/* WPR Management */
+int gsp_wpr_write_mutex_acquire(struct gsp_client *client, u32 wpr_id);
+int gsp_wpr_write_mutex_release(struct gsp_client *client, u32 wpr_id);
+int gsp_wpr_scratch_mutex_acquire(struct gsp_client *client, u32 wpr_id);
+int gsp_wpr_scratch_mutex_release(struct gsp_client *client, u32 wpr_id);
+
#endif /* _GSP_RPC_CLIENT_H_ */
diff --git a/kernel/include/memory.h b/kernel/include/memory.h
index 14a35e9..07e57f2 100644
--- a/kernel/include/memory.h
+++ b/kernel/include/memory.h
@@ -16,6 +16,17 @@ struct nv_vgpu_mapping {
struct list_head list;
};
+struct nv_vgpu_simple_mapping {
+ u64 gpa;
+ u64 size;
+ u64 iova;
+ struct page **pages;
+ int page_count;
+ bool pinned;
+ bool mapped;
+ struct list_head list;
+};
+
struct nv_vgpu_mm_ctx {
struct nv_vgpu_device *vgpu;
struct mutex lock;
diff --git a/kernel/memory.c b/kernel/memory.c
index 710799d..cd81496 100644
--- a/kernel/memory.c
+++ b/kernel/memory.c
@@ -12,17 +12,6 @@
#include "nvidia-vgpu-vfio.h"
#include "memory.h"
-struct nv_vgpu_simple_mapping {
- u64 gpa;
- u64 size;
- u64 iova;
- struct page **pages;
- int page_count;
- bool pinned;
- bool mapped;
- struct list_head list;
-};
-
static LIST_HEAD(simple_mappings);
static DEFINE_MUTEX(simple_lock);
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 11:13:18 +0000