summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorArthur Hanson Rasmusson <arthur@vgpu.io>2026-10-05 12:25:05 -0400
committerArthur Hanson Rasmusson <arthur@vgpu.io>2026-10-05 12:25:05 -0400
commit549d1e8126f96b3515d37037ae22e0dce430c20b (patch)
tree5fc949da9f724b75a84dc8b5fdc4e6462ff94335
parent3b1dcb87279a2f58210b87028a39c9a0932faf2c (diff)
downloadopen-iov-549d1e8126f96b3515d37037ae22e0dce430c20b.tar.gz
open-iov-549d1e8126f96b3515d37037ae22e0dce430c20b.zip
vgsp: per-mdev emulated VRAM size (1.0 multiple mediate sizes)
- nv_vgpu_type_info/ioctl gain vram_mb; nv_vgpu_device gains fb_emul_size bound from the mdev type at create (default 512 MiB). - New nvidia-vgpu-turing-1g type (1024 MiB); turing max_instances 1 -> 2; show_description reports the per-type framebuffer and max_instance. - Thread fb_emul_size through GSCI fb_length/region/frtsOffset, WPR2, NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE, the vidmem allocator bound and the BAR1 shadow; bar_info callback uses the per-device size. - UNTESTED at commit time: builds clean; validation blocked by the Xid 140 incident (reboot pending). See Handoff-vGPU-IO.md ADDENDUM 48.
-rw-r--r--kernel/bar_emulation.c20
-rw-r--r--kernel/bar_init.c5
-rw-r--r--kernel/gsp_emulation.c2
-rw-r--r--kernel/gsp_rm_emulation.c20
-rw-r--r--kernel/gsp_rpc_client.c2
-rw-r--r--kernel/include/ioctl.h2
-rw-r--r--kernel/include/nvidia-vgpu-vfio.h12
-rw-r--r--kernel/ioctl.c1
-rw-r--r--kernel/main.c51
9 files changed, 92 insertions, 23 deletions
diff --git a/kernel/bar_emulation.c b/kernel/bar_emulation.c
index e095d8b..8e3f8cf 100644
--- a/kernel/bar_emulation.c
+++ b/kernel/bar_emulation.c
@@ -279,7 +279,25 @@ int nv_vgpu_bar_emul_init(struct nv_vgpu_device *vgpu)
* "WPR2 initialized at an unexpected location". Must stay in
* lockstep with GSCI fb_length/frtsOffset and wpr2_addr_lo.
*/
- *(u32 *)((u8 *)ctx->bar0_shadow + 0x100ce0) = 0x00000019;
+ {
+ /*
+ * Derive the encoding from the mdev's emulated FB size:
+ * fbSize = MAG << (SCALE + 20), SCALE bits 3:0, MAG bits 9:4.
+ * 512 MiB -> mag=1 scale=9 -> 0x19 (the historical value);
+ * 1 GiB -> 0x1a; 2 GiB -> 0x1b. Sizes that are not exactly
+ * representable round down; the emulation keeps GSCI
+ * fb_length/frtsOffset derived from the same value, so the
+ * guest's fbSize - reserved arithmetic stays consistent.
+ */
+ u64 fb = nv_vgpu_fb_emul_size(ctx->vgpu);
+ u32 scale = 0;
+
+ while (scale < 15 && (fb >> (scale + 20)) > 0x3f)
+ scale++;
+ *(u32 *)((u8 *)ctx->bar0_shadow + 0x100ce0) =
+ ((((u32)(fb >> (scale + 20))) & 0x3f) << 4) |
+ (scale & 0xf);
+ }
/*
* NV_PCFG (BAR0+0x88000) - PCI config space mirror the driver reads
diff --git a/kernel/bar_init.c b/kernel/bar_init.c
index 5e4b0a0..f4c982a 100644
--- a/kernel/bar_init.c
+++ b/kernel/bar_init.c
@@ -62,9 +62,10 @@ int nv_vgpu_bar_init(struct nv_vgpu_device *vgpu)
* accesses must hit our own clean buffer, not the (broken) host
* GPU's VRAM. Allocated lazily on first use to keep mdev create
* cheap; 256MB to match the GSCI fb size. */
- vgpu->bars[1].vaddr = vzalloc(NV_VGPU_BAR1_SIZE);
+ vgpu->bars[1].vaddr = vzalloc(nv_vgpu_fb_emul_size(vgpu));
if (!vgpu->bars[1].vaddr) {
- nv_vgpu_err(vgpu, "Failed to allocate emulated vidmem (256MB)");
+ nv_vgpu_err(vgpu, "Failed to allocate emulated vidmem (%llu MiB)",
+ (unsigned long long)(nv_vgpu_fb_emul_size(vgpu) >> 20));
return -ENOMEM;
}
nv_vgpu_info(vgpu, "BAR1: gpa=0x%llx, size=0x%llx, emulated vidmem=%p",
diff --git a/kernel/gsp_emulation.c b/kernel/gsp_emulation.c
index b1c67a4..47c8151 100644
--- a/kernel/gsp_emulation.c
+++ b/kernel/gsp_emulation.c
@@ -750,7 +750,7 @@ int gsp_emu_init(struct nv_vgpu_device *vgpu)
* with GSCI fb_length/frtsOffset and NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE. */
ctx->boot_stage = 0; /* GSP_BOOT_STAGE_IDLE */
ctx->flcn_kick_count = 0;
- ctx->wpr2_addr_lo = 0x001fe000;
+ ctx->wpr2_addr_lo = (u32)(((nv_vgpu_fb_emul_size(vgpu) - 0x200000) >> 12) << 4);
/* The guest's BAR0 is the emulation shadow (vzalloc'd by
* nv_vgpu_bar_emul_init, mapped to the guest via VFIO mmap) - NOT the
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index f123792..f2656bb 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -1025,7 +1025,7 @@ static void gsp_rm_handle_set_guest_system_info(struct gsp_emu_context *ctx,
* Builds the 1656-byte GspStaticConfigInfo for TU104.
* ============================================================ */
-static void gsp_rm_build_static_info(u8 *buf)
+static void gsp_rm_build_static_info(u8 *buf, u64 fb_size)
{
u64 val64;
u32 val32;
@@ -1058,7 +1058,7 @@ static void gsp_rm_build_static_info(u8 *buf)
{
struct tu104_fb_region *region = (void *)(p + 8); /* 8-aligned */
val64 = 0; memcpy(&region->base, &val64, 8);
- val64 = (NV_VGPU_EMUL_FB_SIZE - 1);
+ val64 = (fb_size - 1);
memcpy(&region->limit, &val64, 8);
val64 = 0; memcpy(&region->reserved, &val64, 8);
val32 = 0; memcpy(&region->performance, &val32, 4);
@@ -1097,9 +1097,10 @@ static void gsp_rm_build_static_info(u8 *buf)
/* poisonFuseEnabled (offset 1216, 1 byte) */
buf[TU104_GSCI_OFF_poisonFuseEnabled] = 0;
- /* fb_length (offset 1224, 8 bytes) - 512MB. Kept in lockstep with
- * NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE and wpr2_addr_lo. */
- val64 = NV_VGPU_EMUL_FB_SIZE;
+ /* fb_length (offset 1224, 8 bytes) - this mdev's emulated FB size.
+ * Kept in lockstep with NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE and
+ * wpr2_addr_lo (all derive from nv_vgpu_fb_emul_size). */
+ val64 = fb_size;
memcpy(buf + TU104_GSCI_OFF_fb_length, &val64, 8);
/* fbio_mask (offset 1232, 8 bytes) - 8 FBIOs for 256-bit bus */
@@ -1167,7 +1168,7 @@ static void gsp_rm_build_static_info(u8 *buf)
/* fwWprLayoutOffset (offset 1640, 16 bytes): nonWprHeapOffset, frtsOffset */
val64 = 0;
memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset, &val64, 8);
- val64 = NV_VGPU_EMUL_FB_SIZE - 0x200000; /* frtsOffset = fbSize - 2MB */
+ val64 = fb_size - 0x200000; /* frtsOffset = fbSize - 2MB */
memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset + 8, &val64, 8);
}
@@ -1241,12 +1242,13 @@ static void gsp_rm_handle_get_static_info(struct gsp_emu_context *ctx,
return;
}
- gsp_rm_build_static_info(reply);
+ gsp_rm_build_static_info(reply, nv_vgpu_fb_emul_size(ctx->vgpu));
gsp_rm_apply_static_overlay(ctx, reply);
nv_vgpu_info(ctx->vgpu, "vgsp-rm: GET_GSP_STATIC_INFO seq=%u -> %u bytes (fb=%lluMB, engines=0x%x/0x%x)",
sequence, TU104_GSCI_SIZE,
- (unsigned long long)(NV_VGPU_EMUL_FB_SIZE >> 20), 0x60003E02, 0x362020);
+ (unsigned long long)(nv_vgpu_fb_emul_size(ctx->vgpu) >> 20),
+ 0x60003E02, 0x362020);
if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO, sequence, 0,
reply, TU104_GSCI_SIZE))
@@ -1993,7 +1995,7 @@ static void gsp_rm_handle_kgr_get_context_buffers_info(
*/
static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index)
{
- u64 fb = NV_VGPU_EMUL_FB_SIZE;
+ u64 fb = nv_vgpu_fb_emul_size(ctx->vgpu);
switch (index) {
case TU104_FB_INFO_IDX_TILE_REGION_COUNT:
diff --git a/kernel/gsp_rpc_client.c b/kernel/gsp_rpc_client.c
index 28659ff..be197cc 100644
--- a/kernel/gsp_rpc_client.c
+++ b/kernel/gsp_rpc_client.c
@@ -592,7 +592,7 @@ int gsp_rpc_alloc_emul(struct gsp_client *client,
u64 va = addr_hint ? addr_hint : 0x100000000ULL; /* Default 4GB */
dma_addr_t iova;
static u64 next_vidmem_va = 0x100000; /* Start after 1MB (skip WPR2/reserved) */
- u64 fb_size = NV_VGPU_EMUL_FB_SIZE;
+ u64 fb_size = nv_vgpu_fb_emul_size(vgpu);
/* For vidmem allocations (golden image, channels, etc.), allocate from
* the emulated GPU's vidmem address range (0 - fb_size). The memmgr
diff --git a/kernel/include/ioctl.h b/kernel/include/ioctl.h
index 36d28ac..58fea37 100644
--- a/kernel/include/ioctl.h
+++ b/kernel/include/ioctl.h
@@ -42,6 +42,7 @@ struct nv_vgpu_type_info {
u64 vdev_id;
u64 pdev_id;
u32 max_instances;
+ u32 vram_mb; /* emulated framebuffer size; 0 = 512 MiB default */
};
struct nv_vgpu_ioctl_get_info {
@@ -111,6 +112,7 @@ struct nv_vgpu_ioctl_register_vgpu_type {
u64 vdev_id;
u64 pdev_id;
u32 max_instances;
+ u32 vram_mb;
};
int nv_vgpu_ioctl_get_info(struct nv_vgpu_device *vgpu, void __user *arg);
diff --git a/kernel/include/nvidia-vgpu-vfio.h b/kernel/include/nvidia-vgpu-vfio.h
index ccd7262..bed90a7 100644
--- a/kernel/include/nvidia-vgpu-vfio.h
+++ b/kernel/include/nvidia-vgpu-vfio.h
@@ -137,6 +137,11 @@ struct nv_vgpu_device {
u64 fb_base;
u64 fb_size;
+ /* Emulated framebuffer/vidmem size for this mdev, from its type's
+ * vram_mb (nv_vgpu_type_info); defaults to NV_VGPU_EMUL_FB_SIZE. All
+ * emulated-GSP consumers (GSCI fb_length/lb region/frtsOffset, WPR2,
+ * LOCAL_MEMORY_RANGE, vidmem allocator, BAR shadow) must agree. */
+ u64 fb_emul_size;
u64 bar0_config_base;
struct list_head list;
@@ -223,6 +228,13 @@ static inline struct nv_vgpu_device *mdev_to_vgpu(struct mdev_device *mdev)
return dev_get_drvdata(&mdev->dev);
}
+/* Per-mdev emulated framebuffer size in bytes (default 512 MiB). */
+static inline u64 nv_vgpu_fb_emul_size(const struct nv_vgpu_device *vgpu)
+{
+ return (vgpu && vgpu->fb_emul_size) ? vgpu->fb_emul_size :
+ NV_VGPU_EMUL_FB_SIZE;
+}
+
/* NULL-tolerant logging: emulation paths may run with a NULL vgpu
* (e.g. unit tests, early init) - fall back to plain pr_*. */
#define nv_vgpu_dbg(vgpu, fmt, ...) do { \
diff --git a/kernel/ioctl.c b/kernel/ioctl.c
index 754857d..48e5253 100644
--- a/kernel/ioctl.c
+++ b/kernel/ioctl.c
@@ -283,6 +283,7 @@ int nv_vgpu_ioctl_register_vgpu_type(void __user *arg)
info.vdev_id = params.vdev_id;
info.pdev_id = params.pdev_id;
info.max_instances = params.max_instances;
+ info.vram_mb = params.vram_mb;
ret = nv_vgpu_register_type_info(&info);
if (ret)
diff --git a/kernel/main.c b/kernel/main.c
index 7028bdd..a5a64fd 100644
--- a/kernel/main.c
+++ b/kernel/main.c
@@ -112,8 +112,22 @@ static unsigned int nv_vgpu_get_available(struct mdev_type *mtype)
*/
static ssize_t nv_vgpu_show_description(struct mdev_type *mtype, char *buf)
{
- return sysfs_emit(buf, "num_heads=1, frl_config=60, framebuffer=256M, "
- "max_resolution=5120x2880, max_instance=1\n");
+ struct nv_vgpu_type_info *info;
+ const char *name = mtype->sysfs_name;
+ u32 vram_mb = 512, max_inst = 1;
+
+ if (name && str_has_prefix(name, "nvidia-"))
+ name += strlen("nvidia-");
+ info = name ? nv_vgpu_find_type_info(name) : NULL;
+ if (info) {
+ if (info->vram_mb)
+ vram_mb = info->vram_mb;
+ if (info->max_instances)
+ max_inst = info->max_instances;
+ }
+ return sysfs_emit(buf, "num_heads=1, frl_config=60, framebuffer=%uM, "
+ "max_resolution=5120x2880, max_instance=%u\n",
+ vram_mb, max_inst);
}
struct mdev_driver nv_vgpu_mdev_driver = {
@@ -199,7 +213,7 @@ static NV_STATUS nv_vgpu_bar_info_callback(struct pci_dev *pdev, const NvU8 *uui
*size = NV_VGPU_BAR0_SIZE; /* 16MB */
break;
case 1:
- *size = NV_VGPU_BAR1_SIZE; /* 256MB */
+ *size = nv_vgpu_fb_emul_size(vgpu_dev); /* per-type FB */
break;
case 2:
*size = NV_VGPU_BAR2_SIZE; /* 32MB */
@@ -370,14 +384,15 @@ EXPORT_SYMBOL(nv_vgpu_find_type_info);
static void nv_vgpu_create_static_types(void)
{
int i;
- struct nv_vgpu_type_info static_types[4] = {
- { .name = "nvidia-vgpu-ampere", .description = "NVIDIA Ampere vGPU", .vdev_id = 0x10de2235, .pdev_id = 0x10de2235, .max_instances = 1 },
- { .name = "nvidia-vgpu-hopper", .description = "NVIDIA Hopper vGPU", .vdev_id = 0x10de2331, .pdev_id = 0x10de2331, .max_instances = 1 },
- { .name = "nvidia-vgpu-turing", .description = "NVIDIA Turing vGPU", .vdev_id = 0x10de1ed3, .pdev_id = 0x10de1ed3, .max_instances = 1 },
- { .name = "nvidia-vgpu-pascal", .description = "NVIDIA Pascal vGPU", .vdev_id = 0x10de1db1, .pdev_id = 0x10de1db1, .max_instances = 1 },
+ struct nv_vgpu_type_info static_types[5] = {
+ { .name = "nvidia-vgpu-ampere", .description = "NVIDIA Ampere vGPU", .vdev_id = 0x10de2235, .pdev_id = 0x10de2235, .max_instances = 1, .vram_mb = 512 },
+ { .name = "nvidia-vgpu-hopper", .description = "NVIDIA Hopper vGPU", .vdev_id = 0x10de2331, .pdev_id = 0x10de2331, .max_instances = 1, .vram_mb = 512 },
+ { .name = "nvidia-vgpu-turing", .description = "NVIDIA Turing vGPU", .vdev_id = 0x10de1ed3, .pdev_id = 0x10de1ed3, .max_instances = 2, .vram_mb = 512 },
+ { .name = "nvidia-vgpu-turing-1g", .description = "NVIDIA Turing vGPU (1 GiB)", .vdev_id = 0x10de1ed3, .pdev_id = 0x10de1ed3, .max_instances = 2, .vram_mb = 1024 },
+ { .name = "nvidia-vgpu-pascal", .description = "NVIDIA Pascal vGPU", .vdev_id = 0x10de1db1, .pdev_id = 0x10de1db1, .max_instances = 1, .vram_mb = 512 },
};
- for (i = 0; i < 4; i++) {
+ for (i = 0; i < 5; i++) {
nv_vgpu_register_type_info(&static_types[i]);
}
}
@@ -546,6 +561,24 @@ static int nv_vgpu_mdev_create(struct mdev_device *mdev)
vgpu->mdev = mdev;
vgpu->state = NV_VGPU_STATE_UNINITIALIZED;
+ /*
+ * Bind the emulated FB size to this mdev's type (the type's vram_mb;
+ * 0/absent keeps the 512 MiB default). The type dir may carry an
+ * extra "nvidia-" prefix vs our registered names.
+ */
+ {
+ const char *tname = mdev->type ? mdev->type->sysfs_name : NULL;
+ struct nv_vgpu_type_info *tinfo;
+
+ if (tname && str_has_prefix(tname, "nvidia-"))
+ tname += strlen("nvidia-");
+ tinfo = tname ? nv_vgpu_find_type_info(tname) : NULL;
+ vgpu->fb_emul_size = (tinfo && tinfo->vram_mb) ?
+ (u64)tinfo->vram_mb << 20 : NV_VGPU_EMUL_FB_SIZE;
+ pr_info("vgsp: mdev create: type=%s fb_emul_size=%llu MiB\n",
+ tname ? tname : "(none)",
+ (unsigned long long)(vgpu->fb_emul_size >> 20));
+ }
mutex_init(&vgpu->lock);
INIT_WORK(&vgpu->init_work, nv_vgpu_device_init_work);
INIT_WORK(&vgpu->cleanup_work, nv_vgpu_device_cleanup_work);