From 708a3802824358c1770d5ad0b3c2e24fae114eb8 Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Mon, 5 Oct 2026 12:25:05 -0400 Subject: vgsp: per-mdev emulated VRAM size (1.0 multiple mediate sizes) - nv_vgpu_type_info/ioctl gain vram_mb; nv_vgpu_device gains fb_emul_size bound from the mdev type at create (default 512 MiB). - New nvidia-vgpu-turing-1g type (1024 MiB); turing max_instances 1 -> 2; show_description reports the per-type framebuffer and max_instance. - Thread fb_emul_size through GSCI fb_length/region/frtsOffset, WPR2, NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE, the vidmem allocator bound and the BAR1 shadow; bar_info callback uses the per-device size. - UNTESTED at commit time: builds clean; validation blocked by the Xid 140 incident (reboot pending). See Handoff-vGPU-IO.md ADDENDUM 48. --- kernel/bar_emulation.c | 20 ++++++++++++++- kernel/bar_init.c | 5 ++-- kernel/gsp_emulation.c | 2 +- kernel/gsp_rm_emulation.c | 20 ++++++++------- kernel/gsp_rpc_client.c | 2 +- kernel/include/ioctl.h | 2 ++ kernel/include/nvidia-vgpu-vfio.h | 12 +++++++++ kernel/ioctl.c | 1 + kernel/main.c | 51 ++++++++++++++++++++++++++++++++------- 9 files changed, 92 insertions(+), 23 deletions(-) (limited to 'kernel') diff --git a/kernel/bar_emulation.c b/kernel/bar_emulation.c index e095d8b..8e3f8cf 100644 --- a/kernel/bar_emulation.c +++ b/kernel/bar_emulation.c @@ -279,7 +279,25 @@ int nv_vgpu_bar_emul_init(struct nv_vgpu_device *vgpu) * "WPR2 initialized at an unexpected location". Must stay in * lockstep with GSCI fb_length/frtsOffset and wpr2_addr_lo. */ - *(u32 *)((u8 *)ctx->bar0_shadow + 0x100ce0) = 0x00000019; + { + /* + * Derive the encoding from the mdev's emulated FB size: + * fbSize = MAG << (SCALE + 20), SCALE bits 3:0, MAG bits 9:4. + * 512 MiB -> mag=1 scale=9 -> 0x19 (the historical value); + * 1 GiB -> 0x1a; 2 GiB -> 0x1b. Sizes that are not exactly + * representable round down; the emulation keeps GSCI + * fb_length/frtsOffset derived from the same value, so the + * guest's fbSize - reserved arithmetic stays consistent. + */ + u64 fb = nv_vgpu_fb_emul_size(ctx->vgpu); + u32 scale = 0; + + while (scale < 15 && (fb >> (scale + 20)) > 0x3f) + scale++; + *(u32 *)((u8 *)ctx->bar0_shadow + 0x100ce0) = + ((((u32)(fb >> (scale + 20))) & 0x3f) << 4) | + (scale & 0xf); + } /* * NV_PCFG (BAR0+0x88000) - PCI config space mirror the driver reads diff --git a/kernel/bar_init.c b/kernel/bar_init.c index 5e4b0a0..f4c982a 100644 --- a/kernel/bar_init.c +++ b/kernel/bar_init.c @@ -62,9 +62,10 @@ int nv_vgpu_bar_init(struct nv_vgpu_device *vgpu) * accesses must hit our own clean buffer, not the (broken) host * GPU's VRAM. Allocated lazily on first use to keep mdev create * cheap; 256MB to match the GSCI fb size. */ - vgpu->bars[1].vaddr = vzalloc(NV_VGPU_BAR1_SIZE); + vgpu->bars[1].vaddr = vzalloc(nv_vgpu_fb_emul_size(vgpu)); if (!vgpu->bars[1].vaddr) { - nv_vgpu_err(vgpu, "Failed to allocate emulated vidmem (256MB)"); + nv_vgpu_err(vgpu, "Failed to allocate emulated vidmem (%llu MiB)", + (unsigned long long)(nv_vgpu_fb_emul_size(vgpu) >> 20)); return -ENOMEM; } nv_vgpu_info(vgpu, "BAR1: gpa=0x%llx, size=0x%llx, emulated vidmem=%p", diff --git a/kernel/gsp_emulation.c b/kernel/gsp_emulation.c index b1c67a4..47c8151 100644 --- a/kernel/gsp_emulation.c +++ b/kernel/gsp_emulation.c @@ -750,7 +750,7 @@ int gsp_emu_init(struct nv_vgpu_device *vgpu) * with GSCI fb_length/frtsOffset and NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE. */ ctx->boot_stage = 0; /* GSP_BOOT_STAGE_IDLE */ ctx->flcn_kick_count = 0; - ctx->wpr2_addr_lo = 0x001fe000; + ctx->wpr2_addr_lo = (u32)(((nv_vgpu_fb_emul_size(vgpu) - 0x200000) >> 12) << 4); /* The guest's BAR0 is the emulation shadow (vzalloc'd by * nv_vgpu_bar_emul_init, mapped to the guest via VFIO mmap) - NOT the diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index f123792..f2656bb 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -1025,7 +1025,7 @@ static void gsp_rm_handle_set_guest_system_info(struct gsp_emu_context *ctx, * Builds the 1656-byte GspStaticConfigInfo for TU104. * ============================================================ */ -static void gsp_rm_build_static_info(u8 *buf) +static void gsp_rm_build_static_info(u8 *buf, u64 fb_size) { u64 val64; u32 val32; @@ -1058,7 +1058,7 @@ static void gsp_rm_build_static_info(u8 *buf) { struct tu104_fb_region *region = (void *)(p + 8); /* 8-aligned */ val64 = 0; memcpy(®ion->base, &val64, 8); - val64 = (NV_VGPU_EMUL_FB_SIZE - 1); + val64 = (fb_size - 1); memcpy(®ion->limit, &val64, 8); val64 = 0; memcpy(®ion->reserved, &val64, 8); val32 = 0; memcpy(®ion->performance, &val32, 4); @@ -1097,9 +1097,10 @@ static void gsp_rm_build_static_info(u8 *buf) /* poisonFuseEnabled (offset 1216, 1 byte) */ buf[TU104_GSCI_OFF_poisonFuseEnabled] = 0; - /* fb_length (offset 1224, 8 bytes) - 512MB. Kept in lockstep with - * NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE and wpr2_addr_lo. */ - val64 = NV_VGPU_EMUL_FB_SIZE; + /* fb_length (offset 1224, 8 bytes) - this mdev's emulated FB size. + * Kept in lockstep with NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE and + * wpr2_addr_lo (all derive from nv_vgpu_fb_emul_size). */ + val64 = fb_size; memcpy(buf + TU104_GSCI_OFF_fb_length, &val64, 8); /* fbio_mask (offset 1232, 8 bytes) - 8 FBIOs for 256-bit bus */ @@ -1167,7 +1168,7 @@ static void gsp_rm_build_static_info(u8 *buf) /* fwWprLayoutOffset (offset 1640, 16 bytes): nonWprHeapOffset, frtsOffset */ val64 = 0; memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset, &val64, 8); - val64 = NV_VGPU_EMUL_FB_SIZE - 0x200000; /* frtsOffset = fbSize - 2MB */ + val64 = fb_size - 0x200000; /* frtsOffset = fbSize - 2MB */ memcpy(buf + TU104_GSCI_OFF_fwWprLayoutOffset + 8, &val64, 8); } @@ -1241,12 +1242,13 @@ static void gsp_rm_handle_get_static_info(struct gsp_emu_context *ctx, return; } - gsp_rm_build_static_info(reply); + gsp_rm_build_static_info(reply, nv_vgpu_fb_emul_size(ctx->vgpu)); gsp_rm_apply_static_overlay(ctx, reply); nv_vgpu_info(ctx->vgpu, "vgsp-rm: GET_GSP_STATIC_INFO seq=%u -> %u bytes (fb=%lluMB, engines=0x%x/0x%x)", sequence, TU104_GSCI_SIZE, - (unsigned long long)(NV_VGPU_EMUL_FB_SIZE >> 20), 0x60003E02, 0x362020); + (unsigned long long)(nv_vgpu_fb_emul_size(ctx->vgpu) >> 20), + 0x60003E02, 0x362020); if (gsp_rm_send(ctx, GSPRM_RPC_FUNC_GET_GSP_STATIC_INFO, sequence, 0, reply, TU104_GSCI_SIZE)) @@ -1993,7 +1995,7 @@ static void gsp_rm_handle_kgr_get_context_buffers_info( */ static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index) { - u64 fb = NV_VGPU_EMUL_FB_SIZE; + u64 fb = nv_vgpu_fb_emul_size(ctx->vgpu); switch (index) { case TU104_FB_INFO_IDX_TILE_REGION_COUNT: diff --git a/kernel/gsp_rpc_client.c b/kernel/gsp_rpc_client.c index 28659ff..be197cc 100644 --- a/kernel/gsp_rpc_client.c +++ b/kernel/gsp_rpc_client.c @@ -592,7 +592,7 @@ int gsp_rpc_alloc_emul(struct gsp_client *client, u64 va = addr_hint ? addr_hint : 0x100000000ULL; /* Default 4GB */ dma_addr_t iova; static u64 next_vidmem_va = 0x100000; /* Start after 1MB (skip WPR2/reserved) */ - u64 fb_size = NV_VGPU_EMUL_FB_SIZE; + u64 fb_size = nv_vgpu_fb_emul_size(vgpu); /* For vidmem allocations (golden image, channels, etc.), allocate from * the emulated GPU's vidmem address range (0 - fb_size). The memmgr diff --git a/kernel/include/ioctl.h b/kernel/include/ioctl.h index 36d28ac..58fea37 100644 --- a/kernel/include/ioctl.h +++ b/kernel/include/ioctl.h @@ -42,6 +42,7 @@ struct nv_vgpu_type_info { u64 vdev_id; u64 pdev_id; u32 max_instances; + u32 vram_mb; /* emulated framebuffer size; 0 = 512 MiB default */ }; struct nv_vgpu_ioctl_get_info { @@ -111,6 +112,7 @@ struct nv_vgpu_ioctl_register_vgpu_type { u64 vdev_id; u64 pdev_id; u32 max_instances; + u32 vram_mb; }; int nv_vgpu_ioctl_get_info(struct nv_vgpu_device *vgpu, void __user *arg); diff --git a/kernel/include/nvidia-vgpu-vfio.h b/kernel/include/nvidia-vgpu-vfio.h index ccd7262..bed90a7 100644 --- a/kernel/include/nvidia-vgpu-vfio.h +++ b/kernel/include/nvidia-vgpu-vfio.h @@ -137,6 +137,11 @@ struct nv_vgpu_device { u64 fb_base; u64 fb_size; + /* Emulated framebuffer/vidmem size for this mdev, from its type's + * vram_mb (nv_vgpu_type_info); defaults to NV_VGPU_EMUL_FB_SIZE. All + * emulated-GSP consumers (GSCI fb_length/lb region/frtsOffset, WPR2, + * LOCAL_MEMORY_RANGE, vidmem allocator, BAR shadow) must agree. */ + u64 fb_emul_size; u64 bar0_config_base; struct list_head list; @@ -223,6 +228,13 @@ static inline struct nv_vgpu_device *mdev_to_vgpu(struct mdev_device *mdev) return dev_get_drvdata(&mdev->dev); } +/* Per-mdev emulated framebuffer size in bytes (default 512 MiB). */ +static inline u64 nv_vgpu_fb_emul_size(const struct nv_vgpu_device *vgpu) +{ + return (vgpu && vgpu->fb_emul_size) ? vgpu->fb_emul_size : + NV_VGPU_EMUL_FB_SIZE; +} + /* NULL-tolerant logging: emulation paths may run with a NULL vgpu * (e.g. unit tests, early init) - fall back to plain pr_*. */ #define nv_vgpu_dbg(vgpu, fmt, ...) do { \ diff --git a/kernel/ioctl.c b/kernel/ioctl.c index 754857d..48e5253 100644 --- a/kernel/ioctl.c +++ b/kernel/ioctl.c @@ -283,6 +283,7 @@ int nv_vgpu_ioctl_register_vgpu_type(void __user *arg) info.vdev_id = params.vdev_id; info.pdev_id = params.pdev_id; info.max_instances = params.max_instances; + info.vram_mb = params.vram_mb; ret = nv_vgpu_register_type_info(&info); if (ret) diff --git a/kernel/main.c b/kernel/main.c index 7028bdd..a5a64fd 100644 --- a/kernel/main.c +++ b/kernel/main.c @@ -112,8 +112,22 @@ static unsigned int nv_vgpu_get_available(struct mdev_type *mtype) */ static ssize_t nv_vgpu_show_description(struct mdev_type *mtype, char *buf) { - return sysfs_emit(buf, "num_heads=1, frl_config=60, framebuffer=256M, " - "max_resolution=5120x2880, max_instance=1\n"); + struct nv_vgpu_type_info *info; + const char *name = mtype->sysfs_name; + u32 vram_mb = 512, max_inst = 1; + + if (name && str_has_prefix(name, "nvidia-")) + name += strlen("nvidia-"); + info = name ? nv_vgpu_find_type_info(name) : NULL; + if (info) { + if (info->vram_mb) + vram_mb = info->vram_mb; + if (info->max_instances) + max_inst = info->max_instances; + } + return sysfs_emit(buf, "num_heads=1, frl_config=60, framebuffer=%uM, " + "max_resolution=5120x2880, max_instance=%u\n", + vram_mb, max_inst); } struct mdev_driver nv_vgpu_mdev_driver = { @@ -199,7 +213,7 @@ static NV_STATUS nv_vgpu_bar_info_callback(struct pci_dev *pdev, const NvU8 *uui *size = NV_VGPU_BAR0_SIZE; /* 16MB */ break; case 1: - *size = NV_VGPU_BAR1_SIZE; /* 256MB */ + *size = nv_vgpu_fb_emul_size(vgpu_dev); /* per-type FB */ break; case 2: *size = NV_VGPU_BAR2_SIZE; /* 32MB */ @@ -370,14 +384,15 @@ EXPORT_SYMBOL(nv_vgpu_find_type_info); static void nv_vgpu_create_static_types(void) { int i; - struct nv_vgpu_type_info static_types[4] = { - { .name = "nvidia-vgpu-ampere", .description = "NVIDIA Ampere vGPU", .vdev_id = 0x10de2235, .pdev_id = 0x10de2235, .max_instances = 1 }, - { .name = "nvidia-vgpu-hopper", .description = "NVIDIA Hopper vGPU", .vdev_id = 0x10de2331, .pdev_id = 0x10de2331, .max_instances = 1 }, - { .name = "nvidia-vgpu-turing", .description = "NVIDIA Turing vGPU", .vdev_id = 0x10de1ed3, .pdev_id = 0x10de1ed3, .max_instances = 1 }, - { .name = "nvidia-vgpu-pascal", .description = "NVIDIA Pascal vGPU", .vdev_id = 0x10de1db1, .pdev_id = 0x10de1db1, .max_instances = 1 }, + struct nv_vgpu_type_info static_types[5] = { + { .name = "nvidia-vgpu-ampere", .description = "NVIDIA Ampere vGPU", .vdev_id = 0x10de2235, .pdev_id = 0x10de2235, .max_instances = 1, .vram_mb = 512 }, + { .name = "nvidia-vgpu-hopper", .description = "NVIDIA Hopper vGPU", .vdev_id = 0x10de2331, .pdev_id = 0x10de2331, .max_instances = 1, .vram_mb = 512 }, + { .name = "nvidia-vgpu-turing", .description = "NVIDIA Turing vGPU", .vdev_id = 0x10de1ed3, .pdev_id = 0x10de1ed3, .max_instances = 2, .vram_mb = 512 }, + { .name = "nvidia-vgpu-turing-1g", .description = "NVIDIA Turing vGPU (1 GiB)", .vdev_id = 0x10de1ed3, .pdev_id = 0x10de1ed3, .max_instances = 2, .vram_mb = 1024 }, + { .name = "nvidia-vgpu-pascal", .description = "NVIDIA Pascal vGPU", .vdev_id = 0x10de1db1, .pdev_id = 0x10de1db1, .max_instances = 1, .vram_mb = 512 }, }; - for (i = 0; i < 4; i++) { + for (i = 0; i < 5; i++) { nv_vgpu_register_type_info(&static_types[i]); } } @@ -546,6 +561,24 @@ static int nv_vgpu_mdev_create(struct mdev_device *mdev) vgpu->mdev = mdev; vgpu->state = NV_VGPU_STATE_UNINITIALIZED; + /* + * Bind the emulated FB size to this mdev's type (the type's vram_mb; + * 0/absent keeps the 512 MiB default). The type dir may carry an + * extra "nvidia-" prefix vs our registered names. + */ + { + const char *tname = mdev->type ? mdev->type->sysfs_name : NULL; + struct nv_vgpu_type_info *tinfo; + + if (tname && str_has_prefix(tname, "nvidia-")) + tname += strlen("nvidia-"); + tinfo = tname ? nv_vgpu_find_type_info(tname) : NULL; + vgpu->fb_emul_size = (tinfo && tinfo->vram_mb) ? + (u64)tinfo->vram_mb << 20 : NV_VGPU_EMUL_FB_SIZE; + pr_info("vgsp: mdev create: type=%s fb_emul_size=%llu MiB\n", + tname ? tname : "(none)", + (unsigned long long)(vgpu->fb_emul_size >> 20)); + } mutex_init(&vgpu->lock); INIT_WORK(&vgpu->init_work, nv_vgpu_device_init_work); INIT_WORK(&vgpu->cleanup_work, nv_vgpu_device_cleanup_work); -- cgit v1.2.3