summaryrefslogtreecommitdiffstats
path: root/kernel/gsp_rm_emulation.c
diff options
context:
space:
mode:
authorArthur Hanson Rasmusson <arthur@vgpu.io>2026-09-22 23:32:44 -0400
committerArthur Hanson Rasmusson <arthur@vgpu.io>2026-09-22 23:32:44 -0400
commit14283fe73fb5e432a7af45de94961655714910f6 (patch)
treee993bf56f3e2ef93195019ac0fd996d82204384a /kernel/gsp_rm_emulation.c
parentb34e4531d8f0b984dc15660d3ab406ae48822931 (diff)
downloadopen-iov-14283fe73fb5e432a7af45de94961655714910f6.tar.gz
open-iov-14283fe73fb5e432a7af45de94961655714910f6.zip
kernel: FB/BUS info-list controls + PCIe link caps for the UVM path
CUDA (cuInit) now reaches UVM's device create: libcuda opens nvidia-uvm, UVM calls nvGpuOpsDeviceCreate/gpuGetGpuInfo, and the driver fails at getPCIELinkRateMBps -> calculatePCIELinkRateMBps ("Unknown PCIe speed", NV_ERR_INVALID_STATE) -> nv_open_device aborts -> cuInit returns CUDA_ERROR_NOT_INITIALIZED (3). Implemented along the way: - FB_GET_INFO_V2 (0x20801303): echo the requested FB info indices and answer TU104 values (256MB RAM/heap/BAR1, GDDR6, 8 LTCs, 4MB L2, ...). Fixes the kern_mem_sys_ctrl.c:911 asserts and gives CUDA/UVM real memory sizes. - BUS_GET_INFO_V2 (0x20801823): echo indices; PCIe Gen3 x16 link caps / GEN info / link status, SYSMEM_CONNECTION_TYPE=PCIE. UVM's gpuGetSysmemLinkInfo reads SYSMEM_CONNECTION_TYPE and the nominal link bandwidth from here. - PCIe Link Capabilities in the emulated config space (0x8C/0x92) and in the BAR0 PCFG mirror (gpuReadBusConfigReg -> NV_PCFG+0x84 = BAR0+ 0x88084): MAX_SPEED=3 (8GT/s Gen3), MAX_WIDTH=0x10 (x16); link status mirror at 0x88088. Also mirrored under the VF-window/low aliases. - Temporary change-watch on PCFG[0x88084] to see if the guest overwrites it (it does not - stays 0x103). Verified: guest nvidia-smi -L still works and reports PCIe Generation Max/Current = 3. cuInit still fails; the remaining question is which address the driver's kbifGetGpuLinkCapabilities read sees as zero (the PCFG shadow holds 0x103). Guest-side kprobe tracing of that function panicked the VM (free_swap_cache page fault on a corrupted pointer) - possible guest memory corruption to investigate separately.
Diffstat (limited to 'kernel/gsp_rm_emulation.c')
-rw-r--r--kernel/gsp_rm_emulation.c201
1 files changed, 201 insertions, 0 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index f18d17c..743a260 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -1269,6 +1269,193 @@ static void gsp_rm_handle_kgr_get_context_buffers_info(
TU104_CTX_GPU_MAX_ENGINES, TU104_CTX_ENGINE_ID_COUNT);
}
+/*
+ * FB_GET_INFO_V2 (0x20801303)
+ *
+ * The client (kern_mem_sys_ctrl.c _fbGetFbInfos) sends a list of FB info
+ * indices to the physical RM and expects the same indices back with data
+ * populated. A zeroed reply trips "pFbInfos[i].index == ..." asserts and
+ * leaves all FB info zero - CUDA/UVM read total/usable memory, BAR1 size,
+ * L2/LTC topology from here. Echo the requested indices and fill in the
+ * emulated TU104 values.
+ */
+static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index)
+{
+ struct nv_vgpu_device *vgpu = ctx->vgpu;
+ u64 fb = vgpu->fb_size ? vgpu->fb_size : NV_VGPU_BAR1_SIZE;
+
+ switch (index) {
+ case TU104_FB_INFO_IDX_TILE_REGION_COUNT:
+ return 0;
+ case TU104_FB_INFO_IDX_COMPRESSION_SIZE:
+ return 0;
+ case TU104_FB_INFO_IDX_PARTITION_COUNT:
+ return 1;
+ case TU104_FB_INFO_IDX_BAR1_SIZE:
+ return (u32)fb;
+ case TU104_FB_INFO_IDX_RAM_SIZE:
+ case TU104_FB_INFO_IDX_TOTAL_RAM_SIZE:
+ case TU104_FB_INFO_IDX_USABLE_RAM_SIZE:
+ return (u32)fb;
+ case TU104_FB_INFO_IDX_HEAP_SIZE:
+ case TU104_FB_INFO_IDX_MAPPABLE_HEAP_SIZE:
+ return (u32)fb;
+ case TU104_FB_INFO_IDX_BUS_WIDTH:
+ return 256;
+ case TU104_FB_INFO_IDX_RAM_TYPE:
+ return TU104_RAM_TYPE_GDDR6;
+ case TU104_FB_INFO_IDX_BANK_COUNT:
+ return 8;
+ case TU104_FB_INFO_IDX_FB_TAX_SIZE_KB:
+ case TU104_FB_INFO_IDX_LARGEST_FREE_REGION_SIZE_KB:
+ return (u32)(fb >> 10);
+ case TU104_FB_INFO_IDX_PARTITION_MASK:
+ return 1;
+ case TU104_FB_INFO_IDX_HEAP_FREE:
+ return (u32)fb;
+ case TU104_FB_INFO_IDX_RAM_LOCATION:
+ return TU104_RAM_LOCATION_GPU_DEDICATED;
+ case TU104_FB_INFO_IDX_FBP_COUNT:
+ return 1;
+ case TU104_FB_INFO_IDX_FBP_MASK:
+ return 1;
+ case TU104_FB_INFO_IDX_L2CACHE_SIZE:
+ return TU104_L2CACHE_SIZE;
+ case TU104_FB_INFO_IDX_MEMORYINFO_VENDOR_ID:
+ return TU104_MEMORYINFO_VENDOR_ID_SAMSUNG;
+ case TU104_FB_INFO_IDX_BAR1_AVAIL_SIZE:
+ case TU104_FB_INFO_IDX_BAR1_MAX_CONTIG_AVAIL_SIZE:
+ return (u32)fb;
+ case TU104_FB_INFO_IDX_LTC_COUNT:
+ return TU104_LTC_COUNT;
+ case TU104_FB_INFO_IDX_LTS_COUNT:
+ return TU104_LTS_COUNT;
+ case TU104_FB_INFO_IDX_PSEUDO_CHANNEL_MODE:
+ return TU104_PSEUDO_CHANNEL_MODE_DISABLED;
+ case TU104_FB_INFO_IDX_ALLOW_PAGE_RETIREMENT:
+ return 0;
+ case TU104_FB_INFO_IDX_LTC_MASK:
+ return (1U << TU104_LTC_COUNT) - 1;
+ default:
+ return 0;
+ }
+}
+
+static void gsp_rm_handle_fb_get_info_v2(struct gsp_emu_context *ctx,
+ const u8 *req_params,
+ u32 req_params_size,
+ u32 *out_status,
+ u8 *out_params,
+ u32 *out_params_size)
+{
+ struct tu104_fb_get_info_v2_params *reply =
+ (struct tu104_fb_get_info_v2_params *)out_params;
+ u32 count = 0;
+ u32 i;
+
+ if (req_params_size >= sizeof(count))
+ memcpy(&count, req_params, sizeof(count));
+ if (count > TU104_FB_INFO_MAX_LIST_SIZE)
+ count = TU104_FB_INFO_MAX_LIST_SIZE;
+
+ memset(out_params, 0, sizeof(*reply));
+ reply->fbInfoListSize = count;
+
+ for (i = 0; i < count; i++) {
+ const u8 *in = req_params + sizeof(count) +
+ i * sizeof(struct tu104_fb_info_entry);
+ u32 index = 0;
+
+ if (sizeof(count) + (i + 1) * sizeof(struct tu104_fb_info_entry) <=
+ req_params_size)
+ memcpy(&index, in, sizeof(index));
+
+ reply->fbInfoList[i].index = index;
+ reply->fbInfoList[i].data = tu104_fb_info_data(ctx, index);
+ }
+
+ *out_params_size = sizeof(*reply);
+ *out_status = 0;
+
+ nv_vgpu_info(ctx->vgpu, "GSPRM: FB_GET_INFO_V2 -> %u entries populated (first idx=0x%x)",
+ count, count ? reply->fbInfoList[0].index : 0);
+}
+
+/*
+ * BUS_GET_INFO_V2 (0x20801823)
+ *
+ * Same flat index/data list pattern as FB_GET_INFO_V2. UVM's
+ * gpuGetSysmemLinkInfo() asks for SYSMEM_CONNECTION_TYPE and then
+ * PCIE_GPU_LINK_CAPS to compute the sysmem link bandwidth; a zeroed reply
+ * makes getPCIELinkRateMBps() return NV_ERR_INVALID_STATE ("Unsupported
+ * sysmem connection type"). Report a PCIe Gen3 x16 link for the emulated
+ * vGPU and echo every requested index.
+ */
+static u32 tu104_bus_info_data(u32 index)
+{
+ switch (index) {
+ case TU104_BUS_INFO_IDX_TYPE:
+ return TU104_BUS_INFO_TYPE_PCI_EXPRESS;
+ case TU104_BUS_INFO_IDX_INTLINE:
+ return 0;
+ case TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CAPS:
+ case TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CAPS:
+ case TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CAPS:
+ case TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CAPS:
+ case TU104_BUS_INFO_IDX_PCIE_GEN_INFO:
+ return TU104_PCIE_LINK_CAP_GEN3_X16;
+ case TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CTRL_STATUS:
+ case TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CTRL_STATUS:
+ case TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CTRL:
+ case TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CTRL:
+ return TU104_PCIE_LINK_CTRL_STATUS_GEN3_X16;
+ case TU104_BUS_INFO_IDX_SYSMEM_CONNECTION_TYPE:
+ return TU104_BUS_INFO_SYSMEM_CONN_PCIE;
+ default:
+ return 0;
+ }
+}
+
+static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx,
+ const u8 *req_params,
+ u32 req_params_size,
+ u32 *out_status,
+ u8 *out_params,
+ u32 *out_params_size)
+{
+ struct tu104_bus_get_info_v2_params *reply =
+ (struct tu104_bus_get_info_v2_params *)out_params;
+ u32 count = 0;
+ u32 i;
+
+ if (req_params_size >= sizeof(count))
+ memcpy(&count, req_params, sizeof(count));
+ if (count > TU104_BUS_INFO_MAX_LIST_SIZE)
+ count = TU104_BUS_INFO_MAX_LIST_SIZE;
+
+ memset(out_params, 0, sizeof(*reply));
+ reply->busInfoListSize = count;
+
+ for (i = 0; i < count; i++) {
+ const u8 *in = req_params + sizeof(count) +
+ i * sizeof(struct tu104_bus_info_entry);
+ u32 index = 0;
+
+ if (sizeof(count) + (i + 1) * sizeof(struct tu104_bus_info_entry) <=
+ req_params_size)
+ memcpy(&index, in, sizeof(index));
+
+ reply->busInfoList[i].index = index;
+ reply->busInfoList[i].data = tu104_bus_info_data(index);
+ }
+
+ *out_params_size = sizeof(*reply);
+ *out_status = 0;
+
+ nv_vgpu_info(ctx->vgpu, "GSPRM: BUS_GET_INFO_V2 -> %u entries populated (first idx=0x%x)",
+ count, count ? reply->busInfoList[0].index : 0);
+}
+
static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
const u8 *elem, u32 sequence)
{
@@ -1375,6 +1562,20 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx,
reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
break;
+ case TU104_CTRL_FB_GET_INFO_V2:
+ gsp_rm_handle_fb_get_info_v2(ctx,
+ req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
+ &out_status,
+ reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+ break;
+
+ case TU104_CTRL_BUS_GET_INFO_V2:
+ gsp_rm_handle_bus_get_info_v2(ctx,
+ req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize,
+ &out_status,
+ reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size);
+ break;
+
case TU104_CTRL_STATIC_KGR_GET_FLOORSWEEPING_MASKS:
gsp_rm_handle_kgr_get_floorsweeping_masks(ctx, sequence,
&out_status,