diff options
| author | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-09-22 23:32:44 -0400 |
|---|---|---|
| committer | Arthur Hanson Rasmusson <arthur@vgpu.io> | 2026-09-22 23:32:44 -0400 |
| commit | 14283fe73fb5e432a7af45de94961655714910f6 (patch) | |
| tree | e993bf56f3e2ef93195019ac0fd996d82204384a /kernel/gsp_rm_emulation.c | |
| parent | b34e4531d8f0b984dc15660d3ab406ae48822931 (diff) | |
| download | open-iov-14283fe73fb5e432a7af45de94961655714910f6.tar.gz open-iov-14283fe73fb5e432a7af45de94961655714910f6.zip | |
kernel: FB/BUS info-list controls + PCIe link caps for the UVM path
CUDA (cuInit) now reaches UVM's device create: libcuda opens nvidia-uvm,
UVM calls nvGpuOpsDeviceCreate/gpuGetGpuInfo, and the driver fails at
getPCIELinkRateMBps -> calculatePCIELinkRateMBps ("Unknown PCIe speed",
NV_ERR_INVALID_STATE) -> nv_open_device aborts -> cuInit returns
CUDA_ERROR_NOT_INITIALIZED (3).
Implemented along the way:
- FB_GET_INFO_V2 (0x20801303): echo the requested FB info indices and
answer TU104 values (256MB RAM/heap/BAR1, GDDR6, 8 LTCs, 4MB L2, ...).
Fixes the kern_mem_sys_ctrl.c:911 asserts and gives CUDA/UVM real
memory sizes.
- BUS_GET_INFO_V2 (0x20801823): echo indices; PCIe Gen3 x16 link caps /
GEN info / link status, SYSMEM_CONNECTION_TYPE=PCIE. UVM's
gpuGetSysmemLinkInfo reads SYSMEM_CONNECTION_TYPE and the nominal link
bandwidth from here.
- PCIe Link Capabilities in the emulated config space (0x8C/0x92) and in
the BAR0 PCFG mirror (gpuReadBusConfigReg -> NV_PCFG+0x84 = BAR0+
0x88084): MAX_SPEED=3 (8GT/s Gen3), MAX_WIDTH=0x10 (x16); link status
mirror at 0x88088. Also mirrored under the VF-window/low aliases.
- Temporary change-watch on PCFG[0x88084] to see if the guest overwrites
it (it does not - stays 0x103).
Verified: guest nvidia-smi -L still works and reports PCIe Generation
Max/Current = 3. cuInit still fails; the remaining question is which
address the driver's kbifGetGpuLinkCapabilities read sees as zero (the
PCFG shadow holds 0x103). Guest-side kprobe tracing of that function
panicked the VM (free_swap_cache page fault on a corrupted pointer) -
possible guest memory corruption to investigate separately.
Diffstat (limited to 'kernel/gsp_rm_emulation.c')
| -rw-r--r-- | kernel/gsp_rm_emulation.c | 201 |
1 files changed, 201 insertions, 0 deletions
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c index f18d17c..743a260 100644 --- a/kernel/gsp_rm_emulation.c +++ b/kernel/gsp_rm_emulation.c @@ -1269,6 +1269,193 @@ static void gsp_rm_handle_kgr_get_context_buffers_info( TU104_CTX_GPU_MAX_ENGINES, TU104_CTX_ENGINE_ID_COUNT); } +/* + * FB_GET_INFO_V2 (0x20801303) + * + * The client (kern_mem_sys_ctrl.c _fbGetFbInfos) sends a list of FB info + * indices to the physical RM and expects the same indices back with data + * populated. A zeroed reply trips "pFbInfos[i].index == ..." asserts and + * leaves all FB info zero - CUDA/UVM read total/usable memory, BAR1 size, + * L2/LTC topology from here. Echo the requested indices and fill in the + * emulated TU104 values. + */ +static u32 tu104_fb_info_data(struct gsp_emu_context *ctx, u32 index) +{ + struct nv_vgpu_device *vgpu = ctx->vgpu; + u64 fb = vgpu->fb_size ? vgpu->fb_size : NV_VGPU_BAR1_SIZE; + + switch (index) { + case TU104_FB_INFO_IDX_TILE_REGION_COUNT: + return 0; + case TU104_FB_INFO_IDX_COMPRESSION_SIZE: + return 0; + case TU104_FB_INFO_IDX_PARTITION_COUNT: + return 1; + case TU104_FB_INFO_IDX_BAR1_SIZE: + return (u32)fb; + case TU104_FB_INFO_IDX_RAM_SIZE: + case TU104_FB_INFO_IDX_TOTAL_RAM_SIZE: + case TU104_FB_INFO_IDX_USABLE_RAM_SIZE: + return (u32)fb; + case TU104_FB_INFO_IDX_HEAP_SIZE: + case TU104_FB_INFO_IDX_MAPPABLE_HEAP_SIZE: + return (u32)fb; + case TU104_FB_INFO_IDX_BUS_WIDTH: + return 256; + case TU104_FB_INFO_IDX_RAM_TYPE: + return TU104_RAM_TYPE_GDDR6; + case TU104_FB_INFO_IDX_BANK_COUNT: + return 8; + case TU104_FB_INFO_IDX_FB_TAX_SIZE_KB: + case TU104_FB_INFO_IDX_LARGEST_FREE_REGION_SIZE_KB: + return (u32)(fb >> 10); + case TU104_FB_INFO_IDX_PARTITION_MASK: + return 1; + case TU104_FB_INFO_IDX_HEAP_FREE: + return (u32)fb; + case TU104_FB_INFO_IDX_RAM_LOCATION: + return TU104_RAM_LOCATION_GPU_DEDICATED; + case TU104_FB_INFO_IDX_FBP_COUNT: + return 1; + case TU104_FB_INFO_IDX_FBP_MASK: + return 1; + case TU104_FB_INFO_IDX_L2CACHE_SIZE: + return TU104_L2CACHE_SIZE; + case TU104_FB_INFO_IDX_MEMORYINFO_VENDOR_ID: + return TU104_MEMORYINFO_VENDOR_ID_SAMSUNG; + case TU104_FB_INFO_IDX_BAR1_AVAIL_SIZE: + case TU104_FB_INFO_IDX_BAR1_MAX_CONTIG_AVAIL_SIZE: + return (u32)fb; + case TU104_FB_INFO_IDX_LTC_COUNT: + return TU104_LTC_COUNT; + case TU104_FB_INFO_IDX_LTS_COUNT: + return TU104_LTS_COUNT; + case TU104_FB_INFO_IDX_PSEUDO_CHANNEL_MODE: + return TU104_PSEUDO_CHANNEL_MODE_DISABLED; + case TU104_FB_INFO_IDX_ALLOW_PAGE_RETIREMENT: + return 0; + case TU104_FB_INFO_IDX_LTC_MASK: + return (1U << TU104_LTC_COUNT) - 1; + default: + return 0; + } +} + +static void gsp_rm_handle_fb_get_info_v2(struct gsp_emu_context *ctx, + const u8 *req_params, + u32 req_params_size, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + struct tu104_fb_get_info_v2_params *reply = + (struct tu104_fb_get_info_v2_params *)out_params; + u32 count = 0; + u32 i; + + if (req_params_size >= sizeof(count)) + memcpy(&count, req_params, sizeof(count)); + if (count > TU104_FB_INFO_MAX_LIST_SIZE) + count = TU104_FB_INFO_MAX_LIST_SIZE; + + memset(out_params, 0, sizeof(*reply)); + reply->fbInfoListSize = count; + + for (i = 0; i < count; i++) { + const u8 *in = req_params + sizeof(count) + + i * sizeof(struct tu104_fb_info_entry); + u32 index = 0; + + if (sizeof(count) + (i + 1) * sizeof(struct tu104_fb_info_entry) <= + req_params_size) + memcpy(&index, in, sizeof(index)); + + reply->fbInfoList[i].index = index; + reply->fbInfoList[i].data = tu104_fb_info_data(ctx, index); + } + + *out_params_size = sizeof(*reply); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: FB_GET_INFO_V2 -> %u entries populated (first idx=0x%x)", + count, count ? reply->fbInfoList[0].index : 0); +} + +/* + * BUS_GET_INFO_V2 (0x20801823) + * + * Same flat index/data list pattern as FB_GET_INFO_V2. UVM's + * gpuGetSysmemLinkInfo() asks for SYSMEM_CONNECTION_TYPE and then + * PCIE_GPU_LINK_CAPS to compute the sysmem link bandwidth; a zeroed reply + * makes getPCIELinkRateMBps() return NV_ERR_INVALID_STATE ("Unsupported + * sysmem connection type"). Report a PCIe Gen3 x16 link for the emulated + * vGPU and echo every requested index. + */ +static u32 tu104_bus_info_data(u32 index) +{ + switch (index) { + case TU104_BUS_INFO_IDX_TYPE: + return TU104_BUS_INFO_TYPE_PCI_EXPRESS; + case TU104_BUS_INFO_IDX_INTLINE: + return 0; + case TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CAPS: + case TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CAPS: + case TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CAPS: + case TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CAPS: + case TU104_BUS_INFO_IDX_PCIE_GEN_INFO: + return TU104_PCIE_LINK_CAP_GEN3_X16; + case TU104_BUS_INFO_IDX_PCIE_GPU_LINK_CTRL_STATUS: + case TU104_BUS_INFO_IDX_PCIE_ROOT_LINK_CTRL_STATUS: + case TU104_BUS_INFO_IDX_PCIE_UPSTREAM_LINK_CTRL: + case TU104_BUS_INFO_IDX_PCIE_DOWNSTREAM_LINK_CTRL: + return TU104_PCIE_LINK_CTRL_STATUS_GEN3_X16; + case TU104_BUS_INFO_IDX_SYSMEM_CONNECTION_TYPE: + return TU104_BUS_INFO_SYSMEM_CONN_PCIE; + default: + return 0; + } +} + +static void gsp_rm_handle_bus_get_info_v2(struct gsp_emu_context *ctx, + const u8 *req_params, + u32 req_params_size, + u32 *out_status, + u8 *out_params, + u32 *out_params_size) +{ + struct tu104_bus_get_info_v2_params *reply = + (struct tu104_bus_get_info_v2_params *)out_params; + u32 count = 0; + u32 i; + + if (req_params_size >= sizeof(count)) + memcpy(&count, req_params, sizeof(count)); + if (count > TU104_BUS_INFO_MAX_LIST_SIZE) + count = TU104_BUS_INFO_MAX_LIST_SIZE; + + memset(out_params, 0, sizeof(*reply)); + reply->busInfoListSize = count; + + for (i = 0; i < count; i++) { + const u8 *in = req_params + sizeof(count) + + i * sizeof(struct tu104_bus_info_entry); + u32 index = 0; + + if (sizeof(count) + (i + 1) * sizeof(struct tu104_bus_info_entry) <= + req_params_size) + memcpy(&index, in, sizeof(index)); + + reply->busInfoList[i].index = index; + reply->busInfoList[i].data = tu104_bus_info_data(index); + } + + *out_params_size = sizeof(*reply); + *out_status = 0; + + nv_vgpu_info(ctx->vgpu, "GSPRM: BUS_GET_INFO_V2 -> %u entries populated (first idx=0x%x)", + count, count ? reply->busInfoList[0].index : 0); +} + static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, const u8 *elem, u32 sequence) { @@ -1375,6 +1562,20 @@ static void gsp_rm_handle_rm_control(struct gsp_emu_context *ctx, reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); break; + case TU104_CTRL_FB_GET_INFO_V2: + gsp_rm_handle_fb_get_info_v2(ctx, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + + case TU104_CTRL_BUS_GET_INFO_V2: + gsp_rm_handle_bus_get_info_v2(ctx, + req_data + GSP_RM_CTRL_HDR_SIZE, req.paramsSize, + &out_status, + reply + GSP_RM_CTRL_HDR_SIZE, &out_params_size); + break; + case TU104_CTRL_STATIC_KGR_GET_FLOORSWEEPING_MASKS: gsp_rm_handle_kgr_get_floorsweeping_masks(ctx, sequence, &out_status, |
