From cc79b478843f0230fe6aa8ecd822de7ba71c891e Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Sat, 3 Oct 2026 14:05:38 -0400 Subject: vgsp: host CUDA bridge + sema release recovery; greeneye simulator wave - kernel: gsp_cuda_bridge (QMD launch capture, debugfs job/out/done, param+output resolution cached from inline writes), sema drain fixes (never hold/starve a channel), GR ring autopin + counter autoreg, per-page push reads, counter-derive gate; module builds clean - tools/cuda-bridge: CUDA driver-API daemon executing mail/vadd kernels on the physical GPU, CUBB2/CBD2 v2 protocol, layout check + self-tests - tools/guest-diag + tools/cuda-ab: scanner/pin fixes, victory push-only mode - debug/risc-v-gsp-rm-vm: Phase S simulator wave (GSP device model, boot ladder, triage/re-demo harnesses, docs 30-41), GPGPU-sim scaffolding - STATE-CURRENT/docs updates --- kernel/gsp_phys_booter.c | 186 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 159 insertions(+), 27 deletions(-) (limited to 'kernel/gsp_phys_booter.c') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 3d1ccea..bead218 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -440,8 +440,34 @@ bool gsp_phys_boot_requested(void) * carries both PA and VA in one entry, on the non-KMD V5 channel. */ #define GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET 19u +/* + * Mode 20 (2026-10-02, untested): the mode-6 no-entry hVirtMemory promote on + * the V6 KMD-managed channel (SKIP_CTXBUFFER_ALLOC). Mode 6 was refused with + * NV_ERR_INVALID_ARGUMENT (0x1f) on the V5 channel because the server's + * client-promote path rejects the hVirtMemory (pVirtMemDesc) branch unless the + * channel is KMD-managed; V6 sets exactly that flag. The object alloc stays on + * the default external VAS, so the shared-management ctx-buffer map path that + * wedged grsvas2/grsvas3 never runs. The promote asks the server to create the + * virtual-context memdesc/vaList entry for the context window itself, after + * which the KMD commit path can validate ENG_CTX_VALID. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT 20u +/* + * Mode 21 (2026-10-02, untested): the mode-7 no-entry bare (virtAddress, size) + * promote on the V6 KMD-managed channel - the counterpart of mode 20 without + * the hVirtMemory object. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA 21u +/* + * Mode 22 (2026-10-02, untested): install the physical context memdesc with + * GPU_INITIALIZE_CTX (accepted alone in modes 10/11), then send mode 20's + * no-entry hVirtMemory promote. If mode 20's plain 0x40 was the promote's + * physical-memdesc precondition, this combination satisfies it before the + * virtual-memdesc promote. + */ +#define GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT 22u #define GSP_PHYS_RMCALL_GR_PROMOTE_MAX \ - (GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET) + (GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) /* * Work-submit token notifier slot. The default index is @@ -483,8 +509,16 @@ bool gsp_phys_boot_requested(void) */ #define GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW 0u #define GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED BIT(3) +#define GSP_PHYS_RMCALL_VAS_FLAG_SHARED_MANAGEMENT BIT(2) #define GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS BIT(8) #define GSP_PHYS_RMCALL_VAS_SIZE 0x10000000ULL +/* + * Shared-management VAS size: the normal KMD/UVM client passes vaBase=0 and + * rm_va_size = 128 TB on Volta (uvm_volta.c:48-49); the split-VAS server-RM + * carveout math (gpu_vaspace.c:440-470) underflows with a small (256 MB) + * size and asserts the alloc with NV_ERR_INVALID_ARGUMENT. + */ +#define GSP_PHYS_RMCALL_VAS_SIZE_SHARED 0x200000000000ULL /* * Turing/TU104 uses the GP10X GMMU root format: the root level has 2 index * bits, i.e. exactly 4 entries covering 2^47 bytes each, and the server caps @@ -2969,6 +3003,16 @@ static void gsp_phys_fault_corroborate(struct gsp_phys_fault_info *out); #define GSP_PHYS_NOCAT_OFF_DIAGBUF 180 #define GSP_PHYS_NOCAT_MIN_PAYLOAD GSP_PHYS_NOCAT_OFF_DIAGBUF #define GSP_PHYS_NOCAT_FULL_PAYLOAD 1208 +/* + * Assert diagnostic buffer (RM_NOCAT_ASSERT_DIAG_BUFFER, g_journal_nvoc.h): + * ten u32 call-stack entries (each already relative to the NOCAT load + * address) followed by a u32 repeat count. The stack entries are the exact + * call chain of the assert, so a dump makes the failing path nameable + * offline without another run. + */ +#define GSP_PHYS_NOCAT_DIAG_STACK 10 +#define GSP_PHYS_NOCAT_OFF_DIAGCOUNT (GSP_PHYS_NOCAT_OFF_DIAGBUF + 4 * GSP_PHYS_NOCAT_DIAG_STACK) +#define GSP_PHYS_NOCAT_DIAG_NEEDED (GSP_PHYS_NOCAT_OFF_DIAGCOUNT + 4) /* Local re-declaration of the reference record, only for the offset asserts. */ struct gsp_phys_nocat_journal_record { @@ -3116,10 +3160,33 @@ static void gsp_phys_nocat_record(const struct gsp_phys_rx_record *rec) gsp_phys_nocat_accumulate(rec_type, error_code, subsystem, bugcheck, source, engine, rec->seq_num); - if (gsp_phys_nocat_param) + if (gsp_phys_nocat_param) { + u32 diag_len = 0; + u32 count = 0; + u32 i; + char stack[GSP_PHYS_NOCAT_DIAG_STACK * 12]; + size_t pos = 0; + pr_info("vgsp-nocat: rec seq=%u recType=%u errorCode=0x%llx subsystem=0x%x bugcheck=0x%x payload=%u source=\"%s\" faultingEngine=\"%s\"\n", rec->seq_num, rec_type, error_code, subsystem, bugcheck, len, source, engine); + if (len >= GSP_PHYS_NOCAT_OFF_DIAGBUFLEN + 4) + diag_len = get_unaligned_le32(p + + GSP_PHYS_NOCAT_OFF_DIAGBUFLEN); + if (rec_type == 5 && diag_len >= 4 * GSP_PHYS_NOCAT_DIAG_STACK + 4 && + len >= GSP_PHYS_NOCAT_DIAG_NEEDED) { + for (i = 0; i < GSP_PHYS_NOCAT_DIAG_STACK; i++) { + u32 v = get_unaligned_le32(p + + GSP_PHYS_NOCAT_OFF_DIAGBUF + 4 * i); + pos += scnprintf(stack + pos, sizeof(stack) - pos, + "%s0x%x", i ? "," : "", v); + } + count = get_unaligned_le32(p + + GSP_PHYS_NOCAT_OFF_DIAGCOUNT); + pr_info("vgsp-nocat: rec seq=%u assertCallStack count=%u [%s] (offsets from load base 0x4000000)\n", + rec->seq_num, count, stack); + } + } } static void gsp_phys_nocat_summary_dump(void) @@ -6732,6 +6799,12 @@ static const char *gsp_phys_rmcall_gr_promote_name(void) return "mode 8 shape plus a client FERMI_CONTEXT_SHARE_A (hContextShare) that allocates the GraphicsSubcontext"; case GSP_PHYS_RMCALL_GR_PROMOTE_GUEST_SET: return "stock guest's 9-entry promote set (MAIN 2 MB PA+VA, PATCH/global/FECS/priv maps) on V5"; + case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT: + return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + no-entry hVirtMemory promote"; + case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA: + return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + no-entry bare VA+size promote"; + case GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT: + return "KMD-managed channel (V6 SKIP_CTXBUFFER_ALLOC) + GPU_INITIALIZE_CTX then no-entry hVirtMemory promote"; default: return "off (PROMOTE_CTX never sent)"; } @@ -6907,7 +6980,13 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) boot_stage.rmcall_gr_promote == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM || boot_stage.rmcall_gr_promote == - GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE)) { + GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA || + boot_stage.rmcall_gr_promote == + GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT)) { vars = &gsp_phys_rmcall_channel_variant_skipctx; attempts = 1; } @@ -7843,7 +7922,10 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD || mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_FB || mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_PM || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE) && + mode == GSP_PHYS_RMCALL_GR_PROMOTE_CTXSHARE || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) && boot_stage.rmcall_channel_variant != gsp_phys_rmcall_channel_variant_skipctx.id) { pr_err("vgsp-boot: rm-call: mode %u requires the SKIP_CTXBUFFER_ALLOC channel (V%u %s); winning variant is V%u; no PROMOTE_CTX is sent and the ladder aborts with no retry\n", @@ -7981,17 +8063,21 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) } boot_stage.rmcall_gr_promote_step = 2; } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA) { + mode == GSP_PHYS_RMCALL_GR_PROMOTE_BARE_VA || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_BARE_VA) { /* - * Modes 6/7: the documented no-entry branches. Mode 6 first - * reserves the context window with an NV01_MEMORY_VIRTUAL - * object on the ladder VAS and passes it as hVirtMemory + * Modes 6/7/20: the documented no-entry branches. Modes 6/20 + * first reserve the context window with an NV01_MEMORY_VIRTUAL + * object on the ladder VAS and pass it as hVirtMemory * (hClient is its owner); mode 7 sends the bare * (virtAddress, size) pair with no hVirtMemory. entryCount=0 - * in both, so the server creates the memdesc/vaList entry - * instead of resolving one of our raw-PTE entries. + * in all, so the server creates the memdesc/vaList entry + * instead of resolving one of our raw-PTE entries. Mode 20 is + * the same shape as mode 6 but on the V6 KMD-managed channel. */ - if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT) { + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) { ret = gsp_phys_rmcall_alloc_gr_hvirt(bar0); if (ret) { kvfree(p); @@ -8009,11 +8095,13 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, p, sizeof(*p), 24, NULL, 0, &rpc_result, &status, - mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ? + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) ? "GPU_PROMOTE_CTX (hVirtMemory)" : "GPU_PROMOTE_CTX (bare VA+size)"); pr_info("vgsp-boot: rm-call: PROMOTE_CTX %s: entryCount=0 hVirtMemory=0x%x virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hClient=0x%x hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n", - mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ? + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) ? "hVirtMemory" : "bare VA+size", le32_to_cpu(p->h_virt_memory), boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size, @@ -8022,7 +8110,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) GSP_PHYS_RMCALL_ENGINE_GR0, mode); if (ret) { pr_err("vgsp-boot: rm-call: PROMOTE_CTX %s failed (status=0x%x rpc_result=0x%x, %s, ret=%d); the GR context is not committed and the ladder aborts before BIND with no retry\n", - mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT ? + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_HVIRT || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_HVIRT) ? "hVirtMemory" : "bare VA+size", status, rpc_result, gsp_phys_rmcall_status_str(status), ret); @@ -8063,9 +8152,10 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) } boot_stage.rmcall_gr_promote_step = 1; } else if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_VA || - mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB) { + mode == GSP_PHYS_RMCALL_GR_PROMOTE_OLDIFACE_FB || + mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) { /* - * Modes 10/11: the old-interface client-managed flow. Step 1 + * Modes 10/11/22: the old-interface client-managed flow. Step 1 * sends GPU_INITIALIZE_CTX (0x2080012d) with the physical * context buffer, which installs the PA/size/physAttr * memdesc on the GraphicsContext (grctxInitializeAPI) that @@ -8076,7 +8166,8 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) * entryCount=0 and the bare (virtAddress, size) pair; the * server builds the virtual memdesc itself and, on the V6 * KMD-managed channel, binds it without the VA/bookkeeping - * resolve. + * resolve. Mode 22 is mode 10's step 1 followed by mode 20's + * hVirtMemory no-entry promote instead of the bare pair. */ struct gsp_phys_rmcall_init_ctx_params *ic; u32 ic_phys_attr = (mode == @@ -8126,6 +8217,16 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) p->virt_address = cpu_to_le64(boot_stage.rmcall_ctx_va); p->size = cpu_to_le64(boot_stage.rmcall_ctx_size); p->entry_count = cpu_to_le32(0); + if (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) { + ret = gsp_phys_rmcall_alloc_gr_hvirt(bar0); + if (ret) { + kvfree(p); + return ret; + } + p->h_client = cpu_to_le32(boot_stage.rmcall_client); + p->h_virt_memory = + cpu_to_le32(boot_stage.rmcall_gr_hvirt); + } rpc_result = 0; status = 0; ret = gsp_phys_rmcall_control(bar0, @@ -8133,14 +8234,21 @@ static int gsp_phys_rmcall_gr_ctx_promote(void __iomem *bar0) GSP_PHYS_RMCALL_CTRL_GPU_PROMOTE_CTX, p, sizeof(*p), 24, NULL, 0, &rpc_result, &status, + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ? + "GPU_PROMOTE_CTX (hVirtMemory after init)" : "GPU_PROMOTE_CTX (KMD bare VA+size after init)"); - pr_info("vgsp-boot: rm-call: PROMOTE_CTX bare VA+size after init: entryCount=0 virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n", + pr_info("vgsp-boot: rm-call: PROMOTE_CTX %s: entryCount=0 hVirtMemory=0x%x virtAddress=0x%llx size=0x%x status=0x%x rpc_result=0x%x (hObject=0x%x hChanClient=0x%x engineType=0x%x gr_promote_param=%u)\n", + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ? + "hVirtMemory after init" : "bare VA+size after init", + le32_to_cpu(p->h_virt_memory), boot_stage.rmcall_ctx_va, boot_stage.rmcall_ctx_size, status, rpc_result, boot_stage.rmcall_channel, boot_stage.rmcall_client, GSP_PHYS_RMCALL_ENGINE_GR0, mode); if (ret) { - pr_err("vgsp-boot: rm-call: PROMOTE_CTX bare VA+size failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: INITIALIZE_CTX accepted (1/2); the GR context is not committed and the ladder aborts before BIND with no retry\n", + pr_err("vgsp-boot: rm-call: PROMOTE_CTX %s failed (status=0x%x rpc_result=0x%x, %s, ret=%d); last completed promote step: INITIALIZE_CTX accepted (1/2); the GR context is not committed and the ladder aborts before BIND with no retry\n", + (mode == GSP_PHYS_RMCALL_GR_PROMOTE_KMD_INIT_HVIRT) ? + "hVirtMemory after init" : "bare VA+size", status, rpc_result, gsp_phys_rmcall_status_str(status), ret); kvfree(p); @@ -8632,17 +8740,41 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) gsp_phys_rmcall_gr_promote_snapshot(); /* - * The TSG below carries this externally-owned FERMI_VASPACE_A, whose - * root page directory is committed from the zeroed VASPDB chunk before - * any channel alloc (the instance-block setup programs that base and - * fails 0x40 when it is NULL, kern_gmmu_gp100.c:150-152). Either call - * aborts the ladder: the variant walk cannot succeed without them. + * The TSG below carries this explicitly allocated FERMI_VASPACE_A, + * whose root page directory is committed from the zeroed VASPDB chunk + * before any channel alloc (the instance-block setup programs that + * base and fails 0x40 when it is NULL, kern_gmmu_gp100.c:150-152). + * Either call aborts the ladder: the variant walk cannot succeed + * without them. + * + * History (2026-10-02): the VAS was allocated IS_EXTERNALLY_OWNED to + * mirror the guest's UVM-owned address space. That flag tells the RM + * that the client owns all page tables and VA/context commitment, so + * the server never books the GR engine-context VA itself and every + * promote either asserts ("Externally owned object not found", + * kchannelMapEngineCtxBuf) or silently leaves the channel + * CTXNOTVALID. The normal KMD client instead allocates its VAS with + * SHARED_MANAGEMENT (nv_gpu_ops.c:2495): the client provides the root + * PDB (SET_PAGE_DIRECTORY still legal, gpu_vaspace.c:3108) while the + * RM manages the lower page tables and maps/commits engine contexts. + * gsp_phys_rmcall_vas_shared=1 selects that variant; the default 0 + * keeps the byte-identical external path. */ memset(&vas, 0, sizeof(vas)); vas.index = cpu_to_le32(GSP_PHYS_RMCALL_VAS_INDEX_GPU_NEW); - vas.flags = cpu_to_le32(GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED | - GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS); - vas.va_size = cpu_to_le64(GSP_PHYS_RMCALL_VAS_SIZE); + vas.flags = cpu_to_le32(gsp_phys_rmcall_vas_shared_value() ? + (GSP_PHYS_RMCALL_VAS_FLAG_SHARED_MANAGEMENT | + GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS) : + (GSP_PHYS_RMCALL_VAS_FLAG_IS_EXTERNALLY_OWNED | + GSP_PHYS_RMCALL_VAS_FLAG_ALLOW_ZERO_ADDRESS)); + vas.va_size = cpu_to_le64(gsp_phys_rmcall_vas_shared_value() ? + GSP_PHYS_RMCALL_VAS_SIZE_SHARED : + GSP_PHYS_RMCALL_VAS_SIZE); + pr_info("vgsp-boot: rm-call: FERMI_VASPACE_A flags=0x%x va_size=0x%llx (%s)\n", + le32_to_cpu(vas.flags), le64_to_cpu(vas.va_size), + gsp_phys_rmcall_vas_shared_value() ? + "shared management (RM owns page tables/commit)" : + "externally owned (client owns page tables/commit)"); ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, boot_stage.rmcall_device, GSP_PHYS_RMCALL_HANDLE_VAS, -- cgit v1.2.3