From 2f70923aaa88bcfb1fd6fef8d5c123f85951cf6c Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 1 Oct 2026 20:30:44 -0400 Subject: gsp-phys: build the channel page directory and submit the CE copy through GPU VAs Fill the external VAS root's GP10X v2 hierarchy (root/PD2/PD1/PD0-small/PT) around a 2 MB VA window so the PDBMA/CE can fetch the GPFIFO and pushbuffer by GPU VA; gpFifoOffset and the GPFIFO ring entry GET now carry those VAs. Allocate the USERD page in FBMEM (class 0x40, matching the reference default) with the sysmem chunk as fallback, and add the advisory TSG-level GPFIFO_SCHEDULE 0xa06c0101 after the channel BIND/schedule. Log the window, table DMAs and entries, and expose them in the debugfs state snapshot. --- kernel/gsp_phys_booter.c | 394 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 366 insertions(+), 28 deletions(-) (limited to 'kernel/gsp_phys_booter.c') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index fa2d3b4..5565159 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -228,6 +228,7 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX 0xc36f010au #define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE 0xa06f0103u #define GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL 0xa06f0104u +#define GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE 0xa06c0101u #define GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY 0x801813u #define GSP_PHYS_RMCALL_ENGINE_COPY0 0x9u @@ -285,6 +286,19 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_PDB_FLAG_ALL_CHANNELS BIT(3) #define GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID 1u +/* + * Channel page-directory GPU VA window. The GPFIFO and the pushbuffer are + * fetched by GPU VA (the GPFIFO from gpFifoOffset, the pushbuffer from the + * GPFIFO ring entry GET), so the channel's external root must translate them. + * Both pages live in one 2 MB PD0 slot below the VAS vaSize limit: with + * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0 + * and the two 4 KB pages take small-PT slots 0x100/0x101 + * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). + */ +#define GSP_PHYS_RMCALL_VA_BASE 0x00100000ULL +#define GSP_PHYS_RMCALL_VA_GPFIFO GSP_PHYS_RMCALL_VA_BASE +#define GSP_PHYS_RMCALL_VA_PB (GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE) + /* * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 = * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188 @@ -330,6 +344,7 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 0xcaf00018u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5 0xcaf00024u #define GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM 0xcaf00028u +#define GSP_PHYS_RMCALL_HANDLE_USERD_MEM 0xcaf0002cu #define GSP_PHYS_RMCALL_HANDLE_MEM_BASE 0xcaf01000u #define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE 4u @@ -362,6 +377,15 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_RAMFC, GSP_PHYS_RMCALL_BUF_MTHDBUF, GSP_PHYS_RMCALL_BUF_VASPDB, + /* + * Client-owned channel page-directory hierarchy behind the external + * root: PD2/PD1 single 8 B PDE levels, PD0 16 B dual PDE and the 4 KB + * small-page PT (GP10X v2, kern_gmmu_fmt_gp10x.c:16-83). + */ + GSP_PHYS_RMCALL_BUF_PD2, + GSP_PHYS_RMCALL_BUF_PD1, + GSP_PHYS_RMCALL_BUF_PD0, + GSP_PHYS_RMCALL_BUF_PT, GSP_PHYS_RMCALL_BUF_COUNT }; @@ -904,6 +928,18 @@ static struct gsp_phys_boot_stage { u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ u64 rmcall_notifier_base; /* its returned frame-buffer address */ bool rmcall_notifier_valid; /* the FBMEM notifier alloc succeeded */ + u32 rmcall_userd_mem; /* FBMEM USERD object handle */ + u64 rmcall_userd_base; /* its returned frame-buffer address */ + bool rmcall_userd_valid; /* the FBMEM USERD alloc succeeded */ + u64 rmcall_va_base; /* channel PDB VA window base */ + u64 rmcall_va_gpfifo; /* GPFIFO GPU VA in that window */ + u64 rmcall_va_pb; /* pushbuffer GPU VA in that window */ + u64 rmcall_pdb_root; /* root entry -> PD2 */ + u64 rmcall_pdb_pd2; /* PD2 entry -> PD1 */ + u64 rmcall_pdb_pd1; /* PD1 entry -> PD0 */ + u64 rmcall_pdb_pd0_small; /* PD0 dual entry, small (4 KB) half */ + u64 rmcall_pt_gpfifo; /* small-PT entry for the GPFIFO page */ + u64 rmcall_pt_pb; /* small-PT entry for the PB page */ struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; @@ -1419,6 +1455,18 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_notifier_mem = 0; boot_stage.rmcall_notifier_base = 0; boot_stage.rmcall_notifier_valid = false; + boot_stage.rmcall_userd_mem = 0; + boot_stage.rmcall_userd_base = 0; + boot_stage.rmcall_userd_valid = false; + boot_stage.rmcall_va_base = 0; + boot_stage.rmcall_va_gpfifo = 0; + boot_stage.rmcall_va_pb = 0; + boot_stage.rmcall_pdb_root = 0; + boot_stage.rmcall_pdb_pd2 = 0; + boot_stage.rmcall_pdb_pd1 = 0; + boot_stage.rmcall_pdb_pd0_small = 0; + boot_stage.rmcall_pt_gpfifo = 0; + boot_stage.rmcall_pt_pb = 0; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; boot_stage.rmcall_mem_registered[i] = false; @@ -4736,6 +4784,83 @@ static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0) return 0; } +/* + * Allocate the channel's USERD page in FBMEM. Same class-0x40 shape as the + * error-notifier allocation above (owner HEAP_OWNER_RM_CLIENT_GENERIC, type + * IMAGE, VIDMEM|UNCACHED, SKIP_SCRUB); the reference defaults USERD to + * VIDMEM (mem_utils_gm107.c:792-804,1132-1161) and the client push path + * allocates NV01_MEMORY_LOCAL_USER with these flags (nvidia-push-init.c: + * 515-554). The server returns the allocated FB offset in + * NV_MEMORY_ALLOCATION_PARAMS.offset, which userdMem.base carries with + * addressSpace FBMEM. + * + * Failure is non-fatal: the channel params then fall back to the sysmem + * USERD chunk and log the fallback. + */ +static int gsp_phys_rmcall_alloc_userd_mem(void __iomem *bar0) +{ + struct gsp_phys_rmcall_memory_alloc_params mem; + struct gsp_phys_rmcall_rm_alloc_req *req; + struct gsp_phys_rx_record rec; + u32 req_len = sizeof(*req) + sizeof(mem); + u32 status; + u64 base; + int ret; + + memset(&mem, 0, sizeof(mem)); + mem.owner = cpu_to_le32(GSP_PHYS_RMCALL_HEAP_OWNER_RM_CLIENT_GENERIC); + mem.type = cpu_to_le32(GSP_PHYS_RMCALL_OS32_TYPE_IMAGE); + mem.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE); + mem.attr = cpu_to_le32(GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED); + mem.internalflags = + cpu_to_le32(GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB); + + req = kvzalloc(req_len, GFP_KERNEL); + if (!req) + return -ENOMEM; + req->h_client = cpu_to_le32(boot_stage.rmcall_client); + req->h_parent = cpu_to_le32(boot_stage.rmcall_device); + req->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_USERD_MEM); + req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER); + req->params_size = cpu_to_le32(sizeof(mem)); + memcpy(req->params, &mem, sizeof(mem)); + + ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC, + req, req_len, 20, &rec, true, NULL); + kvfree(req); + if (ret) { + pr_err("gsp-phys-boot: rm-call: FBMEM USERD alloc (hMemory=0x%x class=0x40 size=0x%x) RPC failed: %d\n", + GSP_PHYS_RMCALL_HANDLE_USERD_MEM, + GSP_PHYS_RMCALL_COPY_SIZE, ret); + gsp_phys_rx_record_free(&rec); + return ret; + } + + status = gsp_phys_rmcall_reply_u32(&rec, 16); + if (status != 0 || + rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset) + 8) { + pr_err("gsp-phys-boot: rm-call: FBMEM USERD alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n", + status, rec.payload_len, + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset) + 8); + gsp_phys_rx_record_free(&rec); + return -EIO; + } + base = get_unaligned_le64(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 32 + + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset)); + gsp_phys_rx_record_free(&rec); + + boot_stage.rmcall_userd_mem = GSP_PHYS_RMCALL_HANDLE_USERD_MEM; + boot_stage.rmcall_userd_base = base; + boot_stage.rmcall_userd_valid = true; + pr_info("gsp-phys-boot: rm-call: FBMEM USERD allocated: hMemory=0x%x class=0x40 size=0x%x FB base=0x%llx\n", + boot_stage.rmcall_userd_mem, GSP_PHYS_RMCALL_COPY_SIZE, + boot_stage.rmcall_userd_base); + return 0; +} + /* * One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). out_status (optional) @@ -4916,7 +5041,7 @@ static int gsp_phys_rmcall_stage2(void __iomem *bar0) } } - pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb zeroed)\n", + pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt zeroed)\n", GSP_PHYS_RMCALL_BUF_COUNT); return 0; } @@ -5005,11 +5130,31 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch, } if (v->server_mem) return; /* zeroed memdescs, gpFifoOffset/gpFifoEntries 0 */ - ch->gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma( - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0)); + /* + * The PBDMA/CE fetch the GPFIFO by GPU VA through the channel PDB + * (NV_RAMFC_GP_BASE from gpFifoOffset, fifo_g100.c:1120-1137), so the + * offset is the VA built in gsp_phys_rmcall_pdb_build, not the host DMA. + */ + ch->gp_fifo_offset = cpu_to_le64(GSP_PHYS_RMCALL_VA_GPFIFO); ch->gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES); ch->instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE); - ch->userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD); + /* + * USERD is the reference default FBMEM page (mem_utils_gm107.c: + * 792-804,1132-1161; nvidia-push-init.c:515-554) when stage 3's + * class-0x40 allocation succeeded; the sysmem chunk remains the + * fallback when it did not. + */ + if (boot_stage.rmcall_userd_valid) { + ch->userd_mem.base = cpu_to_le64(boot_stage.rmcall_userd_base); + ch->userd_mem.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE); + ch->userd_mem.address_space = + cpu_to_le32(GSP_PHYS_RMCALL_ADDR_FBMEM); + ch->userd_mem.cache_attrib = + cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED); + } else { + ch->userd_mem = + gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD); + } ch->ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC); ch->mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF); } @@ -5077,6 +5222,118 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) return ret; } +/* + * GP10X GMMU (format version 2) sysmem-noncoherent PDE/PTE encoders. The + * single PDE is IS_PDE=0 (bit 0), APERTURE=3 in bits 2:1, VOL=1 (bit 3) and + * the physical page number in bits 53:8 (ADDRESS_SYS, shift 12); the leaf PTE + * adds VALID=1 (bit 0) with KIND=PITCH (bits 63:56 zero). The PD0 dual entry + * stores the big (64K/2M) half in qword 0 and the small (4K) half in qword 1 + * with the same aperture/address fields at the same bit positions + * (swref/published/pascal/gp100/dev_mmu.h:97-119,157; + * uvm_pascal_mmu.c:64-141,235-303). + */ +static u64 gsp_phys_rmcall_pde(u64 pa) +{ + return 0xeull | ((pa >> 12) << 8); +} + +static u64 gsp_phys_rmcall_pte(u64 pa) +{ + return 0xfull | ((pa >> 12) << 8); +} + +/* + * Build the channel's page-directory hierarchy around the 2 MB VA window + * GSP_PHYS_RMCALL_VA_BASE that maps the GPFIFO and pushbuffer sysmem pages: + * + * root/PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 dual [28:21] + * -> small PT [20:12] + * + * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,143-161). The window + * puts every parent index at 0 and the two pages at small-PT slots + * 0x100/0x101, so only root[0], PD2[0], PD1[0], the PD0[0] small half and + * those two PT slots are written; the root chunk itself is the + * DMA_SET_PAGE_DIRECTORY payload. The external root merely stores the + * memdesc, so filling before the channel alloc (after a successful + * SET_PAGE_DIRECTORY) is equivalent to filling before the control. + */ +static void gsp_phys_rmcall_pdb_build(void) +{ + struct gsp_phys_boot_chunk *root = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB]; + struct gsp_phys_boot_chunk *pd2 = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD2]; + struct gsp_phys_boot_chunk *pd1 = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD1]; + struct gsp_phys_boot_chunk *pd0 = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD0]; + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + u32 gpfifo_idx = (u32)((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ffu); + u32 pb_idx = (u32)((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ffu); + u64 root_ent, pd2_ent, pd1_ent, pd0_small, pt_gpfifo, pt_pb; + + root_ent = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(pd2, 0)); + pd2_ent = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(pd1, 0)); + pd1_ent = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(pd0, 0)); + pd0_small = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(pt, 0)); + pt_gpfifo = gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], + 0)); + pt_pb = gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0)); + + put_unaligned_le64(root_ent, gsp_phys_chunk_page_va(root, 0)); + put_unaligned_le64(pd2_ent, gsp_phys_chunk_page_va(pd2, 0)); + put_unaligned_le64(pd1_ent, gsp_phys_chunk_page_va(pd1, 0)); + /* 16 B dual PDE: big (64K/2M) half unused, small half -> 4 KB PT. */ + put_unaligned_le64(0, gsp_phys_chunk_page_va(pd0, 0)); + put_unaligned_le64(pd0_small, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + 8); + put_unaligned_le64(pt_gpfifo, + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)gpfifo_idx * 8); + put_unaligned_le64(pt_pb, + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb_idx * 8); + dma_wmb(); + + boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE; + boot_stage.rmcall_va_gpfifo = GSP_PHYS_RMCALL_VA_GPFIFO; + boot_stage.rmcall_va_pb = GSP_PHYS_RMCALL_VA_PB; + boot_stage.rmcall_pdb_root = root_ent; + boot_stage.rmcall_pdb_pd2 = pd2_ent; + boot_stage.rmcall_pdb_pd1 = pd1_ent; + boot_stage.rmcall_pdb_pd0_small = pd0_small; + boot_stage.rmcall_pt_gpfifo = pt_gpfifo; + boot_stage.rmcall_pt_pb = pt_pb; + + pr_info("gsp-phys-boot: rm-call: channel PDB built: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx gpFifoOffset=0x%llx root dma=0x%llx pd2 dma=0x%llx pd1 dma=0x%llx pd0 dma=0x%llx pt dma=0x%llx\n", + GSP_PHYS_RMCALL_VA_BASE, GSP_PHYS_RMCALL_VA_GPFIFO, + GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO, + (u64)gsp_phys_chunk_page_dma(root, 0), + (u64)gsp_phys_chunk_page_dma(pd2, 0), + (u64)gsp_phys_chunk_page_dma(pd1, 0), + (u64)gsp_phys_chunk_page_dma(pd0, 0), + (u64)gsp_phys_chunk_page_dma(pt, 0)); + pr_info("gsp-phys-boot: rm-call: channel PDB entries: root[0]=0x%016llx pd2[0]=0x%016llx pd1[0]=0x%016llx pd0[0].small=0x%016llx\n", + root_ent, pd2_ent, pd1_ent, pd0_small); + pr_info("gsp-phys-boot: rm-call: channel PDB PT: pt[%u]=0x%016llx (gpfifo dma=0x%llx) pt[%u]=0x%016llx (pb dma=0x%llx)\n", + gpfifo_idx, pt_gpfifo, + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], + 0), + pb_idx, pt_pb, + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0)); +} + /* * Stage 3: explicit VAS with its page directory, then channel group (COPY0, * physical) and the channel. The VAS and PDB must both succeed: a channel @@ -5095,7 +5352,10 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) * requested here: its generator requires BIND + GPFIFO_SCHEDULE (runlist * membership, kernel_fifo_tu102.c:110-116), so the whole BIND -> SCHEDULE -> * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN order lives in - * stage 4. + * stage 4. After SET_PAGE_DIRECTORY the root's lower levels are built + * (gsp_phys_rmcall_pdb_build), because the engine fetches the GPFIFO and the + * pushbuffer by GPU VA through that hierarchy; the USERD page is allocated in + * FBMEM alongside the notifier. */ static int gsp_phys_rmcall_stage3(void __iomem *bar0) { @@ -5151,6 +5411,12 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) pdb_rpc); return ret; } + /* + * The server only stored our root memdesc; the GPFIFO and pushbuffer + * fetches translate through it, so fill the hierarchy before the + * channel alloc commits the instance block's page-directory base. + */ + gsp_phys_rmcall_pdb_build(); memset(&tsg, 0, sizeof(tsg)); tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); @@ -5185,15 +5451,26 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) pr_warn("gsp-phys-boot: rm-call: FBMEM error-notifier allocation failed (%d); channel allocs carry a zeroed notifier memdesc and only the stage-4 token recovery can save the token\n", ret); + /* + * Allocate the USERD page in FBMEM as well (reference default); on + * failure the channel variants fall back to the sysmem USERD chunk. + */ + ret = gsp_phys_rmcall_alloc_userd_mem(bar0); + if (ret) + pr_warn("gsp-phys-boot: rm-call: FBMEM USERD allocation failed (%d); channel allocs fall back to the sysmem USERD chunk\n", + ret); + ret = gsp_phys_rmcall_alloc_channel(bar0); if (ret) return ret; - pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n", + pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=%s base=0x%llx engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n", boot_stage.rmcall_vas, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant, boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", - boot_stage.rmcall_notifier_base); + boot_stage.rmcall_notifier_base, + boot_stage.rmcall_userd_valid ? "FBMEM" : "sysmem", + boot_stage.rmcall_userd_base); return 0; } @@ -5254,21 +5531,26 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_dma, u64 src_dma, /* * Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE -> - * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN - then the first - * real work item: write the CE method block to the pushbuffer, publish the - * GPFIFO entry and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the - * PF channel doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem - * semaphore word. Afterwards dst must equal src byte for byte. + * TSG_GPFIFO_SCHEDULE -> SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> + * GET_WORK_SUBMIT_TOKEN - then the first real work item: write the CE method + * block to the pushbuffer, publish the GPFIFO entry (GET = the pushbuffer GPU + * VA) and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the PF channel + * doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem semaphore word. + * Afterwards dst must equal src byte for byte. * * BIND is the precondition the TU102 token generator needs (runlist * membership, kernel_fifo_tu102.c:110-116; GPFIFO_SCHEDULE alone only sets * RUNLIST_SET at kernel_channel.c:3244), and SET_NOTIF_INDEX (index 3 for * subdevice 0, nvidia-push-init.c:303-316) gives the FBMEM notifier write a * target slot; it is advisory here because no error-context object exists. - * Every other control failure logs its rpc_result/status and aborts - * before the doorbell; a 0x40 GET_WORK_SUBMIT_TOKEN whose echoed params carry - * a nonzero token still proceeds (the server writes the token before the - * notifier update, kernel_channel.c:3471-3477). + * The TSG-level NVA06C_CTRL_CMD_GPFIFO_SCHEDULE 0xa06c0101 ("schedules a + * channel group in hardware", ctrla06c.h:82-101) is insurance on top of the + * channel-level schedule, whose internal HAL is a stub in the shipped + * dispatch; it too is advisory. Every other control failure logs its + * rpc_result/status and aborts before the doorbell; a 0x40 + * GET_WORK_SUBMIT_TOKEN whose echoed params carry a nonzero token still + * proceeds (the server writes the token before the notifier update, + * kernel_channel.c:3471-3477). */ static int gsp_phys_rmcall_stage4(void __iomem *bar0) { @@ -5339,6 +5621,30 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) return ret; } + /* + * (b2) TSG-level NVA06C_CTRL_CMD_GPFIFO_SCHEDULE (0xa06c0101, the + * "schedules a channel group in hardware" call, ctrla06c.h:82-101, + * same 2-byte NVA06F params) as insurance: the channel-level + * GPFIFO_SCHEDULE's internal HAL is a stub in the shipped dispatch, so + * the runlist entry is only fully committed by the group schedule. + * Advisory, not fatal: the channel-level schedule already set + * RUNLIST_SET, which is what the token generator consumes + * (kernel_channel.c:3237-3245). + */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, + NULL, 0, &rpc_result, &status, + "TSG_GPFIFO_SCHEDULE"); + if (ret) + pr_warn("gsp-phys-boot: rm-call: TSG_GPFIFO_SCHEDULE (0xa06c0101 bEnable=1 bSkipSubmit=0 on hTsg=0x%x) advisory failure: rpc_result=0x%x status=0x%x; the channel-level schedule already set RUNLIST_SET, continuing\n", + boot_stage.rmcall_tsg, rpc_result, status); + else + pr_info("gsp-phys-boot: rm-call: TSG_GPFIFO_SCHEDULE 0xa06c0101 on hTsg=0x%x accepted (rpc_result=0x%x status=0x%x)\n", + boot_stage.rmcall_tsg, rpc_result, status); + /* * (c) SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (nvidia-push-init.c:303-316). * Advisory, not fatal: with no error-context object (hErrorContext=0) @@ -5412,18 +5718,25 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) slot = gpput; /* - * e0: GET = pushbuffer address >> 2 in bits 31:2, FETCH/NO_CONTEXT_ - * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the address high - * dword and LENGTH 30:10 in dwords, exactly as the reference RM's own - * submission does it (mem_utils_gm107.c:1861-1868: NvU64_HI32(get) in - * GET_HI, NvU64_LO32(get) >> 2 in GET). The physical channel model - * addresses the ring and pushbuffer by their system DMA addresses. + * e0: GET = pushbuffer GPU VA >> 2 in bits 31:2, FETCH/NO_CONTEXT_ + * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the VA high byte + * and LENGTH 30:10 in dwords, exactly the reference RM's own submission + * (mem_utils_gm107.c:1858-1868: get = pbGpuVA + offset, NvU64_HI32(get) + * in GET_HI, NvU64_LO32(get) >> 2 in GET, PRIV USER and LEVEL MAIN). + * The engine translates this VA through the channel PDB built in + * gsp_phys_rmcall_pdb_build. */ - e0 = (u32)(pb_dma >> 2) & 0x3fffffffu; - e1 = (u32)((pb_dma >> 32) & 0xffu) | (words << 10); + e0 = (u32)(GSP_PHYS_RMCALL_VA_PB >> 2) & 0x3fffffffu; + e1 = (u32)((GSP_PHYS_RMCALL_VA_PB >> 32) & 0xffu) | (words << 10); put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4); dma_wmb(); + /* + * GP_PUT stays on the client sysmem USERD chunk, which is the hardware + * USERD only on the FBMEM-alloc fallback path; when the server used the + * FBMEM USERD page this host write cannot reach it (no BAR1/PRAMIN + * aperture here), kept as-is for the fallback. + */ put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); /* @@ -5442,8 +5755,10 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) wmb(); writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF); - pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", - pb_dma, words, slot, gpget, gpput, e0, e1, + pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx userd=%s words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", + pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO, + boot_stage.rmcall_userd_valid ? "FBMEM" : "sysmem", + words, slot, gpget, gpput, e0, e1, boot_stage.rmcall_token, runlist_id, ch_id, doorbell, GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma, GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS); @@ -5730,6 +6045,17 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_token, boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", boot_stage.rmcall_notifier_base); + seq_printf(m, + "rm_call_userd: userd=%s hMemory=0x%x base=0x%llx | rm_call_pdb: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx\n", + boot_stage.rmcall_userd_valid ? "FBMEM" : "sysmem", + boot_stage.rmcall_userd_mem, boot_stage.rmcall_userd_base, + boot_stage.rmcall_va_base, boot_stage.rmcall_va_gpfifo, + boot_stage.rmcall_va_pb); + seq_printf(m, + "rm_call_pdb_entries: root=0x%016llx pd2=0x%016llx pd1=0x%016llx pd0Small=0x%016llx ptGpfifo=0x%016llx ptPb=0x%016llx\n", + boot_stage.rmcall_pdb_root, boot_stage.rmcall_pdb_pd2, + boot_stage.rmcall_pdb_pd1, boot_stage.rmcall_pdb_pd0_small, + boot_stage.rmcall_pt_gpfifo, boot_stage.rmcall_pt_pb); seq_printf(m, "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n", wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0), @@ -6625,12 +6951,24 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 10, - "the rm-call ladder registers exactly ten DMA buffers"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 14, + "the rm-call ladder registers exactly fourteen DMA buffers"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, "rm-call memory handles must stay inside the RM client handle range"); +static_assert(GSP_PHYS_RMCALL_VA_PB + GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_VAS_SIZE, + "the channel PDB VA window must stay below the VAS vaSize limit"); +static_assert(((GSP_PHYS_RMCALL_VA_BASE >> 47) & 0x3) == 0 && + ((GSP_PHYS_RMCALL_VA_BASE >> 38) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_VA_BASE >> 29) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_VA_BASE >> 21) & 0xff) == 0 && + ((GSP_PHYS_RMCALL_VA_PB >> 21) & 0xff) == 0, + "the channel PDB VA window must sit in root/PD2/PD1/PD0 index 0 (GP10X 48:47/46:38/37:29/28:21 splits)"); +static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) != + ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff), + "the GPFIFO and pushbuffer VAs must take distinct small-PT slots"); /* * GspSystemInfo wire ABI, pinned against the reference header compile -- cgit v1.2.3