diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/gsp_phys_booter.c | 394 |
1 files changed, 366 insertions, 28 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index fa2d3b4..5565159 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -228,6 +228,7 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX 0xc36f010au #define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE 0xa06f0103u #define GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL 0xa06f0104u +#define GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE 0xa06c0101u #define GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY 0x801813u #define GSP_PHYS_RMCALL_ENGINE_COPY0 0x9u @@ -286,6 +287,19 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID 1u /* + * Channel page-directory GPU VA window. The GPFIFO and the pushbuffer are + * fetched by GPU VA (the GPFIFO from gpFifoOffset, the pushbuffer from the + * GPFIFO ring entry GET), so the channel's external root must translate them. + * Both pages live in one 2 MB PD0 slot below the VAS vaSize limit: with + * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0 + * and the two 4 KB pages take small-PT slots 0x100/0x101 + * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204). + */ +#define GSP_PHYS_RMCALL_VA_BASE 0x00100000ULL +#define GSP_PHYS_RMCALL_VA_GPFIFO GSP_PHYS_RMCALL_VA_BASE +#define GSP_PHYS_RMCALL_VA_PB (GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE) + +/* * Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 = * NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188 * internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only: @@ -330,6 +344,7 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 0xcaf00018u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5 0xcaf00024u #define GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM 0xcaf00028u +#define GSP_PHYS_RMCALL_HANDLE_USERD_MEM 0xcaf0002cu #define GSP_PHYS_RMCALL_HANDLE_MEM_BASE 0xcaf01000u #define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE 4u @@ -362,6 +377,15 @@ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_RAMFC, GSP_PHYS_RMCALL_BUF_MTHDBUF, GSP_PHYS_RMCALL_BUF_VASPDB, + /* + * Client-owned channel page-directory hierarchy behind the external + * root: PD2/PD1 single 8 B PDE levels, PD0 16 B dual PDE and the 4 KB + * small-page PT (GP10X v2, kern_gmmu_fmt_gp10x.c:16-83). + */ + GSP_PHYS_RMCALL_BUF_PD2, + GSP_PHYS_RMCALL_BUF_PD1, + GSP_PHYS_RMCALL_BUF_PD0, + GSP_PHYS_RMCALL_BUF_PT, GSP_PHYS_RMCALL_BUF_COUNT }; @@ -904,6 +928,18 @@ static struct gsp_phys_boot_stage { u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */ u64 rmcall_notifier_base; /* its returned frame-buffer address */ bool rmcall_notifier_valid; /* the FBMEM notifier alloc succeeded */ + u32 rmcall_userd_mem; /* FBMEM USERD object handle */ + u64 rmcall_userd_base; /* its returned frame-buffer address */ + bool rmcall_userd_valid; /* the FBMEM USERD alloc succeeded */ + u64 rmcall_va_base; /* channel PDB VA window base */ + u64 rmcall_va_gpfifo; /* GPFIFO GPU VA in that window */ + u64 rmcall_va_pb; /* pushbuffer GPU VA in that window */ + u64 rmcall_pdb_root; /* root entry -> PD2 */ + u64 rmcall_pdb_pd2; /* PD2 entry -> PD1 */ + u64 rmcall_pdb_pd1; /* PD1 entry -> PD0 */ + u64 rmcall_pdb_pd0_small; /* PD0 dual entry, small (4 KB) half */ + u64 rmcall_pt_gpfifo; /* small-PT entry for the GPFIFO page */ + u64 rmcall_pt_pb; /* small-PT entry for the PB page */ struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT]; @@ -1419,6 +1455,18 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_notifier_mem = 0; boot_stage.rmcall_notifier_base = 0; boot_stage.rmcall_notifier_valid = false; + boot_stage.rmcall_userd_mem = 0; + boot_stage.rmcall_userd_base = 0; + boot_stage.rmcall_userd_valid = false; + boot_stage.rmcall_va_base = 0; + boot_stage.rmcall_va_gpfifo = 0; + boot_stage.rmcall_va_pb = 0; + boot_stage.rmcall_pdb_root = 0; + boot_stage.rmcall_pdb_pd2 = 0; + boot_stage.rmcall_pdb_pd1 = 0; + boot_stage.rmcall_pdb_pd0_small = 0; + boot_stage.rmcall_pt_gpfifo = 0; + boot_stage.rmcall_pt_pb = 0; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; boot_stage.rmcall_mem_registered[i] = false; @@ -4737,6 +4785,83 @@ static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0) } /* + * Allocate the channel's USERD page in FBMEM. Same class-0x40 shape as the + * error-notifier allocation above (owner HEAP_OWNER_RM_CLIENT_GENERIC, type + * IMAGE, VIDMEM|UNCACHED, SKIP_SCRUB); the reference defaults USERD to + * VIDMEM (mem_utils_gm107.c:792-804,1132-1161) and the client push path + * allocates NV01_MEMORY_LOCAL_USER with these flags (nvidia-push-init.c: + * 515-554). The server returns the allocated FB offset in + * NV_MEMORY_ALLOCATION_PARAMS.offset, which userdMem.base carries with + * addressSpace FBMEM. + * + * Failure is non-fatal: the channel params then fall back to the sysmem + * USERD chunk and log the fallback. + */ +static int gsp_phys_rmcall_alloc_userd_mem(void __iomem *bar0) +{ + struct gsp_phys_rmcall_memory_alloc_params mem; + struct gsp_phys_rmcall_rm_alloc_req *req; + struct gsp_phys_rx_record rec; + u32 req_len = sizeof(*req) + sizeof(mem); + u32 status; + u64 base; + int ret; + + memset(&mem, 0, sizeof(mem)); + mem.owner = cpu_to_le32(GSP_PHYS_RMCALL_HEAP_OWNER_RM_CLIENT_GENERIC); + mem.type = cpu_to_le32(GSP_PHYS_RMCALL_OS32_TYPE_IMAGE); + mem.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE); + mem.attr = cpu_to_le32(GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED); + mem.internalflags = + cpu_to_le32(GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB); + + req = kvzalloc(req_len, GFP_KERNEL); + if (!req) + return -ENOMEM; + req->h_client = cpu_to_le32(boot_stage.rmcall_client); + req->h_parent = cpu_to_le32(boot_stage.rmcall_device); + req->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_USERD_MEM); + req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER); + req->params_size = cpu_to_le32(sizeof(mem)); + memcpy(req->params, &mem, sizeof(mem)); + + ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC, + req, req_len, 20, &rec, true, NULL); + kvfree(req); + if (ret) { + pr_err("gsp-phys-boot: rm-call: FBMEM USERD alloc (hMemory=0x%x class=0x40 size=0x%x) RPC failed: %d\n", + GSP_PHYS_RMCALL_HANDLE_USERD_MEM, + GSP_PHYS_RMCALL_COPY_SIZE, ret); + gsp_phys_rx_record_free(&rec); + return ret; + } + + status = gsp_phys_rmcall_reply_u32(&rec, 16); + if (status != 0 || + rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset) + 8) { + pr_err("gsp-phys-boot: rm-call: FBMEM USERD alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n", + status, rec.payload_len, + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset) + 8); + gsp_phys_rx_record_free(&rec); + return -EIO; + } + base = get_unaligned_le64(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 32 + + offsetof(struct gsp_phys_rmcall_memory_alloc_params, + offset)); + gsp_phys_rx_record_free(&rec); + + boot_stage.rmcall_userd_mem = GSP_PHYS_RMCALL_HANDLE_USERD_MEM; + boot_stage.rmcall_userd_base = base; + boot_stage.rmcall_userd_valid = true; + pr_info("gsp-phys-boot: rm-call: FBMEM USERD allocated: hMemory=0x%x class=0x40 size=0x%x FB base=0x%llx\n", + boot_stage.rmcall_userd_mem, GSP_PHYS_RMCALL_COPY_SIZE, + boot_stage.rmcall_userd_base); + return 0; +} + +/* * One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at * payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). out_status (optional) * receives the server control status even when it is nonzero; the echoed @@ -4916,7 +5041,7 @@ static int gsp_phys_rmcall_stage2(void __iomem *bar0) } } - pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb zeroed)\n", + pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt zeroed)\n", GSP_PHYS_RMCALL_BUF_COUNT); return 0; } @@ -5005,11 +5130,31 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch, } if (v->server_mem) return; /* zeroed memdescs, gpFifoOffset/gpFifoEntries 0 */ - ch->gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma( - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0)); + /* + * The PBDMA/CE fetch the GPFIFO by GPU VA through the channel PDB + * (NV_RAMFC_GP_BASE from gpFifoOffset, fifo_g100.c:1120-1137), so the + * offset is the VA built in gsp_phys_rmcall_pdb_build, not the host DMA. + */ + ch->gp_fifo_offset = cpu_to_le64(GSP_PHYS_RMCALL_VA_GPFIFO); ch->gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES); ch->instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE); - ch->userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD); + /* + * USERD is the reference default FBMEM page (mem_utils_gm107.c: + * 792-804,1132-1161; nvidia-push-init.c:515-554) when stage 3's + * class-0x40 allocation succeeded; the sysmem chunk remains the + * fallback when it did not. + */ + if (boot_stage.rmcall_userd_valid) { + ch->userd_mem.base = cpu_to_le64(boot_stage.rmcall_userd_base); + ch->userd_mem.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE); + ch->userd_mem.address_space = + cpu_to_le32(GSP_PHYS_RMCALL_ADDR_FBMEM); + ch->userd_mem.cache_attrib = + cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED); + } else { + ch->userd_mem = + gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD); + } ch->ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC); ch->mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF); } @@ -5078,6 +5223,118 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) } /* + * GP10X GMMU (format version 2) sysmem-noncoherent PDE/PTE encoders. The + * single PDE is IS_PDE=0 (bit 0), APERTURE=3 in bits 2:1, VOL=1 (bit 3) and + * the physical page number in bits 53:8 (ADDRESS_SYS, shift 12); the leaf PTE + * adds VALID=1 (bit 0) with KIND=PITCH (bits 63:56 zero). The PD0 dual entry + * stores the big (64K/2M) half in qword 0 and the small (4K) half in qword 1 + * with the same aperture/address fields at the same bit positions + * (swref/published/pascal/gp100/dev_mmu.h:97-119,157; + * uvm_pascal_mmu.c:64-141,235-303). + */ +static u64 gsp_phys_rmcall_pde(u64 pa) +{ + return 0xeull | ((pa >> 12) << 8); +} + +static u64 gsp_phys_rmcall_pte(u64 pa) +{ + return 0xfull | ((pa >> 12) << 8); +} + +/* + * Build the channel's page-directory hierarchy around the 2 MB VA window + * GSP_PHYS_RMCALL_VA_BASE that maps the GPFIFO and pushbuffer sysmem pages: + * + * root/PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 dual [28:21] + * -> small PT [20:12] + * + * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,143-161). The window + * puts every parent index at 0 and the two pages at small-PT slots + * 0x100/0x101, so only root[0], PD2[0], PD1[0], the PD0[0] small half and + * those two PT slots are written; the root chunk itself is the + * DMA_SET_PAGE_DIRECTORY payload. The external root merely stores the + * memdesc, so filling before the channel alloc (after a successful + * SET_PAGE_DIRECTORY) is equivalent to filling before the control. + */ +static void gsp_phys_rmcall_pdb_build(void) +{ + struct gsp_phys_boot_chunk *root = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB]; + struct gsp_phys_boot_chunk *pd2 = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD2]; + struct gsp_phys_boot_chunk *pd1 = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD1]; + struct gsp_phys_boot_chunk *pd0 = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD0]; + struct gsp_phys_boot_chunk *pt = + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT]; + u32 gpfifo_idx = (u32)((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ffu); + u32 pb_idx = (u32)((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ffu); + u64 root_ent, pd2_ent, pd1_ent, pd0_small, pt_gpfifo, pt_pb; + + root_ent = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(pd2, 0)); + pd2_ent = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(pd1, 0)); + pd1_ent = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(pd0, 0)); + pd0_small = gsp_phys_rmcall_pde( + (u64)gsp_phys_chunk_page_dma(pt, 0)); + pt_gpfifo = gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], + 0)); + pt_pb = gsp_phys_rmcall_pte( + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0)); + + put_unaligned_le64(root_ent, gsp_phys_chunk_page_va(root, 0)); + put_unaligned_le64(pd2_ent, gsp_phys_chunk_page_va(pd2, 0)); + put_unaligned_le64(pd1_ent, gsp_phys_chunk_page_va(pd1, 0)); + /* 16 B dual PDE: big (64K/2M) half unused, small half -> 4 KB PT. */ + put_unaligned_le64(0, gsp_phys_chunk_page_va(pd0, 0)); + put_unaligned_le64(pd0_small, + (u8 *)gsp_phys_chunk_page_va(pd0, 0) + 8); + put_unaligned_le64(pt_gpfifo, + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)gpfifo_idx * 8); + put_unaligned_le64(pt_pb, + (u8 *)gsp_phys_chunk_page_va(pt, 0) + + (size_t)pb_idx * 8); + dma_wmb(); + + boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE; + boot_stage.rmcall_va_gpfifo = GSP_PHYS_RMCALL_VA_GPFIFO; + boot_stage.rmcall_va_pb = GSP_PHYS_RMCALL_VA_PB; + boot_stage.rmcall_pdb_root = root_ent; + boot_stage.rmcall_pdb_pd2 = pd2_ent; + boot_stage.rmcall_pdb_pd1 = pd1_ent; + boot_stage.rmcall_pdb_pd0_small = pd0_small; + boot_stage.rmcall_pt_gpfifo = pt_gpfifo; + boot_stage.rmcall_pt_pb = pt_pb; + + pr_info("gsp-phys-boot: rm-call: channel PDB built: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx gpFifoOffset=0x%llx root dma=0x%llx pd2 dma=0x%llx pd1 dma=0x%llx pd0 dma=0x%llx pt dma=0x%llx\n", + GSP_PHYS_RMCALL_VA_BASE, GSP_PHYS_RMCALL_VA_GPFIFO, + GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO, + (u64)gsp_phys_chunk_page_dma(root, 0), + (u64)gsp_phys_chunk_page_dma(pd2, 0), + (u64)gsp_phys_chunk_page_dma(pd1, 0), + (u64)gsp_phys_chunk_page_dma(pd0, 0), + (u64)gsp_phys_chunk_page_dma(pt, 0)); + pr_info("gsp-phys-boot: rm-call: channel PDB entries: root[0]=0x%016llx pd2[0]=0x%016llx pd1[0]=0x%016llx pd0[0].small=0x%016llx\n", + root_ent, pd2_ent, pd1_ent, pd0_small); + pr_info("gsp-phys-boot: rm-call: channel PDB PT: pt[%u]=0x%016llx (gpfifo dma=0x%llx) pt[%u]=0x%016llx (pb dma=0x%llx)\n", + gpfifo_idx, pt_gpfifo, + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], + 0), + pb_idx, pt_pb, + (u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0)); +} + +/* * Stage 3: explicit VAS with its page directory, then channel group (COPY0, * physical) and the channel. The VAS and PDB must both succeed: a channel * whose VAS has no page-directory base fails the context-share/instance-block @@ -5095,7 +5352,10 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0) * requested here: its generator requires BIND + GPFIFO_SCHEDULE (runlist * membership, kernel_fifo_tu102.c:110-116), so the whole BIND -> SCHEDULE -> * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN order lives in - * stage 4. + * stage 4. After SET_PAGE_DIRECTORY the root's lower levels are built + * (gsp_phys_rmcall_pdb_build), because the engine fetches the GPFIFO and the + * pushbuffer by GPU VA through that hierarchy; the USERD page is allocated in + * FBMEM alongside the notifier. */ static int gsp_phys_rmcall_stage3(void __iomem *bar0) { @@ -5151,6 +5411,12 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) pdb_rpc); return ret; } + /* + * The server only stored our root memdesc; the GPFIFO and pushbuffer + * fetches translate through it, so fill the hierarchy before the + * channel alloc commits the instance block's page-directory base. + */ + gsp_phys_rmcall_pdb_build(); memset(&tsg, 0, sizeof(tsg)); tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas); @@ -5185,15 +5451,26 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) pr_warn("gsp-phys-boot: rm-call: FBMEM error-notifier allocation failed (%d); channel allocs carry a zeroed notifier memdesc and only the stage-4 token recovery can save the token\n", ret); + /* + * Allocate the USERD page in FBMEM as well (reference default); on + * failure the channel variants fall back to the sysmem USERD chunk. + */ + ret = gsp_phys_rmcall_alloc_userd_mem(bar0); + if (ret) + pr_warn("gsp-phys-boot: rm-call: FBMEM USERD allocation failed (%d); channel allocs fall back to the sysmem USERD chunk\n", + ret); + ret = gsp_phys_rmcall_alloc_channel(bar0); if (ret) return ret; - pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n", + pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=%s base=0x%llx engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n", boot_stage.rmcall_vas, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant, boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", - boot_stage.rmcall_notifier_base); + boot_stage.rmcall_notifier_base, + boot_stage.rmcall_userd_valid ? "FBMEM" : "sysmem", + boot_stage.rmcall_userd_base); return 0; } @@ -5254,21 +5531,26 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_dma, u64 src_dma, /* * Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE -> - * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN - then the first - * real work item: write the CE method block to the pushbuffer, publish the - * GPFIFO entry and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the - * PF channel doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem - * semaphore word. Afterwards dst must equal src byte for byte. + * TSG_GPFIFO_SCHEDULE -> SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> + * GET_WORK_SUBMIT_TOKEN - then the first real work item: write the CE method + * block to the pushbuffer, publish the GPFIFO entry (GET = the pushbuffer GPU + * VA) and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the PF channel + * doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem semaphore word. + * Afterwards dst must equal src byte for byte. * * BIND is the precondition the TU102 token generator needs (runlist * membership, kernel_fifo_tu102.c:110-116; GPFIFO_SCHEDULE alone only sets * RUNLIST_SET at kernel_channel.c:3244), and SET_NOTIF_INDEX (index 3 for * subdevice 0, nvidia-push-init.c:303-316) gives the FBMEM notifier write a * target slot; it is advisory here because no error-context object exists. - * Every other control failure logs its rpc_result/status and aborts - * before the doorbell; a 0x40 GET_WORK_SUBMIT_TOKEN whose echoed params carry - * a nonzero token still proceeds (the server writes the token before the - * notifier update, kernel_channel.c:3471-3477). + * The TSG-level NVA06C_CTRL_CMD_GPFIFO_SCHEDULE 0xa06c0101 ("schedules a + * channel group in hardware", ctrla06c.h:82-101) is insurance on top of the + * channel-level schedule, whose internal HAL is a stub in the shipped + * dispatch; it too is advisory. Every other control failure logs its + * rpc_result/status and aborts before the doorbell; a 0x40 + * GET_WORK_SUBMIT_TOKEN whose echoed params carry a nonzero token still + * proceeds (the server writes the token before the notifier update, + * kernel_channel.c:3471-3477). */ static int gsp_phys_rmcall_stage4(void __iomem *bar0) { @@ -5340,6 +5622,30 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) } /* + * (b2) TSG-level NVA06C_CTRL_CMD_GPFIFO_SCHEDULE (0xa06c0101, the + * "schedules a channel group in hardware" call, ctrla06c.h:82-101, + * same 2-byte NVA06F params) as insurance: the channel-level + * GPFIFO_SCHEDULE's internal HAL is a stub in the shipped dispatch, so + * the runlist entry is only fully committed by the group schedule. + * Advisory, not fatal: the channel-level schedule already set + * RUNLIST_SET, which is what the token generator consumes + * (kernel_channel.c:3237-3245). + */ + rpc_result = 0; + status = 0; + ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg, + GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE, + sched_params, sizeof(sched_params), 16, + NULL, 0, &rpc_result, &status, + "TSG_GPFIFO_SCHEDULE"); + if (ret) + pr_warn("gsp-phys-boot: rm-call: TSG_GPFIFO_SCHEDULE (0xa06c0101 bEnable=1 bSkipSubmit=0 on hTsg=0x%x) advisory failure: rpc_result=0x%x status=0x%x; the channel-level schedule already set RUNLIST_SET, continuing\n", + boot_stage.rmcall_tsg, rpc_result, status); + else + pr_info("gsp-phys-boot: rm-call: TSG_GPFIFO_SCHEDULE 0xa06c0101 on hTsg=0x%x accepted (rpc_result=0x%x status=0x%x)\n", + boot_stage.rmcall_tsg, rpc_result, status); + + /* * (c) SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (nvidia-push-init.c:303-316). * Advisory, not fatal: with no error-context object (hErrorContext=0) * the server's memory and ctxdma lookups both fail and it returns @@ -5412,18 +5718,25 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) slot = gpput; /* - * e0: GET = pushbuffer address >> 2 in bits 31:2, FETCH/NO_CONTEXT_ - * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the address high - * dword and LENGTH 30:10 in dwords, exactly as the reference RM's own - * submission does it (mem_utils_gm107.c:1861-1868: NvU64_HI32(get) in - * GET_HI, NvU64_LO32(get) >> 2 in GET). The physical channel model - * addresses the ring and pushbuffer by their system DMA addresses. + * e0: GET = pushbuffer GPU VA >> 2 in bits 31:2, FETCH/NO_CONTEXT_ + * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the VA high byte + * and LENGTH 30:10 in dwords, exactly the reference RM's own submission + * (mem_utils_gm107.c:1858-1868: get = pbGpuVA + offset, NvU64_HI32(get) + * in GET_HI, NvU64_LO32(get) >> 2 in GET, PRIV USER and LEVEL MAIN). + * The engine translates this VA through the channel PDB built in + * gsp_phys_rmcall_pdb_build. */ - e0 = (u32)(pb_dma >> 2) & 0x3fffffffu; - e1 = (u32)((pb_dma >> 32) & 0xffu) | (words << 10); + e0 = (u32)(GSP_PHYS_RMCALL_VA_PB >> 2) & 0x3fffffffu; + e1 = (u32)((GSP_PHYS_RMCALL_VA_PB >> 32) & 0xffu) | (words << 10); put_unaligned_le32(e0, (u8 *)ring_va + slot * 8); put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4); dma_wmb(); + /* + * GP_PUT stays on the client sysmem USERD chunk, which is the hardware + * USERD only on the FBMEM-alloc fallback path; when the server used the + * FBMEM USERD page this host write cannot reach it (no BAR1/PRAMIN + * aperture here), kept as-is for the fallback. + */ put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES, (u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF); /* @@ -5442,8 +5755,10 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) wmb(); writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF); - pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", - pb_dma, words, slot, gpget, gpput, e0, e1, + pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx userd=%s words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n", + pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO, + boot_stage.rmcall_userd_valid ? "FBMEM" : "sysmem", + words, slot, gpget, gpput, e0, e1, boot_stage.rmcall_token, runlist_id, ch_id, doorbell, GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma, GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS); @@ -5731,6 +6046,17 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rmcall_notifier_valid ? "FBMEM" : "none", boot_stage.rmcall_notifier_base); seq_printf(m, + "rm_call_userd: userd=%s hMemory=0x%x base=0x%llx | rm_call_pdb: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx\n", + boot_stage.rmcall_userd_valid ? "FBMEM" : "sysmem", + boot_stage.rmcall_userd_mem, boot_stage.rmcall_userd_base, + boot_stage.rmcall_va_base, boot_stage.rmcall_va_gpfifo, + boot_stage.rmcall_va_pb); + seq_printf(m, + "rm_call_pdb_entries: root=0x%016llx pd2=0x%016llx pd1=0x%016llx pd0Small=0x%016llx ptGpfifo=0x%016llx ptPb=0x%016llx\n", + boot_stage.rmcall_pdb_root, boot_stage.rmcall_pdb_pd2, + boot_stage.rmcall_pdb_pd1, boot_stage.rmcall_pdb_pd0_small, + boot_stage.rmcall_pt_gpfifo, boot_stage.rmcall_pt_pb); + seq_printf(m, "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n", wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0), wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0), @@ -6625,12 +6951,24 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296 "NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296"); static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360, "NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360"); -static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 10, - "the rm-call ladder registers exactly ten DMA buffers"); +static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 14, + "the rm-call ladder registers exactly fourteen DMA buffers"); static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE + (GSP_PHYS_RMCALL_BUF_COUNT - 1) * GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u, "rm-call memory handles must stay inside the RM client handle range"); +static_assert(GSP_PHYS_RMCALL_VA_PB + GSP_PHYS_PAGE_SIZE <= + GSP_PHYS_RMCALL_VAS_SIZE, + "the channel PDB VA window must stay below the VAS vaSize limit"); +static_assert(((GSP_PHYS_RMCALL_VA_BASE >> 47) & 0x3) == 0 && + ((GSP_PHYS_RMCALL_VA_BASE >> 38) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_VA_BASE >> 29) & 0x1ff) == 0 && + ((GSP_PHYS_RMCALL_VA_BASE >> 21) & 0xff) == 0 && + ((GSP_PHYS_RMCALL_VA_PB >> 21) & 0xff) == 0, + "the channel PDB VA window must sit in root/PD2/PD1/PD0 index 0 (GP10X 48:47/46:38/37:29/28:21 splits)"); +static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) != + ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff), + "the GPFIFO and pushbuffer VAs must take distinct small-PT slots"); /* * GspSystemInfo wire ABI, pinned against the reference header compile |