summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c394
1 files changed, 366 insertions, 28 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index fa2d3b4..5565159 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -228,6 +228,7 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_CTRL_SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX 0xc36f010au
#define GSP_PHYS_RMCALL_CTRL_GPFIFO_SCHEDULE 0xa06f0103u
#define GSP_PHYS_RMCALL_CTRL_BIND_CHANNEL 0xa06f0104u
+#define GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE 0xa06c0101u
#define GSP_PHYS_RMCALL_CTRL_DMA_SET_PAGE_DIRECTORY 0x801813u
#define GSP_PHYS_RMCALL_ENGINE_COPY0 0x9u
@@ -286,6 +287,19 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_PDB_SUBDEVICE_ID 1u
/*
+ * Channel page-directory GPU VA window. The GPFIFO and the pushbuffer are
+ * fetched by GPU VA (the GPFIFO from gpFifoOffset, the pushbuffer from the
+ * GPFIFO ring entry GET), so the channel's external root must translate them.
+ * Both pages live in one 2 MB PD0 slot below the VAS vaSize limit: with
+ * VA_BASE = 0x00100000 every parent index (48:47, 46:38, 37:29, 28:21) is 0
+ * and the two 4 KB pages take small-PT slots 0x100/0x101
+ * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,172-204).
+ */
+#define GSP_PHYS_RMCALL_VA_BASE 0x00100000ULL
+#define GSP_PHYS_RMCALL_VA_GPFIFO GSP_PHYS_RMCALL_VA_BASE
+#define GSP_PHYS_RMCALL_VA_PB (GSP_PHYS_RMCALL_VA_BASE + GSP_PHYS_PAGE_SIZE)
+
+/*
* Channel alloc flags/internal flags (alloc/alloc_channel.h:141-143 bit 5 =
* NVOS04_FLAGS_PRIVILEGED_CHANNEL; generated/g_kernel_channel_nvoc.h:185-188
* internalFlags 1:0 = privilege, KERNEL=2). These are sent as a hint only:
@@ -330,6 +344,7 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 0xcaf00018u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V5 0xcaf00024u
#define GSP_PHYS_RMCALL_HANDLE_NOTIFIER_MEM 0xcaf00028u
+#define GSP_PHYS_RMCALL_HANDLE_USERD_MEM 0xcaf0002cu
#define GSP_PHYS_RMCALL_HANDLE_MEM_BASE 0xcaf01000u
#define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE 4u
@@ -362,6 +377,15 @@ enum gsp_phys_rmcall_buf {
GSP_PHYS_RMCALL_BUF_RAMFC,
GSP_PHYS_RMCALL_BUF_MTHDBUF,
GSP_PHYS_RMCALL_BUF_VASPDB,
+ /*
+ * Client-owned channel page-directory hierarchy behind the external
+ * root: PD2/PD1 single 8 B PDE levels, PD0 16 B dual PDE and the 4 KB
+ * small-page PT (GP10X v2, kern_gmmu_fmt_gp10x.c:16-83).
+ */
+ GSP_PHYS_RMCALL_BUF_PD2,
+ GSP_PHYS_RMCALL_BUF_PD1,
+ GSP_PHYS_RMCALL_BUF_PD0,
+ GSP_PHYS_RMCALL_BUF_PT,
GSP_PHYS_RMCALL_BUF_COUNT
};
@@ -904,6 +928,18 @@ static struct gsp_phys_boot_stage {
u32 rmcall_notifier_mem; /* FBMEM error-notifier object handle */
u64 rmcall_notifier_base; /* its returned frame-buffer address */
bool rmcall_notifier_valid; /* the FBMEM notifier alloc succeeded */
+ u32 rmcall_userd_mem; /* FBMEM USERD object handle */
+ u64 rmcall_userd_base; /* its returned frame-buffer address */
+ bool rmcall_userd_valid; /* the FBMEM USERD alloc succeeded */
+ u64 rmcall_va_base; /* channel PDB VA window base */
+ u64 rmcall_va_gpfifo; /* GPFIFO GPU VA in that window */
+ u64 rmcall_va_pb; /* pushbuffer GPU VA in that window */
+ u64 rmcall_pdb_root; /* root entry -> PD2 */
+ u64 rmcall_pdb_pd2; /* PD2 entry -> PD1 */
+ u64 rmcall_pdb_pd1; /* PD1 entry -> PD0 */
+ u64 rmcall_pdb_pd0_small; /* PD0 dual entry, small (4 KB) half */
+ u64 rmcall_pt_gpfifo; /* small-PT entry for the GPFIFO page */
+ u64 rmcall_pt_pb; /* small-PT entry for the PB page */
struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */
struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
u32 rmcall_mem_handles[GSP_PHYS_RMCALL_BUF_COUNT];
@@ -1419,6 +1455,18 @@ static void gsp_phys_rmcall_reset(void)
boot_stage.rmcall_notifier_mem = 0;
boot_stage.rmcall_notifier_base = 0;
boot_stage.rmcall_notifier_valid = false;
+ boot_stage.rmcall_userd_mem = 0;
+ boot_stage.rmcall_userd_base = 0;
+ boot_stage.rmcall_userd_valid = false;
+ boot_stage.rmcall_va_base = 0;
+ boot_stage.rmcall_va_gpfifo = 0;
+ boot_stage.rmcall_va_pb = 0;
+ boot_stage.rmcall_pdb_root = 0;
+ boot_stage.rmcall_pdb_pd2 = 0;
+ boot_stage.rmcall_pdb_pd1 = 0;
+ boot_stage.rmcall_pdb_pd0_small = 0;
+ boot_stage.rmcall_pt_gpfifo = 0;
+ boot_stage.rmcall_pt_pb = 0;
for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
boot_stage.rmcall_mem_handles[i] = 0;
boot_stage.rmcall_mem_registered[i] = false;
@@ -4737,6 +4785,83 @@ static int gsp_phys_rmcall_alloc_notifier_mem(void __iomem *bar0)
}
/*
+ * Allocate the channel's USERD page in FBMEM. Same class-0x40 shape as the
+ * error-notifier allocation above (owner HEAP_OWNER_RM_CLIENT_GENERIC, type
+ * IMAGE, VIDMEM|UNCACHED, SKIP_SCRUB); the reference defaults USERD to
+ * VIDMEM (mem_utils_gm107.c:792-804,1132-1161) and the client push path
+ * allocates NV01_MEMORY_LOCAL_USER with these flags (nvidia-push-init.c:
+ * 515-554). The server returns the allocated FB offset in
+ * NV_MEMORY_ALLOCATION_PARAMS.offset, which userdMem.base carries with
+ * addressSpace FBMEM.
+ *
+ * Failure is non-fatal: the channel params then fall back to the sysmem
+ * USERD chunk and log the fallback.
+ */
+static int gsp_phys_rmcall_alloc_userd_mem(void __iomem *bar0)
+{
+ struct gsp_phys_rmcall_memory_alloc_params mem;
+ struct gsp_phys_rmcall_rm_alloc_req *req;
+ struct gsp_phys_rx_record rec;
+ u32 req_len = sizeof(*req) + sizeof(mem);
+ u32 status;
+ u64 base;
+ int ret;
+
+ memset(&mem, 0, sizeof(mem));
+ mem.owner = cpu_to_le32(GSP_PHYS_RMCALL_HEAP_OWNER_RM_CLIENT_GENERIC);
+ mem.type = cpu_to_le32(GSP_PHYS_RMCALL_OS32_TYPE_IMAGE);
+ mem.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE);
+ mem.attr = cpu_to_le32(GSP_PHYS_RMCALL_OS32_ATTR_VIDMEM_UNCACHED);
+ mem.internalflags =
+ cpu_to_le32(GSP_PHYS_RMCALL_OS32_INTERNAL_FLAG_SKIP_SCRUB);
+
+ req = kvzalloc(req_len, GFP_KERNEL);
+ if (!req)
+ return -ENOMEM;
+ req->h_client = cpu_to_le32(boot_stage.rmcall_client);
+ req->h_parent = cpu_to_le32(boot_stage.rmcall_device);
+ req->h_object = cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_USERD_MEM);
+ req->h_class = cpu_to_le32(GSP_PHYS_RMCALL_CLASS_MEMORY_LOCAL_USER);
+ req->params_size = cpu_to_le32(sizeof(mem));
+ memcpy(req->params, &mem, sizeof(mem));
+
+ ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC,
+ req, req_len, 20, &rec, true, NULL);
+ kvfree(req);
+ if (ret) {
+ pr_err("gsp-phys-boot: rm-call: FBMEM USERD alloc (hMemory=0x%x class=0x40 size=0x%x) RPC failed: %d\n",
+ GSP_PHYS_RMCALL_HANDLE_USERD_MEM,
+ GSP_PHYS_RMCALL_COPY_SIZE, ret);
+ gsp_phys_rx_record_free(&rec);
+ return ret;
+ }
+
+ status = gsp_phys_rmcall_reply_u32(&rec, 16);
+ if (status != 0 ||
+ rec.payload_len < 32 + offsetof(struct gsp_phys_rmcall_memory_alloc_params,
+ offset) + 8) {
+ pr_err("gsp-phys-boot: rm-call: FBMEM USERD alloc rejected: status=0x%x payload=%u B (need the echoed params through offset at +%zu)\n",
+ status, rec.payload_len,
+ offsetof(struct gsp_phys_rmcall_memory_alloc_params,
+ offset) + 8);
+ gsp_phys_rx_record_free(&rec);
+ return -EIO;
+ }
+ base = get_unaligned_le64(rec.buf + GSP_PHYS_MSG_ELEMENT_PAYLOAD_OFF + 32 +
+ offsetof(struct gsp_phys_rmcall_memory_alloc_params,
+ offset));
+ gsp_phys_rx_record_free(&rec);
+
+ boot_stage.rmcall_userd_mem = GSP_PHYS_RMCALL_HANDLE_USERD_MEM;
+ boot_stage.rmcall_userd_base = base;
+ boot_stage.rmcall_userd_valid = true;
+ pr_info("gsp-phys-boot: rm-call: FBMEM USERD allocated: hMemory=0x%x class=0x40 size=0x%x FB base=0x%llx\n",
+ boot_stage.rmcall_userd_mem, GSP_PHYS_RMCALL_COPY_SIZE,
+ boot_stage.rmcall_userd_base);
+ return 0;
+}
+
+/*
* One GSP_RM_CONTROL (76). Reply status at payload+12, echoed params at
* payload+24 (rpcRmApiControl_GSP, rpc.c:9797-10031). out_status (optional)
* receives the server control status even when it is nonzero; the echoed
@@ -4916,7 +5041,7 @@ static int gsp_phys_rmcall_stage2(void __iomem *bar0)
}
}
- pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb zeroed)\n",
+ pr_info("gsp-phys-boot: rm-call: stage 2 complete: %u buffers registered (src=0xa5, dst/sem/pb/gpfifo/userd/instance/ramfc/mthdbuf/vaspdb/pd2/pd1/pd0/pt zeroed)\n",
GSP_PHYS_RMCALL_BUF_COUNT);
return 0;
}
@@ -5005,11 +5130,31 @@ gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch,
}
if (v->server_mem)
return; /* zeroed memdescs, gpFifoOffset/gpFifoEntries 0 */
- ch->gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
- &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0));
+ /*
+ * The PBDMA/CE fetch the GPFIFO by GPU VA through the channel PDB
+ * (NV_RAMFC_GP_BASE from gpFifoOffset, fifo_g100.c:1120-1137), so the
+ * offset is the VA built in gsp_phys_rmcall_pdb_build, not the host DMA.
+ */
+ ch->gp_fifo_offset = cpu_to_le64(GSP_PHYS_RMCALL_VA_GPFIFO);
ch->gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
ch->instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE);
- ch->userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD);
+ /*
+ * USERD is the reference default FBMEM page (mem_utils_gm107.c:
+ * 792-804,1132-1161; nvidia-push-init.c:515-554) when stage 3's
+ * class-0x40 allocation succeeded; the sysmem chunk remains the
+ * fallback when it did not.
+ */
+ if (boot_stage.rmcall_userd_valid) {
+ ch->userd_mem.base = cpu_to_le64(boot_stage.rmcall_userd_base);
+ ch->userd_mem.size = cpu_to_le64(GSP_PHYS_RMCALL_COPY_SIZE);
+ ch->userd_mem.address_space =
+ cpu_to_le32(GSP_PHYS_RMCALL_ADDR_FBMEM);
+ ch->userd_mem.cache_attrib =
+ cpu_to_le32(GSP_PHYS_RMCALL_CACHE_UNCACHED);
+ } else {
+ ch->userd_mem =
+ gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD);
+ }
ch->ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC);
ch->mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF);
}
@@ -5078,6 +5223,118 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
}
/*
+ * GP10X GMMU (format version 2) sysmem-noncoherent PDE/PTE encoders. The
+ * single PDE is IS_PDE=0 (bit 0), APERTURE=3 in bits 2:1, VOL=1 (bit 3) and
+ * the physical page number in bits 53:8 (ADDRESS_SYS, shift 12); the leaf PTE
+ * adds VALID=1 (bit 0) with KIND=PITCH (bits 63:56 zero). The PD0 dual entry
+ * stores the big (64K/2M) half in qword 0 and the small (4K) half in qword 1
+ * with the same aperture/address fields at the same bit positions
+ * (swref/published/pascal/gp100/dev_mmu.h:97-119,157;
+ * uvm_pascal_mmu.c:64-141,235-303).
+ */
+static u64 gsp_phys_rmcall_pde(u64 pa)
+{
+ return 0xeull | ((pa >> 12) << 8);
+}
+
+static u64 gsp_phys_rmcall_pte(u64 pa)
+{
+ return 0xfull | ((pa >> 12) << 8);
+}
+
+/*
+ * Build the channel's page-directory hierarchy around the 2 MB VA window
+ * GSP_PHYS_RMCALL_VA_BASE that maps the GPFIFO and pushbuffer sysmem pages:
+ *
+ * root/PD3 [48:47] -> PD2 [46:38] -> PD1 [37:29] -> PD0 dual [28:21]
+ * -> small PT [20:12]
+ *
+ * (kern_gmmu_fmt_gp10x.c:16-83; uvm_pascal_mmu.c:25-32,143-161). The window
+ * puts every parent index at 0 and the two pages at small-PT slots
+ * 0x100/0x101, so only root[0], PD2[0], PD1[0], the PD0[0] small half and
+ * those two PT slots are written; the root chunk itself is the
+ * DMA_SET_PAGE_DIRECTORY payload. The external root merely stores the
+ * memdesc, so filling before the channel alloc (after a successful
+ * SET_PAGE_DIRECTORY) is equivalent to filling before the control.
+ */
+static void gsp_phys_rmcall_pdb_build(void)
+{
+ struct gsp_phys_boot_chunk *root =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_VASPDB];
+ struct gsp_phys_boot_chunk *pd2 =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD2];
+ struct gsp_phys_boot_chunk *pd1 =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD1];
+ struct gsp_phys_boot_chunk *pd0 =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PD0];
+ struct gsp_phys_boot_chunk *pt =
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PT];
+ u32 gpfifo_idx = (u32)((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ffu);
+ u32 pb_idx = (u32)((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ffu);
+ u64 root_ent, pd2_ent, pd1_ent, pd0_small, pt_gpfifo, pt_pb;
+
+ root_ent = gsp_phys_rmcall_pde(
+ (u64)gsp_phys_chunk_page_dma(pd2, 0));
+ pd2_ent = gsp_phys_rmcall_pde(
+ (u64)gsp_phys_chunk_page_dma(pd1, 0));
+ pd1_ent = gsp_phys_rmcall_pde(
+ (u64)gsp_phys_chunk_page_dma(pd0, 0));
+ pd0_small = gsp_phys_rmcall_pde(
+ (u64)gsp_phys_chunk_page_dma(pt, 0));
+ pt_gpfifo = gsp_phys_rmcall_pte(
+ (u64)gsp_phys_chunk_page_dma(
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO],
+ 0));
+ pt_pb = gsp_phys_rmcall_pte(
+ (u64)gsp_phys_chunk_page_dma(
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0));
+
+ put_unaligned_le64(root_ent, gsp_phys_chunk_page_va(root, 0));
+ put_unaligned_le64(pd2_ent, gsp_phys_chunk_page_va(pd2, 0));
+ put_unaligned_le64(pd1_ent, gsp_phys_chunk_page_va(pd1, 0));
+ /* 16 B dual PDE: big (64K/2M) half unused, small half -> 4 KB PT. */
+ put_unaligned_le64(0, gsp_phys_chunk_page_va(pd0, 0));
+ put_unaligned_le64(pd0_small,
+ (u8 *)gsp_phys_chunk_page_va(pd0, 0) + 8);
+ put_unaligned_le64(pt_gpfifo,
+ (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+ (size_t)gpfifo_idx * 8);
+ put_unaligned_le64(pt_pb,
+ (u8 *)gsp_phys_chunk_page_va(pt, 0) +
+ (size_t)pb_idx * 8);
+ dma_wmb();
+
+ boot_stage.rmcall_va_base = GSP_PHYS_RMCALL_VA_BASE;
+ boot_stage.rmcall_va_gpfifo = GSP_PHYS_RMCALL_VA_GPFIFO;
+ boot_stage.rmcall_va_pb = GSP_PHYS_RMCALL_VA_PB;
+ boot_stage.rmcall_pdb_root = root_ent;
+ boot_stage.rmcall_pdb_pd2 = pd2_ent;
+ boot_stage.rmcall_pdb_pd1 = pd1_ent;
+ boot_stage.rmcall_pdb_pd0_small = pd0_small;
+ boot_stage.rmcall_pt_gpfifo = pt_gpfifo;
+ boot_stage.rmcall_pt_pb = pt_pb;
+
+ pr_info("gsp-phys-boot: rm-call: channel PDB built: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx gpFifoOffset=0x%llx root dma=0x%llx pd2 dma=0x%llx pd1 dma=0x%llx pd0 dma=0x%llx pt dma=0x%llx\n",
+ GSP_PHYS_RMCALL_VA_BASE, GSP_PHYS_RMCALL_VA_GPFIFO,
+ GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO,
+ (u64)gsp_phys_chunk_page_dma(root, 0),
+ (u64)gsp_phys_chunk_page_dma(pd2, 0),
+ (u64)gsp_phys_chunk_page_dma(pd1, 0),
+ (u64)gsp_phys_chunk_page_dma(pd0, 0),
+ (u64)gsp_phys_chunk_page_dma(pt, 0));
+ pr_info("gsp-phys-boot: rm-call: channel PDB entries: root[0]=0x%016llx pd2[0]=0x%016llx pd1[0]=0x%016llx pd0[0].small=0x%016llx\n",
+ root_ent, pd2_ent, pd1_ent, pd0_small);
+ pr_info("gsp-phys-boot: rm-call: channel PDB PT: pt[%u]=0x%016llx (gpfifo dma=0x%llx) pt[%u]=0x%016llx (pb dma=0x%llx)\n",
+ gpfifo_idx, pt_gpfifo,
+ (u64)gsp_phys_chunk_page_dma(
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO],
+ 0),
+ pb_idx, pt_pb,
+ (u64)gsp_phys_chunk_page_dma(
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_PB], 0));
+}
+
+/*
* Stage 3: explicit VAS with its page directory, then channel group (COPY0,
* physical) and the channel. The VAS and PDB must both succeed: a channel
* whose VAS has no page-directory base fails the context-share/instance-block
@@ -5095,7 +5352,10 @@ static int gsp_phys_rmcall_alloc_channel(void __iomem *bar0)
* requested here: its generator requires BIND + GPFIFO_SCHEDULE (runlist
* membership, kernel_fifo_tu102.c:110-116), so the whole BIND -> SCHEDULE ->
* SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN order lives in
- * stage 4.
+ * stage 4. After SET_PAGE_DIRECTORY the root's lower levels are built
+ * (gsp_phys_rmcall_pdb_build), because the engine fetches the GPFIFO and the
+ * pushbuffer by GPU VA through that hierarchy; the USERD page is allocated in
+ * FBMEM alongside the notifier.
*/
static int gsp_phys_rmcall_stage3(void __iomem *bar0)
{
@@ -5151,6 +5411,12 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
pdb_rpc);
return ret;
}
+ /*
+ * The server only stored our root memdesc; the GPFIFO and pushbuffer
+ * fetches translate through it, so fill the hierarchy before the
+ * channel alloc commits the instance block's page-directory base.
+ */
+ gsp_phys_rmcall_pdb_build();
memset(&tsg, 0, sizeof(tsg));
tsg.h_va_space = cpu_to_le32(boot_stage.rmcall_vas);
@@ -5185,15 +5451,26 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
pr_warn("gsp-phys-boot: rm-call: FBMEM error-notifier allocation failed (%d); channel allocs carry a zeroed notifier memdesc and only the stage-4 token recovery can save the token\n",
ret);
+ /*
+ * Allocate the USERD page in FBMEM as well (reference default); on
+ * failure the channel variants fall back to the sysmem USERD chunk.
+ */
+ ret = gsp_phys_rmcall_alloc_userd_mem(bar0);
+ if (ret)
+ pr_warn("gsp-phys-boot: rm-call: FBMEM USERD allocation failed (%d); channel allocs fall back to the sysmem USERD chunk\n",
+ ret);
+
ret = gsp_phys_rmcall_alloc_channel(bar0);
if (ret)
return ret;
- pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n",
+ pr_info("gsp-phys-boot: rm-call: stage 3 complete: hVas=0x%x hTsg=0x%x hChannel=0x%x variant=V%u notifier=%s base=0x%llx userd=%s base=0x%llx engine=COPY0 (cid is server-assigned, request value ignored; token is requested in stage 4)\n",
boot_stage.rmcall_vas, boot_stage.rmcall_tsg,
boot_stage.rmcall_channel, boot_stage.rmcall_channel_variant,
boot_stage.rmcall_notifier_valid ? "FBMEM" : "none",
- boot_stage.rmcall_notifier_base);
+ boot_stage.rmcall_notifier_base,
+ boot_stage.rmcall_userd_valid ? "FBMEM" : "sysmem",
+ boot_stage.rmcall_userd_base);
return 0;
}
@@ -5254,21 +5531,26 @@ static u32 gsp_phys_rmcall_build_methods(u32 *pb, u64 sem_dma, u64 src_dma,
/*
* Stage 4: the corrected token sequence - BIND (COPY0) -> GPFIFO_SCHEDULE ->
- * SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX -> GET_WORK_SUBMIT_TOKEN - then the first
- * real work item: write the CE method block to the pushbuffer, publish the
- * GPFIFO entry and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the
- * PF channel doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem
- * semaphore word. Afterwards dst must equal src byte for byte.
+ * TSG_GPFIFO_SCHEDULE -> SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX ->
+ * GET_WORK_SUBMIT_TOKEN - then the first real work item: write the CE method
+ * block to the pushbuffer, publish the GPFIFO entry (GET = the pushbuffer GPU
+ * VA) and the USERD GP_PUT (mem_utils_gm107.c:1815-1904), ring the PF channel
+ * doorbell (kernel_fifo_tu102.c:46-59) and poll the sysmem semaphore word.
+ * Afterwards dst must equal src byte for byte.
*
* BIND is the precondition the TU102 token generator needs (runlist
* membership, kernel_fifo_tu102.c:110-116; GPFIFO_SCHEDULE alone only sets
* RUNLIST_SET at kernel_channel.c:3244), and SET_NOTIF_INDEX (index 3 for
* subdevice 0, nvidia-push-init.c:303-316) gives the FBMEM notifier write a
* target slot; it is advisory here because no error-context object exists.
- * Every other control failure logs its rpc_result/status and aborts
- * before the doorbell; a 0x40 GET_WORK_SUBMIT_TOKEN whose echoed params carry
- * a nonzero token still proceeds (the server writes the token before the
- * notifier update, kernel_channel.c:3471-3477).
+ * The TSG-level NVA06C_CTRL_CMD_GPFIFO_SCHEDULE 0xa06c0101 ("schedules a
+ * channel group in hardware", ctrla06c.h:82-101) is insurance on top of the
+ * channel-level schedule, whose internal HAL is a stub in the shipped
+ * dispatch; it too is advisory. Every other control failure logs its
+ * rpc_result/status and aborts before the doorbell; a 0x40
+ * GET_WORK_SUBMIT_TOKEN whose echoed params carry a nonzero token still
+ * proceeds (the server writes the token before the notifier update,
+ * kernel_channel.c:3471-3477).
*/
static int gsp_phys_rmcall_stage4(void __iomem *bar0)
{
@@ -5340,6 +5622,30 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
}
/*
+ * (b2) TSG-level NVA06C_CTRL_CMD_GPFIFO_SCHEDULE (0xa06c0101, the
+ * "schedules a channel group in hardware" call, ctrla06c.h:82-101,
+ * same 2-byte NVA06F params) as insurance: the channel-level
+ * GPFIFO_SCHEDULE's internal HAL is a stub in the shipped dispatch, so
+ * the runlist entry is only fully committed by the group schedule.
+ * Advisory, not fatal: the channel-level schedule already set
+ * RUNLIST_SET, which is what the token generator consumes
+ * (kernel_channel.c:3237-3245).
+ */
+ rpc_result = 0;
+ status = 0;
+ ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_tsg,
+ GSP_PHYS_RMCALL_CTRL_TSG_GPFIFO_SCHEDULE,
+ sched_params, sizeof(sched_params), 16,
+ NULL, 0, &rpc_result, &status,
+ "TSG_GPFIFO_SCHEDULE");
+ if (ret)
+ pr_warn("gsp-phys-boot: rm-call: TSG_GPFIFO_SCHEDULE (0xa06c0101 bEnable=1 bSkipSubmit=0 on hTsg=0x%x) advisory failure: rpc_result=0x%x status=0x%x; the channel-level schedule already set RUNLIST_SET, continuing\n",
+ boot_stage.rmcall_tsg, rpc_result, status);
+ else
+ pr_info("gsp-phys-boot: rm-call: TSG_GPFIFO_SCHEDULE 0xa06c0101 on hTsg=0x%x accepted (rpc_result=0x%x status=0x%x)\n",
+ boot_stage.rmcall_tsg, rpc_result, status);
+
+ /*
* (c) SET_WORK_SUBMIT_TOKEN_NOTIF_INDEX (nvidia-push-init.c:303-316).
* Advisory, not fatal: with no error-context object (hErrorContext=0)
* the server's memory and ctxdma lookups both fail and it returns
@@ -5412,18 +5718,25 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
slot = gpput;
/*
- * e0: GET = pushbuffer address >> 2 in bits 31:2, FETCH/NO_CONTEXT_
- * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the address high
- * dword and LENGTH 30:10 in dwords, exactly as the reference RM's own
- * submission does it (mem_utils_gm107.c:1861-1868: NvU64_HI32(get) in
- * GET_HI, NvU64_LO32(get) >> 2 in GET). The physical channel model
- * addresses the ring and pushbuffer by their system DMA addresses.
+ * e0: GET = pushbuffer GPU VA >> 2 in bits 31:2, FETCH/NO_CONTEXT_
+ * SWITCH 0 (class/cl906f.h:136-142); e1: GET_HI 7:0 = the VA high byte
+ * and LENGTH 30:10 in dwords, exactly the reference RM's own submission
+ * (mem_utils_gm107.c:1858-1868: get = pbGpuVA + offset, NvU64_HI32(get)
+ * in GET_HI, NvU64_LO32(get) >> 2 in GET, PRIV USER and LEVEL MAIN).
+ * The engine translates this VA through the channel PDB built in
+ * gsp_phys_rmcall_pdb_build.
*/
- e0 = (u32)(pb_dma >> 2) & 0x3fffffffu;
- e1 = (u32)((pb_dma >> 32) & 0xffu) | (words << 10);
+ e0 = (u32)(GSP_PHYS_RMCALL_VA_PB >> 2) & 0x3fffffffu;
+ e1 = (u32)((GSP_PHYS_RMCALL_VA_PB >> 32) & 0xffu) | (words << 10);
put_unaligned_le32(e0, (u8 *)ring_va + slot * 8);
put_unaligned_le32(e1, (u8 *)ring_va + slot * 8 + 4);
dma_wmb();
+ /*
+ * GP_PUT stays on the client sysmem USERD chunk, which is the hardware
+ * USERD only on the FBMEM-alloc fallback path; when the server used the
+ * FBMEM USERD page this host write cannot reach it (no BAR1/PRAMIN
+ * aperture here), kept as-is for the fallback.
+ */
put_unaligned_le32((gpput + 1) % GSP_PHYS_RMCALL_GPFIFO_ENTRIES,
(u8 *)userd + GSP_PHYS_RMCALL_USERD_GPPUT_OFF);
/*
@@ -5442,8 +5755,10 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
wmb();
writel(doorbell, bar0 + GSP_PHYS_RMCALL_DOORBELL_OFF);
- pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
- pb_dma, words, slot, gpget, gpput, e0, e1,
+ pr_info("gsp-phys-boot: rm-call: submitted: pb=0x%llx pbVa=0x%llx gpFifoVa=0x%llx userd=%s words=%u slot=%u (was GPGet=%u GPPut=%u) e0=0x%08x e1=0x%08x token=0x%08x (runlistId=%u chId=0x%x) doorbell=0x%08x @0x%x; polling sem=0x%llx for 0x%08x (<=%u ms)\n",
+ pb_dma, GSP_PHYS_RMCALL_VA_PB, GSP_PHYS_RMCALL_VA_GPFIFO,
+ boot_stage.rmcall_userd_valid ? "FBMEM" : "sysmem",
+ words, slot, gpget, gpput, e0, e1,
boot_stage.rmcall_token, runlist_id, ch_id, doorbell,
GSP_PHYS_RMCALL_DOORBELL_OFF, sem_dma,
GSP_PHYS_RMCALL_SEM_PAYLOAD, GSP_PHYS_RMCALL_TIMEOUT_MS);
@@ -5731,6 +6046,17 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
boot_stage.rmcall_notifier_valid ? "FBMEM" : "none",
boot_stage.rmcall_notifier_base);
seq_printf(m,
+ "rm_call_userd: userd=%s hMemory=0x%x base=0x%llx | rm_call_pdb: vaBase=0x%llx vaGpfifo=0x%llx vaPb=0x%llx\n",
+ boot_stage.rmcall_userd_valid ? "FBMEM" : "sysmem",
+ boot_stage.rmcall_userd_mem, boot_stage.rmcall_userd_base,
+ boot_stage.rmcall_va_base, boot_stage.rmcall_va_gpfifo,
+ boot_stage.rmcall_va_pb);
+ seq_printf(m,
+ "rm_call_pdb_entries: root=0x%016llx pd2=0x%016llx pd1=0x%016llx pd0Small=0x%016llx ptGpfifo=0x%016llx ptPb=0x%016llx\n",
+ boot_stage.rmcall_pdb_root, boot_stage.rmcall_pdb_pd2,
+ boot_stage.rmcall_pdb_pd1, boot_stage.rmcall_pdb_pd0_small,
+ boot_stage.rmcall_pt_gpfifo, boot_stage.rmcall_pt_pb);
+ seq_printf(m,
"wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n",
wpr2_lo, gsp_phys_fwsec_wpr2_lo_val(bar0),
wpr2_hi, gsp_phys_fwsec_wpr2_hi_val(bar0),
@@ -6625,12 +6951,24 @@ static_assert(offsetof(struct gsp_phys_rmcall_channel_params, process_id) == 296
"NV_CHANNEL_ALLOC_PARAMS.ProcessID must be at offset 296");
static_assert(offsetof(struct gsp_phys_rmcall_channel_params, tpc_config_id) == 360,
"NV_CHANNEL_ALLOC_PARAMS.tpcConfigID must be at offset 360");
-static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 10,
- "the rm-call ladder registers exactly ten DMA buffers");
+static_assert(GSP_PHYS_RMCALL_BUF_COUNT == 14,
+ "the rm-call ladder registers exactly fourteen DMA buffers");
static_assert((GSP_PHYS_RMCALL_HANDLE_MEM_BASE +
(GSP_PHYS_RMCALL_BUF_COUNT - 1) *
GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE) < 0xcaf80000u,
"rm-call memory handles must stay inside the RM client handle range");
+static_assert(GSP_PHYS_RMCALL_VA_PB + GSP_PHYS_PAGE_SIZE <=
+ GSP_PHYS_RMCALL_VAS_SIZE,
+ "the channel PDB VA window must stay below the VAS vaSize limit");
+static_assert(((GSP_PHYS_RMCALL_VA_BASE >> 47) & 0x3) == 0 &&
+ ((GSP_PHYS_RMCALL_VA_BASE >> 38) & 0x1ff) == 0 &&
+ ((GSP_PHYS_RMCALL_VA_BASE >> 29) & 0x1ff) == 0 &&
+ ((GSP_PHYS_RMCALL_VA_BASE >> 21) & 0xff) == 0 &&
+ ((GSP_PHYS_RMCALL_VA_PB >> 21) & 0xff) == 0,
+ "the channel PDB VA window must sit in root/PD2/PD1/PD0 index 0 (GP10X 48:47/46:38/37:29/28:21 splits)");
+static_assert(((GSP_PHYS_RMCALL_VA_GPFIFO >> 12) & 0x1ff) !=
+ ((GSP_PHYS_RMCALL_VA_PB >> 12) & 0x1ff),
+ "the GPFIFO and pushbuffer VAs must take distinct small-PT slots");
/*
* GspSystemInfo wire ABI, pinned against the reference header compile
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 08:52:49 +0000