From f619b2f5c4f6d16bd10aa384397d5a09e0ee052c Mon Sep 17 00:00:00 2001 From: Arthur Hanson Rasmusson Date: Thu, 1 Oct 2026 18:01:50 -0400 Subject: gsp-phys: channel-alloc variant matrix (device parent, skip-scrubber, gsp-owned, server-allocated memdescs) --- kernel/gsp_phys_booter.c | 224 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 176 insertions(+), 48 deletions(-) (limited to 'kernel/gsp_phys_booter.c') diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c index 033af6e..fa68945 100644 --- a/kernel/gsp_phys_booter.c +++ b/kernel/gsp_phys_booter.c @@ -256,9 +256,23 @@ bool gsp_phys_boot_requested(void) #define GSP_PHYS_RMCALL_HANDLE_SUBDEVICE 0xcaf00004u #define GSP_PHYS_RMCALL_HANDLE_TSG 0xcaf00008u #define GSP_PHYS_RMCALL_HANDLE_CHANNEL 0xcaf0000cu +/* Channel-alloc variant handles (see the stage-3 variant walk). */ +#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1 0xcaf00010u +#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2 0xcaf00014u +#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 0xcaf00018u #define GSP_PHYS_RMCALL_HANDLE_MEM_BASE 0xcaf01000u #define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE 4u +/* + * NVOS04_FLAGS_CHANNEL_SKIP_SCRUBBER bit 23 (alloc/alloc_channel.h:223-225); + * the RM's own internal CE channel sets it + * (mem_mgr/arch/maxwell/mem_utils_gm107.c:1186-1283). Kernel privilege is + * internalFlags 1:0 = 2 and GSP-ownership is internalFlags bit 6 = 0x40 + * (generated/g_kernel_channel_nvoc.h:185-203). + */ +#define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER (1u << 23) +#define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_GSP_OWNED 0x40u + /* Buffers the ladder allocates and registers in stage 2. */ enum gsp_phys_rmcall_buf { GSP_PHYS_RMCALL_BUF_SRC = 0, @@ -803,6 +817,7 @@ static struct gsp_phys_boot_stage { u32 rmcall_subdevice; u32 rmcall_tsg; u32 rmcall_channel; + u32 rmcall_channel_variant; /* winning stage-3 variant (1..4, 0=none) */ u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */ struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */ struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT]; @@ -1312,6 +1327,7 @@ static void gsp_phys_rmcall_reset(void) boot_stage.rmcall_subdevice = 0; boot_stage.rmcall_tsg = 0; boot_stage.rmcall_channel = 0; + boot_stage.rmcall_channel_variant = 0; boot_stage.rmcall_token = 0; for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) { boot_stage.rmcall_mem_handles[i] = 0; @@ -4296,11 +4312,14 @@ gsp_phys_rmcall_dump_failed_reply(const struct gsp_phys_rx_record *rec) * One ladder RPC: submit, then wait up to GSP_PHYS_RMCALL_TIMEOUT_MS for the * reply with the same function id. rec is always owned by the caller (freed * with gsp_phys_rx_record_free(); zeroed when nothing arrived). Aborts on the - * first nonzero rpc_result or a short reply. + * first nonzero rpc_result or a short reply. dump_on_fail=false keeps the + * bounded payload dump for the channel-alloc variant walk; out_rpc_result (if + * given) receives the raw server result for compact summaries. */ static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function, const void *req, u32 req_len, u32 reply_min, - struct gsp_phys_rx_record *rec) + struct gsp_phys_rx_record *rec, bool dump_on_fail, + u32 *out_rpc_result) { struct gsp_phys_status_queue sq; u32 tx_seq = boot_stage.tx_seq_num; @@ -4328,13 +4347,16 @@ static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function, function, tx_seq, rec->rpc_length, rec->payload_len, rec->seq_num, rec->rpc_sequence, rec->rpc_result, rec->rpc_result_private, rec->checksum); + if (out_rpc_result) + *out_rpc_result = rec->rpc_result; gsp_phys_rmcall_dump_reply(rec); if (rec->rpc_result != 0) { pr_err("gsp-phys-boot: rm-call: fn%u server rpc_result=0x%x (%s, not success)\n", function, rec->rpc_result, gsp_phys_rmcall_status_str(rec->rpc_result)); - gsp_phys_rmcall_dump_failed_reply(rec); + if (dump_on_fail) + gsp_phys_rmcall_dump_failed_reply(rec); return -EIO; } if (rec->payload_len < reply_min) { @@ -4354,7 +4376,8 @@ static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client, u32 h_parent, u32 h_object, u32 h_class, const void *params, u32 params_len, bool object_is_out, u32 *out_object, - const char *what) + const char *what, bool dump_on_fail, + u32 *out_rpc_result, u32 *out_status) { struct gsp_phys_rmcall_rm_alloc_req *req; struct gsp_phys_rx_record rec; @@ -4375,7 +4398,8 @@ static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client, memcpy(req->params, params, params_len); ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC, - req, req_len, 20, &rec); + req, req_len, 20, &rec, dump_on_fail, + out_rpc_result); kvfree(req); if (ret) { pr_err("gsp-phys-boot: rm-call: %s (hClass=0x%x) RPC failed: %d\n", @@ -4396,6 +4420,8 @@ static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client, return -EIO; } status = gsp_phys_rmcall_reply_u32(&rec, 16); + if (out_status) + *out_status = status; if (object_is_out) { *out_object = gsp_phys_rmcall_reply_u32(&rec, 32); pr_info("gsp-phys-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x\n", @@ -4446,7 +4472,7 @@ static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index) req->pte[i] = cpu_to_le64(gsp_phys_chunk_page_dma(c, i) >> 12); ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY, - req, (u32)req_len, 0, &rec); + req, (u32)req_len, 0, &rec, true, NULL); kvfree(req); if (ret) { pr_err("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u (hMemory=0x%x %u page(s) dma=0x%llx) failed: %d\n", @@ -4494,7 +4520,7 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd, memcpy(req->params, params, params_len); ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL, - req, req_len, reply_min, &rec); + req, req_len, reply_min, &rec, true, NULL); kvfree(req); if (ret) { pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x) RPC failed: %d\n", @@ -4545,7 +4571,8 @@ static int gsp_phys_rmcall_stage1(void __iomem *bar0) ret = gsp_phys_rmcall_alloc(bar0, 0, 0, 0, GSP_PHYS_RMCALL_CLASS_ROOT, params, sizeof(params), true, - &boot_stage.rmcall_client, "NV01_ROOT"); + &boot_stage.rmcall_client, "NV01_ROOT", + true, NULL, NULL); if (ret) return ret; @@ -4556,7 +4583,8 @@ static int gsp_phys_rmcall_stage1(void __iomem *bar0) boot_stage.rmcall_client, GSP_PHYS_RMCALL_HANDLE_DEVICE, GSP_PHYS_RMCALL_CLASS_DEVICE, - params, 56, false, NULL, "NV01_DEVICE_0"); + params, 56, false, NULL, "NV01_DEVICE_0", + true, NULL, NULL); if (ret) return ret; boot_stage.rmcall_device = GSP_PHYS_RMCALL_HANDLE_DEVICE; @@ -4567,7 +4595,8 @@ static int gsp_phys_rmcall_stage1(void __iomem *bar0) boot_stage.rmcall_device, GSP_PHYS_RMCALL_HANDLE_SUBDEVICE, GSP_PHYS_RMCALL_CLASS_SUBDEVICE, - params, 4, false, NULL, "NV20_SUBDEVICE_0"); + params, 4, false, NULL, "NV20_SUBDEVICE_0", + true, NULL, NULL); if (ret) return ret; boot_stage.rmcall_subdevice = GSP_PHYS_RMCALL_HANDLE_SUBDEVICE; @@ -4653,12 +4682,75 @@ gsp_phys_rmcall_mem_desc(u32 index) } /* - * Stage 3: channel group (COPY0, physical) then the channel with the - * registered physical descriptors, then GET_WORK_SUBMIT_TOKEN. The token is - * requested here as specified; because the server's TU102 generator requires - * the channel to be on a runlist already (kernel_fifo_tu102.c:110-116, which - * GPFIFO_SCHEDULE establishes at kernel_channel.c:3244), a nonzero status is - * logged and retried after the schedule in stage 4 instead of aborting. + * Channel-alloc variants tried in order after the TSG exists. V1/V2 mirror the + * RM's internal CE channel template (mem_utils_gm107.c:1186-1283: device + * parent, hDeviceId, SKIP_SCRUBBER flag, PRIVILEGE_KERNEL internalFlags; + * V2 adds GSP_OWNED). V3 keeps the reference shape but zeroes the memdescs and + * GPFIFO so the server allocates channel memory itself (no client ring to + * fill, so stage 4 refuses). V4 is the previously tried control shape + * (TSG parent, flags 0). + */ +struct gsp_phys_rmcall_channel_variant { + const char *name; + bool tsg_parent; + u32 flags; + u32 internal_flags; + bool server_mem; /* zeroed memdescs + no client GPFIFO */ + u32 handle; +}; + +static const struct gsp_phys_rmcall_channel_variant +gsp_phys_rmcall_channel_variants[] = { + { "ref-device-skip-scrubber", false, + GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER, + GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false, + GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1 }, + { "ref-plus-gsp-owned", false, + GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER, + GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL | + GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_GSP_OWNED, false, + GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2 }, + { "server-allocated-mem", false, + GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER, + GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, true, + GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 }, + { "current-control", true, 0, + GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false, + GSP_PHYS_RMCALL_HANDLE_CHANNEL }, +}; + +static void +gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch, + const struct gsp_phys_rmcall_channel_variant *v) +{ + memset(ch, 0, sizeof(*ch)); + ch->flags = cpu_to_le32(v->flags); + ch->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); + ch->cid = cpu_to_le32(1); /* client hint; server assigns its own */ + ch->sub_device_id = cpu_to_le32(0); + ch->internal_flags = cpu_to_le32(v->internal_flags); + if (v->server_mem) + return; /* zeroed memdescs, gpFifoOffset/gpFifoEntries 0 */ + ch->gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma( + &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0)); + ch->gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES); + ch->instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE); + ch->userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD); + ch->ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC); + ch->mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF); +} + +/* + * Stage 3: channel group (COPY0, physical, parent=device) then the channel. + * The channel alloc walks the variants below in order (same already-held + * gsp_phys_rm_lock and gates); the first rpc_result==0 wins. Failed variants + * log one compact line each and no payload dump, and an all-failed walk ends + * in one summary line with every variant's rpc_result/status. Then + * GET_WORK_SUBMIT_TOKEN. The token is requested before scheduling as + * specified; because the server's TU102 generator requires the channel to be + * on a runlist already (kernel_fifo_tu102.c:110-116, which GPFIFO_SCHEDULE + * establishes at kernel_channel.c:3244), a nonzero status is logged and + * retried after the schedule in stage 4 instead of aborting. */ static int gsp_phys_rmcall_stage3(void __iomem *bar0) { @@ -4682,35 +4774,64 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) GSP_PHYS_RMCALL_HANDLE_TSG, GSP_PHYS_RMCALL_CLASS_TSG, &tsg, sizeof(tsg), false, NULL, - "KEPLER_CHANNEL_GROUP_A"); + "KEPLER_CHANNEL_GROUP_A", true, NULL, NULL); if (ret) return ret; boot_stage.rmcall_tsg = GSP_PHYS_RMCALL_HANDLE_TSG; - memset(&ch, 0, sizeof(ch)); - ch.gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma( - &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0)); - ch.gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES); - ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED); - ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0); - ch.cid = cpu_to_le32(1); /* client hint only: the server assigns - * its own unique cid (kernel_channel.c: - * 212-215) */ - ch.sub_device_id = cpu_to_le32(0); - ch.internal_flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL); - ch.instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE); - ch.userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD); - ch.ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC); - ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF); - ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client, - boot_stage.rmcall_tsg, - GSP_PHYS_RMCALL_HANDLE_CHANNEL, - GSP_PHYS_RMCALL_CLASS_CHANNEL, - &ch, sizeof(ch), false, NULL, - "TURING_CHANNEL_GPFIFO_A"); - if (ret) - return ret; - boot_stage.rmcall_channel = GSP_PHYS_RMCALL_HANDLE_CHANNEL; + { + u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; + u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)]; + u32 i; + + ret = -EIO; + for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) { + const struct gsp_phys_rmcall_channel_variant *v = + &gsp_phys_rmcall_channel_variants[i]; + u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg : + boot_stage.rmcall_device; + + gsp_phys_rmcall_channel_params_init(&ch, v); + v_rpc[i] = 0; + v_status[i] = 0; + ret = gsp_phys_rmcall_alloc(bar0, + boot_stage.rmcall_client, parent, + v->handle, + GSP_PHYS_RMCALL_CLASS_CHANNEL, + &ch, sizeof(ch), false, NULL, v->name, + false, &v_rpc[i], &v_status[i]); + pr_info("gsp-phys-boot: rm-call: stage 3 channel variant %u/%zu (%s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n", + i + 1, + ARRAY_SIZE(gsp_phys_rmcall_channel_variants), + v->name, parent, v->handle, v->flags, + v->internal_flags, ret, v_rpc[i], v_status[i]); + if (!ret) { + boot_stage.rmcall_channel = v->handle; + boot_stage.rmcall_channel_variant = i + 1; + pr_info("gsp-phys-boot: rm-call: stage 3 channel variant %u (%s) WON: hChannel=0x%x\n", + i + 1, v->name, v->handle); + break; + } + } + if (ret) { + char summary[256]; + u32 pos = 0; + + for (i = 0; + i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); + i++) + pos += scnprintf(summary + pos, + sizeof(summary) - pos, + "%sV%u %s rpc=0x%x status=0x%x", + i ? "; " : "", i + 1, + gsp_phys_rmcall_channel_variants[i].name, + v_rpc[i], v_status[i]); + pr_err("gsp-phys-boot: rm-call: stage 3: all %zu channel variants failed: %s\n", + ARRAY_SIZE(gsp_phys_rmcall_channel_variants), + summary); + return ret; + } + } ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel, GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN, @@ -4724,13 +4845,9 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0) boot_stage.rmcall_token = token; } - pr_info("gsp-phys-boot: rm-call: stage 3 complete: hTsg=0x%x hChannel=0x%x token=0x%x engine=COPY0 (cid is server-assigned, request value ignored) descriptors instance=0x%llx userd=0x%llx ramfc=0x%llx mthdbuf=0x%llx\n", + pr_info("gsp-phys-boot: rm-call: stage 3 complete: hTsg=0x%x hChannel=0x%x variant=%u token=0x%x engine=COPY0 (cid is server-assigned, request value ignored)\n", boot_stage.rmcall_tsg, boot_stage.rmcall_channel, - boot_stage.rmcall_token, - (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE], 0), - (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD], 0), - (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC], 0), - (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_MTHDBUF], 0)); + boot_stage.rmcall_channel_variant, boot_stage.rmcall_token); return 0; } @@ -4812,6 +4929,16 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0) unsigned long deadline; int ret; + /* + * V3 won with server-allocated memdescs: there is no client GPFIFO, + * USERD or pushbuffer to fill, so no work item can be submitted. + * Distinct error, nothing is scheduled or rung. + */ + if (boot_stage.rmcall_channel_variant == 3) { + pr_err("gsp-phys-boot: rm-call: stage 4 unavailable: winning channel variant V3 (server-allocated mem) has no client GPFIFO/USERD; submission requires V1, V2 or V4\n"); + return -EOPNOTSUPP; + } + if (!boot_stage.pdev) { pr_err("gsp-phys-boot: rm-call: stage 4 has no claimed pci_dev for the BAR0 length\n"); return -EINVAL; @@ -5156,12 +5283,13 @@ static int gsp_phys_state_show(struct seq_file *m, void *v) boot_stage.rm_probe_last_function, boot_stage.rm_probe_last_payload_len); seq_printf(m, - "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hTsg=0x%x hChannel=0x%x token=0x%x\n", + "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hTsg=0x%x hChannel=0x%x variant=%u token=0x%x\n", boot_stage.rmcall_ran, boot_stage.rmcall_stage, boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret, boot_stage.rmcall_rpcs, boot_stage.rmcall_client, boot_stage.rmcall_device, boot_stage.rmcall_subdevice, boot_stage.rmcall_tsg, boot_stage.rmcall_channel, + boot_stage.rmcall_channel_variant, boot_stage.rmcall_token); seq_printf(m, "wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n", -- cgit v1.2.3