summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c224
1 files changed, 176 insertions, 48 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 033af6e..fa68945 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -256,9 +256,23 @@ bool gsp_phys_boot_requested(void)
#define GSP_PHYS_RMCALL_HANDLE_SUBDEVICE 0xcaf00004u
#define GSP_PHYS_RMCALL_HANDLE_TSG 0xcaf00008u
#define GSP_PHYS_RMCALL_HANDLE_CHANNEL 0xcaf0000cu
+/* Channel-alloc variant handles (see the stage-3 variant walk). */
+#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1 0xcaf00010u
+#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2 0xcaf00014u
+#define GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 0xcaf00018u
#define GSP_PHYS_RMCALL_HANDLE_MEM_BASE 0xcaf01000u
#define GSP_PHYS_RMCALL_HANDLE_MEM_STRIDE 4u
+/*
+ * NVOS04_FLAGS_CHANNEL_SKIP_SCRUBBER bit 23 (alloc/alloc_channel.h:223-225);
+ * the RM's own internal CE channel sets it
+ * (mem_mgr/arch/maxwell/mem_utils_gm107.c:1186-1283). Kernel privilege is
+ * internalFlags 1:0 = 2 and GSP-ownership is internalFlags bit 6 = 0x40
+ * (generated/g_kernel_channel_nvoc.h:185-203).
+ */
+#define GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER (1u << 23)
+#define GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_GSP_OWNED 0x40u
+
/* Buffers the ladder allocates and registers in stage 2. */
enum gsp_phys_rmcall_buf {
GSP_PHYS_RMCALL_BUF_SRC = 0,
@@ -803,6 +817,7 @@ static struct gsp_phys_boot_stage {
u32 rmcall_subdevice;
u32 rmcall_tsg;
u32 rmcall_channel;
+ u32 rmcall_channel_variant; /* winning stage-3 variant (1..4, 0=none) */
u32 rmcall_token; /* GET_WORK_SUBMIT_TOKEN result */
struct pci_dev *pdev; /* borrowed claim pdev (valid while primed) */
struct gsp_phys_boot_chunk rmcall_chunks[GSP_PHYS_RMCALL_BUF_COUNT];
@@ -1312,6 +1327,7 @@ static void gsp_phys_rmcall_reset(void)
boot_stage.rmcall_subdevice = 0;
boot_stage.rmcall_tsg = 0;
boot_stage.rmcall_channel = 0;
+ boot_stage.rmcall_channel_variant = 0;
boot_stage.rmcall_token = 0;
for (i = 0; i < GSP_PHYS_RMCALL_BUF_COUNT; i++) {
boot_stage.rmcall_mem_handles[i] = 0;
@@ -4296,11 +4312,14 @@ gsp_phys_rmcall_dump_failed_reply(const struct gsp_phys_rx_record *rec)
* One ladder RPC: submit, then wait up to GSP_PHYS_RMCALL_TIMEOUT_MS for the
* reply with the same function id. rec is always owned by the caller (freed
* with gsp_phys_rx_record_free(); zeroed when nothing arrived). Aborts on the
- * first nonzero rpc_result or a short reply.
+ * first nonzero rpc_result or a short reply. dump_on_fail=false keeps the
+ * bounded payload dump for the channel-alloc variant walk; out_rpc_result (if
+ * given) receives the raw server result for compact summaries.
*/
static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function,
const void *req, u32 req_len, u32 reply_min,
- struct gsp_phys_rx_record *rec)
+ struct gsp_phys_rx_record *rec, bool dump_on_fail,
+ u32 *out_rpc_result)
{
struct gsp_phys_status_queue sq;
u32 tx_seq = boot_stage.tx_seq_num;
@@ -4328,13 +4347,16 @@ static int gsp_phys_rmcall_rpc(void __iomem *bar0, u32 function,
function, tx_seq, rec->rpc_length, rec->payload_len,
rec->seq_num, rec->rpc_sequence, rec->rpc_result,
rec->rpc_result_private, rec->checksum);
+ if (out_rpc_result)
+ *out_rpc_result = rec->rpc_result;
gsp_phys_rmcall_dump_reply(rec);
if (rec->rpc_result != 0) {
pr_err("gsp-phys-boot: rm-call: fn%u server rpc_result=0x%x (%s, not success)\n",
function, rec->rpc_result,
gsp_phys_rmcall_status_str(rec->rpc_result));
- gsp_phys_rmcall_dump_failed_reply(rec);
+ if (dump_on_fail)
+ gsp_phys_rmcall_dump_failed_reply(rec);
return -EIO;
}
if (rec->payload_len < reply_min) {
@@ -4354,7 +4376,8 @@ static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client,
u32 h_parent, u32 h_object, u32 h_class,
const void *params, u32 params_len,
bool object_is_out, u32 *out_object,
- const char *what)
+ const char *what, bool dump_on_fail,
+ u32 *out_rpc_result, u32 *out_status)
{
struct gsp_phys_rmcall_rm_alloc_req *req;
struct gsp_phys_rx_record rec;
@@ -4375,7 +4398,8 @@ static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client,
memcpy(req->params, params, params_len);
ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_ALLOC,
- req, req_len, 20, &rec);
+ req, req_len, 20, &rec, dump_on_fail,
+ out_rpc_result);
kvfree(req);
if (ret) {
pr_err("gsp-phys-boot: rm-call: %s (hClass=0x%x) RPC failed: %d\n",
@@ -4396,6 +4420,8 @@ static int gsp_phys_rmcall_alloc(void __iomem *bar0, u32 h_client,
return -EIO;
}
status = gsp_phys_rmcall_reply_u32(&rec, 16);
+ if (out_status)
+ *out_status = status;
if (object_is_out) {
*out_object = gsp_phys_rmcall_reply_u32(&rec, 32);
pr_info("gsp-phys-boot: rm-call: %s alloc: hClass=0x%x status=0x%x -> hObject=0x%x\n",
@@ -4446,7 +4472,7 @@ static int gsp_phys_rmcall_register_mem(void __iomem *bar0, u32 index)
req->pte[i] = cpu_to_le64(gsp_phys_chunk_page_dma(c, i) >> 12);
ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_ALLOC_MEMORY,
- req, (u32)req_len, 0, &rec);
+ req, (u32)req_len, 0, &rec, true, NULL);
kvfree(req);
if (ret) {
pr_err("gsp-phys-boot: rm-call: ALLOC_MEMORY buffer %u (hMemory=0x%x %u page(s) dma=0x%llx) failed: %d\n",
@@ -4494,7 +4520,7 @@ static int gsp_phys_rmcall_control(void __iomem *bar0, u32 h_object, u32 cmd,
memcpy(req->params, params, params_len);
ret = gsp_phys_rmcall_rpc(bar0, GSP_PHYS_RMCALL_FUNC_GSP_RM_CONTROL,
- req, req_len, reply_min, &rec);
+ req, req_len, reply_min, &rec, true, NULL);
kvfree(req);
if (ret) {
pr_err("gsp-phys-boot: rm-call: %s ctrl 0x%x (hObject=0x%x) RPC failed: %d\n",
@@ -4545,7 +4571,8 @@ static int gsp_phys_rmcall_stage1(void __iomem *bar0)
ret = gsp_phys_rmcall_alloc(bar0, 0, 0, 0,
GSP_PHYS_RMCALL_CLASS_ROOT,
params, sizeof(params), true,
- &boot_stage.rmcall_client, "NV01_ROOT");
+ &boot_stage.rmcall_client, "NV01_ROOT",
+ true, NULL, NULL);
if (ret)
return ret;
@@ -4556,7 +4583,8 @@ static int gsp_phys_rmcall_stage1(void __iomem *bar0)
boot_stage.rmcall_client,
GSP_PHYS_RMCALL_HANDLE_DEVICE,
GSP_PHYS_RMCALL_CLASS_DEVICE,
- params, 56, false, NULL, "NV01_DEVICE_0");
+ params, 56, false, NULL, "NV01_DEVICE_0",
+ true, NULL, NULL);
if (ret)
return ret;
boot_stage.rmcall_device = GSP_PHYS_RMCALL_HANDLE_DEVICE;
@@ -4567,7 +4595,8 @@ static int gsp_phys_rmcall_stage1(void __iomem *bar0)
boot_stage.rmcall_device,
GSP_PHYS_RMCALL_HANDLE_SUBDEVICE,
GSP_PHYS_RMCALL_CLASS_SUBDEVICE,
- params, 4, false, NULL, "NV20_SUBDEVICE_0");
+ params, 4, false, NULL, "NV20_SUBDEVICE_0",
+ true, NULL, NULL);
if (ret)
return ret;
boot_stage.rmcall_subdevice = GSP_PHYS_RMCALL_HANDLE_SUBDEVICE;
@@ -4653,12 +4682,75 @@ gsp_phys_rmcall_mem_desc(u32 index)
}
/*
- * Stage 3: channel group (COPY0, physical) then the channel with the
- * registered physical descriptors, then GET_WORK_SUBMIT_TOKEN. The token is
- * requested here as specified; because the server's TU102 generator requires
- * the channel to be on a runlist already (kernel_fifo_tu102.c:110-116, which
- * GPFIFO_SCHEDULE establishes at kernel_channel.c:3244), a nonzero status is
- * logged and retried after the schedule in stage 4 instead of aborting.
+ * Channel-alloc variants tried in order after the TSG exists. V1/V2 mirror the
+ * RM's internal CE channel template (mem_utils_gm107.c:1186-1283: device
+ * parent, hDeviceId, SKIP_SCRUBBER flag, PRIVILEGE_KERNEL internalFlags;
+ * V2 adds GSP_OWNED). V3 keeps the reference shape but zeroes the memdescs and
+ * GPFIFO so the server allocates channel memory itself (no client ring to
+ * fill, so stage 4 refuses). V4 is the previously tried control shape
+ * (TSG parent, flags 0).
+ */
+struct gsp_phys_rmcall_channel_variant {
+ const char *name;
+ bool tsg_parent;
+ u32 flags;
+ u32 internal_flags;
+ bool server_mem; /* zeroed memdescs + no client GPFIFO */
+ u32 handle;
+};
+
+static const struct gsp_phys_rmcall_channel_variant
+gsp_phys_rmcall_channel_variants[] = {
+ { "ref-device-skip-scrubber", false,
+ GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
+ GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
+ GSP_PHYS_RMCALL_HANDLE_CHANNEL_V1 },
+ { "ref-plus-gsp-owned", false,
+ GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
+ GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL |
+ GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_GSP_OWNED, false,
+ GSP_PHYS_RMCALL_HANDLE_CHANNEL_V2 },
+ { "server-allocated-mem", false,
+ GSP_PHYS_RMCALL_CHANNEL_FLAG_SKIP_SCRUBBER,
+ GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, true,
+ GSP_PHYS_RMCALL_HANDLE_CHANNEL_V3 },
+ { "current-control", true, 0,
+ GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL, false,
+ GSP_PHYS_RMCALL_HANDLE_CHANNEL },
+};
+
+static void
+gsp_phys_rmcall_channel_params_init(struct gsp_phys_rmcall_channel_params *ch,
+ const struct gsp_phys_rmcall_channel_variant *v)
+{
+ memset(ch, 0, sizeof(*ch));
+ ch->flags = cpu_to_le32(v->flags);
+ ch->engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0);
+ ch->cid = cpu_to_le32(1); /* client hint; server assigns its own */
+ ch->sub_device_id = cpu_to_le32(0);
+ ch->internal_flags = cpu_to_le32(v->internal_flags);
+ if (v->server_mem)
+ return; /* zeroed memdescs, gpFifoOffset/gpFifoEntries 0 */
+ ch->gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
+ &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0));
+ ch->gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
+ ch->instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE);
+ ch->userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD);
+ ch->ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC);
+ ch->mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF);
+}
+
+/*
+ * Stage 3: channel group (COPY0, physical, parent=device) then the channel.
+ * The channel alloc walks the variants below in order (same already-held
+ * gsp_phys_rm_lock and gates); the first rpc_result==0 wins. Failed variants
+ * log one compact line each and no payload dump, and an all-failed walk ends
+ * in one summary line with every variant's rpc_result/status. Then
+ * GET_WORK_SUBMIT_TOKEN. The token is requested before scheduling as
+ * specified; because the server's TU102 generator requires the channel to be
+ * on a runlist already (kernel_fifo_tu102.c:110-116, which GPFIFO_SCHEDULE
+ * establishes at kernel_channel.c:3244), a nonzero status is logged and
+ * retried after the schedule in stage 4 instead of aborting.
*/
static int gsp_phys_rmcall_stage3(void __iomem *bar0)
{
@@ -4682,35 +4774,64 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
GSP_PHYS_RMCALL_HANDLE_TSG,
GSP_PHYS_RMCALL_CLASS_TSG,
&tsg, sizeof(tsg), false, NULL,
- "KEPLER_CHANNEL_GROUP_A");
+ "KEPLER_CHANNEL_GROUP_A", true, NULL, NULL);
if (ret)
return ret;
boot_stage.rmcall_tsg = GSP_PHYS_RMCALL_HANDLE_TSG;
- memset(&ch, 0, sizeof(ch));
- ch.gp_fifo_offset = cpu_to_le64((u64)gsp_phys_chunk_page_dma(
- &boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_GPFIFO], 0));
- ch.gp_fifo_entries = cpu_to_le32(GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
- ch.flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_FLAG_PRIVILEGED);
- ch.engine_type = cpu_to_le32(GSP_PHYS_RMCALL_ENGINE_COPY0);
- ch.cid = cpu_to_le32(1); /* client hint only: the server assigns
- * its own unique cid (kernel_channel.c:
- * 212-215) */
- ch.sub_device_id = cpu_to_le32(0);
- ch.internal_flags = cpu_to_le32(GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_PRIV_KERNEL);
- ch.instance_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE);
- ch.userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD);
- ch.ramfc_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC);
- ch.mthdbuf_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_MTHDBUF);
- ret = gsp_phys_rmcall_alloc(bar0, boot_stage.rmcall_client,
- boot_stage.rmcall_tsg,
- GSP_PHYS_RMCALL_HANDLE_CHANNEL,
- GSP_PHYS_RMCALL_CLASS_CHANNEL,
- &ch, sizeof(ch), false, NULL,
- "TURING_CHANNEL_GPFIFO_A");
- if (ret)
- return ret;
- boot_stage.rmcall_channel = GSP_PHYS_RMCALL_HANDLE_CHANNEL;
+ {
+ u32 v_rpc[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
+ u32 v_status[ARRAY_SIZE(gsp_phys_rmcall_channel_variants)];
+ u32 i;
+
+ ret = -EIO;
+ for (i = 0; i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants); i++) {
+ const struct gsp_phys_rmcall_channel_variant *v =
+ &gsp_phys_rmcall_channel_variants[i];
+ u32 parent = v->tsg_parent ? boot_stage.rmcall_tsg :
+ boot_stage.rmcall_device;
+
+ gsp_phys_rmcall_channel_params_init(&ch, v);
+ v_rpc[i] = 0;
+ v_status[i] = 0;
+ ret = gsp_phys_rmcall_alloc(bar0,
+ boot_stage.rmcall_client, parent,
+ v->handle,
+ GSP_PHYS_RMCALL_CLASS_CHANNEL,
+ &ch, sizeof(ch), false, NULL, v->name,
+ false, &v_rpc[i], &v_status[i]);
+ pr_info("gsp-phys-boot: rm-call: stage 3 channel variant %u/%zu (%s): hParent=0x%x hObject=0x%x flags=0x%x internalFlags=0x%x ret=%d rpc_result=0x%x status=0x%x\n",
+ i + 1,
+ ARRAY_SIZE(gsp_phys_rmcall_channel_variants),
+ v->name, parent, v->handle, v->flags,
+ v->internal_flags, ret, v_rpc[i], v_status[i]);
+ if (!ret) {
+ boot_stage.rmcall_channel = v->handle;
+ boot_stage.rmcall_channel_variant = i + 1;
+ pr_info("gsp-phys-boot: rm-call: stage 3 channel variant %u (%s) WON: hChannel=0x%x\n",
+ i + 1, v->name, v->handle);
+ break;
+ }
+ }
+ if (ret) {
+ char summary[256];
+ u32 pos = 0;
+
+ for (i = 0;
+ i < ARRAY_SIZE(gsp_phys_rmcall_channel_variants);
+ i++)
+ pos += scnprintf(summary + pos,
+ sizeof(summary) - pos,
+ "%sV%u %s rpc=0x%x status=0x%x",
+ i ? "; " : "", i + 1,
+ gsp_phys_rmcall_channel_variants[i].name,
+ v_rpc[i], v_status[i]);
+ pr_err("gsp-phys-boot: rm-call: stage 3: all %zu channel variants failed: %s\n",
+ ARRAY_SIZE(gsp_phys_rmcall_channel_variants),
+ summary);
+ return ret;
+ }
+ }
ret = gsp_phys_rmcall_control(bar0, boot_stage.rmcall_channel,
GSP_PHYS_RMCALL_CTRL_GET_WORK_SUBMIT_TOKEN,
@@ -4724,13 +4845,9 @@ static int gsp_phys_rmcall_stage3(void __iomem *bar0)
boot_stage.rmcall_token = token;
}
- pr_info("gsp-phys-boot: rm-call: stage 3 complete: hTsg=0x%x hChannel=0x%x token=0x%x engine=COPY0 (cid is server-assigned, request value ignored) descriptors instance=0x%llx userd=0x%llx ramfc=0x%llx mthdbuf=0x%llx\n",
+ pr_info("gsp-phys-boot: rm-call: stage 3 complete: hTsg=0x%x hChannel=0x%x variant=%u token=0x%x engine=COPY0 (cid is server-assigned, request value ignored)\n",
boot_stage.rmcall_tsg, boot_stage.rmcall_channel,
- boot_stage.rmcall_token,
- (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_INSTANCE], 0),
- (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_USERD], 0),
- (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_RAMFC], 0),
- (u64)gsp_phys_chunk_page_dma(&boot_stage.rmcall_chunks[GSP_PHYS_RMCALL_BUF_MTHDBUF], 0));
+ boot_stage.rmcall_channel_variant, boot_stage.rmcall_token);
return 0;
}
@@ -4812,6 +4929,16 @@ static int gsp_phys_rmcall_stage4(void __iomem *bar0)
unsigned long deadline;
int ret;
+ /*
+ * V3 won with server-allocated memdescs: there is no client GPFIFO,
+ * USERD or pushbuffer to fill, so no work item can be submitted.
+ * Distinct error, nothing is scheduled or rung.
+ */
+ if (boot_stage.rmcall_channel_variant == 3) {
+ pr_err("gsp-phys-boot: rm-call: stage 4 unavailable: winning channel variant V3 (server-allocated mem) has no client GPFIFO/USERD; submission requires V1, V2 or V4\n");
+ return -EOPNOTSUPP;
+ }
+
if (!boot_stage.pdev) {
pr_err("gsp-phys-boot: rm-call: stage 4 has no claimed pci_dev for the BAR0 length\n");
return -EINVAL;
@@ -5156,12 +5283,13 @@ static int gsp_phys_state_show(struct seq_file *m, void *v)
boot_stage.rm_probe_last_function,
boot_stage.rm_probe_last_payload_len);
seq_printf(m,
- "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hTsg=0x%x hChannel=0x%x token=0x%x\n",
+ "rm_call: ran=%u stage=%u last_stage=%u last_ret=%d rpcs=%u hClient=0x%x hDevice=0x%x hSubdevice=0x%x hTsg=0x%x hChannel=0x%x variant=%u token=0x%x\n",
boot_stage.rmcall_ran, boot_stage.rmcall_stage,
boot_stage.rmcall_last_stage, boot_stage.rmcall_last_ret,
boot_stage.rmcall_rpcs, boot_stage.rmcall_client,
boot_stage.rmcall_device, boot_stage.rmcall_subdevice,
boot_stage.rmcall_tsg, boot_stage.rmcall_channel,
+ boot_stage.rmcall_channel_variant,
boot_stage.rmcall_token);
seq_printf(m,
"wpr2: LO=0x%08x (VAL=0x%x) HI=0x%08x (VAL=0x%x) state=%s\n",
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 21:39:04 +0000