summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c135
-rw-r--r--kernel/gsp_phys_gpu.c87
-rw-r--r--kernel/gsp_phys_gpu.h13
3 files changed, 227 insertions, 8 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 4fb37f4..2273164 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -12843,13 +12843,21 @@ static int gsp_phys_rmcall_alloc_user_lane_channel(void __iomem *bar0)
ch.internal_flags = cpu_to_le32(
GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_USER_LANE);
/*
- * m70: back to the capture-exact FBMEM instance/ramfc slices. The m62
- * sysmem-chunk deviation made the server's writes host-readable (that
- * job is done), but the capture uses FBMEM here and the GR engine
- * core's context restore may require VIDMEM instance/RAMFC.
+ * m70: capture-exact FBMEM instance/ramfc slices by default. m73
+ * selects the m62 host-visible sysmem chunks for A/B runs: the FECS
+ * context load completes with the sysmem instance (VALID=1, TARGET=3)
+ * but never completes with the FBMEM slice.
*/
- ch.instance_mem = gsp_phys_rmcall_mem_desc_fb(fb, 0x1000);
- ch.ramfc_mem = gsp_phys_rmcall_mem_desc_fb(fb, 0x200);
+ if (gsp_phys_rmcall_gr_inst_sys_value()) {
+ ch.instance_mem =
+ gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_INSTANCE);
+ ch.ramfc_mem =
+ gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_RAMFC);
+ ch.ramfc_mem.size = cpu_to_le64(0x200);
+ } else {
+ ch.instance_mem = gsp_phys_rmcall_mem_desc_fb(fb, 0x1000);
+ ch.ramfc_mem = gsp_phys_rmcall_mem_desc_fb(fb, 0x200);
+ }
ch.userd_mem = gsp_phys_rmcall_mem_desc(GSP_PHYS_RMCALL_BUF_USERD);
ch.h_userd_memory[0] =
cpu_to_le32(GSP_PHYS_RMCALL_HANDLE_USER_LANE_USERD);
@@ -12870,7 +12878,7 @@ static int gsp_phys_rmcall_alloc_user_lane_channel(void __iomem *bar0)
false, NULL,
"mode 35 V7 real-client user lane channel", true,
&rpc, &status);
- pr_info("vgsp-boot: rm-call: mode 35 V7 channel alloc: hObject=0x%x hParent=hTsg=0x%x flags=0x%x internalFlags=0x%x engineType=0 hContextShare=0x%x hVASpace=0 gpFifoOffset=0x%llx gpFifoEntries=%u hUserdMemory[0]=0 userdMem=sysmem instance=0x%llx/0x1000 FBMEM ramfc=0x%llx/0x200 FBMEM (m70: capture shape) hObjectError=0x%x errorNotifier=sysmem status=0x%x rpc_result=0x%x ret=%d (deviations: entries=%u vs captured 1024; sysmem USERD vs captured hUserdMemory object)\n",
+ pr_info("vgsp-boot: rm-call: mode 35 V7 channel alloc: hObject=0x%x hParent=hTsg=0x%x flags=0x%x internalFlags=0x%x engineType=0 hContextShare=0x%x hVASpace=0 gpFifoOffset=0x%llx gpFifoEntries=%u hUserdMemory[0]=0 userdMem=sysmem instance=0x%llx/0x1000 ramfc=0x%llx/0x200 (%s) hObjectError=0x%x errorNotifier=sysmem status=0x%x rpc_result=0x%x ret=%d (deviations: entries=%u vs captured 1024; sysmem USERD vs captured hUserdMemory object)\n",
GSP_PHYS_RMCALL_HANDLE_USER_LANE, boot_stage.rmcall_tsg,
GSP_PHYS_RMCALL_CHANNEL_FLAG_USER_LANE,
GSP_PHYS_RMCALL_CHANNEL_INTERNALFLAGS_USER_LANE,
@@ -12879,6 +12887,8 @@ static int gsp_phys_rmcall_alloc_user_lane_channel(void __iomem *bar0)
le32_to_cpu(ch.gp_fifo_entries),
le64_to_cpu(ch.instance_mem.base),
le64_to_cpu(ch.ramfc_mem.base),
+ gsp_phys_rmcall_gr_inst_sys_value() ? "sysmem m73" :
+ "FBMEM m70 shape",
GSP_PHYS_RMCALL_HANDLE_USER_LANE_NOTIFIER,
status, rpc, ret, GSP_PHYS_RMCALL_GPFIFO_ENTRIES);
return ret;
@@ -15194,6 +15204,21 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
#define GSP_PHYS_RMCALL_GR_INIT_NON_THROTTLED_C 0x2ecu
#define GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL 0x250u
/*
+ * m72 local-memory context block (vendor turingLocalMemUpdate, emitted by the
+ * stock driver at context init): SET_SHADER_LOCAL_MEMORY_A/B (VAS lmem base),
+ * SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A/B (lmem size per TPC),
+ * SET_SHADER_LOCAL_MEMORY_WINDOW_A/B (GTAS lmem window base). Without it the
+ * channel's local-memory state is zero and SKED cancels the QMD with
+ * SKEDCHECK05 LOCAL_MEMORY_TOTAL_SIZE (SKED_HWW_ESR bit12) after allocating
+ * and deallocating a CWD entry.
+ */
+#define GSP_PHYS_RMCALL_GR_INIT_LMEM_A 0x790u
+#define GSP_PHYS_RMCALL_GR_INIT_LMEM_B 0x794u
+#define GSP_PHYS_RMCALL_GR_INIT_LMEM_NT_A 0x2e4u
+#define GSP_PHYS_RMCALL_GR_INIT_LMEM_NT_B 0x2e8u
+#define GSP_PHYS_RMCALL_GR_INIT_LMEM_WIN_A 0x7b0u
+#define GSP_PHYS_RMCALL_GR_INIT_LMEM_WIN_B 0x7b4u
+/*
* NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (class/clc5c0.h): emitted between the
* compute init and the inline-QMD window when gsp_phys_rmcall_gr_qmdvirt=1
* (the default). Value enables IQ2M_ADDR_ENABLE (bit 16) and
@@ -15248,6 +15273,27 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p)
(u32)((u64)GSP_PHYS_RMCALL_GR_SHARED_WINDOW_BASE >> 32));
p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SMEM_WINDOW_B,
GSP_PHYS_RMCALL_GR_SHARED_WINDOW_BASE);
+ /* m72: vendor local-memory context block (SKEDCHECK05 fix). */
+ if (gsp_phys_rmcall_gr_lmem_en_value()) {
+ u64 lmem_base = (u64)gsp_phys_rmcall_gr_lmem_base_value();
+ u32 lmem_tpc = gsp_phys_rmcall_gr_lmem_tpc_size_value();
+
+ p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_LMEM_A,
+ (u32)(lmem_base >> 32));
+ p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_LMEM_B,
+ (u32)lmem_base);
+ p = gsp_phys_rmcall_push_gr(p,
+ GSP_PHYS_RMCALL_GR_INIT_LMEM_NT_A, 0u);
+ p = gsp_phys_rmcall_push_gr(p,
+ GSP_PHYS_RMCALL_GR_INIT_LMEM_NT_B,
+ lmem_tpc);
+ p = gsp_phys_rmcall_push_gr(p,
+ GSP_PHYS_RMCALL_GR_INIT_LMEM_WIN_A,
+ (u32)(lmem_base >> 32));
+ p = gsp_phys_rmcall_push_gr(p,
+ GSP_PHYS_RMCALL_GR_INIT_LMEM_WIN_B,
+ (u32)lmem_base);
+ }
p = gsp_phys_rmcall_push_gr(p, GSP_PHYS_RMCALL_GR_INIT_SPA_VERSION,
GSP_PHYS_RMCALL_GR_SPA_VERSION);
for (i = GSP_PHYS_RMCALL_GR_CWD_LEVELS; i-- > 0;)
@@ -15726,6 +15772,12 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
u8 *qmd_dst_mem = NULL;
u32 qmd_rel_off = 0, ch_rel_off = 0;
u32 poll = 0, w0 = 0, w1 = 0, grpoll_n = 0;
+ static u32 sked_prev[11];
+ static u32 sked_sample[11];
+ static u32 sked_tick;
+ static bool sked_first;
+ sked_tick = 0;
+ sked_first = true;
u32 qmd_win_pde = 0, qmd_win_installed = 0;
int qmd_win_ret = 0;
bool qmd_win_diag = false;
@@ -16306,6 +16358,42 @@ gr_page_unmapped:
deadline = jiffies + msecs_to_jiffies(GSP_PHYS_RMCALL_GR_POLL_MS);
for (;;) {
+ /*
+ * m72: SKED transition recorder - sample the CWD/HWW registers
+ * every ~8 ms and log only changes, so the exact allocate,
+ * cancel or dispatch sequence of the launch window is on the
+ * record even when the poll times out.
+ */
+ if (gsp_phys_rmcall_gr_skedrec_value() && bar0 &&
+ (++sked_tick & 7u) == 0u) {
+ sked_sample[0] = ioread32(bar0 + 0x00407060u);
+ sked_sample[1] = ioread32(bar0 + 0x00407064u);
+ sked_sample[2] = ioread32(bar0 + 0x00407068u);
+ sked_sample[3] = ioread32(bar0 + 0x00407028u);
+ sked_sample[4] = ioread32(bar0 + 0x0040702cu);
+ sked_sample[5] = ioread32(bar0 + 0x00407054u);
+ sked_sample[6] = ioread32(bar0 + 0x00407020u);
+ sked_sample[7] = ioread32(bar0 + 0x00400504u);
+ sked_sample[8] = ioread32(bar0 + 0x00409b00u);
+ sked_sample[9] = ioread32(bar0 + 0x00409b04u);
+ sked_sample[10] = ioread32(bar0 + 0x00409400u);
+ if (sked_first ||
+ memcmp(sked_sample, sked_prev,
+ sizeof(sked_sample))) {
+ pr_info("vgsp-gr: sked rec +%ums cwd0=%08x cwd1=%08x cwd2=%08x info1=%08x info2=%08x act=%08x esr=%08x grfifo=%08x fecs_cur=%08x fecs_new=%08x ctxsw1=%08x\n",
+ (sked_tick / 8u) *
+ GSP_PHYS_RMCALL_POLL_MS,
+ sked_sample[0], sked_sample[1],
+ sked_sample[2], sked_sample[3],
+ sked_sample[4], sked_sample[5],
+ sked_sample[6], sked_sample[7],
+ sked_sample[8], sked_sample[9],
+ sked_sample[10]);
+ memcpy(sked_prev, sked_sample,
+ sizeof(sked_sample));
+ sked_first = false;
+ }
+ }
if (qmd_rel) {
if (qmd_rel_hpa)
w0 = READ_ONCE(*(u32 *)((u8 *)__va(qmd_rel_hpa) +
@@ -16537,11 +16625,35 @@ gr_page_unmapped:
ioread32(bar0 + 0x0040a508u),
ioread32(bar0 + 0x0040a50cu));
if (bar0)
- pr_info("vgsp-gr: SKED readback: HWW_ESR=0x%08x CWD_T0=0x%08x CWD_T1=0x%08x FECS_CTX=0x%08x\n",
+ pr_info("vgsp-gr: SKED readback: HWW_ESR=0x%08x HWW_EN=0x%08x INFO1=0x%08x INFO2=0x%08x INFO3=0x%08x CWD_T0=0x%08x CWD_T1=0x%08x CWD_T2=0x%08x ACT=0x%08x FECS_CTX=0x%08x\n",
ioread32(bar0 + 0x00407020u),
+ ioread32(bar0 + 0x00407024u),
+ ioread32(bar0 + 0x00407028u),
+ ioread32(bar0 + 0x0040702cu),
+ ioread32(bar0 + 0x00407030u),
ioread32(bar0 + 0x00407060u),
ioread32(bar0 + 0x00407064u),
+ ioread32(bar0 + 0x00407068u),
+ ioread32(bar0 + 0x00407054u),
ioread32(bar0 + 0x00409b00u));
+ if (bar0)
+ pr_info("vgsp-gr: SKED watermark[0..15]=0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
+ ioread32(bar0 + 0x004070a0u),
+ ioread32(bar0 + 0x004070a4u),
+ ioread32(bar0 + 0x004070a8u),
+ ioread32(bar0 + 0x004070acu),
+ ioread32(bar0 + 0x004070b0u),
+ ioread32(bar0 + 0x004070b4u),
+ ioread32(bar0 + 0x004070b8u),
+ ioread32(bar0 + 0x004070bcu),
+ ioread32(bar0 + 0x004070c0u),
+ ioread32(bar0 + 0x004070c4u),
+ ioread32(bar0 + 0x004070c8u),
+ ioread32(bar0 + 0x004070ccu),
+ ioread32(bar0 + 0x004070d0u),
+ ioread32(bar0 + 0x004070d4u),
+ ioread32(bar0 + 0x004070d8u),
+ ioread32(bar0 + 0x004070dcu));
}
poll = w1;
ret = -ETIMEDOUT;
@@ -16552,6 +16664,13 @@ gr_page_unmapped:
gsp_phys_gr_ok++;
poll = w1;
+ if (bar0)
+ pr_info("vgsp-gr: SKED readback (ok): HWW_ESR=0x%08x INFO2=0x%08x CWD_T0=0x%08x CWD_T1=0x%08x CWD_T2=0x%08x\n",
+ ioread32(bar0 + 0x00407020u),
+ ioread32(bar0 + 0x0040702cu),
+ ioread32(bar0 + 0x00407060u),
+ ioread32(bar0 + 0x00407064u),
+ ioread32(bar0 + 0x00407068u));
pr_info("vgsp-gr: GUEST GR LAUNCH OK h=0x%x push=0x%llx qmd=0x%llx prog/pages=%u qmd_rel=0x%08x ch_rel=0x%08x elapsed_ms=%u ok=%llu\n",
req->h_object, req->push_va, req->qmd_va, req->n_pages, w0, w1,
jiffies_to_msecs(jiffies - start), gsp_phys_gr_ok);
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 1029b2e..ceb85ee 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -746,6 +746,93 @@ int gsp_phys_rmcall_faultbuf_sys_value(void)
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_faultbuf_sys_value);
/*
+ * m72 local-memory context block (gsp_phys_rmcall_gr_lmem_en; default 0).
+ * When set, the forwarded GR launch prelude appends the vendor
+ * turingLocalMemUpdate block (SET_SHADER_LOCAL_MEMORY_A/B 0x790/0x794,
+ * SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A/B 0x2e4/0x2e8,
+ * SET_SHADER_LOCAL_MEMORY_WINDOW_A/B 0x7b0/0x7b4), which the stock CUDA
+ * driver emits at context init. Without it the channel's local-memory
+ * context state is zero and SKED cancels the QMD at validation with
+ * SKEDCHECK05 LOCAL_MEMORY_TOTAL_SIZE (SKED_HWW_ESR bit12) while still
+ * allocating/deallocating a CWD entry (the m64-m71 signature). The base
+ * (default 0x3000000 = CU_TURING_LMEM_WINDOW_BASE) and per-TPC size
+ * (default 0x320000, derived for the TU104 QMD low=0/high=1600 footprint,
+ * 48 SMs / 24 TPCs) are overridable for A/B runs.
+ */
+static int gsp_phys_rmcall_gr_lmem_en_param;
+module_param_named(gsp_phys_rmcall_gr_lmem_en,
+ gsp_phys_rmcall_gr_lmem_en_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_lmem_en,
+ "Append the vendor local-memory context block to the GR prelude (default 0; 1 = m72 SKEDCHECK05 fix)");
+
+int gsp_phys_rmcall_gr_lmem_en_value(void)
+{
+ return gsp_phys_rmcall_gr_lmem_en_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_lmem_en_value);
+
+static unsigned int gsp_phys_rmcall_gr_lmem_base_param = 0x3000000u;
+module_param_named(gsp_phys_rmcall_gr_lmem_base,
+ gsp_phys_rmcall_gr_lmem_base_param, uint, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_lmem_base,
+ "Local-memory window/base VAS address for the m72 lmem block (default 0x3000000)");
+
+unsigned int gsp_phys_rmcall_gr_lmem_base_value(void)
+{
+ return gsp_phys_rmcall_gr_lmem_base_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_lmem_base_value);
+
+static unsigned int gsp_phys_rmcall_gr_lmem_tpc_size_param = 0x320000u;
+module_param_named(gsp_phys_rmcall_gr_lmem_tpc_size,
+ gsp_phys_rmcall_gr_lmem_tpc_size_param, uint, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_lmem_tpc_size,
+ "Local-memory size per TPC for the m72 lmem block (default 0x320000)");
+
+unsigned int gsp_phys_rmcall_gr_lmem_tpc_size_value(void)
+{
+ return gsp_phys_rmcall_gr_lmem_tpc_size_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_lmem_tpc_size_value);
+
+/*
+ * m73 sysmem instance/RAMFC selector (gsp_phys_rmcall_gr_inst_sys; default 0).
+ * The m70 capture-shape FBMEM instance/ramfc makes the FECS context load
+ * never complete (sampler: NEW_CTX=V7 instance with VALID=0, then unloaded;
+ * no GR execution), while the m62 host-visible sysmem chunks showed the load
+ * completing (VALID=1, TARGET=3). 1 = m62 sysmem chunks for A/B runs.
+ */
+static int gsp_phys_rmcall_gr_inst_sys_param;
+module_param_named(gsp_phys_rmcall_gr_inst_sys,
+ gsp_phys_rmcall_gr_inst_sys_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_inst_sys,
+ "Use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape; default 0 = FBMEM capture shape)");
+
+int gsp_phys_rmcall_gr_inst_sys_value(void)
+{
+ return gsp_phys_rmcall_gr_inst_sys_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_inst_sys_value);
+
+/*
+ * m72 SKED transition recorder (gsp_phys_rmcall_gr_skedrec; default 0). When
+ * set, the forwarded GR launch poll loop samples the SKED debug/CWD/HWW
+ * registers every ~8 ms and logs a line only when a value changes, recording
+ * the exact CWD allocate/cancel/dispatch sequence of the launch window.
+ */
+static int gsp_phys_rmcall_gr_skedrec_param;
+module_param_named(gsp_phys_rmcall_gr_skedrec,
+ gsp_phys_rmcall_gr_skedrec_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_skedrec,
+ "Log SKED CWD/HWW register transitions during the GR launch poll (default 0)");
+
+int gsp_phys_rmcall_gr_skedrec_value(void)
+{
+ return gsp_phys_rmcall_gr_skedrec_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_skedrec_value);
+
+/*
* m69 window-0 shadow pool (gsp_phys_rmcall_gr_shadow_win0; default 0). When
* set, the forwarded GR launch copies the QMD's program/param pages into the
* window-0 shadow pool (module pages at VAs 0x108000..0x10f000, PTEs planted
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 196e844..939cf0f 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -337,6 +337,19 @@ int gsp_phys_rmcall_faultbuf_sys_value(void);
* (proven small-PT line) instead of the window-1 scratch range (default 0). */
int gsp_phys_rmcall_gr_shadow_win0_value(void);
+/* m72: append the vendor local-memory context block to the GR prelude so
+ * SKEDCHECK05 LOCAL_MEMORY_TOTAL_SIZE passes (default 0); base VA and
+ * per-TPC size are overridable for A/B runs. */
+int gsp_phys_rmcall_gr_lmem_en_value(void);
+unsigned int gsp_phys_rmcall_gr_lmem_base_value(void);
+unsigned int gsp_phys_rmcall_gr_lmem_tpc_size_value(void);
+
+/* m72: log SKED CWD/HWW register transitions during the launch poll. */
+int gsp_phys_rmcall_gr_skedrec_value(void);
+
+/* m73: use host-visible sysmem chunks for V7 instance/RAMFC (m62 shape). */
+int gsp_phys_rmcall_gr_inst_sys_value(void);
+
/*
* Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
* 0 = FERMI_VASPACE_A is allocated IS_EXTERNALLY_OWNED (historical ladder
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 10:36:59 +0000