summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c50
-rw-r--r--kernel/gsp_phys_booter.h8
-rw-r--r--kernel/gsp_phys_gpu.c36
-rw-r--r--kernel/gsp_phys_gpu.h14
4 files changed, 94 insertions, 14 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 4cbbd40..11f205c 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -13526,12 +13526,21 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
#define GSP_PHYS_RMCALL_GR_INIT_VSPAN_C 0x208u
#define GSP_PHYS_RMCALL_GR_INIT_NON_THROTTLED_C 0x2ecu
#define GSP_PHYS_RMCALL_GR_INIT_SM_DISP_CTRL 0x250u
+/*
+ * NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (class/clc5c0.h): emitted between the
+ * compute init and the inline-QMD window when gsp_phys_rmcall_gr_qmdvirt=1
+ * (the default). Value enables IQ2M_ADDR_ENABLE (bit 16) and
+ * SEND_PCAS_ENABLE (bit 24).
+ */
+#define GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL 0x29cu
+#define GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS 0x01010000u
#define GSP_PHYS_RMCALL_GR_SHARED_WINDOW_BASE 0x1000000u
#define GSP_PHYS_RMCALL_GR_SPA_VERSION 0x0705u
#define GSP_PHYS_RMCALL_GR_CWD_REFCOUNT 1536u
#define GSP_PHYS_RMCALL_GR_CWD_LEVELS 64u
#define GSP_PHYS_RMCALL_GR_MAX_SM_COUNT 24u
-/* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8). */
+/* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) +
+ * the optional QMD-virt pair(2). */
#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u
static u64 gsp_phys_gr_submitted;
@@ -13608,16 +13617,24 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
u64 sem_va, u32 sem_payload)
{
u32 *p = pb;
+ int prelude = gsp_phys_rmcall_gr_prelude_value();
+ int qmdvirt = gsp_phys_rmcall_gr_qmdvirt_value();
p = gsp_phys_rmcall_build_gr_init(p);
+ if (qmdvirt)
+ p = gsp_phys_rmcall_push_gr(p,
+ GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL,
+ GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS);
/*
* Verbatim replay: copy the captured GRCE prelude and the guest's
* own inline-QMD run, keeping the method headers, and relocate only
* the ADDRESS_A/B data words and the 64 QMD words exactly like the
* rebuilt window. Only a contiguous 0x318 run is replayed; anything
- * else falls back to the rebuilt window.
+ * else falls back to the rebuilt window. With
+ * gsp_phys_rmcall_gr_prelude=0 (default) the rebuilt window is always
+ * used even when a verbatim prelude is available.
*/
- if (req && req->raw &&
+ if (prelude && req && req->raw &&
req->raw_words >=
(u32)req->raw_qmd_hdr + GSP_PHYS_RMCALL_GR_QMD_WINDOW_WORDS &&
req->raw_qmd_end ==
@@ -13639,7 +13656,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
pr_info("vgsp-gr: verbatim replay prelude+window words=%u qmd_hdr=%u\n",
pre, req->raw_qmd_hdr);
} else {
- if (req && req->raw)
+ if (prelude && req && req->raw)
pr_info("vgsp-gr: raw prelude unusable n=%u hdr=%u end=%u; rebuilt window\n",
req->raw_words, req->raw_qmd_hdr,
req->raw_qmd_end);
@@ -14450,10 +14467,13 @@ gr_page_unmapped:
*/
if (qmd_dst_mem) {
memcpy(qmd_dst_mem, qmd, GSP_PHYS_GR_QMD_WORDS * 4);
- put_unaligned_le32(GSP_PHYS_GR_QMD_SENTINEL, qmd_dst_mem);
+ put_unaligned_le32(GSP_PHYS_GR_QMD_SENTINEL,
+ qmd_dst_mem +
+ GSP_PHYS_GR_QMD_SENTINEL_WORD * 4);
wmb();
- pr_info("vgsp-gr: QMD prewritten to dest 0x%llx (%u words, w0 sentinel=0x%08x)\n",
+ pr_info("vgsp-gr: QMD prewritten to dest 0x%llx (%u words, w%u sentinel=0x%08x; w0 left authentic)\n",
qmd_dst, GSP_PHYS_GR_QMD_WORDS,
+ GSP_PHYS_GR_QMD_SENTINEL_WORD,
GSP_PHYS_GR_QMD_SENTINEL);
}
@@ -14475,9 +14495,12 @@ gr_page_unmapped:
goto out;
}
- pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n",
+ pr_info("vgsp-gr: GR LAUNCH submitted h=0x%x push=0x%llx len=%u pages=%u shadow=%u qmd_va=0x%llx slot=%u words=%u prelude=%d qmdvirt=%d token=0x%x; polling qmd_rel=%u ch_rel=0x%08x (<=%u ms)\n",
req->h_object, req->push_va, req->push_len, req->n_pages,
- n_shadow, req->qmd_va, sub.slot, words, boot_stage.rmcall_token,
+ n_shadow, req->qmd_va, sub.slot, words,
+ gsp_phys_rmcall_gr_prelude_value(),
+ gsp_phys_rmcall_gr_qmdvirt_value(),
+ boot_stage.rmcall_token,
qmd_rel, want_ch, GSP_PHYS_RMCALL_GR_POLL_MS);
if (qmd_win_diag)
pr_info("vgsp-gr: qmd dest va=0x%llx pdeIndex=%u installed=%u win_mask=0x%x ensure_ret=%d\n",
@@ -14540,6 +14563,7 @@ gr_page_unmapped:
ud + GSP_PHYS_RMCALL_USERD_GPGET_OFF)
: 0;
u32 q0 = 0, q1 = 0, q23 = 0, q24 = 0, q25 = 0;
+ u32 q63 = 0;
u32 relw = 0;
{
@@ -14554,16 +14578,20 @@ gr_page_unmapped:
q23 = get_unaligned_le32(qp + 92);
q24 = get_unaligned_le32(qp + 96);
q25 = get_unaligned_le32(qp + 100);
+ q63 = get_unaligned_le32(
+ qp +
+ GSP_PHYS_GR_QMD_SENTINEL_WORD *
+ 4);
}
}
if (qmd_rel_hpa)
relw = READ_ONCE(
*(u32 *)((u8 *)__va(qmd_rel_hpa) +
qmd_rel_off));
- pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x relw=%08x iq2m=%s\n",
+ pr_info("vgsp-gr: post-mortem gpget=%u->%u qmd_page w0=%08x w1=%08x w23..25=%08x %08x %08x w63=%08x relw=%08x iq2m=%s\n",
sub.gpget, gpget_post, q0, q1, q23, q24,
- q25, relw,
- q0 == GSP_PHYS_GR_QMD_SENTINEL ?
+ q25, q63, relw,
+ q63 == GSP_PHYS_GR_QMD_SENTINEL ?
"not-run (sentinel intact)" :
"ran (page overwritten)");
}
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index 67d5d5c..e46fb86 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -336,14 +336,16 @@ void gsp_phys_backend_relay_counters(u64 *requested, u64 *ok, u64 *verify_fail,
#define GSP_PHYS_GR_INLINE_BYTES 64u
#define GSP_PHYS_GR_QMD_WORDS 64u
/*
- * IQ2M sentinel: written to the inline-QMD destination's word 0 after the
- * prewrite. If the engine processes the inline window, the copy overwrites
- * the page (word 0 -> the inline value, 0) and the post-mortem sees the page
+ * IQ2M sentinel: written to the inline-QMD destination's word 63 after the
+ * prewrite (a word SKED does not consume; word 0 stays the guest's authentic
+ * OUTER_PUT). If the engine processes the inline window, the copy overwrites
+ * the page (word 63 -> the inline value) and the post-mortem sees the page
* changed; if the sentinel survives, the engine stalled before the inline
* window and the page was never touched. Distinguishes "IQ2M never ran" from
* "QMD deposited but not executed" without a server-side capture.
*/
#define GSP_PHYS_GR_QMD_SENTINEL 0xdeadbeefu
+#define GSP_PHYS_GR_QMD_SENTINEL_WORD 63u
#define GSP_PHYS_GR_QMD_WORD_NONE 0xffffffffu
#define GSP_PHYS_GR_RAW_WORDS 256u
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 2f441a0..ad2244e 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -580,6 +580,42 @@ int gsp_phys_rmcall_gr_promote_value(void)
EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_promote_value);
/*
+ * GR launch verbatim-prelude gate (default 0). 0 = skip the captured
+ * 136-word verbatim guest prelude and emit the rebuilt inline-QMD window,
+ * i.e. the proven init(152)+window(67)+PCAS(4)+sem(8) shape; 1 = replay the
+ * verbatim prelude exactly as before. Read at each launch.
+ */
+static int gsp_phys_rmcall_gr_prelude_param;
+module_param_named(gsp_phys_rmcall_gr_prelude,
+ gsp_phys_rmcall_gr_prelude_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_prelude,
+ "GR launch verbatim-prelude gate (default 0: skip the captured guest prelude and emit the rebuilt inline-QMD window; 1: replay the verbatim prelude)");
+
+int gsp_phys_rmcall_gr_prelude_value(void)
+{
+ return gsp_phys_rmcall_gr_prelude_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_prelude_value);
+
+/*
+ * QMD virtualization control gate (default 1). When set, the GR launch block
+ * emits NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c) = IQ2M_ADDR_ENABLE
+ * (bit 16) | SEND_PCAS_ENABLE (bit 24) after the compute init and before the
+ * inline-QMD window; 0 omits the method pair. Read at each launch.
+ */
+static int gsp_phys_rmcall_gr_qmdvirt_param = 1;
+module_param_named(gsp_phys_rmcall_gr_qmdvirt,
+ gsp_phys_rmcall_gr_qmdvirt_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_qmdvirt,
+ "Emit NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c = IQ2M_ADDR_ENABLE|SEND_PCAS_ENABLE) in the GR launch block (default 1; 0 omits the method pair)");
+
+int gsp_phys_rmcall_gr_qmdvirt_value(void)
+{
+ return gsp_phys_rmcall_gr_qmdvirt_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_qmdvirt_value);
+
+/*
* Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
* The H1 GR promote wall is a direct consequence of the ladder's
* externally-owned FERMI_VASPACE_A: with IS_EXTERNALLY_OWNED the RM treats
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 65e1e1f..3dce1a6 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -280,6 +280,20 @@ int gsp_phys_rmcall_engine_value(void);
int gsp_phys_rmcall_gr_promote_value(void);
/*
+ * gsp_phys_rmcall_gr_prelude (default 0): 0 skips the captured verbatim
+ * guest prelude and emits the rebuilt inline-QMD window (proven
+ * init+window+PCAS+sem shape); 1 replays the verbatim prelude.
+ */
+int gsp_phys_rmcall_gr_prelude_value(void);
+
+/*
+ * gsp_phys_rmcall_gr_qmdvirt (default 1): emit
+ * NVC5C0_SET_QMD_VIRTUALIZATION_CONTROL (0x29c = IQ2M_ADDR_ENABLE |
+ * SEND_PCAS_ENABLE) in the GR launch block; 0 omits the method pair.
+ */
+int gsp_phys_rmcall_gr_qmdvirt_value(void);
+
+/*
* Ladder VAS ownership selector (gsp_phys_rmcall_vas_shared; default 0).
* 0 = FERMI_VASPACE_A is allocated IS_EXTERNALLY_OWNED (historical ladder
* behaviour: the client owns page tables, VA bookings and engine-context
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-12 11:13:26 +0000