summaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/gsp_phys_booter.c80
-rw-r--r--kernel/gsp_sema_service.c164
-rw-r--r--kernel/gsp_sema_service.h14
3 files changed, 256 insertions, 2 deletions
diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 96d4c59..bb75a8c 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -16789,8 +16789,12 @@ EXPORT_SYMBOL_GPL(gsp_phys_backend_guest_ce_copy);
#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1 0x00407040u
#define GSP_PHYS_RMCALL_PRI_SKED_DEBUG_1_EN_CWD_SLOT_COUNT 0x800u
/* Method block bound: init(152) + inline run(67) + PCAS(4) + REPORT_SEM(8) +
- * the optional QMD-virt pair(2). */
-#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 512u
+ * the optional QMD-virt pair(2). Raised 512 -> 1024 (SESSION 153-
+ * CONTINUED-53) so a 3D stream replay can carry the captured MME setup
+ * prelude (<=512 words) ahead of the largest chunked window (<=448) in one
+ * block; the block still lands inside the one 4 KB chunk page it is copied
+ * to. */
+#define GSP_PHYS_RMCALL_GR_METHOD_WORDS 1024u
static u64 gsp_phys_gr_submitted;
static u64 gsp_phys_gr_ok;
@@ -17106,6 +17110,26 @@ MODULE_PARM_DESC(gsp_phys_rmcall_gr_3d,
"Replay a captured raw C597 method window verbatim instead of the compute prelude (default 1)");
/*
+ * SESSION 153-CONTINUED-53: MME context-setup prelude. The r1073-r1077
+ * in-flight capture proved the draw-window replay raises a real MME
+ * ILLEGAL_OPCODE on CALL_MME_MACRO(2)=0x3810 (mme_pc=0, mme_ir=1, mme_mac
+ * =2) because the ladder context never received the client's macro
+ * program. The client does upload it (0x11c/0x114 pointer pairs, 0x118
+ * code, 0x120 entry addresses, 0x124 shadow control; see
+ * svc_mme_prelude_capture); with this knob the launch builder preplays
+ * those captured runs verbatim - once per capture generation - right
+ * after the SET_OBJECT C597 bind and ahead of the window, so the macros
+ * the window calls are resident. Declared before build_gr_launch (its
+ * reader); the matching capture-side knob is sema_mme_prelude_cap.
+ * Default 1; A/B with 0.
+ */
+static bool gsp_phys_rmcall_gr_mme_prelude = true;
+module_param_named(sema_gl_mme_prelude, gsp_phys_rmcall_gr_mme_prelude, bool,
+ 0644);
+MODULE_PARM_DESC(sema_gl_mme_prelude,
+ "Preplay the captured client MME context-setup runs once per capture generation ahead of the first 3D window replay (default 1)");
+
+/*
* GR launch method block: the inline QMD run at its original shape and the
* channel REPORT_SEMAPHORE release. The stock guest push does NOT emit
* SET_OBJECT before the inline run (the compute class is bound on the
@@ -17137,10 +17161,59 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
*/
if (gsp_phys_rmcall_gr_3d && req && win3d && win3d_n &&
win3d_n + 2u + 8u <= GSP_PHYS_RMCALL_GR_METHOD_WORDS) {
+ /*
+ * SESSION 153-CONTINUED-53: MME context-setup prelude (see the
+ * sema_gl_mme_prelude knob above). Preplay the captured client
+ * setup runs once per capture generation, right after the
+ * SET_OBJECT C597 bind and ahead of the window, so
+ * CALL_MME_MACRO() runs execute against a loaded macro
+ * program instead of pc=0 -> ILLEGAL_OPCODE. The read is
+ * seqchecked (generation before/after): a concurrent append
+ * only defers emission to the next launch. A window too large
+ * to also carry the prelude replays without it (logged) - the
+ * emission then happens on the next launch that fits.
+ */
+ static u32 mme_prelude_emitted_gen;
+ bool emit_prelude = false;
+ const u32 *pl = NULL;
+ u32 pl_n = 0, pl_gen = 0;
+
+ if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) {
+ pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen);
+ if (pl && pl_n && pl_gen &&
+ pl_gen != READ_ONCE(mme_prelude_emitted_gen)) {
+ if (win3d_n + 2u + 8u + pl_n <=
+ GSP_PHYS_RMCALL_GR_METHOD_WORDS) {
+ emit_prelude = true;
+ } else {
+ pr_info("vgsp-gr: MME-PRELUDE deferred (window=%u prelude=%u exceeds block; replaying window alone) h=0x%x\n",
+ win3d_n, pl_n, req->h_object);
+ }
+ }
+ }
+
*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) | (1u << 16) |
(0u << 13) |
(GSP_PHYS_RMCALL_GR_SET_OBJECT >> 2);
*p++ = GSP_PHYS_RMCALL_CLASS_GR_3D;
+ if (emit_prelude) {
+ u32 g0 = pl_gen, g1 = 0;
+
+ memcpy(p, pl, (size_t)pl_n * 4);
+ p += pl_n;
+ /* seqcheck: a mid-copy append must not be recorded as
+ * emitted (the copy may be a torn prefix). */
+ gsp_sema_mme_prelude_get(NULL, &g1);
+ if (g1 == g0) {
+ WRITE_ONCE(mme_prelude_emitted_gen, g0);
+ pr_info("vgsp-gr: MME-PRELUDE emitted gen=%u words=%u window=%u h=0x%x push=0x%llx\n",
+ g0, pl_n, win3d_n, req->h_object,
+ req->push_va);
+ } else {
+ pr_info("vgsp-gr: MME-PRELUDE copy raced an append (gen %u->%u); emitted words=%u, will re-emit next launch\n",
+ g0, g1, pl_n);
+ }
+ }
memcpy(p, win3d, (size_t)win3d_n * 4);
p += win3d_n;
p = gsp_phys_rmcall_push_gr(p,
@@ -19728,6 +19801,9 @@ module_param_named(sema_gl_mme_reloc, gsp_phys_rmcall_gr_mme_reloc, bool, 0644);
MODULE_PARM_DESC(sema_gl_mme_reloc,
"Relocate CALL_MME_MACRO/DATA(j) run payload words whose page resolves in the launch page set to shadow VAs in 3D replays (default 0 = off)");
+/* (SESSION 153-CONTINUED-53 MME prelude knob is declared before
+ * gsp_phys_rmcall_build_gr_launch, its reader.) */
+
/*
* SESSION 153-CONTINUED-48: ladder-wedge recovery. CONT-47 measured that
* the FIRST beef launch per boot whose window carries an unresolvable raw
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 8fd0603..f94a768 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -13821,6 +13821,162 @@ static void svc_mme_census(const u8 *buf, u32 len, u64 push_va, u32 h_object)
}
}
+/*
+ * SESSION 153-CONTINUED-53: MME context-setup prelude buffer.
+ *
+ * The draw-window replay raises a real MME ILLEGAL_OPCODE
+ * (NV_PGRAPH_PRI_MME_HWW_ESR bit 2) on CALL_MME_MACRO(2)=0x3810 with
+ * mme_pc=0/mme_ir=1: the MME executed a macro call with no macro program
+ * loaded. The client DOES upload the program (the census records the
+ * 0x11c/0x114 pointer pairs and the 0x124 shadow-control runs), but the
+ * replay only ever forwards draw-window slices, so the ladder context
+ * never sees the setup. This buffer captures the setup runs verbatim -
+ * every method run whose id is in the C597 MME setup families:
+ *
+ * 0x114 LOAD_MME_INSTRUCTION_RAM_POINTER
+ * 0x118 LOAD_MME_INSTRUCTION_RAM (the macro code words)
+ * 0x11c LOAD_MME_START_ADDRESS_RAM_POINTER
+ * 0x120 LOAD_MME_START_ADDRESS_RAM (per-macro entry pc)
+ * 0x124 SET_MME_SHADOW_RAM_CONTROL (METHOD_TRACK[_WITH_FILTER])
+ * 0x234 SET_MME_VERSION
+ * 0x550..0x568 MME mem address / data RAM / DMA / reduction
+ * 0x3400..0x34fc SET_MME_SHADOW_SCRATCH
+ *
+ * - in first-encounter order, so the booter can preplay them ahead of the
+ * first 3D window replay of a capture generation (see
+ * gsp_sema_mme_prelude_get and the gsp_phys_rmcall_gr_mme_prelude knob).
+ * Identical runs (a re-upload of the same program) are skipped by an
+ * exact-run signature table, so repeated setup pushes do not grow the
+ * buffer; genuinely new content appends and bumps the generation so the
+ * next window re-preplays. The writer is the single service thread and
+ * appends strictly in order; the generation only increments after a
+ * capture that appended, giving the booter a seqcheck against torn views.
+ */
+#define SVC_MME_PRELUDE_WORDS 512u
+#define SVC_MME_PRELUDE_SIGS 256u
+static u32 svc_mme_prelude_buf[SVC_MME_PRELUDE_WORDS];
+static u32 svc_mme_prelude_n;
+static u32 svc_mme_prelude_gen;
+static u64 svc_mme_prelude_sigs[SVC_MME_PRELUDE_SIGS];
+static u32 svc_mme_prelude_sig_n;
+static u32 svc_mme_prelude_dump_n;
+static u32 svc_mme_prelude_flag = 1;
+module_param_named(sema_mme_prelude_cap, svc_mme_prelude_flag, uint, 0644);
+MODULE_PARM_DESC(sema_mme_prelude_cap,
+ "Capture the client's MME context-setup method runs (0x114..0x124, 0x234, 0x550..0x568, 0x3400..0x34fc) from trusted beef pushes for the GR launch builder's one-time preplay (default 1)");
+
+static bool svc_mme_setup_run_addr(u32 addr)
+{
+ return (addr >= 0x114u && addr <= 0x124u) || addr == 0x234u ||
+ (addr >= 0x550u && addr <= 0x568u) ||
+ (addr >= 0x3400u && addr <= 0x34fcu);
+}
+
+static u64 svc_mme_run_sig(const u8 *p, u32 span, u32 hdr)
+{
+ u64 h = 1469598103934665603ULL ^ hdr;
+ u32 i;
+
+ for (i = 1u; i < span; i++) {
+ u32 v;
+
+ memcpy(&v, p + 4u * i, 4);
+ h ^= v;
+ h *= 1099511628211ULL;
+ }
+ return h;
+}
+
+const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen)
+{
+ if (n)
+ *n = READ_ONCE(svc_mme_prelude_n);
+ if (gen)
+ *gen = READ_ONCE(svc_mme_prelude_gen);
+ return svc_mme_prelude_buf;
+}
+
+static void svc_mme_prelude_capture(const u8 *buf, u32 len, u64 push_va,
+ u32 h_object)
+{
+ u32 off = 0, added = 0, dups = 0, full = 0;
+ u32 words0 = svc_mme_prelude_n;
+
+ if (!READ_ONCE(svc_mme_prelude_flag))
+ return;
+ if ((h_object & 0xfffff000u) != 0xbeef0000u)
+ return;
+ while (off + 4u <= len) {
+ u32 hdr, op, cnt, addr, span = 1u, i;
+ u64 sig;
+ bool dup = false;
+
+ memcpy(&hdr, buf + off, 4);
+ op = (hdr >> 29) & 7u;
+ cnt = (hdr >> 16) & 0xfffu;
+ addr = (hdr & 0x1fffu) << 2;
+ if ((op == 1u || op == 3u || op == 5u) && cnt) {
+ if ((u64)off + 4u + (u64)cnt * 4u > (u64)len)
+ break;
+ span = 1u + cnt;
+ }
+ if (!svc_mme_setup_run_addr(addr)) {
+ off += span * 4u;
+ continue;
+ }
+ sig = svc_mme_run_sig(buf + off, span, hdr);
+ for (i = 0; i < svc_mme_prelude_sig_n; i++) {
+ if (svc_mme_prelude_sigs[i] == sig) {
+ dup = true;
+ break;
+ }
+ }
+ if (dup) {
+ dups++;
+ off += span * 4u;
+ continue;
+ }
+ if (svc_mme_prelude_n + span > SVC_MME_PRELUDE_WORDS ||
+ svc_mme_prelude_sig_n >= SVC_MME_PRELUDE_SIGS) {
+ full++;
+ off += span * 4u;
+ continue;
+ }
+ memcpy(&svc_mme_prelude_buf[svc_mme_prelude_n], buf + off,
+ span * 4u);
+ svc_mme_prelude_n += span;
+ svc_mme_prelude_sigs[svc_mme_prelude_sig_n++] = sig;
+ added++;
+ off += span * 4u;
+ }
+ if (added) {
+ WRITE_ONCE(svc_mme_prelude_gen, svc_mme_prelude_gen + 1);
+ if (svc_mme_prelude_dump_n < 4) {
+ u32 k;
+
+ svc_mme_prelude_dump_n++;
+ pr_info("sema-mme-prelude: capture h=0x%x va=0x%llx len=%u added=%u words=%u gen=%u (was %u)\n",
+ h_object, push_va, len, added,
+ svc_mme_prelude_n, svc_mme_prelude_gen,
+ words0);
+ for (k = 0; k + 4u <= svc_mme_prelude_n &&
+ k < 512u; k += 4u)
+ pr_info("sema-mme-prelude[%03u] %08x %08x %08x %08x\n",
+ k, svc_mme_prelude_buf[k],
+ svc_mme_prelude_buf[k + 1],
+ svc_mme_prelude_buf[k + 2],
+ svc_mme_prelude_buf[k + 3]);
+ }
+ if (full)
+ pr_info("sema-mme-prelude: buffer full=%u words=%u dup=%u\n",
+ full, svc_mme_prelude_n, dups);
+ } else if (dups && svc_mme_prelude_dump_n < 8) {
+ svc_mme_prelude_dump_n++;
+ pr_info("sema-mme-prelude: re-upload all-duplicate h=0x%x va=0x%llx dup=%u words=%u\n",
+ h_object, push_va, dups, svc_mme_prelude_n);
+ }
+}
+
static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va,
u32 h_object)
{
@@ -14551,6 +14707,14 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
if (ch)
svc_mme_census(buf, len, push_va, ch->h_object);
+ /*
+ * SESSION 153-CONTINUED-53: MME context-setup prelude capture. Only a
+ * trusted resolution feeds the buffer (the bytes are replayed into a
+ * hardware context); identical re-uploads are deduped inside.
+ */
+ if (ch && trusted)
+ svc_mme_prelude_capture(buf, len, push_va, ch->h_object);
+
memset(&st, 0, sizeof(st));
st.push_va = push_va;
/*
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 69d6f33..12a785c 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -448,6 +448,20 @@ struct gsp_sema_gr_reloc {
int gsp_sema_gr_reloc_resolve(u64 va, struct gsp_sema_gr_reloc *out);
/*
+ * SESSION 153-CONTINUED-53: MME context-setup prelude accessor for the GR
+ * launch builder. Returns the captured setup runs (C597 MME family
+ * 0x114..0x124/0x234/0x550..0x568/0x3400..0x34fc, verbatim, first-
+ * encounter order), the word count in *n and the capture generation in
+ * *gen. The builder preplays the words once per generation, ahead of the
+ * first 3D window replay, so the ladder context's MME has its macro
+ * program before CALL_MME_MACRO runs. Reading the generation before and
+ * after the copy detects a concurrent append (the service thread only
+ * appends; a mismatch just defers emission to the next launch). *n is 0
+ * until the client's setup push has been decoded.
+ */
+const u32 *gsp_sema_mme_prelude_get(u32 *n, u32 *gen);
+
+/*
* r378 write guards. Both re-validate a guest-memory write target that was
* resolved earlier (ledger/pin/walk/counter registration, or the GR submit's
* per-page resolution) against the CURRENT resolution before the bytes land,
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-31 13:46:02 +0000