vgsp: CONT-107 - IRAM READBACK GATE THAT FIXES THE ZERO-IR FETCH: gsp_phys_mme_iram_read3() spec-correct MME debug read per NV_PGRAPH_PRI_MME_DEBUG (ENABLE bit1 -> PC|RD-trigger bit2 -> poll DONE -> data in HWW_ESR_INFO4/3/2); gsp_phys_gr_mme_iram_gate() reads every covered SA entry compares to packed image auto-calibrates port address unit (resolved INSTRUCTION not word); gate at poll-break (before live-gen promotion) + builder (before releasing calls); on fail: hold calls clear prelude latch rewind pack/drain cursors; MME-LOAD-ONLY carriers (window elided while unverified so 0x114/0x118 IRAM load cannot be interrupted by window's DS/SPH poison); knobs sema_gl_mme_iram_gate/_max/_probe sema_gl_mme_hold_fe_idle - MILESTONE r1374: MME-IRAM-BASE resolved=instruction (entry) at entry=0x5; MME-IRAM-GATE pass break h=0xbeef0100 gen=3 base=1 checked=54 + pass build run=1,2,3; MME-LOAD-ONLY carrier window dropped until gate verifies; ILLEGAL_OPCODE = 0 OCCURRENCES (was every prior boot); live readback 4044d0=0x01B20006 (IRAM_PC=0x364) -> 0x40449c=31c28300 0x4044a8=1a05834a 0x4044ac=00000003 EXACTLY THE GOLDEN MACRO-57 GROUP; MME fetch latch mme_esr_info=100003a3(pcv=1 pc=931) ir=71cc0301,18c0034a,1c34238f MATCHING reconstructed image at 931; new wall: calls released yet beef launches park grfifo=07071602 (FIFO not-empty+FULL count 22) MME at pc=931 (slot 58 entry ENDNEXT waiting on LOAD0 INFO_EX=4); GATE1 black 0/900 - next: explain LOAD0 stall at slot 58 entry 0x3a3 (decode full released window call runs win_dec.py on MME-WINRUN dump: is slot 58 genuinely called or macro 57 falls through predicated ENDNEXT 0x3a0/0x3a1 not taken); if fall-through snapshot/seed MME STATE predicate reads; if real call feed params or stub slot 58 to proven stub PC; r1374 first boot where full program runs this deep - SESSION 153 r1369-r1374

1 files changed, 403 insertions, 2 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index a94bbef..5bb1fec 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -5132,6 +5132,55 @@ static void gsp_phys_mme_regs_log(void __iomem *bar0, const char *tag)
 }
 
 /*
+ * SESSION 153-CONTINUED-107: MME IRAM debug port readback.
+ *
+ * CONT-106 found the debug port at 0x4044d0 host-side; the definitive field
+ * spec (tu104 dev_graphics_nobundle.h:2166 NV_PGRAPH_PRI_MME_DEBUG) is
+ *   bit 0    STATUS    (R; 0=RUNNING, 1=STOPPED)
+ *   bit 1    ENABLE    (RW; init DISABLED)
+ *   bit 2    IRAM_RD   (write 1 = TRIGGER, reads 0 = DONE)
+ *   bits 26:15 IRAM_PC (RW, 12 bits = the IRAM word address)
+ * and the fetched group lands in NV_PGRAPH_PRI_MME_HWW_ESR_INFO4/3/2
+ * (0x4044ac/a8/9c, field IR 31:0 each). CONT-106's r1364 read at PC 0
+ * returned the reconstructed image's first three words
+ * (0x00003463,0x190d82f0,0x39804300), so the data path is the INFO
+ * triplet. The read is a two-step hardware handshake: set ENABLE, write
+ * the PC with IRAM_RD=1 (trigger), WAIT for IRAM_RD to read back 0
+ * (DONE), then read the group. `a` is an IRAM WORD address (callers pass
+ * instruction * 3, matching gsp_phys_gr_mme_ir_pack[] indexing).
+ * Read-only diagnostic; bounded by its call sites.
+ */
+static void gsp_phys_mme_iram_read3(void __iomem *bar0, u32 a, u32 out[3])
+{
+	u32 i;
+
+	iowrite32(0x2u, bar0 + 0x004044d0u);	/* ENABLE */
+	iowrite32(((a & 0xfffu) << 15) | 6u, bar0 + 0x004044d0u);
+	for (i = 0; i < 1000u; i++) {
+		if (!(ioread32(bar0 + 0x004044d0u) & 0x4u))
+			break;
+		cpu_relax();
+	}
+	out[0] = ioread32(bar0 + 0x004044acu);
+	out[1] = ioread32(bar0 + 0x004044a8u);
+	out[2] = ioread32(bar0 + 0x0040449cu);
+}
+
+/*
+ * Defined next to the packed-image array (gsp_phys_gr_mme_ir_pack); forward
+ * declaration so the post-doorbell forensics spin can call it.
+ */
+static void gsp_phys_mme_iram_probe(void __iomem *bar0, const char *tag,
+				    u32 h_object);
+
+/*
+ * SESSION 153-CONTINUED-107: probe enable (see the sema_gl_mme_iram_probe
+ * knob beside sema_gl_mme_iram_gate). Declared here for the forensics spin
+ * above; the matching module_param sits with the gate knob.
+ */
+static bool gsp_phys_rmcall_gr_mme_iram_probe;
+
+/*
  * SESSION 153-CONTINUED-69: pre-reset MME fault capture. Everything sampled
  * so far was post-mortem: by the time the 5 ms launch poll ticks, the
  * GSP-RM's FECS exception handler has already read NV_PGRAPH_PRI_MME_HWW_ESR
@@ -5319,6 +5368,16 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag,
 	fed1_p = ioread32(bar0 + 0x0040a634u);
 	pr_info("vgsp-gr: MME-FAST begin %s h=0x%x esr=%08x ds=%08x grfifo=%08x trap=%08x fe0=%08x fe1=%08x (spin %u ms)\n",
 		tag, h_object, esr_p, ds_p, gf_p, tr_p, fed0_p, fed1_p, ms);
+	/*
+	 * SESSION 153-CONTINUED-107: IRAM chronology probe (see the
+	 * gsp_phys_mme_iram_probe comment). Bounded to the first 8 fast
+	 * samples of the boot so the launch train cannot flood; knob-gated
+	 * because the debug-port writes disturb a live launch (see the
+	 * sema_gl_mme_iram_probe comment).
+	 */
+	if (fast_boot_n <= 8u &&
+	    READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe))
+		gsp_phys_mme_iram_probe(bar0, "begin", h_object);
 	t_end = ktime_get() + (s64)ms * 1000000ll;
 	for (;;) {
 		u32 esr = ioread32(bar0 + 0x00404490u);
@@ -5387,6 +5446,16 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag,
 					mmemac & 0xfffu,
 					(mmemac >> 12) & 0xffu,
 					(mmemac >> 20) & 1u);
+				/*
+				 * SESSION 153-CONTINUED-107: sample the live
+				 * IRAM at every fault-register transition, so
+				 * the chronology (words loaded early vs wiped
+				 * by the poison/DROP machinery) is on record
+				 * for the same instant the ESR changes.
+				 */
+				if (READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe))
+					gsp_phys_mme_iram_probe(bar0, "trans",
+								h_object);
 			}
 			/*
 			 * SESSION 153-CONTINUED-106: when the DS latch CHANGES
@@ -5483,6 +5552,14 @@ static bool gsp_phys_mme_fast_sample(void __iomem *bar0, const char *tag,
 	pr_info("vgsp-gr: MME-FAST end %s h=0x%x events=%u%s\n",
 		tag, h_object, n, poisoned ? " (poison)" : "");
 	/*
+	 * SESSION 153-CONTINUED-107: post-spin IRAM sample - the state left
+	 * when the poison (if any) has just cleared, before the poll/retire
+	 * work touches anything.
+	 */
+	if (fast_boot_n <= 8u &&
+	    READ_ONCE(gsp_phys_rmcall_gr_mme_iram_probe))
+		gsp_phys_mme_iram_probe(bar0, "end", h_object);
+	/*
 	 * SESSION 153-CONTINUED-83 store-side emission forensics: right after
 	 * the launch's retire spin, snapshot the DS shader-program-header
 	 * latches (which pipeline stages failed SPH validation), the MME
@@ -18218,6 +18295,20 @@ MODULE_PARM_DESC(sema_gl_mme_hold_calls,
 	"Hold (elide) window CALL_MME_MACRO/DATA runs until the MME prelude latches for the current generation+channel (default 1; 0 = legacy, real calls run against a partial prelude)");
 
 /*
+ * SESSION 153-CONTINUED-107: separately control the CONT-104 FE-idle clause
+ * of the hold (see the call site). With the IRAM gate proving residency the
+ * clause's only value is the r1359 race, and r1373 showed the FE wedged at
+ * 0x07071602 for minutes while every calls-kept window was held. Default 1
+ * (pre-CONT-107 behavior); 0 releases the calls as soon as the prelude
+ * latch + IRAM gate allow.
+ */
+static bool gsp_phys_rmcall_gr_mme_hold_fe_idle = true;
+module_param_named(sema_gl_mme_hold_fe_idle,
+		   gsp_phys_rmcall_gr_mme_hold_fe_idle, bool, 0644);
+MODULE_PARM_DESC(sema_gl_mme_hold_fe_idle,
+	"Also hold the real CALL runs while the front-end is not settled (0x400504 != 1; default 1)");
+
+/*
  * SESSION 153-CONTINUED-84: emit NVC597_SET_MME_VERSION=1 (0x234) with the
  * prelude step. The channel-init defaults the client pushes once per channel
  * (Turing+) are never replayed, so the replayed context's MME stays in its
@@ -18287,6 +18378,173 @@ MODULE_PARM_DESC(sema_gl_mme_prelude_pack,
 static u32 gsp_phys_gr_mme_ir_pack[4096];
 
 /*
+ * SESSION 153-CONTINUED-107: IRAM chronology + load-gate helpers.
+ *
+ * CONT-106 proved the trap-time fetch at PC 0x364 (macro 57 entry) reads
+ * ZERO IR words while the host-side pack is verbatim (CONT-105), and the
+ * IRAM debug port at 0x4044d0 also reads zero at 0x364 on that boot while
+ * the same port read the golden first words at pc=0 on r1364. The one
+ * unproven link is the LOAD: did the emitted 0x114 pack writes land in the
+ * MME's IRAM before the macro executed? These helpers read the live IRAM
+ * back and (a) log the timeline around each launch's fault-register
+ * transitions, (b) gate the real CALL runs on the readback matching the
+ * packed image at every covered SA entry.
+ *
+ * The probe reads the macro-57 entry group (0x364) under BOTH port address
+ * conventions - instruction base (write 0x364) and word base (write
+ * 3*0x364 = 0xA2C) - plus word 0, and prints the live words next to the
+ * pack-image expectation, so one line answers "is the program resident
+ * right now and which address unit does the port use".
+ */
+static void gsp_phys_mme_iram_probe(void __iomem *bar0, const char *tag,
+				    u32 h_object)
+{
+	u32 a[3], bi[3], bw[3];
+
+	if (!bar0)
+		return;
+	gsp_phys_mme_iram_read3(bar0, 0u, a);
+	gsp_phys_mme_iram_read3(bar0, 0x364u, bi);
+	gsp_phys_mme_iram_read3(bar0, 3u * 0x364u, bw);
+	pr_info("vgsp-gr: MME-IRAM %s h=0x%x w0=%08x,%08x,%08x pack0=%08x,%08x,%08x i364=%08x,%08x,%08x w364=%08x,%08x,%08x pack364=%08x,%08x,%08x\n",
+		tag, h_object, a[0], a[1], a[2],
+		gsp_phys_gr_mme_ir_pack[0],
+		gsp_phys_gr_mme_ir_pack[1],
+		gsp_phys_gr_mme_ir_pack[2],
+		bi[0], bi[1], bi[2],
+		bw[0], bw[1], bw[2],
+		gsp_phys_gr_mme_ir_pack[3u * 0x364u],
+		gsp_phys_gr_mme_ir_pack[3u * 0x364u + 1u],
+		gsp_phys_gr_mme_ir_pack[3u * 0x364u + 2u]);
+}
+
+/*
+ * SESSION 153-CONTINUED-107 gate knob (see the helper comment above).
+ * Default 0: A/B flip - with the gate on, a launch whose readback shows the
+ * IRAM not holding the packed image holds its real CALL runs and owes a
+ * full re-emission (prelude latch cleared; pack/drain cursors rewound), so
+ * the calls only ever execute against a verified-resident program.
+ */
+static bool gsp_phys_rmcall_gr_mme_iram_gate;
+module_param_named(sema_gl_mme_iram_gate, gsp_phys_rmcall_gr_mme_iram_gate,
+		   bool, 0644);
+MODULE_PARM_DESC(sema_gl_mme_iram_gate,
+	"Read the MME IRAM back (0x4044d0 port) and hold/re-emit the packed prelude until it reads back the packed image at the SA entries (default 0; the CONT-106 all-zero fetch)");
+
+/*
+ * SESSION 153-CONTINUED-107: separate knob for the per-launch chronology
+ * probe lines. They write the MME debug port DURING a live launch, and the
+ * first three probe-bearing cycles (r1369..r1371) all parked the engine on
+ * the first X-channel launches while the last probe-free cycle (r1368) ran
+ * the full lane; until the port's side effects are cleared, keep the probe
+ * off by default and enable it only on a known-good boot (A/B).
+ */
+module_param_named(sema_gl_mme_iram_probe, gsp_phys_rmcall_gr_mme_iram_probe,
+		   bool, 0644);
+MODULE_PARM_DESC(sema_gl_mme_iram_probe,
+	"Per-launch MME-IRAM chronology probe lines in the post-doorbell spin (writes the debug port during a live launch; default 0)");
+MODULE_PARM_DESC(sema_gl_mme_iram_probe,
+	"Per-launch MME-IRAM chronology probe lines in the post-doorbell spin (writes the debug port during a live launch; default 0)");
+
+/*
+ * Read every covered SA entry from the live IRAM and compare with the packed
+ * image. Auto-calibrates the port's address unit once (word base 3*entry vs
+ * instruction base entry - the r1364 validation could only settle address 0).
+ * Returns true when every checked entry matches (or there is nothing to
+ * check); false on any mismatch (all-zero included). Bounded logging.
+ */
+static bool gsp_phys_gr_mme_iram_gate(void __iomem *bar0, u32 h_object,
+				      u32 gen, const char *tag)
+{
+	static u32 gate_run_n, gate_fail_n, gate_pass_n;
+	static int base_sel = -1;	/* -1 unresolved; 0 word; 1 instruction */
+	u32 sa_ent[64], i;
+	u32 checked = 0, match = 0, zero = 0;
+	u32 bad_slot = 0xffffffffu, bad_entry = 0, checked_to_bad = 0;
+	u32 got0 = 0, got1 = 0, got2 = 0, exp0 = 0, exp1 = 0, exp2 = 0;
+	u64 sa_mask;
+
+	if (!bar0)
+		return true;
+	sa_mask = gsp_sema_mme_sa_get(sa_ent, 64);
+	for (i = 0; i < 64u; i++) {
+		u32 entry, base, g0, g1, g2, got[3];
+
+		if (!(sa_mask & (1ull << i)) || !sa_ent[i])
+			continue;
+		entry = sa_ent[i];
+		if (entry * 3u + 2u >= 4096u)
+			continue;
+		g0 = gsp_phys_gr_mme_ir_pack[entry * 3u];
+		g1 = gsp_phys_gr_mme_ir_pack[entry * 3u + 1u];
+		g2 = gsp_phys_gr_mme_ir_pack[entry * 3u + 2u];
+		/*
+		 * Zero-gold entries are stub-covered (no captured code) or
+		 * not part of the packed image; they carry nothing to gate.
+		 */
+		if (!g0 && !g1 && !g2)
+			continue;
+		if (base_sel < 0 && entry) {
+			/*
+			 * One-time unit calibration on the first non-zero
+			 * entry: try the word base, then the instruction
+			 * base. Leaving it unresolved simply keeps the word
+			 * base (the documented form).
+			 */
+			gsp_phys_mme_iram_read3(bar0, 3u * entry, got);
+			if (got[0] == g0 && got[1] == g1 && got[2] == g2) {
+				base_sel = 0;
+				pr_info("vgsp-gr: MME-IRAM-BASE resolved=word (3*entry) at entry=0x%x\n",
+					entry);
+			} else {
+				gsp_phys_mme_iram_read3(bar0, entry, got);
+				if (got[0] == g0 && got[1] == g1 &&
+				    got[2] == g2) {
+					base_sel = 1;
+					pr_info("vgsp-gr: MME-IRAM-BASE resolved=instruction (entry) at entry=0x%x\n",
+						entry);
+				}
+			}
+		}
+		base = (base_sel == 1) ? entry : 3u * entry;
+		gsp_phys_mme_iram_read3(bar0, base, got);
+		checked++;
+		if (got[0] == g0 && got[1] == g1 && got[2] == g2) {
+			match++;
+			continue;
+		}
+		if (!got[0] && !got[1] && !got[2])
+			zero++;
+		if (bad_slot == 0xffffffffu) {
+			bad_slot = i;
+			bad_entry = entry;
+			checked_to_bad = checked;
+			got0 = got[0]; got1 = got[1]; got2 = got[2];
+			exp0 = g0; exp1 = g1; exp2 = g2;
+		}
+	}
+	if (!checked || match == checked) {
+		if (gate_pass_n < 8u) {
+			gate_pass_n++;
+			pr_info("vgsp-gr: MME-IRAM-GATE pass %s h=0x%x gen=%u base=%d checked=%u run=%u\n",
+				tag, h_object, gen, base_sel, checked,
+				gate_run_n);
+		}
+		gate_run_n++;
+		return true;
+	}
+	gate_run_n++;
+	if (gate_fail_n < 24u) {
+		gate_fail_n++;
+		pr_warn("vgsp-gr: MME-IRAM-GATE FAIL %s h=0x%x gen=%u base=%d checked=%u match=%u zero=%u bad slot=%u(entry=0x%x @%u) got=%08x,%08x,%08x want=%08x,%08x,%08x run=%u\n",
+			tag, h_object, gen, base_sel, checked, match, zero,
+			bad_slot, bad_entry, checked_to_bad,
+			got0, got1, got2, exp0, exp1, exp2, gate_run_n);
+	}
+	return false;
+}
+
+/*
  * SESSION 153-CONTINUED-86: the missing SPH (shader-program-header) state.
  *
  * The executed beef launches fault `Graphics Exception: Shader Program
@@ -18765,6 +19023,45 @@ static u32 gsp_phys_gr_mme_prelude_h;
 static u32 gsp_phys_gr_mme_pre_full_dump;
 
 /*
+ * SESSION 153-CONTINUED-107: readback-failure counter + give-up bound for
+ * the load-only starvation (see the mme_load_only comment). 0 = unlimited;
+ * default 24.
+ */
+static u32 gsp_phys_gr_mme_iram_gate_fails;
+static u32 gsp_phys_rmcall_gr_mme_iram_gate_max = 24u;
+
+/*
+ * SESSION 153-CONTINUED-107: gate decision shared by both IRAM-gate call
+ * sites (see gsp_phys_gr_mme_iram_gate). The readback failed, so hold the
+ * calls and owe a full re-emission. Clearing the prelude latch makes the
+ * next launch emit again (the builder's emit condition); rewinding the pack
+ * and drain cursors makes that re-emission restart from the first word. The
+ * pending-gen latch is cleared so the poll break cannot promote the
+ * generation to "resident".
+ */
+static void gsp_phys_gr_mme_iram_gate_fail(void)
+{
+	gsp_phys_gr_mme_iram_gate_fails++;
+	WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false);
+}
+
+module_param_named(sema_gl_mme_iram_gate_max,
+		   gsp_phys_rmcall_gr_mme_iram_gate_max, uint, 0644);
+MODULE_PARM_DESC(sema_gl_mme_iram_gate_max,
+	"Give up the IRAM-gate load-only starvation after this many readback failures (0 = never; default 24)");
+
+static bool gsp_phys_gr_mme_iram_gate_giveup(void)
+{
+	u32 m = READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate_max);
+
+	return m && READ_ONCE(gsp_phys_gr_mme_iram_gate_fails) >= m;
+}
+
+/*
  * SESSION 153-CONTINUED-57: attr/pipeline state prelude. The r1091
  * full-MME-prelude boot cleared MME (mme_esr=80000000) but the first beef
  * window then latched a draw-shader program-header error
@@ -18947,6 +19244,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 		bool pre_done = false;
 		bool mme_hold_calls = false;
 		bool mme_hold_fe_busy = false;
+		bool mme_load_only = false;
 		bool latch_before = false;
 		u32 pre_words_out = 0;
 		const u32 *pl = NULL;
@@ -19896,6 +20194,43 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 			}
 		}
 		/*
+		 * SESSION 153-CONTINUED-107: prelude-only LOAD carriers while
+		 * the IRAM gate has not verified the packed program.
+		 *
+		 * r1372's first instruction (decisive): the calls-held pack
+		 * carrier still poisoned (`DS-FAST ... ds a2040a00->badf1201
+		 * ... MME-FAST end launch events=1 (poison)` -> WEDGE-RECOVER
+		 * wipes the context) because the block also carried the WINDOW
+		 * - the window's own SPH state trips the DS validation and the
+		 * FECS poison episode interrupts the 0x114/0x118 IRAM load
+		 * mid-flight. Holding the calls is not enough; the LOAD must
+		 * ride a carrier whose methods cannot fault, so while the
+		 * generation is not verified live (live_gen != pl_gen) emit
+		 * the pack/SA/release WITHOUT the window. The carrier retires,
+		 * the poll-break gate reads the IRAM back; on a pass live_gen
+		 * is promoted and the very next launch carries the window
+		 * again (latch && live). Bounded by
+		 * sema_gl_mme_iram_gate_max so a broken readback cannot starve
+		 * the lane forever.
+		 */
+		if (READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) &&
+		    READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) &&
+		    READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls) &&
+		    pl && pl_n && pl_gen &&
+		    !gsp_phys_gr_mme_iram_gate_giveup()) {
+			bool latch_now =
+				(pl_gen ==
+				 READ_ONCE(gsp_phys_gr_mme_prelude_gen) &&
+				 req->h_object ==
+				 READ_ONCE(gsp_phys_gr_mme_prelude_h));
+			bool live_now =
+				(READ_ONCE(gsp_phys_rmcall_gr_mme_live_skip) &&
+				 READ_ONCE(gsp_phys_gr_mme_live_gen) == pl_gen);
+
+			if (!(latch_now && live_now))
+				mme_load_only = true;
+		}
+		/*
 		 * SESSION 153-CONTINUED-82: hold real CALL runs until the
 		 * prelude latch for this generation+channel is set (see the
 		 * sema_gl_mme_hold_calls comment). Evaluated AFTER the
@@ -19929,13 +20264,46 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 			 * settled (the r1359 MME ILLEGAL_OPCODE race; see
 			 * gsp_phys_gr_fe_idle). The held launch retires and
 			 * a later window retries once 0x400504 reads idle.
+			 * SESSION 153-CONTINUED-107: separately knob-gated -
+			 * with the IRAM gate proving residency, the residual
+			 * value of this clause is only the r1359 race; r1373
+			 * left the FE at 0x07071602 for minutes and starved
+			 * every calls-kept window, so allow an explicit A/B
+			 * (sema_gl_mme_hold_fe_idle=0).
 			 */
-			if (!mme_hold_calls && !gsp_phys_gr_fe_idle()) {
+			if (!mme_hold_calls &&
+			    READ_ONCE(gsp_phys_rmcall_gr_mme_hold_fe_idle) &&
+			    !gsp_phys_gr_fe_idle()) {
 				mme_hold_calls = true;
 				mme_hold_fe_busy = true;
 			}
 		}
 		/*
+		 * SESSION 153-CONTINUED-107: IRAM readback gate (see the
+		 * gsp_phys_gr_mme_iram_gate comment). The calls would run this
+		 * launch; verify the live IRAM actually holds the packed image
+		 * at the SA entries first. On a mismatch (the CONT-106
+		 * all-zero fetch) hold the calls and owe a full re-emission -
+		 * the r1368 evidence says the real CALL runs against a
+		 * non-resident program poison the lane (ILLEGAL_OPCODE at the
+		 * macro entry) instead of drawing. Only meaningful on the
+		 * packed path (the pack image is the comparator).
+		 */
+		if (!mme_hold_calls && pl && pl_n && pl_gen &&
+		    READ_ONCE(gsp_phys_rmcall_gr_mme_prelude) &&
+		    READ_ONCE(gsp_phys_rmcall_gr_mme_hold_calls) &&
+		    READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) &&
+		    READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate)) {
+			void __iomem *gbar0 = gsp_phys_gpu_bar0();
+
+			if (gbar0 &&
+			    !gsp_phys_gr_mme_iram_gate(gbar0, req->h_object,
+						       pl_gen, "build")) {
+				mme_hold_calls = true;
+				gsp_phys_gr_mme_iram_gate_fail();
+			}
+		}
+		/*
 		 * SESSION 153-CONTINUED-63: emit the SA-RAM completion computed
 		 * above - a one-instruction end-noop stub in the IR plus one
 		 * ONE_INC 0x11c run loading [slot, stub] for every called slot
@@ -20174,7 +20542,23 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 				pr_info("vgsp-gr: MME-SA-REARM %u slot(s) re-pointed before the window (h=0x%x push=0x%llx)\n",
 					rearm, req->h_object, req->push_va);
 		}
-		if (mme_hold_calls) {
+		if (mme_load_only) {
+			/*
+			 * SESSION 153-CONTINUED-107: no window on a LOAD
+			 * carrier (see the mme_load_only comment). The block
+			 * is SET_OBJECT + prelude/SA/pack + release only;
+			 * the engine can retire it without any DS/shader
+			 * state to validate, so the IRAM load cannot be
+			 * interrupted by the SPH poison.
+			 */
+			static u32 load_only_log;
+
+			load_only_log++;
+			if (load_only_log <= 24u)
+				pr_info("vgsp-gr: MME-LOAD-ONLY carrier h=0x%x gen=%u window=%u dropped until the IRAM gate verifies (log %u) push=0x%llx\n",
+					req->h_object, pl_gen, win3d_n,
+					load_only_log, req->push_va);
+		} else if (mme_hold_calls) {
 			/*
 			 * SESSION 153-CONTINUED-82: the prelude latch for
 			 * this generation+channel is not set - elide every
@@ -27623,6 +28007,23 @@ gr_page_unmapped:
 		else
 			w1 = READ_ONCE(*(u32 *)(sem_va + 4));
 		if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) {
+			/*
+			 * SESSION 153-CONTINUED-107: if THIS launch carried
+			 * the final packed prelude part, gate its residency on
+			 * the IRAM readback BEFORE the break log promotes the
+			 * generation to live (see gsp_phys_gr_mme_iram_gate).
+			 * A failed readback clears the pending gen, the
+			 * prelude latch and both emission cursors: the calls
+			 * stay held and the next launch re-emits from word 0.
+			 */
+			if (bar0 &&
+			    READ_ONCE(gsp_phys_gr_mme_pending_gen) &&
+			    READ_ONCE(gsp_phys_rmcall_gr_mme_prelude_pack) &&
+			    READ_ONCE(gsp_phys_rmcall_gr_mme_iram_gate) &&
+			    !gsp_phys_gr_mme_iram_gate(bar0, req->h_object,
+						       READ_ONCE(gsp_phys_gr_mme_pending_gen),
+						       "break"))
+				gsp_phys_gr_mme_iram_gate_fail();
 			/* F4-a: first poll break + USERD witnesses. */
 			gsp_phys_gr_poll_break_log(w0, want_qmd, w1,
 						   poll_start_us, &sub,