vgsp: per-channel MME pack completion accounting + rearm release + context-reset invalidation (eng40, validation pending)

After the multi-part beef pack completes (2799/2799; keep_gen=1 works), the
prelude generation drifts and the pack branch refuses re-emission (poff==maxa),
so calls stay held forever (pcv=0, shadow draws 0, wrote=0). This adds:
(1) per-channel contiguous-load accounting + pack_done_h so the completion is
remembered for the loaded channel; (2) an MME-PACK-REARM call release when a
completed pack's channel launches again; (3) gsp_phys_gr_mme_chan_ctx_reset(h)
invalidation on context wipes/channel re-registers (the client recycles the
beef channel between pushes, c496), shown firing safely. Validation: the fix
was not yet observed completing on a green window with a same-incarnation draw;
committed for reproducibility per house style.

3 files changed, 152 insertions, 0 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index bfc9149..35cfc79 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -18505,6 +18505,23 @@ static u32 gsp_phys_gr_mme_pack_off;
 static bool gsp_phys_gr_mme_pack_sa;
 
 /*
+ * eng40: per-channel pack-load accounting. The pack cursor above is global
+ * (the packed image is byte-identical across channels and generations under
+ * sema_gl_mme_golden_force), but the 0x114/0x11c IRAM writes land in EACH
+ * window's own channel context. A channel that emitted part 1 and then saw
+ * the completion emitted elsewhere holds only a PREFIX of the program; only
+ * a channel whose emissions were contiguous from word 0 to maxa holds the
+ * whole program (c489: beef 1692 + 1107 = 2799; c487: X-lane completion
+ * after a beef part 1 -> the X context held only the last chunk). pack_done_h
+ * names the channel whose context holds the complete program; the post-
+ * completion call release keys on it, and every context wipe / channel
+ * re-register invalidates it (see gsp_phys_gr_mme_chan_ctx_reset).
+ */
+static u32 gsp_phys_gr_mme_pack_load_h;
+static u32 gsp_phys_gr_mme_pack_load_words;
+static u32 gsp_phys_gr_mme_pack_done_h;
+
+/*
  * CONT-103: prelude residency witness + calls-kept block shrink.
  * The CONT-79 split-emission re-carries the whole prelude prefix on EVERY
  * launch while keep_mask is set (idempotent re-emission, CONT-72), so the
@@ -20588,6 +20605,9 @@ static void gsp_phys_gr_mme_iram_gate_fail(void)
 	WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
 	WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
 	WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0);
 	/*
 	 * CONT-118: residency is unproven again, so a settled release armed
 	 * for this generation is void; the re-emission re-earns it.
@@ -20805,6 +20825,9 @@ static void gsp_phys_gr_ctx_wipe_latches(void)
 	WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
 	WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
 	WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0);
 	WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0);
 	WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0);
 	WRITE_ONCE(gsp_phys_gr_mme_release_h, 0);
@@ -20819,6 +20842,45 @@ static void gsp_phys_gr_ctx_wipe_latches(void)
 }
 
 /*
+ * eng40: a channel re-registration gives that channel a FRESH engine
+ * context - the MME program loaded earlier is gone. Invalidate the
+ * per-channel completion state along with the pack cursors and the
+ * latch/live/release witnesses, so the next window on the new context
+ * re-emits the packed program from word 0 (the pre-eng40 behaviour) and
+ * the post-completion call release cannot fire against a non-resident
+ * program. Gated by the same A/B knobs as the release itself; with the
+ * knobs off this is a no-op (legacy).
+ */
+void gsp_phys_gr_mme_chan_ctx_reset(u32 h)
+{
+	if (!(READ_ONCE(gsp_phys_rmcall_gr_mme_pack_keep_gen) &&
+	      READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force)))
+		return;
+	/*
+	 * Only the channel that owns the in-flight or completed packed load
+	 * loses its context on (re-)registration; an unrelated channel's
+	 * recycle (the client re-registers its whole family per fire) must
+	 * not rewind a pack another channel holds.
+	 */
+	if (h != READ_ONCE(gsp_phys_gr_mme_pack_load_h) &&
+	    h != READ_ONCE(gsp_phys_gr_mme_pack_done_h))
+		return;
+	WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_prelude_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_pending_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_release_h, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_window_hold_gen, 0);
+	WRITE_ONCE(gsp_phys_gr_mme_window_hold_h, 0);
+}
+
+/*
  * CONT.474 CWD-CLEAR: pre-replay priv-error clear for the noQMD stream3d lane.
  * The c240-c242 receipts show the FE already poisoned when a stream3d replay is
  * about to be emitted (`MME-FAST begin launch ... esr=0xbadf1201`,
@@ -21521,6 +21583,12 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 						WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
 						WRITE_ONCE(gsp_phys_gr_mme_pack_sa,
 							   false);
+						WRITE_ONCE(gsp_phys_gr_mme_pack_load_h,
+							   0);
+						WRITE_ONCE(gsp_phys_gr_mme_pack_load_words,
+							   0);
+						WRITE_ONCE(gsp_phys_gr_mme_pack_done_h,
+							   0);
 					}
 				}
 				/*
@@ -22476,6 +22544,46 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 							WRITE_ONCE(
 							 gsp_phys_gr_mme_pack_off,
 							 poff);
+							/*
+							 * eng40: contiguous-load
+							 * accounting (see
+							 * gsp_phys_gr_mme_pack_done_h).
+							 */
+							if (poff - chunk == 0u) {
+								WRITE_ONCE(
+								 gsp_phys_gr_mme_pack_load_h,
+								 req->h_object);
+								WRITE_ONCE(
+								 gsp_phys_gr_mme_pack_load_words,
+								 chunk);
+							} else if (
+								READ_ONCE(
+								 gsp_phys_gr_mme_pack_load_h) ==
+								 req->h_object &&
+								poff - chunk ==
+								READ_ONCE(
+								 gsp_phys_gr_mme_pack_load_words)) {
+								WRITE_ONCE(
+								 gsp_phys_gr_mme_pack_load_words,
+								 poff);
+							} else {
+								WRITE_ONCE(
+								 gsp_phys_gr_mme_pack_load_h,
+								 0);
+								WRITE_ONCE(
+								 gsp_phys_gr_mme_pack_load_words,
+								 0);
+							}
+							if (poff >= maxa)
+								WRITE_ONCE(
+								 gsp_phys_gr_mme_pack_done_h,
+								 (READ_ONCE(
+								  gsp_phys_gr_mme_pack_load_h) ==
+								  req->h_object &&
+								  READ_ONCE(
+								  gsp_phys_gr_mme_pack_load_words) >=
+								  maxa) ?
+								 req->h_object : 0);
 						}
 						pwords = (u32)(pp - p);
 						p = pp;
@@ -22836,6 +22944,34 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 			}
 		}
 		/*
+		 * eng40: post-completion per-channel re-arm. Once the packed
+		 * program has completed ON THIS CHANNEL - contiguous load,
+		 * deterministic golden image - the channel context keeps it
+		 * until a wipe or re-register. Every later frame bumps the
+		 * capture generation, the pack branch cannot re-emit (poff ==
+		 * maxa), and the completion carrier dropped its own window, so
+		 * without this release the real calls stay held forever (c489:
+		 * pcv=0 on every post-completion draw). Only non-emitting
+		 * launches release (pre_words_out == 0 - the r1222 same-launch
+		 * clause stays intact), and the per-channel flag is invalidated
+		 * by any context wipe or channel registration, so a fresh context
+		 * can never run calls against a non-resident program.
+		 */
+		if (mme_hold_calls && pre_words_out == 0 &&
+		    READ_ONCE(gsp_phys_rmcall_gr_mme_pack_keep_gen) &&
+		    READ_ONCE(gsp_phys_rmcall_gr_mme_golden_force) &&
+		    READ_ONCE(gsp_phys_gr_mme_pack_done_h) == req->h_object) {
+			static u32 pack_rearm_log;
+
+			if (pack_rearm_log < 8u) {
+				pack_rearm_log++;
+				pr_info("vgsp-gr: MME-PACK-REARM h=0x%x gen=%u (pack complete + contiguous load; calls released) push=0x%llx\n",
+					req->h_object, pl_gen,
+					(unsigned long long)req->push_va);
+			}
+			mme_hold_calls = false;
+		}
+		/*
 		 * r1487 STREAM-DAMAGE: an earlier chunk of this same push was
 		 * dropped (LOAD-ONLY carrier or the arming window hold), so
 		 * the channel no longer holds the state that chunk carried.
@@ -32345,6 +32481,9 @@ gr_page_unmapped:
 						WRITE_ONCE(gsp_phys_gr_mme_pre_emit_off, 0);
 						WRITE_ONCE(gsp_phys_gr_mme_pack_off, 0);
 						WRITE_ONCE(gsp_phys_gr_mme_pack_sa, false);
+						WRITE_ONCE(gsp_phys_gr_mme_pack_load_h, 0);
+						WRITE_ONCE(gsp_phys_gr_mme_pack_load_words, 0);
+						WRITE_ONCE(gsp_phys_gr_mme_pack_done_h, 0);
 						WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0);
 						WRITE_ONCE(
 						 gsp_phys_gr_mme_release_gen,
diff --git a/kernel/gsp_phys_booter.h b/kernel/gsp_phys_booter.h
index f821a85..92c8e69 100644
--- a/kernel/gsp_phys_booter.h
+++ b/kernel/gsp_phys_booter.h
@@ -714,4 +714,9 @@ int gsp_phys_boot_rm_unload(void __iomem *bar0);
 int gsp_phys_boot_debugfs_init(void);
 void gsp_phys_boot_debugfs_fini(void);
 
+/* eng40: invalidate the per-channel MME pack-completion state on a channel
+ * registration (fresh engine context); no-op unless the pack_keep_gen +
+ * golden_force A/B knobs are on. */
+void gsp_phys_gr_mme_chan_ctx_reset(u32 h);
+
 #endif /* GSP_PHYS_BOOTER_H */
diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index b928faf..d4390b9 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -3675,6 +3675,14 @@ static void gsp_rm_register_channel(struct gsp_emu_context *ctx, u32 h_client,
 	u64 userd_wire_base = 0;
 
 	/*
+	 * eng40: a (re-)registration gives this channel a fresh engine
+	 * context; the MME program it may have held is gone, so invalidate
+	 * the per-channel pack-completion state (no-op unless the
+	 * pack_keep_gen + golden_force knobs are on).
+	 */
+	gsp_phys_gr_mme_chan_ctx_reset(h_object);
+
+	/*
 	 * CONT.405: name the registration attempt for the direct-GL client's
 	 * channel families. A "beef"/"bfef" handle that appears more than
 	 * once is the client's alloc -> setup -> free -> retry loop; the