vgsp-mme: sema_gl_pair_chain (default 1) - EC0x0d = BEGIN_END_ACTIVE decode: drain (WAIT_FOR_IDLE<-0) before each following normal/normal BEGIN so each END closes exactly one pair; held-call branch pair elision gated on pairs_gen==pl_gen; per-launch/context-wipe resets; guest BEGIN data carried verbatim (no normalisation site exists); 0=legacy bytes

1 files changed, 281 insertions, 6 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 9d5fb71..b9e247c 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -18412,6 +18412,179 @@ MODULE_PARM_DESC(sema_gl_mme_nop_pair,
 	"Elide program-address pair runs (0x2054/0x2094/0x2154 families) from a calls-kept 3D window (default 0; the engine traps 0x2058 on the relocated shadow operand)");
 
 /*
+ * r1453 PAIR-CHAIN (sema_gl_pair_chain): the offline EC 0x0d decode
+ * (BEGIN_END_ACTIVE, nv50.c:173) and the r1452b window dumps show the
+ * guest's draws arriving as back-to-back normal/normal BEGIN (0x1618)
+ * ... END (0x1614) pairs with no WAIT_FOR_IDLE (0x110) / SEND_GO_IDLE
+ * (0x13c) drain between them, and the FE then refuses the closing END
+ * (trap 0x80001614, Data 0, EC 0x0d) - and every later state write
+ * (0x2058 PROGRAM_ADDRESS_B) while a pair stays active.
+ *
+ * The guest's BEGIN data word is carried VERBATIM end to end (the
+ * window is a raw-push slice: svc_gl_flush -> req->stream ->
+ * gsp_phys_rmcall_gr_3d_patch_window memcpy). Nothing in the kernel
+ * tree constructs or normalises a 0x586/0x585 word: the only 32-bit
+ * BEGIN data form is a count-carrying run at method 0x1618 whose first
+ * payload word is the full data (SPLIT_MODE is bits 30:29 of THAT
+ * word); the 13-bit IMMD header form (op 4, data 8/7) cannot express
+ * the split bits and is what the observed raw pairs carry. This pass
+ * therefore never rebuilds a run: it (a) receipts the full BEGIN data
+ * word it carries, (b) inserts ONE WAIT_FOR_IDLE<-0 (INC 0x110, 1 data
+ * word) ahead of a following normal-begin pair so each END closes a
+ * drained pair, and (c) leaves split-mode chains (BEGIN 1/2/3) alone.
+ * 0 = legacy bytes (pass not run; held-call elision unchanged).
+ */
+static bool gsp_phys_rmcall_gr_pair_chain = true;
+module_param_named(sema_gl_pair_chain, gsp_phys_rmcall_gr_pair_chain,
+		   bool, 0644);
+MODULE_PARM_DESC(sema_gl_pair_chain,
+	"One logical BEGIN..END chain per launch: carry the full 32-bit BEGIN data (SPLIT_MODE 30:29), insert WAIT_FOR_IDLE(0x110) drains between back-to-back normal/normal pairs, and extend pair elision to held-call windows (default 1; 0 = legacy bytes)");
+static bool gsp_phys_gr_mme_pairs_emitted;	/* launch emitted a pair run */
+static u32 gsp_phys_gr_mme_pairs_gen;		/* gen whose pairs retired OK */
+static u32 gsp_phys_gr_mme_launch_gen;		/* last built launch's gen */
+
+/*
+ * True when a method run in [w, w+n) writes NVC597_BEGIN (0x1618) or
+ * NVC597_END (0x1614) - the per-launch proof the pair-elision extension
+ * gates on (pairs must have reached the FE once for the generation).
+ */
+static bool gsp_phys_gr_pair_present(const u32 *w, u32 n)
+{
+	u32 r = 0;
+
+	while (r < n) {
+		u32 h = w[r];
+		u32 op = (h >> 29) & 7u;
+		u32 cnt = (h >> 16) & 0xfffu;
+		u32 span = 1u;
+		u32 m = (h & 0xfffu) << 2;
+
+		if ((op == 1u || op == 3u || op == 5u) && cnt &&
+		    r + 1u + cnt <= n)
+			span = 1u + cnt;
+		if ((op == 1u || op == 3u || op == 4u || op == 5u) &&
+		    (m == 0x1618u || m == 0x1614u))
+			return true;
+		r += span;
+	}
+	return false;
+}
+
+/*
+ * Apply the pair-chain rule to the final emitted window (prepends
+ * included). Returns the number of inserted drains; *np is updated.
+ * The scan is method-run aligned and insertion copies whole runs, so a
+ * run is never split and the BEGIN data word is never rewritten.
+ */
+static u32 gsp_phys_gr_pair_chain_fix(u32 *w, u32 *np, u32 cap)
+{
+	u32 n = *np, r = 0, begins = 0, ends = 0, drains = 0;
+	u32 chains = 0, orphans = 0, nested = 0, badv = 0;
+	bool open = false, seen = false, begin_logged = false;
+	u32 open_split = 0;
+
+	while (r < n) {
+		u32 h = w[r];
+		u32 op = (h >> 29) & 7u;
+		u32 cnt = (h >> 16) & 0xfffu;
+		u32 span = 1u;
+		u32 m = (h & 0xfffu) << 2;
+		bool is_begin = false, is_end = false;
+		u32 data = 0, split = 0, ev = 0;
+
+		if ((op == 1u || op == 3u || op == 5u) && cnt &&
+		    r + 1u + cnt <= n)
+			span = 1u + cnt;
+		if (op == 4u) {
+			if (m == 0x1618u) {
+				is_begin = true;
+				data = (h >> 16) & 0x1fffu;
+			} else if (m == 0x1614u) {
+				is_end = true;
+				ev = (h >> 16) & 1u;
+			}
+		} else if ((op == 1u || op == 3u || op == 5u) && cnt &&
+			   m == 0x1618u) {
+			is_begin = true;
+			data = w[r + 1u];
+			split = (data >> 29) & 3u;
+		} else if ((op == 1u || op == 3u || op == 5u) && cnt &&
+			   m == 0x1614u) {
+			is_end = true;
+			ev = w[r + 1u] & 1u;
+		}
+		if (is_begin) {
+			if (!begin_logged) {
+				begin_logged = true;
+				pr_info_ratelimited("vgsp-gr: PAIR-BEGIN-DATA off=%u data=%08x split=%u form=%s (full 32-bit data carried verbatim)\n",
+					r, data, split,
+					(op == 4u) ? "immd13(no-split)" :
+						     "run32");
+			}
+			if (!open && seen && split == 0u) {
+				/*
+				 * EC 0x0d: a normal/normal pair begun while
+				 * the previous pair is not drained is refused
+				 * at its END. Insert the class's own stall
+				 * (WAIT_FOR_IDLE <- 0) ahead of it.
+				 */
+				if (n + 2u <= cap) {
+					memmove(&w[r + 2u], &w[r],
+						(size_t)(n - r) * 4u);
+					w[r] = ((u32)GSP_PHYS_RMCALL_METHOD_IMM
+						<< 29) | (1u << 16) |
+					       (0x110u >> 2);
+					w[r + 1u] = 0u;
+					n += 2u;
+					r += 2u;
+					drains++;
+					pr_info_ratelimited("vgsp-gr: PAIR-DRAIN inserted WAIT_FOR_IDLE(0x110)<-0 before pair #%u at off=%u window=%u\n",
+						begins + 1u, r, n);
+				} else {
+					pr_warn_ratelimited("vgsp-gr: PAIR-DRAIN skipped (window=%u cap=%u): back-to-back pairs can still raise EC0x0d\n",
+						n, cap);
+				}
+			}
+			if (open) {
+				/*
+				 * Legal only as a split chain (the previous
+				 * BEGIN left its end open).
+				 */
+				if (open_split == 1u || open_split == 2u)
+					chains++;
+				else
+					nested++;
+			} else {
+				open = true;
+				seen = true;
+			}
+			open_split = split;
+			begins++;
+		} else if (is_end) {
+			if (!open) {
+				orphans++;
+			} else {
+				u32 expect = ((open_split == 1u ||
+					       open_split == 2u) ? 1u : 0u);
+
+				if (ev != expect)
+					badv++;
+				open = false;
+				ends++;
+			}
+		}
+		r += span;
+	}
+	if (begins > 1u || drains || chains || orphans || nested || badv ||
+	    open)
+		pr_info_ratelimited("vgsp-gr: PAIR-CHAIN begin=%u end=%u drains=%u chain=%u orphan_end=%u nested=%u endv_mismatch=%u open_at_end=%u window=%u\n",
+			begins, ends, drains, chains, orphans, nested,
+			badv, open ? 1u : 0u, n);
+	*np = n;
+	return drains;
+}
+
+/*
  * SESSION 153-CONTINUED-79: parameter-conserving NOP for kept parking slots.
  * The r1177 bisect: keep={0,2,57} (full 3089-word program resident, split
  * emission) parks the PBDMA on the emitted block (MME-FAST events=0, grfifo
@@ -19775,6 +19948,7 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 		bool mme_load_only = false;
 		bool latch_before = false;
 		u32 pre_words_out = 0;
+		u32 win_emit_off = 0;
 		const u32 *pl = NULL;
 		u32 pl_n = 0, pl_gen = 0, pl_room = 0;
 		u32 block_cap = READ_ONCE(gsp_phys_rmcall_gr_block_cap);
@@ -20017,6 +20191,14 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 		if (READ_ONCE(gsp_phys_rmcall_gr_mme_prelude)) {
 			pl = gsp_sema_mme_prelude_get(&pl_n, &pl_gen);
 			/*
+			 * r1453 PAIR-CHAIN: remember the generation this
+			 * launch was built against; the submit's success
+			 * edge promotes it to gsp_phys_gr_mme_pairs_gen
+			 * once a pair-bearing window retires (the held
+			 * elision extension's install proof).
+			 */
+			gsp_phys_gr_mme_launch_gen = pl_gen;
+			/*
 			 * SESSION 153-CONTINUED-72: with keep_mask bits set a
 			 * kept macro can execute, and it needs the MME program
 			 * resident. The r1152 evidence: the prelude was carried
@@ -21577,6 +21759,14 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 			WRITE_ONCE(gsp_phys_gr_mme_strict_holds, 0);
 			mme_hold_window = false;
 		}
+		/*
+		 * r1453 PAIR-CHAIN: the window was already drained in the
+		 * submit path (after the SPH-BIND/SURF-PRELUDE prepends,
+		 * once per launch - see gsp_phys_gr_pair_chain_fix). The
+		 * emitted-pair proof is recomputed from the final branch
+		 * output below.
+		 */
+		win_emit_off = (u32)(p - pb);
 		if (mme_load_only || mme_hold_window) {
 			if (mme_hold_window && !mme_load_only) {
 				pr_info_ratelimited("vgsp-gr: MME-WINDOW-HOLD gen=%u h=%#x window=%u dropped (r1447 arming hold) push=0x%llx\n",
@@ -21611,21 +21801,46 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 			 * the MME macro method space 0x3800..0x39fc are the
 			 * ones that execute a macro; the drop decision is
 			 * the same walk gsp_phys_gr_mme_drop_calls uses.
+			 *
+			 * r1453 PAIR-CHAIN: the held window is the 0x2058
+			 * "other writer" (r1453 strategy §3.2) - it copies
+			 * everything except CALL runs, including the
+			 * program-address pair runs, and the r1452b #33
+			 * launch then traps 0x2058 at Data 0x19b0900 /
+			 * EC 0x0d. Extend the sema_gl_mme_nop_pair elision
+			 * here ONLY once a pair-bearing window of this
+			 * same generation has retired (pairs_gen == pl_gen,
+			 * set on the submit success edge): eliding before
+			 * that starves the binds (r1452 D1, HUBCLIENT_PD
+			 * @ 0x0). The elision still requires the nop_pair
+			 * opt-in, so the default held shape is unchanged.
 			 */
-			u32 r = 0, w = 0, held = 0;
+			u32 r = 0, w = 0, held = 0, pelided = 0;
+			bool pair_installed =
+				READ_ONCE(gsp_phys_rmcall_gr_mme_nop_pair) &&
+				READ_ONCE(gsp_phys_rmcall_gr_pair_chain) &&
+				pl_gen && READ_ONCE(gsp_phys_gr_mme_pairs_gen) ==
+				pl_gen;
 
 			while (r < win3d_n) {
 				u32 h = win3d[r];
 				u32 op = (h >> 29) & 7u;
 				u32 cnt = (h >> 16) & 0xfffu;
 				u32 span = 1u;
-				bool iscall = false;
-				u32 k;
+				bool iscall = false, ispair = false;
+				u32 k, m0, mend;
 
 				if ((op == 1u || op == 3u || op == 5u) &&
 				    cnt && r + 1u + cnt <= win3d_n)
 					span = 1u + cnt;
-				for (k = 0; k < span; k++) {
+				m0 = (h & 0xfffu) << 2;
+				mend = m0 + 4u * (span - 1u);
+				if (pair_installed &&
+				    (op == 0u || op == 1u || op == 3u ||
+				     op == 4u || op == 5u) &&
+				    m0 <= 0x2158u && mend >= 0x2054u)
+					ispair = true;
+				for (k = 0; k < span && !ispair; k++) {
 					u32 m = ((h & 0xfffu) << 2) +
 						4u * k;
 
@@ -21636,8 +21851,11 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 						break;
 					}
 				}
-				if (iscall) {
-					held += span;
+				if (iscall || ispair) {
+					if (iscall)
+						held += span;
+					if (ispair)
+						pelided += span;
 					r += span;
 					continue;
 				}
@@ -21651,6 +21869,10 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 					win3d_n, w, req->push_va,
 					mme_hold_fe_busy ?
 						" [FE-not-idle (CONT-104)]" : "");
+			if (pelided)
+				pr_info_ratelimited("vgsp-gr: PAIR-CHAIN held-elide %u word(s) of program-pair runs dropped (pairs installed gen=%u h=0x%x; 0x2058 other-writer) push=0x%llx\n",
+					pelided, pl_gen, req->h_object,
+					req->push_va);
 			p += w;
 		} else if (READ_ONCE(gsp_phys_rmcall_gr_mme_nop_pair)) {
 			/*
@@ -21705,6 +21927,18 @@ if (gsp_phys_gr_slot_kept(j, sa_mask,
 			p += win3d_n;
 		}
 		/*
+		 * r1453 PAIR-CHAIN: the install proof for the held-call
+		 * elision extension. Recomputed from the FINAL emitted window
+		 * (after the CALL/pair elision branches above), so a launch
+		 * that dropped every pair never marks its generation as
+		 * pair-installed.
+		 */
+		if (READ_ONCE(gsp_phys_rmcall_gr_pair_chain) &&
+		    (u32)(p - pb) > win_emit_off)
+			gsp_phys_gr_mme_pairs_emitted =
+				gsp_phys_gr_pair_present(&pb[win_emit_off],
+					(u32)(p - pb) - win_emit_off);
+		/*
 		 * SESSION 153-CONTINUED-84: the SA re-arm block used to be
 		 * emitted HERE (after the window). It was moved ahead of the
 		 * window above; see that block's comment for the r1236
@@ -26554,6 +26788,8 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 
 	fast_rearmed = false;
 	deposit_rb_done = false;
+	gsp_phys_gr_mme_pairs_emitted = false;	/* r1453 PAIR-CHAIN */
+	gsp_phys_gr_mme_launch_gen = 0;
 	gsp_phys_gr_codehash_done = false;	/* CODEHASH: once per launch */
 	/*
 	 * M91 FAULT-CAP: snapshot the fault epoch per launch, so the capture
@@ -28356,6 +28592,21 @@ gr_page_unmapped:
 		}
 	}
 
+	/*
+	 * r1453 PAIR-CHAIN: drain the final window ONCE per launch, after
+	 * every prepend (SPH-BIND, SURF-PRELUDE) and the relocation pass,
+	 * so the same bytes feed both the first build and the outcome
+	 * retry. The pass only inserts WAIT_FOR_IDLE(0x110) runs ahead of
+	 * back-to-back normal/normal BEGIN pairs; the guest's BEGIN data
+	 * words are carried verbatim. Knob 0 = no pass (legacy bytes).
+	 */
+	if (READ_ONCE(gsp_phys_rmcall_gr_pair_chain) && win3d && win3d_n) {
+		u32 pn = win3d_n;
+
+		(void)gsp_phys_gr_pair_chain_fix((u32 *)win3d, &pn,
+					GSP_PHYS_RMCALL_GR_METHOD_WORDS);
+		win3d_n = pn;
+	}
 	words = gsp_phys_rmcall_build_gr_launch(methods, req, qmd,
 						qmd_dst,
 						ch_sem_va, want_ch,
@@ -29019,6 +29270,9 @@ gr_page_unmapped:
 				WRITE_ONCE(gsp_phys_gr_mme_live_gen, 0);
 				WRITE_ONCE(gsp_phys_gr_mme_release_gen, 0);
 				WRITE_ONCE(gsp_phys_gr_mme_release_h, 0);
+				/* r1453 PAIR-CHAIN: context wipe voids the
+				 * pair-install proof; re-earn it. */
+				WRITE_ONCE(gsp_phys_gr_mme_pairs_gen, 0);
 				/*
 				 * r1447: the wiped context voids the arming
 				 * window-hold and the strict grace budget too.
@@ -29155,6 +29409,21 @@ gr_page_unmapped:
 			w1 = READ_ONCE(*(u32 *)(sem_va + 4));
 		if ((!qmd_rel || w0 == want_qmd) && w1 == want_ch) {
 			/*
+			 * r1453 PAIR-CHAIN: this launch retired. If its
+			 * emitted window carried BEGIN/END runs, the pair
+			 * runs are installed on the channel for this
+			 * prelude generation - the proof the held-call
+			 * elision extension waits for. A context wipe
+			 * (WEDGE-RECOVER / park recovery) clears this so a
+			 * later generation re-earns it from a drain-carrying
+			 * window.
+			 */
+			if (gsp_phys_gr_mme_pairs_emitted &&
+			    gsp_phys_gr_mme_launch_gen &&
+			    READ_ONCE(gsp_phys_rmcall_gr_pair_chain))
+				WRITE_ONCE(gsp_phys_gr_mme_pairs_gen,
+					   gsp_phys_gr_mme_launch_gen);
+			/*
 			 * SESSION 153-CONTINUED-107: if THIS launch carried
 			 * the final packed prelude part, gate its residency on
 			 * the IRAM readback BEFORE the break log promotes the
@@ -29364,6 +29633,12 @@ gr_page_unmapped:
 						WRITE_ONCE(
 						 gsp_phys_gr_mme_release_h,
 						 0);
+						/* r1453 PAIR-CHAIN: same
+						 * context wipe - the pair
+						 * install proof is void. */
+						WRITE_ONCE(
+						 gsp_phys_gr_mme_pairs_gen,
+						 0);
 						/*
 						 * r1447: same context wipe -
 						 * void the arming window-hold