vgsp-gr r597: emit the captured prelude's SEM_EXECUTE ACQUIRE groups ahead of the rebuilt window (default off)

The verbatim prelude replay breaks the launch (r599: qmd_rel=0 after an
8.9 s stall), so the acquire cannot be taken by replaying the prelude.
This takes only the wait method groups - C46F addr 0x5c count>=5 with
EXECUTE&7 != 1, and C5C0 SET_REPORT_SEMAPHORE A..D with a non-release op -
and copies them verbatim before the proven init+window+PCAS+release
block, leaving the window untouched.

Gate: gsp_phys_rmcall_gr_acq (default 0). Live A/B on r601: the mail
launch's prelude carries ZERO acquire groups (the C5C0 prelude is inline
parameter data + release only), so emitted=0 and the VADD race is
unchanged - the driver assumes its relocation copies precede the launch
without an engine-visible wait.

3 files changed, 94 insertions, 0 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 46060b1..d43430b 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -16744,6 +16744,61 @@ static u32 *gsp_phys_rmcall_build_gr_init(u32 *p)
 }
 
 /*
+ * r597: extract the SEM_EXECUTE ACQUIRE groups from the captured guest
+ * prelude and copy them, verbatim, ahead of the rebuilt inline-QMD window.
+ * Two encodings are recognised:
+ *   - C46F host-semaphore group: INC_METHOD addr 0x5c count>=5, words
+ *     {LO, HI, PAYLOAD_LO, PAYLOAD_HI, EXECUTE}; EXECUTE & 7 != 1 is an
+ *     acquire (the pump's own op decode).
+ *   - C5C0 SET_REPORT_SEMAPHORE A..D: addr 0x1b00 count>=4, word D's low
+ *     two bits select release (0) / acquire (non-zero).
+ * Only the wait methods are taken, so the proven window (init + inline QMD
+ * + PCAS + release) is untouched. Returns the number of words emitted.
+ */
+static u32 gsp_phys_rmcall_gr_emit_acquires(u32 *p,
+					    const struct gsp_phys_gr_submit *req,
+					    u32 budget)
+{
+	u32 off = 0, words = 0;
+
+	if (!gsp_phys_rmcall_gr_acq_value() || !req || !req->raw)
+		return 0;
+	while (off < req->raw_qmd_hdr && words + 8 < budget) {
+		u32 hdr = req->raw[off];
+		u32 op = (hdr >> 29) & 0x7;
+		u32 count = (hdr >> 16) & 0xfff;
+		u32 addr = (hdr & 0x1fff) << 2;
+		u32 n = 1 + count;
+
+		if ((op != 1 && op != 3) || !count ||
+		    off + n > req->raw_qmd_hdr) {
+			off++;
+			continue;
+		}
+		if (addr == 0x5c && count >= 5) {
+			u32 v4 = req->raw[off + 5];
+			u64 va = ((u64)(req->raw[off + 2] & 0xff) << 32) |
+				 (u64)(req->raw[off + 1] & 0xfffffffc);
+
+			if ((v4 & 0x7) != 1 && va) {
+				memcpy(p + words, &req->raw[off], n * 4);
+				words += n;
+			}
+		} else if (addr >= 0x1b00 && addr <= 0x1b0c &&
+			   count >= 4) {
+			u32 d = req->raw[off + 4] & 0x3;
+
+			if (d != 0) {
+				memcpy(p + words, &req->raw[off], n * 4);
+				words += n;
+			}
+		}
+		off += n;
+	}
+	return words;
+}
+
+/*
  * GR launch method block: the inline QMD run at its original shape and the
  * channel REPORT_SEMAPHORE release. The stock guest push does NOT emit
  * SET_OBJECT before the inline run (the compute class is bound on the
@@ -16776,6 +16831,25 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 				GSP_PHYS_RMCALL_GR_SET_QMD_VIRT_CONTROL,
 				GSP_PHYS_RMCALL_GR_QMD_VIRT_IQ2M_SEND_PCAS);
 	/*
+	 * r597: the prelude's acquire groups (if any) go before the QMD run
+	 * so the engine cannot read the launch's data pages until the copy
+	 * channel's semaphore has advanced.
+	 */
+	{
+		u32 w = gsp_phys_rmcall_gr_emit_acquires(p, req,
+							 128u);
+		static u32 acq_log;
+
+		if (w) {
+			p += w;
+			if (acq_log < 16) {
+				acq_log++;
+				pr_info("vgsp-gr: prelude acquire groups emitted words=%u h=0x%x push=0x%llx\n",
+					w, req->h_object, req->push_va);
+			}
+		}
+	}
+	/*
 	 * Verbatim replay: copy the captured GRCE prelude and the guest's
 	 * own inline-QMD run, keeping the method headers, and relocate only
 	 * the ADDRESS_A/B data words and the 64 QMD words exactly like the
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index 74062dd..e9c0c5a 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -617,6 +617,25 @@ int gsp_phys_rmcall_gr_prelude_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_prelude_value);
 
 /*
+ * r597: emit the SEM_EXECUTE ACQUIRE groups found in the captured guest
+ * prelude ahead of the rebuilt inline-QMD window (default 0). The verbatim
+ * prelude replay breaks the launch (qmd_rel=0 after an 8.9 s stall, r599),
+ * so the acquire cannot be taken by replaying the prelude; this gate takes
+ * only the wait method groups and leaves the proven window in place.
+ */
+static int gsp_phys_rmcall_gr_acq_param;
+module_param_named(gsp_phys_rmcall_gr_acq, gsp_phys_rmcall_gr_acq_param,
+		   int, 0644);
+MODULE_PARM_DESC(gsp_phys_rmcall_gr_acq,
+		 "Emit the captured prelude's SEM_EXECUTE ACQUIRE groups ahead of the rebuilt GR window (default 0)");
+
+int gsp_phys_rmcall_gr_acq_value(void)
+{
+	return gsp_phys_rmcall_gr_acq_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_acq_value);
+
+/*
  * QMD virtualization control gate (default 0 after m26 falsified the
  * hypothesis: on Turing the context init value is 0, native CUDA never
  * writes 0x29c, and the nonzero enable alone does not schedule). When set,
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index 36c2220..3993c6e 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -295,6 +295,7 @@ int gsp_phys_rmcall_gr_promote_value(void);
  * init+window+PCAS+sem shape); 1 replays the verbatim prelude.
  */
 int gsp_phys_rmcall_gr_prelude_value(void);
+int gsp_phys_rmcall_gr_acq_value(void);
 
 /*
  * gsp_phys_rmcall_gr_qmdvirt (default 1): emit