vgsp: relocate a CODESRC-SUSPECT GR program page to the engine-mapped window-0 shadow instead of leaving the QMD pointer at an unmapped guest VA (eng33)

The client's MME-expanded draw builds a QMD whose program page (qmd[48])
classifies CODESRC-SUSPECT (empty-page): the engine's GPCCLIENT_GCC I-fetch
then read the guest VA above every engine-PDB window, raised FAULT_PDE and
the FIFO parked for the rest of the boot (the TSG re-arm cannot revive a
parked PBDMA). c361-c368 receipts.

With gsp_phys_gr_codesrc_reloc=1 (default) the builder copies the page into
the window-0 shadow (empty sources only; PTE/module pages are never
republished), overlays the client's own known-good SPH+SASS stage image at
the original QMD program offset (gsp_sema_gl_prog_img, stage-typed from the
section bind; truncated at a 128-bit boundary with a terminal EXIT when the
image is larger than the remaining page), and the existing QMD w48/49 +
prefetch rewrite publishes the shadow VA. If the shadow mapping cannot be
installed the page stays unmapped and the existing refusal drops the launch,
so an unmapped QMD program pointer is never published. 0 restores the old
guest-VA behaviour for a one-flip A/B.

Validation (c373, params-r1624): 31 CODESRC-RELOC events (img_len=384,
off=0xb00, 0 truncations), 51 GR LAUNCH submitted, 11 MME-FAST end
events=3, 79 FETCH-WIT with GP_GET advancing engine-fetched, 0 FAULT_PDE,
0 vgsp-fault, 0 WEDGE-REARM/PARK-RECOVER; work_gr backend_ok=35 fault=0.
The draws still write nothing (NOWRITE, no clear run in the emitted
window), so px0 stays 0 - the store wall is unchanged and is the next
target.

1 files changed, 98 insertions, 11 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 8756d83..c8e224b 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -25725,14 +25725,41 @@ gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req,
  * (gsp_sema_gr_reloc_resolve: pins/map/walks before the ledger, the
  * guest-walk-first counterpart of the fill's ledger-first order). Only a
  * different, plausibly-code page is adopted (mode ALT; the engine-visible
- * copy is taken from it); anything else leaves the QMD pair at the guest
- * VA (mode GUEST_VA, the ptr_guest behaviour for this pair). One
- * rate-limited CODESRC-SUSPECT line per detection.
+ * copy is taken from it).
+ *
+ * r1625 (the c361-c368 wedge): mode GUEST_VA used to leave the QMD pair at
+ * the guest VA, which is above every engine-PDB window - the GPCCLIENT_GCC
+ * I-fetch then raised FAULT_PDE and parked the FIFO permanently (the
+ * WEDGE-REARM TSG probe cannot revive a dead PBDMA). With
+ * gsp_phys_gr_codesrc_reloc (default 1) the refused page is instead
+ * relocated to the engine-mapped window-0 shadow and the QMD pair is
+ * rewritten there. The source page is copied when it is only empty (so any
+ * content elsewhere in the page is preserved and inline records still land
+ * on top); PTE/module pages are never republished. The program entry
+ * (original QMD offset preserved) is then overwritten with the client's own
+ * known-good SPH+SASS image for the bound stage (gsp_sema_gl_prog_img) -
+ * a raw EXIT stub would fail the engine's SPH validation - truncated at a
+ * 128-bit instruction boundary with a terminal EXIT appended. If the shadow
+ * mapping cannot be installed the launch is refused (the existing unmapped-
+ * page path), so a QMD whose program pointer is not engine-mapped is never
+ * published. One rate-limited CODESRC-SUSPECT line per detection, plus a
+ * CODESRC-RELOC line for each relocation.
  */
 #define GSP_PHYS_GR_CODESRC_OK		0
 #define GSP_PHYS_GR_CODESRC_ALT		1
 #define GSP_PHYS_GR_CODESRC_GUEST_VA	2
 
+/*
+ * r1625: terminal 16 B sm_75 SASS (MOV R1, c[0x0][0x0]; EXIT), taken from
+ * the first two instructions of a compiled sm_75 kernel's .text. Appended
+ * after a truncated relocation image so control can never run into the
+ * zero-filled tail of the shadow page.
+ */
+static const u8 gsp_phys_gr_codesrc_exit[16] = {
+	0x02, 0x7a, 0x01, 0x00, 0x00, 0x0a, 0x00, 0x00,
+	0x00, 0x0f, 0x00, 0x00, 0x00, 0xc4, 0x0f, 0x00,
+};
+
 static bool
 gsp_phys_gr_codesrc_chunk_has(const struct gsp_phys_boot_chunk *c, u64 hpa)
 {
@@ -25816,7 +25843,8 @@ static bool gsp_phys_gr_codesrc_empty(const u8 *p, u32 off)
 
 static noinline int
 gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req,
-		     const struct gsp_phys_gr_page *pg, u64 *alt_hpa)
+		     const struct gsp_phys_gr_page *pg, u64 *alt_hpa,
+		     const char **why_out, bool *unsafe_out)
 {
 	u64 page = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
 	u32 off = (u32)(req->qmd[48] & (GSP_PHYS_PAGE_SIZE - 1));
@@ -25826,6 +25854,10 @@ gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req,
 	int mode = GSP_PHYS_GR_CODESRC_GUEST_VA;
 
 	*alt_hpa = 0;
+	if (why_out)
+		*why_out = NULL;
+	if (unsafe_out)
+		*unsafe_out = false;
 	if (off > GSP_PHYS_PAGE_SIZE - 16u)
 		off = GSP_PHYS_PAGE_SIZE - 16u;	/* keep 16 B in-page */
 	if (gsp_phys_gr_codesrc_empty(p, off))
@@ -25836,6 +25868,14 @@ gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req,
 		why = "module-pt-page";
 	if (!why)
 		return GSP_PHYS_GR_CODESRC_OK;
+	if (why_out)
+		*why_out = why;
+	/* The page content must not be republished as code: PTE/filler
+	 * bytes would execute as SASS. Empty is safe to copy (all zero)
+	 * but is republished with a known-good image at the entry. */
+	if (unsafe_out && why &&
+	    (!strcmp(why, "pte-page") || !strcmp(why, "module-pt-page")))
+		*unsafe_out = true;
 	/*
 	 * Alternative resolution order; only a different page that itself
 	 * passes the content checks is adopted.
@@ -29208,16 +29248,63 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 			 * CODESRC: the program page (qmd_word 48) is the page
 			 * the engine's I-fetch reads; screen its source
 			 * before publishing it. A refused source is either
-			 * replaced by the alternative resolver's page or the
-			 * pair is left at the guest VA (no copy, no rewrite,
-			 * no mapping - the page is fully withdrawn from the
-			 * submission).
+			 * replaced by the alternative resolver's page or
+			 * (r1625) relocated to this shadow with the client's
+			 * known-good stage image at the entry, so the QMD
+			 * pair is rewritten to an engine-mapped VA. Leaving
+			 * it at the guest VA makes the I-fetch fault (PDE)
+			 * and parks the FIFO; that legacy withdrawal is now
+			 * gated behind gsp_phys_gr_codesrc_reloc=0.
 			 */
 			if (pg->hpa && pg->qmd_word == 48u) {
 				u64 alt_hpa = 0;
-				int cs = gsp_phys_gr_code_src(req, pg, &alt_hpa);
-
-				if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA) {
+				const char *why = NULL;
+				bool unsafe = false;
+				int cs = gsp_phys_gr_code_src(req, pg,
+							      &alt_hpa, &why,
+							      &unsafe);
+
+				if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA &&
+				    gsp_phys_gr_codesrc_reloc_value()) {
+					u32 po = qmd[48] &
+						 (GSP_PHYS_PAGE_SIZE - 1);
+					const u8 *img = NULL;
+					u32 imlen = 0, cp = 0, fit;
+
+					if (po > GSP_PHYS_PAGE_SIZE - 16u)
+						po = GSP_PHYS_PAGE_SIZE - 16u;
+					if (!unsafe)
+						memcpy(sh_page,
+						       (u8 *)__va(pg->hpa),
+						       GSP_PHYS_PAGE_SIZE);
+					img = gsp_sema_gl_prog_img(
+						pg->guest_va, &imlen);
+					fit = GSP_PHYS_PAGE_SIZE - po;
+					if (imlen > fit)
+						fit = fit >
+							  sizeof(gsp_phys_gr_codesrc_exit) ?
+							  fit -
+							  sizeof(gsp_phys_gr_codesrc_exit) :
+							  0u;
+					if (img && imlen) {
+						cp = imlen < fit ? imlen : fit;
+						memcpy(sh_page + po, img, cp);
+					}
+					if (cp < imlen || !img || !imlen)
+						memcpy(sh_page + po + cp,
+						       gsp_phys_gr_codesrc_exit,
+						       sizeof(gsp_phys_gr_codesrc_exit));
+					copied++;
+					pr_info_ratelimited("vgsp-gr: CODESRC-RELOC va=0x%llx pa=0x%llx why=%s src=%s img_len=%u cp=%u off=0x%x trunc=%u h=0x%x push=0x%llx (engine copy + QMD pair rewrite)\n",
+						pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1),
+						pg->hpa, why ? why : "none",
+						unsafe ? "stub-only" :
+						img ? "image" : "exit",
+						imlen, cp, po,
+						cp < imlen ? 1u : 0u,
+						req->h_object,
+						req->push_va);
+				} else if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA) {
 					code_keep_guest = true;
 					mapped = true;
 				} else {