vgsp: relocate a CODESRC-SUSPECT GR program page to the engine-mapped window-0 shadow instead of leaving the QMD pointer at an unmapped guest VA (eng33)

The client's MME-expanded draw builds a QMD whose program page (qmd[48])
classifies CODESRC-SUSPECT (empty-page): the engine's GPCCLIENT_GCC I-fetch
then read the guest VA above every engine-PDB window, raised FAULT_PDE and
the FIFO parked for the rest of the boot (the TSG re-arm cannot revive a
parked PBDMA). c361-c368 receipts.

With gsp_phys_gr_codesrc_reloc=1 (default) the builder copies the page into
the window-0 shadow (empty sources only; PTE/module pages are never
republished), overlays the client's own known-good SPH+SASS stage image at
the original QMD program offset (gsp_sema_gl_prog_img, stage-typed from the
section bind; truncated at a 128-bit boundary with a terminal EXIT when the
image is larger than the remaining page), and the existing QMD w48/49 +
prefetch rewrite publishes the shadow VA. If the shadow mapping cannot be
installed the page stays unmapped and the existing refusal drops the launch,
so an unmapped QMD program pointer is never published. 0 restores the old
guest-VA behaviour for a one-flip A/B.

Validation (c373, params-r1624): 31 CODESRC-RELOC events (img_len=384,
off=0xb00, 0 truncations), 51 GR LAUNCH submitted, 11 MME-FAST end
events=3, 79 FETCH-WIT with GP_GET advancing engine-fetched, 0 FAULT_PDE,
0 vgsp-fault, 0 WEDGE-REARM/PARK-RECOVER; work_gr backend_ok=35 fault=0.
The draws still write nothing (NOWRITE, no clear run in the emitted
window), so px0 stays 0 - the store wall is unchanged and is the next
target.

5 files changed, 182 insertions, 11 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index 8756d83..c8e224b 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -25725,14 +25725,41 @@ gsp_phys_gr_m91_fault_cap(const struct gsp_phys_gr_submit *req,
  * (gsp_sema_gr_reloc_resolve: pins/map/walks before the ledger, the
  * guest-walk-first counterpart of the fill's ledger-first order). Only a
  * different, plausibly-code page is adopted (mode ALT; the engine-visible
- * copy is taken from it); anything else leaves the QMD pair at the guest
- * VA (mode GUEST_VA, the ptr_guest behaviour for this pair). One
- * rate-limited CODESRC-SUSPECT line per detection.
+ * copy is taken from it).
+ *
+ * r1625 (the c361-c368 wedge): mode GUEST_VA used to leave the QMD pair at
+ * the guest VA, which is above every engine-PDB window - the GPCCLIENT_GCC
+ * I-fetch then raised FAULT_PDE and parked the FIFO permanently (the
+ * WEDGE-REARM TSG probe cannot revive a dead PBDMA). With
+ * gsp_phys_gr_codesrc_reloc (default 1) the refused page is instead
+ * relocated to the engine-mapped window-0 shadow and the QMD pair is
+ * rewritten there. The source page is copied when it is only empty (so any
+ * content elsewhere in the page is preserved and inline records still land
+ * on top); PTE/module pages are never republished. The program entry
+ * (original QMD offset preserved) is then overwritten with the client's own
+ * known-good SPH+SASS image for the bound stage (gsp_sema_gl_prog_img) -
+ * a raw EXIT stub would fail the engine's SPH validation - truncated at a
+ * 128-bit instruction boundary with a terminal EXIT appended. If the shadow
+ * mapping cannot be installed the launch is refused (the existing unmapped-
+ * page path), so a QMD whose program pointer is not engine-mapped is never
+ * published. One rate-limited CODESRC-SUSPECT line per detection, plus a
+ * CODESRC-RELOC line for each relocation.
  */
 #define GSP_PHYS_GR_CODESRC_OK		0
 #define GSP_PHYS_GR_CODESRC_ALT		1
 #define GSP_PHYS_GR_CODESRC_GUEST_VA	2
 
+/*
+ * r1625: terminal 16 B sm_75 SASS (MOV R1, c[0x0][0x0]; EXIT), taken from
+ * the first two instructions of a compiled sm_75 kernel's .text. Appended
+ * after a truncated relocation image so control can never run into the
+ * zero-filled tail of the shadow page.
+ */
+static const u8 gsp_phys_gr_codesrc_exit[16] = {
+	0x02, 0x7a, 0x01, 0x00, 0x00, 0x0a, 0x00, 0x00,
+	0x00, 0x0f, 0x00, 0x00, 0x00, 0xc4, 0x0f, 0x00,
+};
+
 static bool
 gsp_phys_gr_codesrc_chunk_has(const struct gsp_phys_boot_chunk *c, u64 hpa)
 {
@@ -25816,7 +25843,8 @@ static bool gsp_phys_gr_codesrc_empty(const u8 *p, u32 off)
 
 static noinline int
 gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req,
-		     const struct gsp_phys_gr_page *pg, u64 *alt_hpa)
+		     const struct gsp_phys_gr_page *pg, u64 *alt_hpa,
+		     const char **why_out, bool *unsafe_out)
 {
 	u64 page = pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
 	u32 off = (u32)(req->qmd[48] & (GSP_PHYS_PAGE_SIZE - 1));
@@ -25826,6 +25854,10 @@ gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req,
 	int mode = GSP_PHYS_GR_CODESRC_GUEST_VA;
 
 	*alt_hpa = 0;
+	if (why_out)
+		*why_out = NULL;
+	if (unsafe_out)
+		*unsafe_out = false;
 	if (off > GSP_PHYS_PAGE_SIZE - 16u)
 		off = GSP_PHYS_PAGE_SIZE - 16u;	/* keep 16 B in-page */
 	if (gsp_phys_gr_codesrc_empty(p, off))
@@ -25836,6 +25868,14 @@ gsp_phys_gr_code_src(const struct gsp_phys_gr_submit *req,
 		why = "module-pt-page";
 	if (!why)
 		return GSP_PHYS_GR_CODESRC_OK;
+	if (why_out)
+		*why_out = why;
+	/* The page content must not be republished as code: PTE/filler
+	 * bytes would execute as SASS. Empty is safe to copy (all zero)
+	 * but is republished with a known-good image at the entry. */
+	if (unsafe_out && why &&
+	    (!strcmp(why, "pte-page") || !strcmp(why, "module-pt-page")))
+		*unsafe_out = true;
 	/*
 	 * Alternative resolution order; only a different page that itself
 	 * passes the content checks is adopted.
@@ -29208,16 +29248,63 @@ int gsp_phys_backend_guest_gr_submit(const struct gsp_phys_gr_submit *req,
 			 * CODESRC: the program page (qmd_word 48) is the page
 			 * the engine's I-fetch reads; screen its source
 			 * before publishing it. A refused source is either
-			 * replaced by the alternative resolver's page or the
-			 * pair is left at the guest VA (no copy, no rewrite,
-			 * no mapping - the page is fully withdrawn from the
-			 * submission).
+			 * replaced by the alternative resolver's page or
+			 * (r1625) relocated to this shadow with the client's
+			 * known-good stage image at the entry, so the QMD
+			 * pair is rewritten to an engine-mapped VA. Leaving
+			 * it at the guest VA makes the I-fetch fault (PDE)
+			 * and parks the FIFO; that legacy withdrawal is now
+			 * gated behind gsp_phys_gr_codesrc_reloc=0.
 			 */
 			if (pg->hpa && pg->qmd_word == 48u) {
 				u64 alt_hpa = 0;
-				int cs = gsp_phys_gr_code_src(req, pg, &alt_hpa);
-
-				if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA) {
+				const char *why = NULL;
+				bool unsafe = false;
+				int cs = gsp_phys_gr_code_src(req, pg,
+							      &alt_hpa, &why,
+							      &unsafe);
+
+				if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA &&
+				    gsp_phys_gr_codesrc_reloc_value()) {
+					u32 po = qmd[48] &
+						 (GSP_PHYS_PAGE_SIZE - 1);
+					const u8 *img = NULL;
+					u32 imlen = 0, cp = 0, fit;
+
+					if (po > GSP_PHYS_PAGE_SIZE - 16u)
+						po = GSP_PHYS_PAGE_SIZE - 16u;
+					if (!unsafe)
+						memcpy(sh_page,
+						       (u8 *)__va(pg->hpa),
+						       GSP_PHYS_PAGE_SIZE);
+					img = gsp_sema_gl_prog_img(
+						pg->guest_va, &imlen);
+					fit = GSP_PHYS_PAGE_SIZE - po;
+					if (imlen > fit)
+						fit = fit >
+							  sizeof(gsp_phys_gr_codesrc_exit) ?
+							  fit -
+							  sizeof(gsp_phys_gr_codesrc_exit) :
+							  0u;
+					if (img && imlen) {
+						cp = imlen < fit ? imlen : fit;
+						memcpy(sh_page + po, img, cp);
+					}
+					if (cp < imlen || !img || !imlen)
+						memcpy(sh_page + po + cp,
+						       gsp_phys_gr_codesrc_exit,
+						       sizeof(gsp_phys_gr_codesrc_exit));
+					copied++;
+					pr_info_ratelimited("vgsp-gr: CODESRC-RELOC va=0x%llx pa=0x%llx why=%s src=%s img_len=%u cp=%u off=0x%x trunc=%u h=0x%x push=0x%llx (engine copy + QMD pair rewrite)\n",
+						pg->guest_va & ~(u64)(GSP_PHYS_PAGE_SIZE - 1),
+						pg->hpa, why ? why : "none",
+						unsafe ? "stub-only" :
+						img ? "image" : "exit",
+						imlen, cp, po,
+						cp < imlen ? 1u : 0u,
+						req->h_object,
+						req->push_va);
+				} else if (cs == GSP_PHYS_GR_CODESRC_GUEST_VA) {
 					code_keep_guest = true;
 					mapped = true;
 				} else {
diff --git a/kernel/gsp_phys_gpu.c b/kernel/gsp_phys_gpu.c
index f1d72f7..33fae68 100644
--- a/kernel/gsp_phys_gpu.c
+++ b/kernel/gsp_phys_gpu.c
@@ -1333,6 +1333,32 @@ int gsp_phys_rmcall_gr_ptr_guest_value(void)
 EXPORT_SYMBOL_GPL(gsp_phys_rmcall_gr_ptr_guest_value);
 
 /*
+ * r1625: relocate a CODESRC-SUSPECT program page (gsp_phys_gr_codesrc_reloc;
+ * default 1). The old conservative refusal left the QMD program pointer at
+ * the guest VA when the program page read empty/PTE-shaped. That VA is above
+ * every engine-PDB window, so the GPCCLIENT_GCC I-fetch raised a FAULT_PDE
+ * and the FIFO parked (the c361-c368 wedge). With this set, the builder
+ * instead copies the page into the window-0 shadow and rewrites
+ * qmd[48]/[49] to the shadow VA; an empty/unsafe source is replaced by the
+ * client's own known-good SPH+SASS image at the program offset (a raw
+ * EXIT stub would be rejected by the SPH validation). If the shadow mapping
+ * cannot be installed the launch is refused (fail-closed), so a QMD whose
+ * program pointer is not engine-mapped is never published. 0 restores the
+ * old guest-VA behaviour for a one-flip A/B.
+ */
+static int gsp_phys_gr_codesrc_reloc_param = 1;
+module_param_named(gsp_phys_gr_codesrc_reloc,
+		   gsp_phys_gr_codesrc_reloc_param, int, 0644);
+MODULE_PARM_DESC(gsp_phys_gr_codesrc_reloc,
+		 "Relocate a CODESRC-SUSPECT GR program page to an engine-mapped shadow page and rewrite the QMD pointer (default 1; 0 = keep the old guest-VA behaviour)");
+
+int gsp_phys_gr_codesrc_reloc_value(void)
+{
+	return gsp_phys_gr_codesrc_reloc_param;
+}
+EXPORT_SYMBOL_GPL(gsp_phys_gr_codesrc_reloc_value);
+
+/*
  * m96: KIND (PTE bits 63:56) for the window-0 shadow pool PTEs. On Turing
  * the v2 PTE has no PITCH kind; UVM sets GENERIC_MEMORY (0x06). A kind=0
  * PTE is a candidate for the SM I-fetch MMU_NACK while the FE tolerates it.
diff --git a/kernel/gsp_phys_gpu.h b/kernel/gsp_phys_gpu.h
index d2faa1e..7b95409 100644
--- a/kernel/gsp_phys_gpu.h
+++ b/kernel/gsp_phys_gpu.h
@@ -413,6 +413,11 @@ void gsp_phys_gr_vp_state_get(int *mode, u32 *words, u32 *n);
 /* m91: keep the forwarded QMD pointer pairs at the guest VAs (default 0). */
 int gsp_phys_rmcall_gr_ptr_guest_value(void);
 
+/* r1625: relocate a CODESRC-SUSPECT program page to an engine-mapped
+ * shadow page and rewrite the QMD pointer pair (default 1; 0 = the old
+ * guest-VA behaviour). */
+int gsp_phys_gr_codesrc_reloc_value(void);
+
 /* m95: program GPC MMU_CTRL from PFB MMU_CTRL before a forwarded launch. */
 int gsp_phys_rmcall_gr_gpc_mmu_init_value(void);
 
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 5071314..e7a0172 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -10447,6 +10447,51 @@ int gsp_sema_gl_sph_blob_version(void)
 }
 
 /*
+ * r1625: the engine-visible known-good program image for a refused
+ * program page (CODESRC-SUSPECT relocation in the GR launch builder).
+ * A program page the engine is about to I-fetch must contain an SPH+SASS
+ * image, not raw filler: the client's per-window program upload never
+ * ran on this lane, so the page reads empty while the client's own
+ * SPH+SASS image is already published (`svc_gl_vs`/`svc_gl_ps`, the
+ * captured upload or the built-in known-good image). Return the image
+ * for the pipeline stage the section bound at this VA (the sticky
+ * SET_PIPELINE_SHADER TYPE: 5 = PIXEL, otherwise VTG), falling back to
+ * the PIXEL image - the observed empty-program class is the pixel stage
+ * (its 0x2154 upload was never captured). NULL when no image is loaded.
+ */
+const u8 *gsp_sema_gl_prog_img(u64 va, u32 *len_out)
+{
+	u64 page = va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u32 i, slot = 0, t = 0xff;
+
+	if (len_out)
+		*len_out = 0;
+	for (i = 0; i < svc_gl.n_progs && i < SVC_GL_PROG_MAX; i++) {
+		if ((svc_gl.progs[i] & ~(u64)(SEMA_PAGE_SIZE - 1)) != page)
+			continue;
+		slot = svc_gl.prog_slot[i] < 32 ? svc_gl.prog_slot[i] : 0;
+		t = svc_gl_shader_type[slot];
+		break;
+	}
+	if (t != 5 && t != 0xff && svc_gl_vs_len) {
+		if (len_out)
+			*len_out = svc_gl_vs_len;
+		return svc_gl_vs;
+	}
+	if (svc_gl_ps_len) {
+		if (len_out)
+			*len_out = svc_gl_ps_len;
+		return svc_gl_ps;
+	}
+	if (svc_gl_vs_len) {
+		if (len_out)
+			*len_out = svc_gl_vs_len;
+		return svc_gl_vs;
+	}
+	return NULL;
+}
+
+/*
  * SESSION 153 (GL client draw sections): the GL client's own per-frame
  * pushes bind TURING_A (0xC597) exactly like the X server's one-shot
  * section, but carry no program/param/release operands at all (measured:
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 6f07209..b40dad2 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -676,6 +676,14 @@ bool gsp_sema_mme_scratch25_get(u32 *val);
 int gsp_sema_gl_sph_blob_version(void);
 
 /*
+ * r1625: the engine-visible known-good SPH+SASS image for a refused
+ * program page (GR CODESRC-SUSPECT relocation). Stage-typed from the
+ * section's own pipeline bind, falling back to the PIXEL image; NULL
+ * when no image is loaded. *len_out receives the byte count.
+ */
+const u8 *gsp_sema_gl_prog_img(u64 va, u32 *len_out);
+
+/*
  * SESSION 153-CONTINUED-68: furthest IR instruction covered by the CAPTURED
  * 0x114 uploads (pc + ceil(code words / 3)). The GR launch builder places
  * its end-noop macro stub at this address (aligned past the replayed