vgsp: eng56 - rewrite the window's own slot-1 SPH bind to the authored VTG image, and harden the author window-head decode

- gsp_phys_booter.c: after authoring the VTG SPH at family+0x900, scan the
  patched 3D window for its own 0x2054 slot-1 pair and rewrite an operand
  whose low 12 bits are 0x600/0x300 (client family VA or pair-cache shadow
  line) to <page>|0x900. The window's own later pair otherwise overwrites
  the synthesized prelude j1 and the DS validates the client's SASS page
  (c743: DS-IMAGE st1=0b0b7208). New SPH-SLOT1-REWRITE / SPH-SLOT1 receipts.
- gsp_phys_booter.c: reject an inline VTG candidate whose IMAP word (w6,
  offset 24) is zero; fall back to the builtin image. A 640 B w6=0 record
  authored a zero-map SPH and the DS went to the FECS/PRI sentinel
  (c749 fam 0x4350000: ds 80000000->badf1201).
- gsp_sema_service.c: the DDX ring-head decode scans past zero-payload
  and unreadable entries within the 8-entry bound instead of stopping
  (c746: the ring head is a zero dummy and the C36F release sits later);
  it still never fabricates a payload.

Validation: c749 is the first boot with DS-IMAGE st0=st1=020a0481
(code ff007321, the authored VSA/VSB image), a clean slot-1 rewrite
(old=0x01e00300 new=0x01e00900), beef MME-PACK-REARM, and the DS latch
reduced from 86040626 to 84000420; stores are still 0 and the DRAW_IAB
macro is the next gap.

2 files changed, 71 insertions, 4 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index babd3c0..0cd45f5 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -31167,6 +31167,17 @@ gr_page_unmapped:
 					if ((w0 & 0x1fu) != 1u ||
 					    ((w0 >> 10) & 0xfu) != 1u)
 						continue;
+					/*
+					 * eng56: a 640 B candidate with IMAP w6=0 is
+					 * not the live VTG SPH (c749 fam 0x4350000:
+					 * it authored a zero-map image and the DS
+					 * went badf1201). Require a nonzero w6; fall
+					 * back to the builtin image when no record
+					 * qualifies.
+					 */
+					if (inl->len >= 28u &&
+					    get_unaligned_le32(inl->data + 24u) == 0u)
+						continue;
 					vinl = inl;
 					vj = vj2;
 				}
@@ -31290,6 +31301,59 @@ gr_page_unmapped:
 				  !READ_ONCE(gsp_phys_rmcall_gr_sph_vtg600)) ?
 				 (fam + 0x900u) : (fam + 0x600u);
 
+			/*
+			 * eng56: the window's own slot-1 pair (0x2054)
+			 * executes AFTER the synthesized prelude and
+			 * overwrites j1 with the client's +0x600/+0x300
+			 * SASS pointer (c743: DS-IMAGE st1=0b0b7208
+			 * while st0=the authored 020a0481). Rewrite that
+			 * operand in the patched window to the authored
+			 * +0x900 image, in the same form the window's
+			 * relocation chose (client family VA or its
+			 * pair-cache shadow line), so the window's own
+			 * bind and the prelude agree.
+			 */
+			if (vtg_auth && win3d && win3d_n) {
+				u32 wi, wi2, nrew = 0;
+
+				for (wi = 0; wi + 2u < win3d_n; wi++) {
+					u32 lo, page, nlo = 0;
+
+					if (win3d[wi] != 0x20020815u)
+						continue;
+					lo = win3d[wi + 2u];
+					if ((lo & 0xfffu) != 0x600u &&
+					    (lo & 0xfffu) != 0x300u)
+						continue;
+					page = lo & ~0xfffu;
+					if (fam &&
+					    page == (u32)(fam & ~0xfffu))
+						nlo = (u32)fam | 0x900u;
+					else
+						for (wi2 = 0;
+						     wi2 < gsp_phys_gr_pair_n;
+						     wi2++)
+							if ((u32)
+							    gsp_phys_gr_pair_va[wi2]
+							    == page) {
+								nlo = page |
+								      0x900u;
+								break;
+							}
+					if (!nlo)
+						continue;
+					((u32 *)win3d)[wi + 2u] = nlo;
+					nrew++;
+					pr_info_ratelimited("vgsp-gr: SPH-SLOT1-REWRITE old=0x%08x new=0x%08x h=0x%x push=0x%llx\n",
+						lo, nlo, req->h_object,
+						req->push_va);
+				}
+				if (nrew)
+					pr_info_ratelimited("vgsp-gr: SPH-SLOT1 done rewrites=%u fam=0x%llx h=0x%x push=0x%llx\n",
+						nrew, (unsigned long long)fam,
+						req->h_object, req->push_va);
+			}
+
 			bool bind_binding =
 				READ_ONCE(gsp_phys_rmcall_gr_sph_bind_binding);
 			bool pair_safe =
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 4655e77..6cb7034 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -27987,7 +27987,7 @@ static void svc_ddx_head_decode(struct nv_vgpu_device *vgpu,
 				ring_ap, ent, sizeof(ent))) {
 			reason = "unreadable";
 			head_unreadable = true;
-			break;
+			continue;
 		}
 		memcpy(&e0, ent, 4);
 		memcpy(&e1, ent + 4, 4);
@@ -28017,17 +28017,20 @@ static void svc_ddx_head_decode(struct nv_vgpu_device *vgpu,
 		if (svc_read_va_push(vgpu, ch, va, buf, len)) {
 			reason = "unreadable";
 			head_unreadable = true;
-			break;
+			continue;
 		}
 		if (!memchr_inv(buf, 0, len)) {
 			/*
 			 * The push read succeeded but answered an all-zero
 			 * page (a stale/zero translation): there is no
-			 * release to decode and no value to apply.
+			 * release to decode and no value to apply. c746:
+			 * the ring head is a zero dummy entry and the real
+			 * C36F release sits at a later slot, so scan past
+			 * it (bounded) instead of stopping the pass.
 			 */
 			reason = "zero-payload";
 			head_unreadable = true;
-			break;
+			continue;
 		}
 		mark = svc_rel_count;
 		/*