vgsp: CONT-91 - svc_gr_inline_retarget re-roots captured program-family inline records (page not in launch prog_va family nor prog_bind_va[]) onto launch family offset-preserved, page-crossing tails follow head, wired into svc_gr_req_fill; svc_gr_page_inline_covers widened; sema_ds_mask_clear=1 clears DS_HWW_REPORT_MASK/_2 (0x405844/c) ahead of every 3D window; gsp-phys-ds-spy debugfs; GR-STATE retire logs sph= (STAGE0..5 START/UPPER) + rr= (PROGRAM_REGION) + MME shadow readback; sema_gl_sph_region + sema_sph_region_base emit Volta+ program region (SET_MME_SHADOW_SCRATCH 0x78/0x79 = 0x35e0/0x35e4) - r1298 real V4 image at exact bind VA (INLINE-APPLY len=1536 first=020a0481 + INLINE-RB rb=020a0481); same launch DS flags sph=0,00620900,00620200,0,0,00620000 rr=0,0 ds=a4000e20 w/ correct bytes + region A/B 0/0 unchanged; DS rejects all bound stages -> unresolved variable = address space DS validates in (our shadow VAs 0x6xxxxx vs client guest VA region); next A/B: knob leaving program operands at ORIGINAL guest VAs when page identity-mappable (skip shadow rewrite for that pair only) - SESSION 153 r1298-r1306

2 files changed, 384 insertions, 1 deletions

diff --git a/kernel/gsp_phys_booter.c b/kernel/gsp_phys_booter.c
index fe1f7b0..454c3e7 100644
--- a/kernel/gsp_phys_booter.c
+++ b/kernel/gsp_phys_booter.c
@@ -17930,6 +17930,47 @@ MODULE_PARM_DESC(sema_gl_shader_exc,
 	"Emit SET_SHADER_EXCEPTIONS (0x1528) <- 0 after SET_OBJECT like _nv3dInitChannelTuring (default 1)");
 
 /*
+ * SESSION 153-CONTINUED-91: clear DS_HWW_REPORT_MASK/_2 (0x405844/0x40584c)
+ * ahead of every replayed 3D window (host write + in-stream MME PRI R/W
+ * macro 5) so a DS SPH validation miss latches without poisoning the PRI
+ * block and the GSP-RM exception handler killing the channel. See the
+ * emission-site comment. Default 1; A/B with 0 restores the old behavior
+ * (every SPH miss = priv-error poison + channel wedge).
+ */
+static bool gsp_phys_rmcall_gr_ds_mask_clear = true;
+module_param_named(sema_ds_mask_clear, gsp_phys_rmcall_gr_ds_mask_clear,
+		   bool, 0644);
+MODULE_PARM_DESC(sema_ds_mask_clear,
+	"Clear DS_HWW_REPORT_MASK/_2 ahead of replayed 3D windows (default 1; 0 = leave the all-report mask)");
+
+/*
+ * SESSION 153-CONTINUED-91: emit the Turing program-region pair (the MME
+ * shadow scratch 0x78/0x79 = NVC397_SET_MME_PROGRAM_REGION) with the
+ * launch's own program family page as the shader-heap base, so the DS's
+ * SPH validation has a region to accept the fetched SPHs against. See the
+ * emission-site comment. Default 1; A/B with 0.
+ */
+static bool gsp_phys_rmcall_gr_sph_region = true;
+module_param_named(sema_gl_sph_region, gsp_phys_rmcall_gr_sph_region,
+		   bool, 0644);
+MODULE_PARM_DESC(sema_gl_sph_region,
+	"Emit the Volta+ program-region pair (SET_MME_SHADOW_SCRATCH 0x78/0x79 <- program family page) ahead of replayed 3D windows (default 1)");
+
+/*
+ * Runtime-selectable program-region base. 0 = auto (the launch's own
+ * program family page, the guest-VA heap base the real driver would use);
+ * a nonzero value forces that base - the engine-side shadow-VA space needs
+ * its own experiment (the SPH_INFO snapshot shows the DS fetched through
+ * shadow VAs 0x620900/0x620200/0x620000, so a guest-VA base may be the
+ * mismatch). Writable at runtime for a live A/B.
+ */
+static u64 gsp_phys_rmcall_gr_sph_region_base;
+module_param_named(sema_sph_region_base, gsp_phys_rmcall_gr_sph_region_base,
+		   ullong, 0644);
+MODULE_PARM_DESC(sema_sph_region_base,
+	"Program-region base override (0 = launch program family page; nonzero = explicit engine-side base)");
+
+/*
  * SESSION 153-CONTINUED-87: the missing TESS_CONTROL/PIXEL program binds.
  * The client's real channel state binds every pipeline stage once per
  * context; the captured init-push image (guest RAM scrape) carries
@@ -18616,6 +18657,53 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 			}
 		}
 		/*
+		 * SESSION 153-CONTINUED-91: DS HWW report suppression.
+		 *
+		 * The DS validates the SPH of every bound pipeline program at
+		 * draw time and reports failures through DS_HWW_REPORT_MASK
+		 * (0x405844, observed live = 0x00ffffff = every SPH<n>_ERR
+		 * reported; mask_2 = 0x77). A reported failure poisons the
+		 * whole PRI block (reads return 0xbadf1201 - the priv-error
+		 * marker already documented for the MME latch) and the GSP-RM
+		 * exception handler kills the channel: every draw attempt
+		 * wedges at the first SPH error storm even when the SPH bytes
+		 * at the bound VAs are the client's own real V4 images (r1298:
+		 * INLINE-RB rb=020a0481 at the bind VA, DS still 5/9/10/11).
+		 * The reference driver's own diagnostic path (Aftermath)
+		 * explicitly ENABLES these reports to capture them, i.e. the
+		 * reports are a diagnostic tap, not the shader-execute gate.
+		 * Clear the mask so a SPH miss latches (readable between
+		 * faults) without poisoning the channel. Host write only:
+		 * an in-stream CALL_MME_MACRO(5) (the NVN PRI R/W macro)
+		 * faulted with MME ILLEGAL_OPCODE (pc=513 mac=0xf1) on
+		 * r1300 - the Turing MME's loaded macro program does not
+		 * carry that GF100-era slot - so the register is written
+		 * directly (it persists: the next beef launch reads 0/0).
+		 * A/B with sema_ds_mask_clear=0.
+		 */
+		if (READ_ONCE(gsp_phys_rmcall_gr_ds_mask_clear)) {
+			static u32 ds_mask_log;
+			void __iomem *ds_bar0 = gsp_phys_gpu_bar0();
+			u32 m0 = 0, m2 = 0;
+
+			if (ds_bar0) {
+				m0 = ioread32(ds_bar0 + 0x00405844u);
+				m2 = ioread32(ds_bar0 + 0x0040584cu);
+				if (m0 || m2) {
+					iowrite32(0u, ds_bar0 + 0x00405844u);
+					iowrite32(0u, ds_bar0 + 0x0040584cu);
+				}
+			}
+			if (ds_mask_log < 24u) {
+				ds_mask_log++;
+				pr_info("vgsp-gr: DS-MASK clear mask=%08x/%08x rb=%08x/%08x h=0x%x push=0x%llx\n",
+					m0, m2,
+					ds_bar0 ? ioread32(ds_bar0 + 0x00405844u) : 0u,
+					ds_bar0 ? ioread32(ds_bar0 + 0x0040584cu) : 0u,
+					req->h_object, req->push_va);
+			}
+		}
+		/*
 		 * SESSION 153-CONTINUED-86: the Turing channel-init SPH version
 		 * pair (see the sema_gl_sph_version knob). Emitted on EVERY
 		 * replayed 3D window (3 words, idempotent): the DS validates
@@ -18649,6 +18737,47 @@ static u32 gsp_phys_rmcall_build_gr_launch(u32 *pb,
 			*p++ = val;
 		}
 		/*
+		 * SESSION 153-CONTINUED-91: the Turing program region.
+		 *
+		 * The DS's SPH validation checks each fetched SPH against the
+		 * channel's PROGRAM REGION. On pre-Volta that was
+		 * SET_PROGRAM_REGION_A/B (0x1608/0x160c); on Volta+ those
+		 * methods are obsolete and the driver writes the same 64-bit
+		 * shader-code-heap base through the MME shadow scratch pair
+		 * (nvTuring: NVC397_SET_MME_PROGRAM_REGION = SET_MME_SHADOW_
+		 * SCRATCH(MME_SCRATCH_PROGRAM_REGION_A=0x78/B=0x79),
+		 * gv100ResendProgramAddresses/`gf100Init`; method 0x3400 +
+		 * i*4 -> 0x35e0/0x35e4). Observed live at the faulting draw:
+		 * DS_HWW_ESR_SPH_INFO_PROGRAM_REGION_A/B = 0/0 and DS latches
+		 * SPH errors for every bound stage whose bytes are provably
+		 * correct (r1298 INLINE-RB rb=020a0481 at the bind VA;
+		 * r1303 GR-STATE sph stages 1/2/5 = 0x620900/0x620200/0x620000
+		 * with ds=a4000e20 and rr=0/0) - a region mismatch rejects all
+		 * of them regardless of content. Emit the pair (A = upper
+		 * 7:0, B = lower 31:0) with the launch's own program family
+		 * page as the heap base. A/B with sema_gl_sph_region=0.
+		 */
+		if (READ_ONCE(gsp_phys_rmcall_gr_sph_region) && req->prog_va) {
+			static u32 sph_reg_log;
+			u64 base = READ_ONCE(gsp_phys_rmcall_gr_sph_region_base);
+
+			if (!base)
+				base = req->prog_va &
+				       ~(u64)(GSP_PHYS_PAGE_SIZE - 1);
+
+			*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+			       (1u << 16) | (0x35e0u >> 2);
+			*p++ = (u32)(base >> 32) & 0xffu;
+			*p++ = ((u32)GSP_PHYS_RMCALL_METHOD_IMM << 29) |
+			       (1u << 16) | (0x35e4u >> 2);
+			*p++ = (u32)base;
+			if (sph_reg_log < 16u) {
+				sph_reg_log++;
+				pr_info("vgsp-gr: SPH-REGION emitted base=0x%llx (scratch 0x78/0x79) h=0x%x push=0x%llx\n",
+					base, req->h_object, req->push_va);
+			}
+		}
+		/*
 		 * SESSION 153-CONTINUED-84: SET_MME_VERSION=1 (0x234).
 		 *
 		 * The Turing channel-init defaults push
@@ -21096,6 +21225,17 @@ static noinline void gsp_phys_gr_state_rb_log(u32 h_object)
 	void __iomem *bar0 = gsp_phys_gpu_bar0();
 	bool beef = (h_object & 0xfffff000u) == 0xbeef0000u;
 	u32 fed0, fed1, fed2, fed3, esr, info, ir2, trap, data, ds, gf;
+	/*
+	 * SESSION 153-CONTINUED-91: with the DS HWW report mask cleared
+	 * (sema_ds_mask_clear) a SPH validation miss latches in DS_HWW_ESR
+	 * without the GSP-RM fault handler poisoning the PRI block, so the
+	 * per-stage SPH_INFO snapshot is finally readable: STAGE0..5 START
+	 * (0x405c00+si*4) / UPPER (0x405c24+si*4) name the exact SPH VA each
+	 * pipeline stage fetched, and PROGRAM_REGION_A/B (0x405c1c/c20) the
+	 * region the DS validated against (the r1277+ wall's missing datum).
+	 */
+	u32 sph[6], sphu[6], rr_a, rr_b;
+	u32 si;
 
 	if (!bar0)
 		return;
@@ -21105,6 +21245,12 @@ static noinline void gsp_phys_gr_state_rb_log(u32 h_object)
 		gs_beef_n++;
 	else
 		gs_other_n++;
+	for (si = 0; si < 6u; si++) {
+		sph[si] = ioread32(bar0 + 0x00405c00u + si * 4u);
+		sphu[si] = ioread32(bar0 + 0x00405c24u + si * 4u);
+	}
+	rr_a = ioread32(bar0 + 0x00405c1cu);
+	rr_b = ioread32(bar0 + 0x00405c20u);
 	fed0 = ioread32(bar0 + 0x0040a630u);
 	fed1 = ioread32(bar0 + 0x0040a634u);
 	fed2 = ioread32(bar0 + 0x0040a638u);
@@ -21121,6 +21267,25 @@ static noinline void gsp_phys_gr_state_rb_log(u32 h_object)
 		fed0 & 0xfffu, (fed0 >> 12) & 0x3ffu, (fed0 >> 31) & 1u,
 		esr, info, ir2, trap, trap & 0x3ffcu, (trap >> 20) & 1u,
 		data, ds, gf);
+	pr_info("vgsp-gr: GR-STATE sph h=0x%x ds2=%08x sph=%08x,%08x,%08x,%08x,%08x,%08x sphU=%02x,%02x,%02x,%02x,%02x,%02x rr=%08x,%08x\n",
+		h_object, ioread32(bar0 + 0x00405848u),
+		sph[0], sph[1], sph[2], sph[3], sph[4], sph[5],
+		sphu[0] & 0xffu, sphu[1] & 0xffu, sphu[2] & 0xffu,
+		sphu[3] & 0xffu, sphu[4] & 0xffu, sphu[5] & 0xffu,
+		rr_a, rr_b);
+	/*
+	 * SESSION 153-CONTINUED-91: did the emitted program-region pair
+	 * (SET_MME_SHADOW_SCRATCH 0x78/0x79 = method 0x35e0/0x35e4 through
+	 * class 0xc597) actually land in the MME shadow? The shadow read
+	 * port is the same one the store-side probe uses; a zero readback
+	 * means the write never took effect and every region experiment so
+	 * far was a no-op.
+	 */
+	if (beef)
+		pr_info("vgsp-gr: GR-STATE region-shadow h=0x%x scratch[0x78]=%08x scratch[0x79]=%08x\n",
+			h_object,
+			gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x35e0u),
+			gsp_phys_mme_shadow_read(bar0, 0xc597u, 0x35e4u));
 }
 
 /*
@@ -29814,6 +29979,109 @@ static const struct file_operations gsp_phys_reg_probe_fops = {
 	.release = single_release,
 };
 
+/*
+ * SESSION 153-CONTINUED-91: DS SPH fault spy.
+ *
+ * The DS_HWW_ESR SPH<n>_ERR bits (bit n = "the SPH with index n failed
+ * validation", hwref NV_PGRAPH_PRI_DS_HWW_ESR_SPH0_ERR..SPH23_ERR) and the
+ * per-stage SPH_INFO snapshot (STAGE0..STAGE5 START/UPPER at 0x405c00+si*4 /
+ * 0x405c24+si*4, PROGRAM_REGION_A/B at 0x405c1c/c20) are latched by the DS
+ * at fault time and consumed+cleared by the real GSP-RM exception handler
+ * before our relayed fault packet arrives (the fault-time register read in
+ * gsp_phys_fault_regs_log sees ESR=0x80000000 and all-zero stage starts
+ * one millisecond later). The RM's own diagnostic prints those STAGE
+ * registers (grErrorPrintDsSphInfo_GM200), so they name the exact SPH VAs
+ * the DS rejected - the missing datum behind the r1277+ Shader Program
+ * Header wall (CONT-90/91 could only move the fault SET by content, never
+ * identify the SPH entries).
+ *
+ * Reading this file runs a bounded tight poll of DS_HWW_ESR and snapshots
+ * the ESR pair, the six stage starts and the region pair on every sample
+ * that has any SPH bit set. Development-only, read-only, no state touched;
+ * run `cat /sys/kernel/debug/vgsp/gsp-phys-ds-spy` while the client draws.
+ * Duration via sema_ds_spy_ms (default 20 s).
+ */
+static u32 gsp_phys_ds_spy_ms = 20000;
+module_param_named(sema_ds_spy_ms, gsp_phys_ds_spy_ms, uint, 0644);
+MODULE_PARM_DESC(sema_ds_spy_ms,
+	"gsp-phys-ds-spy sampling duration in milliseconds (default 20000)");
+
+#define GSP_PHYS_DS_ESR			0x00405840u
+#define GSP_PHYS_DS_ESR_2		0x00405848u
+#define GSP_PHYS_DS_REPORT_MASK		0x00405844u
+#define GSP_PHYS_DS_REPORT_MASK_2	0x0040584cu
+#define GSP_PHYS_DS_SPH_STAGE_LO	0x00405c00u
+#define GSP_PHYS_DS_SPH_STAGE_UP	0x00405c24u
+#define GSP_PHYS_DS_SPH_REGION_A	0x00405c1cu
+#define GSP_PHYS_DS_SPH_REGION_B	0x00405c20u
+
+static int gsp_phys_ds_spy_show(struct seq_file *m, void *unused)
+{
+	void __iomem *bar0 = gsp_phys_gpu_bar0();
+	u64 t0;
+	u32 hits = 0, samples = 0;
+	u32 ms = READ_ONCE(gsp_phys_ds_spy_ms);
+
+	if (!bar0)
+		return -ENODEV;
+	if (!ms)
+		ms = 20000;
+	seq_printf(m, "# DS-SPY begin duration_ms=%u (poll DS_HWW_ESR 0x%08x)\n",
+		   ms, GSP_PHYS_DS_ESR);
+	seq_printf(m, "# DS-SPY masks esr=%08x esr2=%08x mask=%08x mask2=%08x\n",
+		   readl(bar0 + GSP_PHYS_DS_ESR),
+		   readl(bar0 + GSP_PHYS_DS_ESR_2),
+		   readl(bar0 + GSP_PHYS_DS_REPORT_MASK),
+		   readl(bar0 + GSP_PHYS_DS_REPORT_MASK_2));
+	t0 = ktime_get_ns();
+	while (ktime_get_ns() - t0 < (u64)ms * 1000000ull) {
+		u32 esr = readl(bar0 + GSP_PHYS_DS_ESR);
+
+		samples++;
+		if (esr & 0x7fffffffu) {
+			u32 esr2 = readl(bar0 + GSP_PHYS_DS_ESR_2);
+			u32 esr_a, s[6], u[6], ra, rb;
+			u32 si;
+
+			for (si = 0; si < 6u; si++) {
+				s[si] = readl(bar0 +
+					      GSP_PHYS_DS_SPH_STAGE_LO +
+					      si * 4u);
+				u[si] = readl(bar0 +
+					      GSP_PHYS_DS_SPH_STAGE_UP +
+					      si * 4u);
+			}
+			ra = readl(bar0 + GSP_PHYS_DS_SPH_REGION_A);
+			rb = readl(bar0 + GSP_PHYS_DS_SPH_REGION_B);
+			esr_a = readl(bar0 + GSP_PHYS_DS_ESR);
+			hits++;
+			seq_printf(m,
+				   "DS-SPY hit %u t=%lldus esr=%08x esr2=%08x esr_after=%08x region=%08x/%08x stage=%08x,%08x,%08x,%08x,%08x,%08x upper=%02x,%02x,%02x,%02x,%02x,%02x\n",
+				   hits, div_u64(ktime_get_ns() - t0, 1000),
+				   esr, esr2, esr_a, ra, rb,
+				   s[0], s[1], s[2], s[3], s[4], s[5],
+				   u[0] & 0xffu, u[1] & 0xffu, u[2] & 0xffu,
+				   u[3] & 0xffu, u[4] & 0xffu, u[5] & 0xffu);
+			udelay(20);
+		}
+	}
+	seq_printf(m, "# DS-SPY end samples=%u hits=%u\n", samples, hits);
+	return 0;
+}
+
+static int gsp_phys_ds_spy_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, gsp_phys_ds_spy_show, NULL);
+}
+
+static const struct file_operations gsp_phys_ds_spy_fops = {
+	.owner = THIS_MODULE,
+	.open = gsp_phys_ds_spy_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
 int gsp_phys_boot_debugfs_init(void)
 {
 	struct dentry *file;
@@ -29976,7 +30244,16 @@ int gsp_phys_boot_debugfs_init(void)
 			err);
 		return (int)err;
 	}
-	pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n");
+	/*
+	 * SESSION 153-CONTINUED-91: the DS SPH fault spy (see the show fn).
+	 * Diagnostic only; failure to create it must not fail the init.
+	 */
+	file = debugfs_create_file("gsp-phys-ds-spy", 0444,
+				   gsp_phys_debugfs_root, NULL,
+				   &gsp_phys_ds_spy_fops);
+	if (IS_ERR_OR_NULL(file))
+		pr_warn("vgsp-boot: debugfs file gsp-phys-ds-spy creation failed (DS SPH fault spy unavailable)\n");
+	pr_info("vgsp-boot: debugfs files /sys/kernel/debug/vgsp/{gsp-phys-state read-only, gsp-phys-log-dump read-only, gsp-phys-status-drain read-only, gsp-phys-reg-probe read-only, gsp-phys-ds-spy read-only, gsp-phys-unload-rpc write-only, gsp-phys-rm-probe write-only, gsp-phys-rm-call write-only, gsp-phys-guest-gpa write-only, gsp-phys-guest-ram write-only} ready\n");
 	return 0;
 }
 
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 8fc663f..e96b929 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -30688,6 +30688,17 @@ static bool svc_gr_page_inline_covers(u64 page)
 		     ~(u64)(SEMA_PAGE_SIZE - 1)) == page)
 			return true;
 	}
+	/*
+	 * SESSION 153-CONTINUED-91: the request's records may have been
+	 * re-rooted onto this launch's bind targets (svc_gr_inline_retarget),
+	 * so a page the request names can be covered by an inline write even
+	 * when the candidate's original record page differs.
+	 */
+	for (i = 0; i < svc_fwd_req.gr_n_inline; i++) {
+		if ((svc_fwd_req.gr_inline[i].dst_va &
+		     ~(u64)(SEMA_PAGE_SIZE - 1)) == page)
+			return true;
+	}
 	return false;
 }
 
@@ -32627,6 +32638,92 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu,
 			n_cand, svc_fwd_req.gr_n_data, reloc_base);
 }
 
+static u32 svc_gr_inline_retarget_log;
+
+/*
+ * SESSION 153-CONTINUED-91: delivery follows the launch.
+ *
+ * The captured program-family inline records (the real-upload payloads and
+ * the authored stage images) are keyed to the program family of the section
+ * that captured them, but that family is sticky guest state: the client
+ * cycles push slots whose content names a different program family per slot
+ * (observed live: records at 0x4350000/0x4300000 while the launched window
+ * binds 0x42d0000/0x41c0000/0x4400000). The record pages are noted into the
+ * launch page set, so the delivery lands on pages no bind names, the
+ * engine's own bind VAs keep whatever stale content the guest memory holds,
+ * and the DS latches `Shader Program Header N Error` (r1297: zero
+ * INLINE-APPLY, fetch page all zeros).
+ *
+ * Re-root every record onto the launch's own bind targets by in-page
+ * offset: the window's own binds (prog_bind_va) first, then the launch's
+ * program family at the record's own offset (which is also where the
+ * synthesized j=5/j=2 stage binds sit, family+0x0 / family+0x200). A record
+ * that continues the previous record's page (the page-crossing tail of a
+ * split real-upload) follows its head's new target so the split delivery
+ * stays contiguous. A record already on a build-bind page is left alone;
+ * the pass is idempotent and a no-op whenever the families already agree.
+ */
+static void svc_gr_inline_retarget(u64 prog_va, const u64 *binds, u32 n_binds)
+{
+	u64 lfam = prog_va & ~(u64)(SEMA_PAGE_SIZE - 1);
+	u64 prev_page = 0, prev_new = 0;
+	u32 i, moved = 0;
+
+	if (!lfam || !svc_gr_cand.n_inline)
+		return;
+	for (i = 0; i < svc_gr_cand.n_inline && i < SVC_GR_INLINE_MAX; i++) {
+		struct svc_gr_inline *inl = &svc_gr_cand.inline_recs[i];
+		u64 page = inl->dst_va & ~(u64)(SEMA_PAGE_SIZE - 1);
+		u64 off = inl->dst_va & (SEMA_PAGE_SIZE - 1);
+		u64 target = 0;
+		u32 b;
+
+		if (!page)
+			continue;
+		/* Page-crossing tail: follow the head's new location. */
+		if (prev_page && page == prev_page + SEMA_PAGE_SIZE && !off) {
+			u64 nv = (prev_new & ~(u64)(SEMA_PAGE_SIZE - 1)) +
+				 SEMA_PAGE_SIZE;
+
+			if (nv != inl->dst_va) {
+				inl->dst_va = nv;
+				moved++;
+			}
+			prev_page = page;
+			prev_new = inl->dst_va;
+			continue;
+		}
+		/* A build bind whose in-page offset matches this record. */
+		for (b = 0; b < n_binds && b < SVC_GR_CAND_PROG_BINDS; b++) {
+			if (binds[b] &&
+			    (binds[b] & (SEMA_PAGE_SIZE - 1)) == off) {
+				target = binds[b];
+				break;
+			}
+		}
+		/* Else the launch's own family at the record's offset. */
+		if (!target)
+			target = lfam + off;
+		if ((target & ~(u64)(SEMA_PAGE_SIZE - 1)) != page) {
+			inl->dst_va = (target & ~(u64)(SEMA_PAGE_SIZE - 1)) +
+				      (inl->dst_va - page);
+			moved++;
+		}
+		prev_page = page;
+		prev_new = inl->dst_va;
+	}
+	if (moved && svc_gr_inline_retarget_log < 16u) {
+		svc_gr_inline_retarget_log++;
+		pr_info("sema-gr: INLINE-RETARGET prog=0x%llx lfam=0x%llx binds=%u moved=%u/%u\n",
+			prog_va, lfam, n_binds, moved,
+			svc_gr_cand.n_inline);
+		for (i = 0; i < svc_gr_cand.n_inline && i < 8u; i++)
+			pr_info("sema-gr:   inl[%u] dst=0x%llx len=%u\n", i,
+				svc_gr_cand.inline_recs[i].dst_va,
+				svc_gr_cand.inline_recs[i].len);
+	}
+}
+
 static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 			   const struct svc_channel *ch)
 {
@@ -32736,6 +32833,15 @@ static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
 	svc_fwd_req.gr_param_size = svc_gr_cand.param_size;
 	memcpy(svc_fwd_req.gr_qmd, svc_gr_cand.qmd,
 	       sizeof(svc_fwd_req.gr_qmd));
+	/*
+	 * SESSION 153-CONTINUED-91: re-root the captured program-family inline
+	 * records onto this launch's own bind targets before they are copied
+	 * into the request (see svc_gr_inline_retarget). The candidate's
+	 * records are retargeted in place so the page-note loop below and
+	 * svc_gr_scan_inline() both follow the delivered VAs.
+	 */
+	svc_gr_inline_retarget(svc_gr_cand.prog_va, svc_gr_cand.prog_bind_va,
+			       svc_gr_cand.n_prog_binds);
 	svc_fwd_req.gr_n_inline = svc_gr_cand.n_inline;
 	if (svc_fwd_req.gr_n_inline > SVC_GR_INLINE_MAX)
 		svc_fwd_req.gr_n_inline = SVC_GR_INLINE_MAX;