vgsp: direct-GL ring scan hardening + USERD orientation overrides

- per-rank GR shadow-scan cadence (the global 1.5 s window was consumed by
  the first channel in pump order, starving the direct-GL ring scan)
- low-aperture ring content gate: entry0 within +/-8 MB of the ring VA and
  a zero terminus within the first 128 slots (dense data pages and transient
  copies were being latched as the ring and fed phantom entries)
- stop skipping the guest-RAM content scan when the shadow-scan window is
  closed; only the shadow scan is rate-limited now
- strict shared-surface author: write lastKickoffRefCount only, never the raw
  payload into an unvalidated translation
- dev-only sema_userd_swap / sema_userd_override to test the direct-GL USERD
  orientation and a known control page; both default off

1 files changed, 116 insertions, 11 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b2a3803..5f8bf07 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1319,7 +1319,15 @@ static unsigned long svc_pb_gr_done;
 static unsigned long svc_gr_ring_done;	/* GR ranks whose ring is pinned */
 /* SESSION 133: ranks that already consumed their one shadow ring scan. */
 static unsigned long svc_gr_ring_scan_done;
-static unsigned long svc_gr_ring_scan_last;	/* rate limit for the scan */
+/*
+ * r1780 (CONT.430c): per-rank scan cadence. A single global timestamp let
+ * the first GR channel in pump order (descending handles; the RC watchdog
+ * 0x31415900 precedes the direct-GL 0xbeef0100) consume the window every
+ * 1.5 s, so the direct-GL call always returned at this gate and its ring
+ * content scan never ran - the ring stayed unresolved and the X DDX hit
+ * its idle timeout. Per-rank keeps the same bound per channel.
+ */
+static unsigned long svc_gr_ring_scan_last[32];	/* per-rank scan rate limit */
 /* SESSION 138: bounded witness for the GR request builder's page list and
  * the fill's candidate set (one launch's wiring printed per budget slice). */
 static u32 svc_gr_pages_log;
@@ -6318,15 +6326,23 @@ static bool svc_gl_shsurf_sync(struct nv_vgpu_device *vgpu, u32 h, u64 va,
 			   (lkr >> 32) == 0 &&
 			   (lkr & 0xffffffffULL) >= (u64)want &&
 			   (lkr & 0xffffffffULL) <= 0x1000000ULL;
-		if (have_lkr && cur != 0 && cur <= (u32)lkr)
-			want = (u32)lkr;
+		/*
+		 * r1780 (CONT.430e): strict author. Only a page that
+		 * carries the client's own lastKickoffRefCount is the
+		 * shared surface; never fall back to writing the raw
+		 * payload into an unvalidated translation (that branch
+		 * was the one that could hit a data page). The DDX
+		 * asserts refCount <= lkr, so write exactly lkr and only
+		 * forward.
+		 */
+		if (!have_lkr || cur >= (u32)lkr)
+			continue;
+		want = (u32)lkr;
 		if (gsp_sema_gl_shsurf_dbg && dbg_log < 32) {
 			dbg_log++;
 			pr_info("sema-gl: shsurf-cand h=0x%x va=0x%llx pa=0x%llx cur=%u lkr=0x%llx ok=%d want=%u\n",
 				h, va, a, cur, lkr, have_lkr ? 1 : 0, want);
 		}
-		if (cur >= want)
-			continue;
 		if (svc_write_ap(vgpu, a, svc_ap_of(aps[i]), &want,
 				 sizeof(want)))
 			continue;
@@ -26085,6 +26101,66 @@ static int svc_read_va_push(struct nv_vgpu_device *vgpu,
 }
 
 /*
+ * r1780 (CONT.430f): dev overrides for the direct-GL USERD experiment.
+ * `sema_userd_swap` reads GP_PUT at +0x88 and GP_GET at +0x8c (the
+ * orientation measured on the direct-GL client's control page), and
+ * `sema_userd_override` ("0x<handle>,0x<pa>") forces a channel's USERD
+ * candidate to a known sysmem page. Both are diagnostic-only defaults
+ * off; the normal path is unchanged.
+ */
+static bool gsp_sema_userd_swap;
+module_param_named(sema_userd_swap, gsp_sema_userd_swap, bool, 0644);
+MODULE_PARM_DESC(sema_userd_swap,
+	"Read GP_PUT at +0x88 and GP_GET at +0x8c for the USERD probe (direct-GL layout experiment); default 0");
+
+static int svc_userd_override_set(const char *val, const struct kernel_param *kp)
+{
+	u32 h = 0;
+	u64 pa = 0;
+	unsigned long flags;
+	int i, hits = 0;
+	char *buf, *comma;
+
+	buf = kstrdup(val, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+	comma = strchr(buf, ',');
+	if (!comma) {
+		kfree(buf);
+		return -EINVAL;
+	}
+	*comma = '\0';
+	h = (u32)simple_strtoul(buf, NULL, 0);
+	pa = simple_strtoull(comma + 1, NULL, 0);
+	kfree(buf);
+	if (!h || !pa)
+		return -EINVAL;
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		struct svc_channel *c = &svc.chan[i];
+
+		if (!c->valid || (u32)c->h_object != h)
+			continue;
+		c->userd_pa = pa;
+		c->userd_as = SVC_APERTURE_SYS;
+		c->userd_ok = true;
+		c->userd_reg = false;
+		hits++;
+		pr_info("sema-chan: USERD override h=0x%x pa=0x%llx (swap=%d)\n",
+			h, pa, gsp_sema_userd_swap ? 1 : 0);
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+	return hits ? 0 : -ENOENT;
+}
+
+static const struct kernel_param_ops svc_userd_override_ops = {
+	.set = svc_userd_override_set,
+};
+module_param_cb(sema_userd_override, &svc_userd_override_ops, NULL, 0200);
+MODULE_PARM_DESC(sema_userd_override,
+	"Force a channel USERD page (handle,pa), diagnostic only");
+
+/*
  * Read the GPFIFO get/put pair from a candidate USERD. Returns 0 when the
  * values are plausible for a channel with `entries` GPFIFO slots.
  */
@@ -26092,12 +26168,14 @@ static int svc_userd_probe_ex(struct nv_vgpu_device *vgpu, u64 pa, u32 as,
 			      u32 entries, u32 *put, u32 *get, bool loose_get)
 {
 	u32 p = 0, g = 0;
+	u32 put_off = gsp_sema_userd_swap ? SVC_USERD_GP_GET : SVC_USERD_GP_PUT;
+	u32 get_off = gsp_sema_userd_swap ? SVC_USERD_GP_PUT : SVC_USERD_GP_GET;
 
 	if (!pa)
 		return -EINVAL;
-	if (svc_read_ap(vgpu, pa + SVC_USERD_GP_PUT, as, &p, sizeof(p)))
+	if (svc_read_ap(vgpu, pa + put_off, as, &p, sizeof(p)))
 		return -ENOENT;
-	if (svc_read_ap(vgpu, pa + SVC_USERD_GP_GET, as, &g, sizeof(g)))
+	if (svc_read_ap(vgpu, pa + get_off, as, &g, sizeof(g)))
 		return -ENOENT;
 	*put = p;
 	*get = g;
@@ -29746,6 +29824,19 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
 				if (va0 < SVC_LOW_REL_VA_LO ||
 				    va0 >= SVC_LOW_REL_VA_LO + 0x10000000ULL)
 					continue;
+				/*
+				 * r1780 (CONT.430d): entry0 is the main
+				 * pushbuffer next to the ring. Bound it to
+				 * +/-8 MB of the registered ring VA so a
+				 * dense/transient page elsewhere in the
+				 * 64 MB low arena cannot masquerade as the
+				 * ring (measured false latches at
+				 * PA 0x108587000/0x1100b6000 while the real
+				 * ring sat at PA 0x35331b000).
+				 */
+				if (va0 + 0x800000ULL < ch->gpfifo_va ||
+				    va0 > ch->gpfifo_va + 0x800000ULL)
+					continue;
 			} else {
 				if (va0 + SVC_GR_RAM_WIN / 2 < ch->gpfifo_va ||
 				    va0 > ch->gpfifo_va + SVC_GR_RAM_WIN / 2)
@@ -29814,7 +29905,19 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
 			 * Require an all-zero entry slot within the first 128
 			 * entries of the run.
 			 */
-			if (sparse) {
+			/*
+			 * r1780 (CONT.430d): apply the terminus gate to the
+			 * low-aperture family too. The direct-GL ring page
+			 * carries a bounded run of queued entries and then
+			 * zero words (measured: entry0 va=0x4000000 len=2260
+			 * then zeros at slot 23), while a dense data page in
+			 * the same low arena decoded on every slot and was
+			 * latched first at PA 0x108587000 - the pump then
+			 * read phantom entries, the GET/PUT mismatched and
+			 * the guest corrupted itself (bad pmd/OOM) chasing
+			 * them. A real ring cannot be dense past slot 128.
+			 */
+			if (sparse || low) {
 				u32 z;
 				bool term = false;
 
@@ -31848,10 +31951,11 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 		 * pass can run before the client has written the ring or the
 		 * USERD, and a consumed once-bit would never rescan.
 		 */
-		if (time_before(jiffies, (unsigned long)svc_gr_ring_scan_last +
+		if (time_before(jiffies,
+				(unsigned long)svc_gr_ring_scan_last[ch->gr_index] +
 				msecs_to_jiffies(1500)))
-			return;
-		svc_gr_ring_scan_last = jiffies;
+			goto skip_shadow_scan;
+		svc_gr_ring_scan_last[ch->gr_index] = jiffies;
 		if (sbase && lim > 0x1000) {
 			static const u64 udeltas[] = { 0x10000, SVC_GR_USERD_OFF,
 						       0x1000 };
@@ -32087,6 +32191,7 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 		}
 	}
 
+skip_shadow_scan:
 	/*
 	 * r496b: guest-RAM content scan for the ring (see the gr_ram_cursor
 	 * field comment). The driver's kickoff writes GPFIFO entries whose