r496j VAS candidates are registered only after a content-verified ring walk (+3 fixes)

The r496g blast registration of every COPY_SERVER_PDES directory let a
stale kernel directory answer another client's VA in the walker's
first-success order (the r459 stale-directory class) - measured as a CUDA
client hanging after DEVICE. Candidates are now parked and registered only
when a walk of the client's own content-validated ring VA through them
reproduces that ring's PA; the r496f vidmem sweep remains as a fallback.

Also in this batch:
- svc_chan_userd's VRAM shape scan now prefers a block with work PENDING
  (get < put) and then the one nearest the registered memdesc offset: on a
  channel re-registration the stale previous USERD still validates (it
  holds the same put/entry values) and the old first-match adoption made
  the pump write GP_GET into the dead block (r504 second DMA init).
- The USERD/FB scan and the CE caps byte-1 BL bits (r496i) are unchanged;
  TU104_CE_CAPS1 = 0x03 needs the CUDA battery re-verification.

3 files changed, 165 insertions, 31 deletions

diff --git a/kernel/gsp_rm_emulation.c b/kernel/gsp_rm_emulation.c
index 4c848b8..eddfab5 100644
--- a/kernel/gsp_rm_emulation.c
+++ b/kernel/gsp_rm_emulation.c
@@ -2509,25 +2509,24 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx,
 	}
 
 	/*
-	 * r496g: this control IS the kernel RM publishing the page-table
+	 * r496j: this control IS the kernel RM publishing the page-table
 	 * chain of its internal VAS (the level[0..] instances with their
-	 * phys addresses and page shifts). Register the root with the
-	 * progress service so the walker can translate every VA of this
-	 * client - the X/GL channel's pushbuffer pages and its progress
-	 * semaphore, for which no other translation source exists (the
-	 * fn-54 SET_PAGE_DIRECTORY path is external-VAS/UVM only). The
-	 * level values are sanity-gated; a wrong chain only costs failed
-	 * walks.
+	 * phys addresses and page shifts). The chain can name the X/GL
+	 * channel's pushbuffer pages and progress semaphore, for which no
+	 * other translation source exists (the fn-54 SET_PAGE_DIRECTORY
+	 * path is external-VAS/UVM only). It is NOT registered directly:
+	 * a blast registration of every client's directory let a stale
+	 * kernel directory answer another client's VA in the walker's
+	 * first-success order (the r459 stale-directory class; observed
+	 * as a CUDA client hanging after DEVICE with r496g). Instead the
+	 * root is offered as a CANDIDATE; the progress service registers
+	 * it only after it has been VERIFIED against a content-validated
+	 * pair of that very client (the ring found by the RAM content
+	 * scan).
 	 */
 	if (num_levels >= 1 && rm->pt.levels[0].phys &&
 	    rm->pt.levels[0].size >= 16 &&
 	    rm->pt.levels[0].size <= 4096 * 8) {
-		struct sema_vaspace_desc d;
-
-		memset(&d, 0, sizeof(d));
-		d.h_vaspace = h_object;
-		d.pdb = rm->pt.levels[0].phys;
-		d.num_entries = (u32)(rm->pt.levels[0].size / 8);
 		/*
 		 * The level phys values are guest PAs on this platform
 		 * (observed 4+ GiB, above any FB shadow); aperture 3 in the
@@ -2536,13 +2535,17 @@ static void gsp_rm_handle_copy_server_pdes(struct gsp_emu_context *ctx,
 		 * FB, everything else sysmem. The walker also retries the
 		 * other aperture, so a misclassification only costs a walk.
 		 */
-		d.b_vidmem = rm->pt.levels[0].phys <
-			     (u64)ctx->vgpu->bars[1].size;
-		gsp_sema_service_vaspace_add(&d);
+		bool vidmem = rm->pt.levels[0].phys <
+			      (u64)ctx->vgpu->bars[1].size;
+
+		gsp_sema_service_vaspace_candidate_add(
+			h_object, rm->pt.levels[0].phys,
+			(u32)(rm->pt.levels[0].size / 8), vidmem);
 		nv_vgpu_info(ctx->vgpu,
-			     "vgsp-rm: VAS 0x%x page directory registered from COPY_SERVER_PDES pdb=0x%llx entries=%u (%s)",
-			     h_object, d.pdb, d.num_entries,
-			     d.b_vidmem ? "vidmem" : "sysmem");
+			     "vgsp-rm: VAS 0x%x page directory offered as candidate pdb=0x%llx entries=%u (%s; registered only after a content-verified ring walk)",
+			     h_object, (u64)rm->pt.levels[0].phys,
+			     (u32)(rm->pt.levels[0].size / 8),
+			     vidmem ? "vidmem" : "sysmem");
 	}
 }
 
diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index b7ce1df..1e9f981 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -1640,6 +1640,90 @@ void gsp_sema_service_channel_del(struct nv_vgpu_device *vgpu, u32 h_object)
  * the page tables itself and hands their base address to the GPU). Store it so
  * the service can translate that space's VAs directly.
  */
+/*
+ * r496j: unverified page-directory candidates from COPY_SERVER_PDES. A
+ * kernel-RM internal VAS publishes its chain there, but registering every
+ * client's directory directly lets a stale kernel directory answer another
+ * client's VA in the walker's first-success order (the r459 stale-directory
+ * class; observed as a CUDA client hanging after DEVICE). Candidates are
+ * parked here and registered only after a walk of the client's own
+ * content-validated ring VA through the candidate reproduces that ring's PA
+ * (svc_gr_ram_ring_scan's latch calls svc_vas_candidates_verify).
+ */
+#define SVC_VAS_CAND_MAX	16
+static struct {
+	bool valid;
+	u32 h_vaspace;
+	u64 pdb;
+	u32 entries;
+	bool vidmem;
+} svc_vas_cand[SVC_VAS_CAND_MAX];
+
+static int svc_gmmu_walk(struct nv_vgpu_device *vgpu, u64 pdb, u32 entries,
+			 u64 va, int tap, u64 *addr, int *ap, bool trace);
+
+void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb,
+					    u32 entries, bool vidmem)
+{
+	int i;
+
+	if (!pdb || entries < 2 || entries > 4096)
+		return;
+	for (i = 0; i < SVC_VAS_CAND_MAX; i++) {
+		if (svc_vas_cand[i].valid &&
+		    svc_vas_cand[i].h_vaspace == h_vaspace &&
+		    svc_vas_cand[i].pdb == pdb)
+			return;	/* already parked */
+		if (!svc_vas_cand[i].valid) {
+			svc_vas_cand[i].valid = true;
+			svc_vas_cand[i].h_vaspace = h_vaspace;
+			svc_vas_cand[i].pdb = pdb;
+			svc_vas_cand[i].entries = entries;
+			svc_vas_cand[i].vidmem = vidmem;
+			return;
+		}
+	}
+}
+
+/*
+ * Verify each parked candidate against one content-validated translation
+ * (the ring VA -> PA the RAM scan just proved): walk the ring page through
+ * the candidate and register only the directory that reproduces it. Runs on
+ * the service thread from the ring-scan latch; the walk itself is cheap (a
+ * few table reads).
+ */
+static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va,
+				      u64 pa)
+{
+	int i;
+
+	for (i = 0; i < SVC_VAS_CAND_MAX; i++) {
+		u64 wpa = 0;
+		int wap = 0;
+		int ap = svc_vas_cand[i].vidmem ? SVC_APERTURE_FB
+						: SVC_APERTURE_SYS;
+		struct sema_vaspace_desc d;
+
+		if (!svc_vas_cand[i].valid)
+			continue;
+		if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb,
+				  svc_vas_cand[i].entries, va & ~0xfffULL,
+				  ap, &wpa, &wap, false) ||
+		    (wpa & ~0xfffULL) != (pa & ~0xfffULL))
+			continue;
+		memset(&d, 0, sizeof(d));
+		d.h_vaspace = svc_vas_cand[i].h_vaspace;
+		d.pdb = svc_vas_cand[i].pdb;
+		d.num_entries = svc_vas_cand[i].entries;
+		d.b_vidmem = svc_vas_cand[i].vidmem;
+		gsp_sema_service_vaspace_add(&d);
+		svc_vas_cand[i].valid = false;
+		pr_info("sema-svc: VAS 0x%x page directory registered after verified ring walk (pdb=0x%llx entries=%u ap=%d)\n",
+			d.h_vaspace, d.pdb, d.num_entries, ap);
+		break;
+	}
+}
+
 void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc)
 {
 	struct svc_vaspace *vs = NULL;
@@ -13056,6 +13140,9 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) {
 		u64 off, lim;
 		u8 *base = vgpu->bars[1].vaddr;
+		u64 best = 0, best_dist = ~0ULL;
+		u32 best_put = 0, best_get = 0;
+		bool best_pending = false;
 
 		ch->userd_scans++;
 		lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL);
@@ -13064,6 +13151,8 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 				u32 hp, hg;
 				bool ok = true;
 				u32 k;
+				u64 dist;
+				bool pending;
 
 				memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4);
 				memcpy(&hg, base + off + SVC_USERD_GP_GET, 4);
@@ -13106,13 +13195,43 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 				}
 				if (!ok)
 					continue;
-				ch->userd_pa = off;
+				/*
+				 * Several blocks can validate: a previous
+				 * session leaves its old USERD holding the same
+				 * put/entry values while the new channel's block
+				 * sits elsewhere (observed r504: the re-
+				 * registered channel re-scanned, adopted the
+				 * stale block, and its DMA test timed out).
+				 * Prefer a block with work PENDING (get < put);
+				 * among equals prefer the one nearest the
+				 * registered memdesc offset (the driver's own
+				 * hint). The pump's GP_GET write-back goes to
+				 * the adopted block, so a stale adoption strands
+				 * the client's poll.
+				 */
+				dist = off > ch->userd_pa ? off - ch->userd_pa
+							  : ch->userd_pa - off;
+				pending = hg < hp;
+				if (best &&
+				    (best_pending > pending ||
+				     (best_pending == pending && dist >= best_dist)))
+					continue;
+				best = off;
+				best_dist = dist;
+				best_put = hp;
+				best_get = hg;
+				best_pending = pending;
+			}
+			if (best) {
+				ch->userd_pa = best;
 				ch->userd_as = SVC_APERTURE_FB;
 				ch->userd_ok = true;
-				*put = hp;
-				*get = hg;
-				pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u, ring-validated)\n",
-					ch->h_object, ch->gr_index, off, hp, hg);
+				*put = best_put;
+				*get = best_get;
+				pr_info("sema-chan: GR USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n",
+					ch->h_object, ch->gr_index, best,
+					best_put, best_get,
+					best_pending ? "" : ", idle");
 				return 0;
 			}
 		}
@@ -14638,13 +14757,17 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 			set_bit(ch->gr_index, &svc_gr_ring_done);
 			svc_ring_n++;
 			/*
-			 * r496f: with the ring content-validated, discover the
-			 * client VAS's page directory in vidmem so the
-			 * channel's other VAs (pushbuffer pages, the progress
-			 * semaphore the driver polls) resolve through the
-			 * walker. Runs before the pins below matter and only
-			 * once per channel (the bitmap is set).
+			 * r496j: with the ring content-validated, verify the
+			 * parked COPY_SERVER_PDES candidates against it and
+			 * register the directory that reproduces the ring's
+			 * PA; its walk then resolves the client's other VAs
+			 * (pushbuffer pages, the progress semaphore the
+			 * driver polls). r496f's blind vidmem sweep stays as
+			 * a fallback for directories the control never
+			 * published.
 			 */
+			svc_vas_candidates_verify(vgpu, ch->gpfifo_va,
+						  hit_pa);
 			svc_xvas_pdb_discover(vgpu, ch->gpfifo_va, hit_pa);
 			/*
 			 * Persist the latch immediately: a later stalled
diff --git a/kernel/gsp_sema_service.h b/kernel/gsp_sema_service.h
index 305a027..ee5c48a 100644
--- a/kernel/gsp_sema_service.h
+++ b/kernel/gsp_sema_service.h
@@ -81,6 +81,14 @@ struct sema_vaspace_desc {
 
 void gsp_sema_service_vaspace_add(const struct sema_vaspace_desc *desc);
 void gsp_sema_service_vaspace_del(u32 h_vaspace);
+/*
+ * r496j: park a page-directory candidate (from COPY_SERVER_PDES) until a
+ * content-validated walk proves it belongs to a live client; only then is
+ * it registered (a blind registration let stale kernel directories answer
+ * other clients' VAs in the walker's first-success order).
+ */
+void gsp_sema_service_vaspace_candidate_add(u32 h_vaspace, u64 pdb,
+					    u32 entries, bool vidmem);
 /* m177: guest RM reload = new guest boot; wipe boot-scoped discovery state
  * (direct-map base, content-scanned tables) on the svc thread. Client
  * session flushes keep that state - see svc_session_flush. */