vgsp: client-VAS CE ring probe + L0 sanity gate (V12, default off)HEADmaster

- CE siblings (0xbeef0101/0102): extend the beef-family ring discovery
  with an identity probe and the GR guest-RAM content scan plus a
  sparse-run gate and a RAM ring-page ownership table; keep >=2 fresh
  same-VAS park candidates during the client's VAS churn.
- Add svc_pdb_l0_sane() and apply it to unverified directory candidates
  so a freed/reused incarnation is skipped instead of walked.
- Both behaviors are gated behind sema_ce_ram (default 0): the live A/B
  latched false positives on dense data pages and the L0 predicate needs
  tightening before it may guard the GR lane (the validated V9/V10 GR
  behavior is preserved by the default).

1 files changed, 521 insertions, 15 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 7b629ed..3225008 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -251,6 +251,13 @@ struct sema_map_entry {
  */
 unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL;
 module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644);
+
+/*
+ * r1777 (V12): direct-GL CE-family ring probe gate. Defined here because
+ * the demote/promote paths use it before the probe block; the parameter
+ * description lives with the probe implementation. Default off.
+ */
+static bool gsp_sema_ce_ram;
 #define SVC_GR_SCAN_TOP_MAX	0x400000000ULL	/* 16 GiB sanity clamp */
 
 /*
@@ -2485,6 +2492,34 @@ static bool svc_vaspace_verified(u32 h_vaspace)
 }
 
 /*
+ * r1776 (V11): is the stored directory for this handle already this pdb
+ * AND verified? The candidate-promote success path re-walks kept
+ * candidates every sweep (the keep>=2 rule below); re-registering an
+ * unchanged verified directory would clear svc.lat and log the
+ * "VA space ... page directory" line every 100 ms, while an unverified
+ * stored entry (age-out) must still be upgraded.
+ */
+static bool svc_vaspace_pdb_match(u32 h_vaspace, u64 pdb)
+{
+	unsigned long flags;
+	bool found = false;
+	int i;
+
+	if (!h_vaspace || !pdb)
+		return false;
+	spin_lock_irqsave(&svc.vas_lock, flags);
+	for (i = 0; i < SVC_MAX_VASPACES; i++)
+		if (svc.vas[i].valid && svc.vas[i].verified &&
+		    svc.vas[i].h_vaspace == h_vaspace &&
+		    svc.vas[i].pdb == pdb) {
+			found = true;
+			break;
+		}
+	spin_unlock_irqrestore(&svc.vas_lock, flags);
+	return found;
+}
+
+/*
  * Register a guest channel. Called from the GSP-RM emulation when the guest
  * driver allocates a channel (the allocation carries the GPFIFO GPU VA, its
  * entry count and the guest-physical USERD/instance blocks), and dropped when
@@ -3288,6 +3323,49 @@ static void svc_vas_candidates_age_out(struct nv_vgpu_device *vgpu)
 }
 
 /*
+ * r1777 (V12): parked-directory L0 sanity, the directory-incarnation gate.
+ *
+ * The direct-GL client rebuilds its VAS every few seconds (~19 pdbs per
+ * cycle parked under churning handles). A parked pdb whose page the guest
+ * kernel has since freed and reused reads kernel pointers (r1776:
+ * L0=0xffff8a0f8ea410a8) and its walk answers fabricated mappings - the
+ * rc=-6 "mapped but all-zero" ring page class came through exactly such a
+ * directory. Screen the L0 before walking: at least one plausible PDE
+ * (valid bit set, not a kernel-pointer pattern); an all-zero or
+ * kernel-pointer directory is skipped, so the freshest sane incarnation
+ * wins. An unreadable directory stays "unknown" and is walked as before -
+ * this gate can only reject positive evidence of a freed directory.
+ */
+static bool svc_pdb_l0_sane(struct nv_vgpu_device *vgpu, u64 pdb, int ap)
+{
+	u64 ent[64];
+	bool any = false;
+	bool hostile = false;
+	int i;
+
+	if (svc_read_ap(vgpu, pdb, ap, ent, sizeof(ent)))
+		return true;
+	for (i = 0; i < ARRAY_SIZE(ent); i++) {
+		if (!ent[i])
+			continue;
+		if ((ent[i] >> 48) == 0xffffu) {
+			hostile = true;
+			continue;
+		}
+		if (ent[i] & 1u)
+			any = true;
+	}
+	/*
+	 * Reject only on positive evidence of a freed/reused directory:
+	 * kernel-pointer garbage with NO plausible PDE anywhere in the first
+	 * 512 bytes. A directory that has valid PDEs passes even if stale
+	 * entries linger in its unused slots (r1777: the strict
+	 * reject-on-first-kernel-pointer variant skipped real directories).
+	 */
+	return any || !hostile;
+}
+
+/*
  * Verify each parked candidate against one content-validated translation
  * (the ring VA -> PA the RAM scan just proved): walk the ring page through
  * the candidate and register only the directory that reproduces it. Runs on
@@ -3308,6 +3386,9 @@ static void svc_vas_candidates_verify(struct nv_vgpu_device *vgpu, u64 va,
 
 		if (!svc_vas_cand[i].valid)
 			continue;
+		if (gsp_sema_ce_ram &&
+		    !svc_pdb_l0_sane(vgpu, svc_vas_cand[i].pdb, ap))
+			continue;	/* r1777 (V12): freed/reused directory */
 		if (svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb,
 				  svc_vas_cand[i].entries, va & ~0xfffULL,
 				  ap, &wpa, &wap, false) ||
@@ -15279,6 +15360,62 @@ static void svc_fb_claim_reset(void)
 }
 
 /*
+ * r1777 (V12): guest-RAM ring-page ownership, the RAM-side counterpart of
+ * the FB claim table above. A ring page found by the guest-RAM content scan
+ * (svc_gr_ram_ring_scan) or the low-aperture identity probe is recorded per
+ * channel so a sibling's scan cannot latch a page that is already another
+ * channel's ring: the direct-GL GR ring (0xbeef0100) and its CE siblings
+ * (0xbeef0101/0102) all queue low-aperture push entries, so shape alone
+ * cannot tell their ring pages apart (r1776: the CE sweep must not steal
+ * the GR page). One page per channel is enough - a channel's ring page is
+ * a single page. Reset per session by svc_session_flush like the FB table.
+ */
+static struct {
+	u32 h;
+	u64 pa;
+} svc_ram_ring_owner[SVC_MAX_CHANNELS];
+static u32 svc_ram_owner_n;
+
+static void svc_ram_ring_owner_add(u32 h, u64 pa)
+{
+	u32 i;
+
+	if (!pa)
+		return;
+	for (i = 0; i < svc_ram_owner_n; i++) {
+		if (svc_ram_ring_owner[i].h == h) {
+			svc_ram_ring_owner[i].pa = pa;
+			return;
+		}
+	}
+	if (svc_ram_owner_n < ARRAY_SIZE(svc_ram_ring_owner)) {
+		svc_ram_ring_owner[svc_ram_owner_n].h = h;
+		svc_ram_ring_owner[svc_ram_owner_n].pa = pa;
+		svc_ram_owner_n++;
+	}
+}
+
+static void svc_ram_ring_owner_reset(void)
+{
+	svc_ram_owner_n = 0;
+	memset(svc_ram_ring_owner, 0, sizeof(svc_ram_ring_owner));
+}
+
+/* true = the page is already recorded as a DIFFERENT channel's RAM ring. */
+static bool svc_ram_ring_owner_foreign(u32 h, u64 pa)
+{
+	u64 page = pa & ~((u64)SEMA_PAGE_SIZE - 1);
+	u32 i;
+
+	for (i = 0; i < svc_ram_owner_n; i++)
+		if (svc_ram_ring_owner[i].h != h &&
+		    (svc_ram_ring_owner[i].pa & ~((u64)SEMA_PAGE_SIZE - 1)) ==
+			page)
+			return true;
+	return false;
+}
+
+/*
  * ISSUE-0002 (r47): deferred FB sweep for a CE channel whose layout USERD
  * probe fails at discovery. Session/reset-scoped, scheduled on the first
  * failure (+3 s) and re-run once more (+30 s, after the client's follow-up
@@ -23949,6 +24086,7 @@ static void svc_session_flush(void)
 	memset(svc_ce_initrel_val, 0, sizeof(svc_ce_initrel_val));
 	svc_ce_initrel_log = 0;
 	svc_fb_claim_reset();
+	svc_ram_ring_owner_reset();
 	svc_userd_sweep_log = 0;
 	memset(svc_userd_sweep_at, 0, sizeof(svc_userd_sweep_at));
 	memset(svc_userd_sweep_n, 0, sizeof(svc_userd_sweep_n));
@@ -29220,7 +29358,8 @@ static void svc_gr_setup_hold_clear(struct svc_channel *ch)
 }
 
 static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
-				 struct svc_channel *ch, u64 *pa_out)
+				 struct svc_channel *ch, u64 *pa_out,
+				 bool owner_filter, bool sparse)
 {
 	static u8 *buf;
 	u64 top = gsp_sema_gr_scan_top;
@@ -29348,6 +29487,47 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
 			    va0 != SVC_GL_KICK_PUSH_VA &&
 			    va1 != SVC_GL_KICK_PUSH_VA)
 				continue;
+			/*
+			 * r1777 (V12): a CE-sibling scan must not latch a page
+			 * another channel's ring already owns (the GR page
+			 * passes the same low-family shape). Skip and keep
+			 * scanning this slice; the true page may sit later.
+			 */
+			if (owner_filter &&
+			    svc_ram_ring_owner_foreign(ch->h_object, gpa + o))
+				continue;
+			/*
+			 * r1777 (V12): sparse-page gate for the CE family. A
+			 * real GPFIFO ring page carries a bounded run of
+			 * queued entries and then zero words (the r1774 GR
+			 * page: ~17 entries then zeros); the false-positive
+			 * class the loose shape+window rule admits (r1777 c1:
+			 * CE 0xbeef0101 latched va=0x1131860c/393216) is a
+			 * DENSE data array where every 8-byte slot decodes.
+			 * Require an all-zero entry slot within the first 128
+			 * entries of the run.
+			 */
+			if (sparse) {
+				u32 z;
+				bool term = false;
+
+				for (z = 2; z < 128; z++) {
+					u32 zoff = o + eoff +
+						   z * SVC_GPFIFO_ENTRY_SIZE;
+					u32 q0, q1;
+
+					if ((u64)zoff + 8 > chunk)
+						break;
+					memcpy(&q0, buf + zoff, 4);
+					memcpy(&q1, buf + zoff + 4, 4);
+					if (!q0 && !q1) {
+						term = true;
+						break;
+					}
+				}
+				if (!term)
+					continue;
+			}
 			*pa_out = gpa + o;
 			return true;
 		}
@@ -29356,9 +29536,14 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
 	ch->gr_ram_passes++;
 	if (ch->gr_ram_cursor == 0 && !ch->gr_ram_logged) {
 		ch->gr_ram_logged = true;
-		pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n",
-			ch->h_object, ch->gr_index, ch->gr_ram_passes,
-			ch->gpfifo_va);
+		if (ch->engine_type)
+			pr_info("sema-chan: CE ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n",
+				ch->h_object, ch->ce_index, ch->gr_ram_passes,
+				ch->gpfifo_va);
+		else
+			pr_info("sema-chan: GR ring h=0x%x rank=%u RAM content scan found no candidate in %u slice(s) (va=0x%llx)\n",
+				ch->h_object, ch->gr_index, ch->gr_ram_passes,
+				ch->gpfifo_va);
 	}
 	/*
 	 * Persist the cursor immediately: the pump's stalled-userd path
@@ -30593,7 +30778,7 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 {
 	u32 h = ch->h_vaspace;
 	bool beef = ((ch->h_object & 0xfffff000u) == 0xbeef0000u);
-	int i;
+	int i, pass;
 
 	/*
 	 * X-DMA-FIX: the GR client channel declares hVASpace==0 (attach to
@@ -30644,7 +30829,16 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 			}
 			return;
 		}
-		if (svc_vaspace_stale_candidate(h)) {
+		/*
+		 * r1776 (V11): with the always-probe the direct-GL family
+		 * re-validates its ring content every sweep, so demoting the
+		 * stored directory on a churning parked candidate is redundant
+		 * - and now harmful: keeping >=2 parks alive means a
+		 * different-pdb candidate would re-trigger demote (and the
+		 * svc.lat reset) on every sweep. Other clients keep CONT.401.
+		 */
+		if (svc_vaspace_stale_candidate(h) &&
+		    !(beef && gsp_sema_vas_client_ring)) {
 			pr_info("sema-svc: VAS 0x%x verified directory is stale (rebuilt pdb parked); demoting for re-validation\n",
 				h);
 			svc_vaspace_demote(h);
@@ -30667,6 +30861,35 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 	if (beef && gsp_sema_vas_client_ring &&
 	    !svc_vaspromote_rl_allow(ch->h_object))
 		return;
+	/*
+	 * r1777 (V12): directory-incarnation handling. The client parks ~19
+	 * pdbs per churn cycle; the freshest incarnation is the one most
+	 * likely to map the ring VA NOW, so walk it first (pass 0) and the
+	 * rest afterwards (pass 1). A parked pdb whose L0 reads as a freed/
+	 * reused page (kernel-pointer pattern or all-zero) is skipped before
+	 * the walk - r1776: walking such a directory fabricated the rc=-6
+	 * "mapped but all-zero" ring page.
+	 */
+	for (pass = 0; pass < 2; pass++) {
+		int newest = -1;
+
+		if (pass == 0) {
+			unsigned long best = 0;
+			int j;
+
+			for (j = 0; j < SVC_VAS_CAND_MAX; j++) {
+				if (!svc_vas_cand[j].valid ||
+				    svc_vas_cand[j].h_vaspace != h)
+					continue;
+				if (newest < 0 ||
+				    time_after(svc_vas_cand[j].stamp, best)) {
+					best = svc_vas_cand[j].stamp;
+					newest = j;
+				}
+			}
+			if (newest < 0)
+				break;
+		}
 	for (i = 0; i < SVC_VAS_CAND_MAX; i++) {
 		u64 wpa = 0;
 		int wap = SVC_APERTURE_SYS;
@@ -30678,6 +30901,23 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 
 		if (!svc_vas_cand[i].valid || svc_vas_cand[i].h_vaspace != h)
 			continue;
+		if (pass == 0 && i != newest)
+			continue;
+		if (pass == 1 && i == newest)
+			continue;
+		if (gsp_sema_ce_ram &&
+		    !svc_pdb_l0_sane(vgpu, svc_vas_cand[i].pdb,
+				     svc_vas_cand[i].vidmem ? SVC_APERTURE_FB
+							    : SVC_APERTURE_SYS)) {
+			if ((ch->h_object & 0xfffff000u) == 0xbeef0000u &&
+			    time_after(jiffies, svc_vaspromote_diag_at)) {
+				svc_vaspromote_diag_at = jiffies +
+							msecs_to_jiffies(2000);
+				pr_info("sema-svc: VAS-PROMOTE h=0x%x cand pdb=0x%llx L0 sanity failed (freed/reused directory)\n",
+					ch->h_object, svc_vas_cand[i].pdb);
+			}
+			continue;
+		}
 		wrc = svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb,
 				    svc_vas_cand[i].entries,
 				    ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1),
@@ -30719,6 +30959,54 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 		d.num_entries = svc_vas_cand[i].entries;
 		d.b_vidmem = svc_vas_cand[i].vidmem;
 		d.verified = true;	/* CONT.400: content-proven ring walk */
+		/*
+		 * r1776 (V11): keep >=2 fresh same-VAS parks. r1775 CE wall:
+		 * the GR channel content-validated and CONSUMED the only fresh
+		 * directory ~0.1 s before the CE ring page content landed; the
+		 * CE siblings were then left with the stale parked pdb and
+		 * spun "cand ... walk rc=-2" until teardown (submissions=0).
+		 * Keep this candidate parked while fewer than two other
+		 * same-VAS candidates remain so a sibling can re-validate
+		 * through the same directory later. Re-register only when the
+		 * stored directory differs (avoid clearing svc.lat/log spam).
+		 */
+		if (beef) {
+			unsigned nsame = 0;
+			bool reg_log = false;
+			int j;
+
+			for (j = 0; j < SVC_VAS_CAND_MAX; j++)
+				if (j != i && svc_vas_cand[j].valid &&
+				    svc_vas_cand[j].h_vaspace == h)
+					nsame++;
+			if (nsame < 2) {
+				/*
+				 * Keep this candidate parked so a sibling can
+				 * re-validate through the same directory after
+				 * this channel moves on. Re-register only when
+				 * the stored (verified) directory is not
+				 * already this pdb.
+				 */
+				if (!svc_vaspace_pdb_match(h,
+							   svc_vas_cand[i].pdb)) {
+					gsp_sema_service_vaspace_add(&d);
+					reg_log = true;
+				} else
+					svc_vas_cand[i].stamp = jiffies;
+			} else {
+				gsp_sema_service_vaspace_add(&d);
+				svc_vas_cand[i].valid = false;
+				reg_log = true;
+			}
+			svc_vaspromote_backoff_at = 0;
+			svc_vaspromote_backoff_h = 0;
+			if (reg_log)
+				pr_info("sema-svc: VAS 0x%x registered at channel 0x%x turn (parked candidate content-validated%s: gpfifo 0x%llx entry0 va=0x%llx len=%u)\n",
+					h, ch->h_object,
+					nsame < 2 ? ", kept parked" : "",
+					ch->gpfifo_va, eva, len);
+			return;
+		}
 		gsp_sema_service_vaspace_add(&d);
 		svc_vas_cand[i].valid = false;
 		svc_vaspromote_backoff_at = 0;	/* r1773: promotion clears the backoff */
@@ -30727,6 +31015,7 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 			h, ch->h_object, ch->gpfifo_va, eva, len);
 		return;
 	}
+	}	/* end two-pass (newest-first) candidate walk */
 	/*
 	 * r1773: no parked candidate mapped the ring this sweep; back the
 	 * beef-family probe off for 250 ms so a persistently-unresolvable
@@ -30740,6 +31029,71 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 	}
 
 	/*
+	 * r1776 (V11): CE SIBLING SWEEP - when a beef-family channel (the
+	 * direct-GL CE rings 0xbeef0101/0102, and the GR sibling as a
+	 * fallback) misses its ring through every same-VAS parked candidate,
+	 * walk ALL parked candidates regardless of owner VAS. The client
+	 * rebuilds its page directory every few seconds and parks ~19 pdbs
+	 * per cycle under churning handles, so the directory that maps the
+	 * CE ring VA can sit in a slot whose h_vaspace does not match the
+	 * channel's (r1775: CE 0x4100000 never mapped; GR 0x4020000 mapped
+	 * only through a later fresh candidate). The walk stays content-
+	 * gated (entry-0 push plausibility + window bound), so a foreign
+	 * directory can only win by actually mapping THIS channel's ring.
+	 * The candidate is NOT consumed: its owner still needs it, and the
+	 * CE siblings re-walk it until their ring page content lands.
+	 */
+	if (beef) {
+		for (i = 0; i < SVC_VAS_CAND_MAX; i++) {
+			u64 wpa = 0;
+			int wap = SVC_APERTURE_SYS;
+			u8 e[SVC_GPFIFO_ENTRY_SIZE];
+			u32 e0 = 0, e1 = 0, len;
+			u64 eva;
+			int wrc;
+
+			if (!svc_vas_cand[i].valid ||
+			    svc_vas_cand[i].h_vaspace == h)
+				continue;
+			wrc = svc_gmmu_walk(vgpu, svc_vas_cand[i].pdb,
+					    svc_vas_cand[i].entries,
+					    ch->gpfifo_va &
+						~(u64)(SEMA_PAGE_SIZE - 1),
+					    svc_vas_cand[i].vidmem
+							? SVC_APERTURE_FB
+							: SVC_APERTURE_SYS,
+					    &wpa, &wap, false);
+			if (wrc)
+				continue;
+			if (svc_read_ap(vgpu, wpa, wap, e, sizeof(e)))
+				continue;
+			memcpy(&e0, e, 4);
+			memcpy(&e1, e + 4, 4);
+			eva = ((u64)(e1 & 0xff) << 32) |
+			      (u64)(e0 & 0xfffffffc);
+			len = ((e1 >> 10) & 0x1fffff) << 2;
+			if (!eva || len < 4 || len > 0x10000 ||
+			    !svc_gr_entry_win_ok(ch->gpfifo_va, eva))
+				continue;
+			svc_pin_store(ch->gpfifo_va, wpa, wap);
+			svc_lat_store(ch->gpfifo_va, wpa, wap);
+			svc_cur_vas = svc_vas_cand[i].h_vaspace;
+			svc_vaspromote_backoff_at = 0;	/* r1773 */
+			svc_vaspromote_backoff_h = 0;	/* r1774 (V9) */
+			if (time_after(jiffies, svc_ringvas_diag_at)) {
+				svc_ringvas_diag_at = jiffies +
+						      msecs_to_jiffies(2000);
+				pr_info("sema-svc: RING-CAND h=0x%x gpfifo=0x%llx mapped by parked VAS 0x%x cand (pdb=0x%llx ap=%d) -> pa=0x%llx entry0 va=0x%llx len=%u\n",
+					ch->h_object, ch->gpfifo_va,
+					svc_vas_cand[i].h_vaspace,
+					svc_vas_cand[i].pdb, wap, wpa,
+					eva, len);
+			}
+			return;
+		}
+	}
+
+	/*
 	 * CONT.400: the parked candidate is not always the directory that maps
 	 * the ring (the GL client's GR ring 0x4020000 lives in the RM low
 	 * aperture; its client VAS 0xbeef0300 maps the pushbuffers but not
@@ -30751,6 +31105,18 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 	{
 		unsigned long flags;
 		int k;
+		/*
+		 * r1776 (V11): keep the LAST per-directory failure reason for
+		 * the miss line below. r1775 saw only "NOT mapped" - the CE
+		 * ring walk through the fresh directory actually returned rc=0
+		 * with an all-zero entry page during the early window, which
+		 * the bare line hid. rc: walk error, -5 read error, -6 mapped
+		 * but entry implausible (e0/e1 recorded).
+		 */
+		int last_rc = 0;
+		u32 last_vas = 0;
+		u64 last_wpa = 0;
+		u32 last_e0 = 0, last_e1 = 0;
 
 		spin_lock_irqsave(&svc.vas_lock, flags);
 		for (k = 0; k < SVC_MAX_VASPACES; k++) {
@@ -30759,17 +31125,45 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 			u8 e[SVC_GPFIFO_ENTRY_SIZE];
 			u32 e0 = 0, e1 = 0, elen;
 			u64 eva;
+			int wrc;
 			struct svc_vaspace snap;
 
 			if (!svc.vas[k].valid)
 				continue;
 			snap = svc.vas[k];
 			spin_unlock_irqrestore(&svc.vas_lock, flags);
-			if (!svc_gmmu_walk(vgpu, snap.pdb, snap.entries,
-					   ch->gpfifo_va &
+			/*
+			 * r1777 (V12): a registered directory can still be a
+			 * freed/reused page (the UNVERIFIED age-out
+			 * registrations). Skip walking an L0 that reads as
+			 * a kernel pointer or all-zero: the r1776 rc=-6
+			 * "mapped but all-zero" ring page came through
+			 * exactly such an incarnation.
+			 */
+			if (gsp_sema_ce_ram &&
+			    !svc_pdb_l0_sane(vgpu, snap.pdb, snap.ap)) {
+				last_rc = -7;
+				last_vas = snap.h_vaspace;
+				last_wpa = 0;
+				last_e0 = last_e1 = 0;
+				spin_lock_irqsave(&svc.vas_lock, flags);
+				continue;
+			}
+			wrc = svc_gmmu_walk(vgpu, snap.pdb, snap.entries,
+					    ch->gpfifo_va &
 						~(u64)(SEMA_PAGE_SIZE - 1),
-					   snap.ap, &wpa, &wap, false) &&
-			    !svc_read_ap(vgpu, wpa, wap, e, sizeof(e))) {
+					    snap.ap, &wpa, &wap, false);
+			if (wrc) {
+				last_rc = wrc;
+				last_vas = snap.h_vaspace;
+				last_wpa = 0;
+				last_e0 = last_e1 = 0;
+			} else if (svc_read_ap(vgpu, wpa, wap, e, sizeof(e))) {
+				last_rc = -5;
+				last_vas = snap.h_vaspace;
+				last_wpa = wpa;
+				last_e0 = last_e1 = 0;
+			} else {
 				memcpy(&e0, e, 4);
 				memcpy(&e1, e + 4, 4);
 				eva = ((u64)(e1 & 0xff) << 32) |
@@ -30797,6 +31191,11 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 					svc_vaspromote_backoff_h = 0;	/* r1774 (V9) */
 					return;
 				}
+				last_rc = -6;
+				last_vas = snap.h_vaspace;
+				last_wpa = wpa;
+				last_e0 = e0;
+				last_e1 = e1;
 			}
 			spin_lock_irqsave(&svc.vas_lock, flags);
 		}
@@ -30804,8 +31203,9 @@ static void svc_chan_vas_candidate_promote(struct nv_vgpu_device *vgpu,
 		if ((ch->h_object & 0xfffff000u) == 0xbeef0000u &&
 		    time_after(jiffies, svc_promote_diag_at)) {
 			svc_promote_diag_at = jiffies + msecs_to_jiffies(2000);
-			pr_info("sema-svc: RING-VAS h=0x%x gpfifo=0x%llx NOT mapped by any registered VAS\n",
-				ch->h_object, ch->gpfifo_va);
+			pr_info("sema-svc: RING-VAS h=0x%x gpfifo=0x%llx NOT mapped by any registered VAS (last vas=0x%x walk rc=%d pa=0x%llx e0=%08x e1=%08x)\n",
+				ch->h_object, ch->gpfifo_va, last_vas,
+				last_rc, last_wpa, last_e0, last_e1);
 		}
 	}
 }
@@ -31418,7 +31818,7 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 	    !test_bit(ch->gr_index, &svc_gr_ring_done)) {
 		u64 hit_pa = 0;
 
-		if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa)) {
+		if (svc_gr_ram_ring_scan(vgpu, ch, &hit_pa, false, false)) {
 			/*
 			 * Pin ONLY the content-validated ring page (validated,
 			 * not a window guess). The pushbuffer pages and the
@@ -31435,6 +31835,7 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 				      SVC_APERTURE_SYS);
 			svc_lat_store(ch->gpfifo_va, hit_pa,
 				      SVC_APERTURE_SYS);
+			svc_ram_ring_owner_add(ch->h_object, hit_pa);
 			ch->gr_ram_logged = true;
 			ch->gr_ram_ring = true;
 			set_bit(ch->gr_index, &svc_gr_ring_done);
@@ -31484,6 +31885,95 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 	}
 }
 
+/*
+ * r1777 (V12): gate for the direct-GL CE-family ring probe below. Default
+ * OFF: the live A/B latched false positives on dense data pages and the
+ * L0-sanity predicate still needs tightening before it may guard the GR
+ * lane (see the wedge/CE RCA notes). sema_ce_ram=1 enables the probe; the
+ * committed default preserves the validated pre-V12 behaviour.
+ */
+module_param_named(sema_ce_ram, gsp_sema_ce_ram, bool, 0644);
+MODULE_PARM_DESC(sema_ce_ram,
+	"Discover the direct-GL CE sibling rings by guest-RAM content scan + identity probe (default 0 = pre-V12 A/B)");
+
+/*
+ * r1777 (V12): direct-GL CE-family ring discovery (beef 0xbeef01xx).
+ *
+ * The direct-GL client registers a GR channel (0xbeef0100, ring 0x4020000)
+ * and CE siblings (0xbeef0101/0102, rings 0x4100000/0x4160000) that share
+ * VAS 0xbeef0300. The GR ring resolves (registered directory + RAM content
+ * scan + auto-pin); the CE rings never did (r1775/r1776: every parked or
+ * registered directory incarnation either misses the ring VA - walk rc=-2 -
+ * or maps a zeroed page, and the CE branch of svc_ring_autopin never tried
+ * either of the two mechanisms that resolved the GR ring). This probe runs
+ * in that branch (beef CE channels only, gated by sema_ce_ram) and:
+ *   1. reads the ring VA raw - the RM low aperture is an identity-mapped
+ *      VA==PA window (the pump's GR path uses the same trick) - and pins
+ *      va->va when entry0 decodes as a plausible low-family push;
+ *   2. reuses the same guest-RAM content scan loop the GR family uses
+ *      (svc_gr_ram_ring_scan) with the low-aperture push validator, but
+ *      refuses pages another channel's RAM ring already owns
+ *      (svc_ram_ring_owner_*: the GR page passes the same shape, r1776).
+ * On a latch the ring VA is pinned like the GR scan latch, so the pump's
+ * entry/USERD reads resolve through the page.
+ */
+static bool svc_ce_beef_ring_probe(struct nv_vgpu_device *vgpu,
+				   struct svc_channel *ch)
+{
+	u8 e[2 * SVC_GPFIFO_ENTRY_SIZE];
+	u32 e0, e1;
+	u64 hit_pa = 0;
+
+	if (!ch->gpfifo_va)
+		return false;
+	/* 1. identity raw read (the low aperture is a VA==PA window). */
+	if (!svc_read(vgpu, ch->gpfifo_va, e, sizeof(e))) {
+		memcpy(&e0, e, 4);
+		memcpy(&e1, e + 4, 4);
+		if ((e0 || e1) && svc_gr_entry_plausible(ch, e0, e1) &&
+		    !svc_ram_ring_owner_foreign(ch->h_object, ch->gpfifo_va)) {
+			svc_pin_store(ch->gpfifo_va, ch->gpfifo_va,
+				      SVC_APERTURE_SYS);
+			svc_lat_store(ch->gpfifo_va, ch->gpfifo_va,
+				      SVC_APERTURE_SYS);
+			svc_ram_ring_owner_add(ch->h_object, ch->gpfifo_va);
+			set_bit(ch->ce_index, &svc_ring_done);
+			svc_ring_n++;
+			pr_info("sema-chan: CE ring h=0x%x rank=%u identity-probed va=0x%llx -> pa=0x%llx (entry0 va=0x%llx len=%u)\n",
+				ch->h_object, ch->ce_index, ch->gpfifo_va,
+				ch->gpfifo_va,
+				((u64)(e1 & 0xff) << 32) |
+					(u64)(e0 & 0xfffffffc),
+				((e1 >> 10) & 0x1fffff) << 2);
+			return true;
+		}
+	}
+	/* 2. guest-RAM content scan (same loop the GR family uses). */
+	if (ch->gr_ram_passes < svc_gr_ram_pass_budget() &&
+	    svc_gr_ram_ring_scan(vgpu, ch, &hit_pa, true, true)) {
+		u64 eva = 0;
+		u32 elen = 0;
+
+		if (!svc_read(vgpu, hit_pa, e, 8)) {
+			memcpy(&e0, e, 4);
+			memcpy(&e1, e + 4, 4);
+			eva = ((u64)(e1 & 0xff) << 32) |
+			      (u64)(e0 & 0xfffffffc);
+			elen = ((e1 >> 10) & 0x1fffff) << 2;
+		}
+		svc_pin_store(ch->gpfifo_va, hit_pa, SVC_APERTURE_SYS);
+		svc_lat_store(ch->gpfifo_va, hit_pa, SVC_APERTURE_SYS);
+		svc_ram_ring_owner_add(ch->h_object, hit_pa);
+		set_bit(ch->ce_index, &svc_ring_done);
+		svc_ring_n++;
+		pr_info("sema-chan: CE ring h=0x%x rank=%u found by RAM content scan va=0x%llx -> pa=0x%llx (sibling pin; entry0 va=0x%llx len=%u)\n",
+			ch->h_object, ch->ce_index, ch->gpfifo_va, hit_pa,
+			eva, elen);
+		return true;
+	}
+	return false;
+}
+
 static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
 			     struct svc_channel *ch)
 {
@@ -31504,6 +31994,16 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
 		svc_ring_n++;
 		return;
 	}
+	/*
+	 * r1777 (V12): the direct-GL CE siblings get the same two discovery
+	 * sources the GR ring has (content scan + identity probe) before the
+	 * FB convention probe below, which can never describe their
+	 * low-aperture rings.
+	 */
+	if (gsp_sema_ce_ram &&
+	    (ch->h_object & 0xfffff000u) == 0xbeef0000u &&
+	    svc_ce_beef_ring_probe(vgpu, ch))
+		return;
 
 	pa = SVC_CE_RING_FB_BASE + SVC_CE_RING_FB_STRIDE * ch->ce_index;
 	if (svc_read_ap(vgpu, pa, SVC_APERTURE_FB, e, sizeof(e)))
@@ -42637,11 +43137,15 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		 * Re-run every pass: the ring VA churns, so a new VA simply
 		 * re-proves on the next turn (the stale pin is keyed to the
 		 * old VA and is ignored).
+		 * r1777 (V12): extended to the beef CE siblings (same low-
+		 * aperture family, same identity window), with the RAM-ring
+		 * ownership gate so the GR/CE pages cannot alias each other.
 		 */
-		if (rrc && !ch->engine_type &&
+		if (rrc &&
 		    (ch->h_object & 0xfffff000u) == 0xbeef0000u &&
 		    ch->gpfifo_va >= SVC_LOW_REL_VA_LO &&
-		    ch->gpfifo_va < 0x10000000ULL) {
+		    ch->gpfifo_va < 0x10000000ULL &&
+		    !svc_ram_ring_owner_foreign(ch->h_object, ch->gpfifo_va)) {
 			u8 e[2 * SVC_GPFIFO_ENTRY_SIZE];
 			u32 e0, e1;
 
@@ -42656,6 +43160,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 					svc_lat_store(ch->gpfifo_va,
 						      ch->gpfifo_va,
 						      SVC_APERTURE_SYS);
+					svc_ram_ring_owner_add(
+						ch->h_object, ch->gpfifo_va);
 					ring_pa = ch->gpfifo_va;
 					ring_ap = SVC_APERTURE_SYS;
 					rrc = 0;