vgsp: cross-class FB ring-block claims stop the GR autopin stealing a live foreign ring

The GR ring autopin fallback (SESSION 133c userd-shape scan) and the CE
first-submission capture each guarded block ownership only within their own
engine class, so a channel whose own ring was not yet written latched another
channel's live block permanently: the GR channels drained the CE rank-0 ring
(or an unlatched CE rank's convention block), the real launch entry was never
read, and CUDA clients hung after their first device query (victory_mail
stalled in UVM channel_pool_add; every post-reboot boot r805-r811 showed the
same steal, e.g. 'auto-pinned GR ring h=0x5c000019 rank=0 ... -> fb 0x80000
(userd-shape scan ... entry0 va=0x120000000 len=40 put=35 get=35)' while the
CE handle decoded the identical entry[35] raw words from pa=0x10350641c).

Changes:
- shared FB block claim table (svc_fb_claim_add/_foreign/_reset), session
  scoped, consulted by both GR scan passes and the CE fallback scan; every
  live CE rank's convention block is reserved even before that rank latches
  (r809: CE rank 1's block at fb 0xa0000 was stolen pre-latch);
- every GR autopin latch (userd-derived, FB convention, pin, trusted walk,
  scan pass 1/2) and both CE latch paths record claims;
- CE first-submission capture: a rank's own convention block is latched even
  when its stale content probes incoherent (the previous session's consumed
  state), instead of falling through to a never-used zero block (r812/r813:
  client 2's 0x90000 block read put=1/get=146 at the capture instant and the
  zero 0x82000 block was latched, hanging the client).

No module parameters or defaults change; the guards are additive.

Validated (plain mdev, script defaults):
- victory_mail GREEN repeatedly: 'VADD n=32768 bad=0 checksum=12204516.000000'
  / 'STATUS OK', all six GR ranks latched at their own fb 0x140000+0x3000*rank
  rings with 'launch captured h=0x5c000019 push=0x20043be94 len=840';
- a second sequential client on the same boot now progresses (vecadd_loop:
  'VERIFY launch=10 bad=0' - the 100-launch loop needs a long budget).

1 files changed, 198 insertions, 7 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 55ad4b2..939d1bb 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -9116,6 +9116,107 @@ static unsigned long svc_ce_hold_since[32];
  */
 static u64 svc_ce_firstsub_off[32];
 /*
+ * SESSION 144: FB ring-block ownership ACROSS engine classes. The GR
+ * autopin fallback scan (SESSION 133c) and the CE first-submission
+ * capture (SESSION 134c) each solved the same problem only within their
+ * own class; a live-but-foreign ring block passes every shape gate, so a
+ * channel whose own ring is not written yet latches another channel's
+ * block permanently and never reads its own entries. Observed on
+ * r805-r808: every CUDA GR rank's userd-shape scan latch resolved the CE
+ * rank-0 block (fb 0x80000, userd 0x90000; e.g.
+ * "auto-pinned GR ring h=0x5c000019 rank=0 ... -> fb 0x80000 (userd-shape
+ * scan ... entry0 va=0x120000000 len=40 put=35 get=35)" while the CE
+ * handle decoded the identical entry[35] raw words from pa=0x10350641c),
+ * so no GR launch was ever captured and victory_mail hung after DEVICE.
+ * A channel that cannot see its own block must stay unlatched (and retry
+ * next pass) rather than steal a foreign one. Keyed by the FB ring
+ * offset; claims are session state, reset by svc_session_flush.
+ */
+struct svc_fb_claim {
+	u32 h;			/* owning channel handle */
+	u64 ring_off;		/* FB ring offset */
+	u64 userd_off;		/* FB USERD offset (0 = unknown) */
+};
+#define SVC_CE_RING_FB_BASE	0x80000ULL
+#define SVC_CE_RING_FB_STRIDE	0x20000ULL
+static struct svc_fb_claim svc_fb_claim_tab[40];
+static u32 svc_fb_claim_n;
+
+static void svc_fb_claim_add(u32 h, u64 ring_off, u64 userd_off)
+{
+	u32 i;
+
+	for (i = 0; i < svc_fb_claim_n; i++) {
+		if (svc_fb_claim_tab[i].h == h) {
+			svc_fb_claim_tab[i].ring_off = ring_off;
+			svc_fb_claim_tab[i].userd_off = userd_off;
+			return;
+		}
+	}
+	if (svc_fb_claim_n < ARRAY_SIZE(svc_fb_claim_tab)) {
+		svc_fb_claim_tab[svc_fb_claim_n].h = h;
+		svc_fb_claim_tab[svc_fb_claim_n].ring_off = ring_off;
+		svc_fb_claim_tab[svc_fb_claim_n].userd_off = userd_off;
+		svc_fb_claim_n++;
+	}
+}
+
+/*
+ * true = the candidate FB block (ring at ring_off, USERD at userd_off if
+ * known) is already claimed by a DIFFERENT channel, or is a live CE
+ * rank's convention block even before that rank latched, i.e. it must not
+ * be latched or stolen. Same-channel re-latches pass.
+ */
+static bool svc_fb_claim_foreign(struct nv_vgpu_device *vgpu, u32 self_h,
+				 u64 ring_off, u64 userd_off)
+{
+	u32 i;
+	unsigned long flags;
+	int k;
+
+	for (i = 0; i < svc_fb_claim_n; i++) {
+		if (svc_fb_claim_tab[i].h == self_h)
+			continue;
+		if (svc_fb_claim_tab[i].ring_off == ring_off)
+			return true;
+		if (userd_off && svc_fb_claim_tab[i].userd_off == userd_off)
+			return true;
+	}
+	/*
+	 * SESSION 144b: reserve every live CE rank's convention block even
+	 * before that rank has latched. r809: CE rank 1's block at fb
+	 * 0xa0000 was still unclaimed when GR rank 0's userd-shape scan ran
+	 * (rank 1 latched through the plain autopin path, not the
+	 * first-submission capture) and the GR channel stole it again.
+	 */
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (k = 0; k < SVC_MAX_CHANNELS; k++) {
+		u64 conv;
+
+		if (!svc.chan[k].valid || svc.chan[k].vgpu != vgpu ||
+		    !svc.chan[k].engine_type ||
+		    svc.chan[k].h_object == self_h ||
+		    svc.chan[k].ce_index >= 32)
+			continue;
+		conv = SVC_CE_RING_FB_BASE +
+		       SVC_CE_RING_FB_STRIDE * svc.chan[k].ce_index;
+		if (ring_off == conv ||
+		    (userd_off && userd_off == conv + 0x10000)) {
+			spin_unlock_irqrestore(&svc.chan_lock, flags);
+			return true;
+		}
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+	return false;
+}
+
+static void svc_fb_claim_reset(void)
+{
+	svc_fb_claim_n = 0;
+	memset(svc_fb_claim_tab, 0, sizeof(svc_fb_claim_tab));
+}
+
+/*
  * ISSUE-0002 (r47): deferred FB sweep for a CE channel whose layout USERD
  * probe fails at discovery. Session/reset-scoped, scheduled on the first
  * failure (+3 s) and re-run once more (+30 s, after the client's follow-up
@@ -13133,6 +13234,7 @@ static void svc_session_flush(void)
 	memset(svc_ce_firstsub_n, 0, sizeof(svc_ce_firstsub_n));
 	memset(svc_ce_hold_since, 0, sizeof(svc_ce_hold_since));
 	memset(svc_ce_firstsub_off, 0, sizeof(svc_ce_firstsub_off));
+	svc_fb_claim_reset();
 	svc_userd_sweep_log = 0;
 	memset(svc_userd_sweep_at, 0, sizeof(svc_userd_sweep_at));
 	memset(svc_userd_sweep_n, 0, sizeof(svc_userd_sweep_n));
@@ -14778,7 +14880,8 @@ static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu,
 				    struct svc_channel *ch, u8 *sbase,
 				    u64 lim, u64 off, u32 *scan_hits,
 				    u64 *latch_off, u64 *latch_upa,
-				    u32 *latch_put, u32 *latch_get)
+				    u32 *latch_put, u32 *latch_get,
+				    bool own_block)
 {
 	static const u64 udeltas[] = { 0x10000, 0x2000, 0x1000, 0x0 };
 	u32 h0, h1, hlen;
@@ -14804,8 +14907,24 @@ static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu,
 				      SVC_APERTURE_FB,
 				      ch->gpfifo_entries,
 				      &p3, &g3);
-		if (prc && prc != -EAGAIN)
-			continue;
+		/*
+		 * SESSION 144d: the rank's OWN convention block (own_block,
+		 * the +0x10000 candidate at the rank's convention offset) is
+		 * latched even when its stale content probes incoherent
+		 * (g > p) or idle - the guest rewrites put/get on its next
+		 * submission and the pump's per-channel last_get processes
+		 * entry0 exactly once. r813: client #2's convention block at
+		 * fb 0x90000 read put=1/get=146 (g > p -> -EINVAL) at the
+		 * capture instant, the coherence gate rejected it, and the
+		 * never-used 0x82000 zero block was latched instead, so the
+		 * client's first push never drained and it hung. The gates
+		 * below stay for every heuristic candidate (other deltas and
+		 * the fallback scan), preserving the SESSION 135 protection.
+		 */
+		if (!own_block || ui != 0) {
+			if (prc && prc != -EAGAIN)
+				continue;
+		}
 		/*
 		 * SESSION 135: at the FIRST-SUBMISSION instant the channel's
 		 * submission is by definition unconsumed - the notifier clear
@@ -14819,7 +14938,20 @@ static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu,
 		 * old session's, the client stalled on its expectations).
 		 * Idle 0/0 stays acceptable (svc_userd_probe's own rule).
 		 */
-		if (p3 && p3 == g3)
+		/*
+		 * SESSION 144c: this consumed-state rejection is for FOREIGN
+		 * blocks found by the heuristic deltas. The rank's OWN
+		 * convention block (+0x10000) legitimately carries the
+		 * previous session's consumed state until the guest
+		 * re-initialises the reused allocation - rejecting it made
+		 * the r812 second client fall through to a never-used
+		 * 0x82000 block and stall forever (cefirst row: off=0x80000
+		 * upa=0x82000 put=0 get=0 while the client waited for the
+		 * first push's completion). The pump's per-channel last_get
+		 * still processes entry0 exactly once, so latching the
+		 * convention block with stale put/get is correct.
+		 */
+		if (!own_block && p3 && p3 == g3)
 			continue;
 		/* latch exactly like the CE autopin */
 		svc_pin_store(ch->gpfifo_va, off, SVC_APERTURE_FB);
@@ -14904,7 +15036,8 @@ static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu,
 			latched = svc_ce_firstsub_try_off(vgpu, ch, sbase,
 							  lim, exp, &scan_hits,
 							  &latch_off, &latch_upa,
-							  &latch_put, &latch_get);
+							  &latch_put, &latch_get,
+							  true);
 			for (off = 0; !latched && off + 0x90 <= lim;
 			     off += 0x100) {
 				unsigned int ri;
@@ -14923,16 +15056,35 @@ static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu,
 				}
 				if (foreign)
 					continue;
+				/*
+				 * SESSION 144: symmetric protection - a CE
+				 * fallback scan must not steal a block another
+				 * channel (CE rank or GR) already claims.
+				 */
+				if (svc_fb_claim_foreign(vgpu, ch->h_object, off,
+							 off + 0x10000))
+					continue;
 				latched = svc_ce_firstsub_try_off(vgpu, ch,
 								  sbase, lim,
 								  off, &scan_hits,
 								  &latch_off,
 								  &latch_upa,
 								  &latch_put,
-								  &latch_get);
+								  &latch_get,
+								  false);
 			}
-			if (latched)
+			if (latched) {
 				svc_ce_firstsub_off[ch->ce_index] = latch_off;
+				/*
+				 * SESSION 144: publish the claim so a GR rank
+				 * whose own ring is not written yet cannot
+				 * latch this CE block (r805-r808: every CUDA
+				 * GR rank stole the CE rank-0 block and the
+				 * launch was never captured).
+				 */
+				svc_fb_claim_add(ch->h_object, latch_off,
+						 latch_upa);
+			}
 		}
 	}
 
@@ -16458,6 +16610,9 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 				ch->userd_as = cap;
 				ch->userd_ok = true;
 				svc_gr_ring_base = cand;
+				if (cap == SVC_APERTURE_FB)
+					svc_fb_claim_add(ch->h_object, cand,
+							 ch->userd_pa);
 				set_bit(ch->gr_index, &svc_gr_ring_done);
 				svc_ring_n++;
 				pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (userd-derived d=0x%llx; entry0 va=0x%llx len=%u put=%u get=%u)\n",
@@ -16497,6 +16652,8 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 			ch->userd_as = SVC_APERTURE_FB;
 			ch->userd_ok = true;
 			svc_gr_ring_base = base;
+			svc_fb_claim_add(ch->h_object, pa,
+					 pa + SVC_GR_USERD_OFF);
 			set_bit(ch->gr_index, &svc_gr_ring_done);
 			svc_ring_n++;
 			pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u put=%u get=%u)\n",
@@ -16549,6 +16706,10 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 				ch->userd_as = svc_ap_of(pin_ap);
 				ch->userd_ok = true;
 				svc_gr_ring_base = base;
+				if (svc_ap_of(pin_ap) == SVC_APERTURE_FB)
+					svc_fb_claim_add(ch->h_object, pin_pa,
+							 pin_pa +
+							 SVC_GR_USERD_OFF);
 				set_bit(ch->gr_index, &svc_gr_ring_done);
 				svc_ring_n++;
 				pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (entry0 va=0x%llx len=%u put=%u get=%u)\n",
@@ -16584,6 +16745,9 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 			ch->userd_as = wap;
 			ch->userd_ok = true;
 			svc_gr_ring_base = wpa;
+			if (wap == SVC_APERTURE_FB)
+				svc_fb_claim_add(ch->h_object, wpa,
+						 wpa + SVC_GR_USERD_OFF);
 			set_bit(ch->gr_index, &svc_gr_ring_done);
 			svc_ring_n++;
 			pr_info("sema-chan: auto-pinned GR ring h=0x%x rank=%u va=0x%llx -> pa=0x%llx ap=%d (trusted walk; entry0 va=0x%llx len=%u put=%u get=%u)\n",
@@ -16673,6 +16837,19 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 						continue;
 					if (svc_ring_entry_dead(vgpu, h0, h1))
 						continue;
+					/*
+					 * SESSION 144: never latch a foreign
+					 * block. Without this every CUDA GR
+					 * rank stole the CE rank-0 block while
+					 * its own ring was unwritten
+					 * (r805-r808); staying unlatched here
+					 * lets the FB-convention path latch the
+					 * real ring once the client writes it.
+					 */
+					if (svc_fb_claim_foreign(vgpu,
+								 ch->h_object,
+								 roff, off))
+						continue;
 					if (svc_userd_probe(vgpu, off,
 							    SVC_APERTURE_FB,
 							    ch->gpfifo_entries,
@@ -16686,6 +16863,7 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 					ch->userd_as = SVC_APERTURE_FB;
 					ch->userd_ok = true;
 					svc_gr_ring_base = roff;
+					svc_fb_claim_add(ch->h_object, roff, off);
 					set_bit(ch->gr_index,
 						&svc_gr_ring_done);
 					set_bit(ch->gr_index,
@@ -16723,6 +16901,10 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 
 					if (upa + 4 > lim)
 						continue;
+					if (svc_fb_claim_foreign(vgpu,
+								 ch->h_object,
+								 off, upa))
+						continue;
 					if (svc_userd_probe(vgpu, upa,
 							    SVC_APERTURE_FB,
 							    ch->gpfifo_entries,
@@ -16736,6 +16918,8 @@ static void svc_gr_ring_autopin(struct nv_vgpu_device *vgpu,
 					ch->userd_as = SVC_APERTURE_FB;
 					ch->userd_ok = true;
 					svc_gr_ring_base = off;
+					svc_fb_claim_add(ch->h_object, off,
+							 upa);
 					set_bit(ch->gr_index,
 						&svc_gr_ring_done);
 					set_bit(ch->gr_index,
@@ -16960,6 +17144,13 @@ static void svc_ring_autopin(struct nv_vgpu_device *vgpu,
 
 	svc_pin_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
 	svc_lat_store(ch->gpfifo_va, pa, SVC_APERTURE_FB);
+	/*
+	 * SESSION 144b: record the block ownership here too - CE ranks that
+	 * latch through this plain autopin path (not the first-submission
+	 * capture) were invisible to the GR fallback scan, which then stole
+	 * their block (r809: CE rank 1 at fb 0xa0000).
+	 */
+	svc_fb_claim_add(ch->h_object, pa, pa + 0x10000);
 	set_bit(ch->ce_index, &svc_ring_done);
 	svc_ring_n++;
 	pr_info("sema-chan: auto-pinned ring h=0x%x rank=%u va=0x%llx -> fb 0x%llx (entry0 va=0x%llx len=%u)\n",