vgsp-sema: author the UVM init-push release at the channel's own rank slot

r1645 live proof fixes the channel-pool slot mapping: UVM allocates one
tracking semaphore per channel in channel-creation order, so channel rank N
owns pool slot N, and channel_init() waits for payload 1 at that slot. The
r1642 rank+1 fallback pre-satisfied the next rank before its
uvm_gpu_semaphore_alloc() zeroed the slot; the value was lost and the guest
spun in uvm_push_end_and_wait (r1644/r1645: manual write of 1 to the rank-1
slot cleared the spin).

- pool-tracker apply now targets slot = ce_index and fans the value out to
  the secondary pool-page candidate, which flaps between the RM memdesc and
  the guest's VAS walk (r1645: the guest polled the walk page while the boot
  cache named a stale, reused page).
- first-submission authoring no longer requires an unlatched ring. A latch
  with put=get=0 still needs the release (rank 5, r1646), so every UVM CE
  pool channel authors its own rank slot once when the work-submit notifier
  clears; one-shot per rank, forward-only, idempotent.
- widen the UVM CE GPFIFO window filter to 0x121800000.

Evidence: backups/evidence-20261002-victory/r1645 (channels advanced once the
rank slot was written) and r1646 (ranks 0-4 green; rank 5 exposed the
latched-but-undecoded gap).

1 files changed, 268 insertions, 10 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index ca848c6..090bf5b 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -955,6 +955,21 @@ static u32 svc_sess_gen;	/* m176: bumped by every executed session flush;
 				 * state is never inherited by a recycled handle */
 static struct completion svc_sess_flush_done;	/* a pending flush has executed */
 static unsigned long svc_ring_done;	/* CE ranks whose ring is pinned (session state) */
+/*
+ * r1644: one-shot authoring of the UVM channel-pool init-push tracker
+ * release, keyed by CE rank. A UVM CE channel's first submission releases
+ * its own tracking semaphore -- pool slot rank (channel-creation order) in
+ * the page mapped at 0x121000000 -- with payload 1, and channel_init()
+ * waits on that value.
+ * The GPFIFO entry is regularly invisible at the registered VA/PA at the
+ * instant the guest waits for it (r1644: rank 1 read zero at 0x22b0000 and
+ * only later translated to 0x2640000, after the notifier clear), so the
+ * release is authored from the rank + pool mapping when the work-submit
+ * notifier edge proves the submission exists. Session state; re-armed per
+ * channel registration and cleared by svc_session_flush.
+ */
+static unsigned long svc_ce_initrel_done;
+static u32 svc_ce_initrel_log;
 
 /* Per-guest session records (one per open mdev; see SVC_MAX_CHANNELS for the channel cap). */
 #define SVC_MAX_GUESTS		8
@@ -2002,6 +2017,7 @@ static void svc_chan_rearm_rank(struct svc_channel *ch)
 	if (ch->engine_type) {
 		clear_bit(ch->ce_index, &svc_ring_done);
 		clear_bit(ch->ce_index, &svc_pb_done);
+		clear_bit(ch->ce_index, &svc_ce_initrel_done);
 	} else {
 		clear_bit(ch->gr_index, &svc_gr_ring_done);
 		/*
@@ -5569,13 +5585,18 @@ static bool svc_gr_rel_match(u64 va, u32 *payload);
 static bool svc_gr_rel_inflight(u64 va, u32 *payload);
 
 /*
- * r1642: UVM's channel-pool semaphore map reserves slot 0 for the pool
- * release; each channel's own tracking semaphore lives at map_base +
- * 4*(rank+1).  Gdbstub proof on the guest: the rank-0 UVM channel's
- * uvm_gpu_tracking_semaphore polled map_base+4 while its push released
- * map_base+0, and writing the payload into map_base+4 cleared the
- * uvm_push_end_and_wait spin.  When a CE channel releases the pool slot,
- * apply the same forward-only value to its tracker slot as well.
+ * r1645: UVM allocates one tracking semaphore per channel from the GPU
+ * semaphore pool, in channel-creation order: channel rank N owns pool slot
+ * N (slot 0 is rank 0's semaphore, not a reserved pool slot).  The r1642
+ * gdbstub observation ("push released map_base+0 while the tracker polled
+ * map_base+4") was rank 0 already satisfied and rank 1 waiting: writing
+ * rank+1 pre-satisfied the NEXT rank before its uvm_gpu_semaphore_alloc()
+ * zeroed the slot, so the value was lost and rank 1 stalled.  r1645 live
+ * proof: the host's rank+1 write to slot 1 raced rank 1's allocation, the
+ * guest read 0, and a manual write of 1 to slot 1 (rank 1's own slot)
+ * cleared uvm_push_end_and_wait.  A CE channel's release therefore belongs
+ * at its own rank slot; apply the same forward-only value there as a
+ * fallback when the decoded release address was masked to the page base.
  */
 #define SVC_UVM_POOL_MAP_BASE	0x121000000ULL
 static void svc_pool_tracker_apply(struct nv_vgpu_device *vgpu, u32 h,
@@ -5596,8 +5617,7 @@ static void svc_pool_tracker_apply(struct nv_vgpu_device *vgpu, u32 h,
 			break;
 	if (i == SVC_MAX_CHANNELS)
 		return;
-	tva = SVC_UVM_POOL_MAP_BASE + 4ULL *
-	      ((u64)svc.chan[i].ce_index + 1);
+	tva = SVC_UVM_POOL_MAP_BASE + 4ULL * (u64)svc.chan[i].ce_index;
 	if (tva == va)
 		return;
 	if (svc_pool_pa_resolve(vgpu, h, tva, &addr, &ap, &src))
@@ -5611,6 +5631,11 @@ static void svc_pool_tracker_apply(struct nv_vgpu_device *vgpu, u32 h,
 	if (svc_read_ap(vgpu, addr, svc_ap_of(ap), &back, sizeof(back)) ||
 	    back != val)
 		return;
+	/* The pool page can flap between the RM memdesc and the VAS walk
+	 * (r1645: the guest polled the walk page while the boot cache named
+	 * a stale, reused page). Fan the tracker value out to the secondary
+	 * candidate as well, exactly like the decoded release path. */
+	svc_pool_boot_fanout(vgpu, h, tva, val);
 	if (svc.pool_tracker_log < 64) {
 		svc.pool_tracker_log++;
 		pr_info("sema-svc: pool-tracker-wr ch=0x%x rank=%u va=0x%llx pa=0x%llx val=%u\n",
@@ -22504,6 +22529,8 @@ static void svc_session_flush(void)
 	memset(svc_ce_firstsub_n, 0, sizeof(svc_ce_firstsub_n));
 	memset(svc_ce_hold_since, 0, sizeof(svc_ce_hold_since));
 	memset(svc_ce_firstsub_off, 0, sizeof(svc_ce_firstsub_off));
+	svc_ce_initrel_done = 0;
+	svc_ce_initrel_log = 0;
 	svc_fb_claim_reset();
 	svc_userd_sweep_log = 0;
 	memset(svc_userd_sweep_at, 0, sizeof(svc_userd_sweep_at));
@@ -25202,6 +25229,223 @@ static bool svc_ce_firstsub_try_off(struct nv_vgpu_device *vgpu,
 	return false;
 }
 
+/*
+ * r1644: only the guest's UVM channel pool polls the semaphore page at
+ * 0x121000000 -- its CE rings are registered at 0x121010000 + 0x30000*rank
+ * (the CE GPFIFO window). Other CE channels (the RC
+ * watchdog at 0x120000400, the CeUtils scrubber at 0x120064000, ...) never
+ * wait on a pool slot, so authoring their rank slot would corrupt UVM's
+ * tracker words. Keep the fallback to the pool's own ring window.
+ */
+static bool svc_ce_uvm_pool_channel(const struct svc_channel *ch)
+{
+	return ch && ch->engine_type && ch->ce_index < 32 &&
+	       ch->gpfifo_va >= 0x121000000ULL &&
+	       ch->gpfifo_va < 0x121800000ULL;
+}
+
+/* Decode one GPFIFO entry as a CE push: 1 MB-aligned VA in the CE window. */
+static bool svc_ce_push_entry(const u8 *e, u64 *va, u32 *len)
+{
+	u32 e0, e1;
+
+	memcpy(&e0, e, 4);
+	memcpy(&e1, e + 4, 4);
+	*va = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+	*len = ((e1 >> 10) & 0x1fffff) << 2;
+	return !(*va & 0xfffff) && *va >= 0x120000000ULL &&
+	       *va < 0x121000000ULL && *len >= 4 && *len <= 0x10000;
+}
+
+/*
+ * r1644: resolve a CE channel's init GPFIFO entry without relying on the
+ * ring VA's translation alone. Candidates, in order:
+ *   1. the client's own page tables (walk_vaspaces / walk_discovered) for
+ *      the registered GPFIFO VA: the learned/pin layers can answer with a
+ *      stale page at the first submission (r1644: the cached answer was
+ *      0x22b0000 while the VAS walk named 0x2640000), and the walks are the
+ *      ground truth for the VA the guest actually submits through;
+ *   2. the registered GPFIFO VA through the trusted resolver (the historical
+ *      source; may answer a zero page when the guest's BAR1 ring writes and
+ *      its GMMU mapping disagree, the SESSION 134 class);
+ *   3. the allocation's own USERD memdesc minus the CE layout deltas
+ *      (0x10000 is the observed ring-to-USERD distance; 0x2000/0x1000/0
+ *      cover the combined-allocation layouts). The USERD is the guest's
+ *      declaration, so this is the one source that uses alloc data.
+ * Entry 0 is tried first (channel_init is a pristine channel's first push);
+ * when the registered USERD names a put, the newest claimed entry (put-1)
+ * is tried as well. A hit is evidence for the log and names the exact
+ * entry/ring; the release itself is authored from the rank (the entry read
+ * is not a latch gate).
+ */
+static bool svc_ce_init_entry(struct nv_vgpu_device *vgpu,
+			      struct svc_channel *ch, u32 put,
+			      u64 *ring_pa, int *ring_ap,
+			      u64 *entry_va, u32 *entry_len,
+			      u32 *entry_idx)
+{
+	static const u64 udeltas[] = { 0x10000, 0x2000, 0x1000, 0x0 };
+	u64 cand[ARRAY_SIZE(udeltas) + 1];
+	u32 as[ARRAY_SIZE(udeltas) + 1];
+	u32 nc = 0, i, k;
+	u64 va = 0;
+	u32 len = 0;
+	u8 e[SVC_GPFIFO_ENTRY_SIZE];
+	u64 wpa = 0;
+	int wap = SVC_APERTURE_SYS;
+	u64 vpage = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1);
+
+	if (!svc_walk_vaspaces(vgpu, vpage, &wpa, &wap) && wpa) {
+		cand[nc] = wpa + (ch->gpfifo_va & (SEMA_PAGE_SIZE - 1));
+		as[nc++] = (u32)wap;
+	}
+	wpa = 0;
+	wap = SVC_APERTURE_SYS;
+	if (!svc_walk_discovered(vgpu, vpage, &wpa, &wap) && wpa) {
+		cand[nc] = wpa + (ch->gpfifo_va & (SEMA_PAGE_SIZE - 1));
+		as[nc++] = (u32)wap;
+	}
+	if (!svc_lookup(vgpu, ch->gpfifo_va, ring_pa, ring_ap) && *ring_pa) {
+		cand[nc] = *ring_pa;
+		as[nc++] = (u32)*ring_ap;
+	}
+	if (ch->userd_pa) {
+		for (i = 0; i < ARRAY_SIZE(udeltas); i++) {
+			if (ch->userd_pa < udeltas[i])
+				continue;
+			cand[nc] = ch->userd_pa - udeltas[i];
+			as[nc++] = ch->userd_as;
+		}
+	}
+	for (i = 0; i < nc; i++) {
+		u32 idxs[2];
+		u32 nidx = 0;
+
+		idxs[nidx++] = 0;
+		if (put && put <= ch->gpfifo_entries) {
+			k = (put - 1) % ch->gpfifo_entries;
+			if (k)
+				idxs[nidx++] = k;
+		}
+		for (k = 0; k < nidx; k++) {
+			if (svc_read_ap(vgpu, cand[i] +
+					(u64)idxs[k] * SVC_GPFIFO_ENTRY_SIZE,
+					as[i], e, sizeof(e)))
+				continue;
+			if (!svc_ce_push_entry(e, &va, &len))
+				continue;
+			*ring_pa = cand[i];
+			*ring_ap = (int)as[i];
+			*entry_va = va;
+			*entry_len = len;
+			*entry_idx = idxs[k];
+			return true;
+		}
+	}
+	return false;
+}
+
+/*
+ * r1644: author the UVM init push's tracker release for a CE channel whose
+ * ring/USERD discovery failed. The work-submit notifier edge (or a put read
+ * from the allocation's USERD) is the submission evidence; the release is
+ * then the channel's own pool slot (rank) with payload 1, exactly what
+ * channel_init() waits for. svc_sem_write() runs the pool resolver with its
+ * pin / boot-cache / walk candidates and the dual-page fan-out, and queues
+ * the release for the retry pass when no translation exists yet, so the
+ * write reaches the page the guest polls (observed 0x110b44000) even when
+ * the FB candidate (0x2600000 family) is the one the translations name.
+ * One-shot per CE rank per session. Diagnostic names rank, USERD PA/AP and
+ * put on both the found and not-found paths.
+ */
+static void svc_ce_init_rel_author(struct nv_vgpu_device *vgpu,
+				   struct svc_channel *ch, bool submitted,
+				   const char *why)
+{
+	u64 tva, ring_pa = 0, entry_va = 0, upa;
+	int ring_ap = SVC_APERTURE_SYS;
+	u32 entry_len = 0, entry_idx = 0;
+	u32 put = 0, get = 0, uas;
+	int prc = -1;
+	bool put_ok = false, found;
+	int wrc;
+
+	/*
+	 * r1645: do not gate on ch->userd_ok. A latched/known USERD does not
+	 * imply the init push was decoded (rank 5 latched with put=get=0);
+	 * the one-shot rank bit is the only duplicate guard needed.
+	 */
+	if (!svc_ce_uvm_pool_channel(ch))
+		return;
+	if (test_bit(ch->ce_index, &svc_ce_initrel_done))
+		return;
+
+	upa = ch->userd_pa;
+	uas = ch->userd_as;
+	/*
+	 * The allocation's USERD is the guest's own declaration of the
+	 * channel's GP_PUT block. It can be pool-relative on some boots (the
+	 * r46 lesson), so its put is evidence, never a latch gate.
+	 */
+	if (upa && !svc_read_ap(vgpu, upa + SVC_USERD_GP_PUT, uas,
+				&put, sizeof(put))) {
+		(void)svc_read_ap(vgpu, upa + SVC_USERD_GP_GET, uas,
+				  &get, sizeof(get));
+		if (put <= ch->gpfifo_entries)
+			put_ok = true;
+	}
+	if (!put_ok) {
+		/* The CE layout keeps the control block at ring + 0x10000. */
+		u64 rpa = 0;
+		int rap = SVC_APERTURE_SYS;
+
+		if (!svc_lookup(vgpu, ch->gpfifo_va, &rpa, &rap) && rpa) {
+			prc = svc_userd_probe(vgpu, rpa + 0x10000, rap,
+					      ch->gpfifo_entries, &put, &get);
+			if (prc != -ENOENT && prc != -EINVAL) {
+				upa = rpa + 0x10000;
+				uas = (u32)rap;
+				put_ok = true;
+			}
+		}
+	}
+	if (!submitted && put_ok && put)
+		submitted = true;
+
+	found = svc_ce_init_entry(vgpu, ch, put_ok ? put : 0, &ring_pa,
+				  &ring_ap, &entry_va, &entry_len, &entry_idx);
+
+	if (svc_ce_initrel_log < 64) {
+		svc_ce_initrel_log++;
+		if (found)
+			pr_info("sema-ceinit: FOUND h=0x%x rank=%u gpfifo=0x%llx userd=0x%llx/%u put=%u get=%u ring=0x%llx/%d entry=%u va=0x%llx len=%u (%s)\n",
+				ch->h_object, ch->ce_index, ch->gpfifo_va,
+				upa, uas, put, get, ring_pa, ring_ap,
+				entry_idx, entry_va, entry_len, why);
+		else
+			pr_info("sema-ceinit: NOTFOUND h=0x%x rank=%u gpfifo=0x%llx userd=0x%llx/%u put=%u get=%u (%s%s)\n",
+				ch->h_object, ch->ce_index, ch->gpfifo_va,
+				upa, uas, put, get, why,
+				submitted ? "" : ", no submission edge");
+	}
+	if (!submitted)
+		return;
+	if (test_and_set_bit(ch->ce_index, &svc_ce_initrel_done))
+		return;
+
+	/*
+	 * UVM's per-channel tracking semaphore is pool slot rank (allocation
+	 * order); the init push's value is 1. Forward-only and idempotent: a
+	 * slot that already reads >= 1 is left alone by the applier.
+	 */
+	tva = SVC_UVM_POOL_MAP_BASE + 4ULL * (u64)ch->ce_index;
+	svc.cur_ch = ch->h_object;
+	wrc = svc_sem_write(vgpu, ch->h_object, tva, 1);
+	pr_info("sema-ceinit: h=0x%x rank=%u tracker=0x%llx val=1 %s (%s; init push release)\n",
+		ch->h_object, ch->ce_index, tva,
+		wrc == 0 ? "ok" : (wrc == -ENOENT ? "queued" : "failed"), why);
+}
+
 static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu,
 				    struct svc_channel *ch)
 {
@@ -25329,10 +25573,21 @@ static void svc_ce_firstsub_capture(struct nv_vgpu_device *vgpu,
 			scan_hits, latched, latch_off, latch_upa,
 			latch_put, latch_get);
 	}
-	if (latched)
+	if (latched) {
 		pr_info("sema-cefirst: LATCHED h=0x%x rank=%u va=0x%llx -> fb 0x%llx (userd 0x%llx put=%u get=%u; first-submission capture)\n",
 			ch->h_object, ch->ce_index, ch->gpfifo_va,
 			latch_off, latch_upa, latch_put, latch_get);
+	}
+	/*
+	 * r1645: a latch is not a release. Rank 5 latched its fb ring
+	 * (0xa50000) while put/get stayed 0 and the pinned push buffer never
+	 * produced a decode, so the guest spun on slot 5 forever. The
+	 * first-submission notifier edge proves the guest submitted the init
+	 * push, so author the rank-slot release for every UVM pool channel,
+	 * latched or not. One-shot per rank and forward-only, so the normal
+	 * decode path is unaffected when it does land.
+	 */
+	svc_ce_init_rel_author(vgpu, ch, true, "first-submission");
 }
 
 /*
@@ -39963,6 +40218,9 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				 * Treat as stalled so the content sweep looks for
 				 * the releases in RAM.
 				 */
+				if (ch->engine_type && ch->notified)
+					svc_ce_init_rel_author(vgpu, ch, true,
+						"userd-miss");
 				stalled = true;
 				continue;
 			}