vgsp: TSG-sibling mode 30, UVM-stall Patch C, QMD shadow-window ensure + launch diagnostics

1 files changed, 147 insertions, 33 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 83ca6bd..0605692 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -275,6 +275,13 @@ struct svc_channel {
 				 * counter page can land a moment after the
 				 * first push decode); flushed as soon as the
 				 * slot is registered. 0 = none */
+	u32	pending_rel_inc;	/* 1 = the queued release is an increment
+				 * (add payload to the slot), 0 = absolute */
+	u32	relearn_n;	/* counter-slot re-registrations seen for this
+				 * channel: a healthy GR setup re-learns each
+				 * slot a bounded number of times, while a
+				 * clobbered slot loops forever (storm
+				 * detector) */
 };
 
 /*
@@ -516,6 +523,9 @@ struct sema_svc {
 						 * deliverable exact slot */
 	u64			counter_rel_applied;	/* releases to registered counters */
 	u64			counter_rel_dbg;	/* rel-apply diagnostic budget */
+	u64			counter_noslot_dbg;	/* "no slot registered" budget,
+						 * separate so it cannot hide the
+						 * written/skip lines */
 	u64			counter_pub_dbg;	/* counter-publish diagnostic budget */
 	u64			counter_pub_calls;	/* publishes that found a registered counter */
 	u64			gr_rel_applied;	/* release writes into the GR counter page */
@@ -4238,14 +4248,20 @@ static inline bool svc_counter_va(u64 va)
  * through the sema_counter parameter from the client's own pagemap. Writing
  * the payload there is what lets the client's wait observe the release
  * regardless of the VA map. Forward-only, so a stale copy of an
- * already-completed push cannot rewind the counter.
+ * already-completed push cannot rewind the counter. `inc` selects UVM's
+ * increment-release semantics (write cur + payload) instead of the absolute
+ * semantics (write payload); GR tracking-semaphore pushes release an
+ * increment, and an absolute-only guard drops every increment after the
+ * first one because payload == cur.
  */
 static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
-				      u64 sem_va, u32 payload)
+				      u64 sem_va, u32 payload, bool inc)
 {
 	unsigned long flags;
 	u64 pa = 0;
 	u32 cur = 0;
+	u32 nv;
+	bool skip;
 	int i;
 	u32 dbg = 0;
 
@@ -4284,9 +4300,12 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 				if (svc.chan[i].valid &&
 				    svc.chan[i].h_object == h) {
 					if (payload >
-					    svc.chan[i].pending_rel_pay)
+					    svc.chan[i].pending_rel_pay) {
 						svc.chan[i].pending_rel_pay =
 							payload;
+						svc.chan[i].pending_rel_inc =
+							inc ? 1 : 0;
+					}
 					break;
 				}
 			}
@@ -4296,8 +4315,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 				pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> slot pending\n",
 					h, sem_va, payload);
 			}
-		} else if (gsp_sema_dbg && svc.counter_rel_dbg < 96) {
-			svc.counter_rel_dbg++;
+		} else if (gsp_sema_dbg && svc.counter_noslot_dbg < 16) {
+			svc.counter_noslot_dbg++;
 			pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> no slot registered\n",
 				h, sem_va, payload);
 		}
@@ -4312,7 +4331,17 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 		}
 		return;
 	}
-	if (payload <= cur || payload - cur > 0x100000) {
+	/*
+	 * Absolute releases must be forward-only (a stale copy of a completed
+	 * push must not rewind the counter); increment releases add the
+	 * payload, like the generic LAUNCH path does. The jump guard keeps a
+	 * bogus payload from teleporting the counter past the client's wait.
+	 */
+	if (inc)
+		skip = payload > 0x100000;
+	else
+		skip = payload <= cur || payload - cur > 0x100000;
+	if (skip) {
 		if (gsp_sema_dbg && svc.counter_rel_dbg < 32) {
 			svc.counter_rel_dbg++;
 			pr_info("sema-chan: rel-apply h=0x%x pa=0x%llx pay=%u cur=%u -> skip\n",
@@ -4320,8 +4349,8 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 		}
 		return;
 	}
-	if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &payload,
-			 sizeof(payload)))
+	nv = inc ? cur + payload : payload;
+	if (svc_write_ap(vgpu, pa, SVC_APERTURE_SYS, &nv, sizeof(nv)))
 		return;
 	svc.releases_applied++;
 	svc.counter_rel_applied++;
@@ -4336,34 +4365,74 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 }
 
 /*
+ * Clear a channel's queued release in the authoritative array. The caller of
+ * svc_counter_flush_pending() usually works on a pass snapshot, so clearing
+ * only the snapshot would let the next pass flush the same payload again -
+ * harmless for an absolute release (its guard drops it), but an increment
+ * would advance the slot twice.
+ */
+static void svc_counter_pending_clear(u32 h)
+{
+	unsigned long flags;
+	int i;
+
+	if (!h)
+		return;
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
+			svc.chan[i].pending_rel_pay = 0;
+			svc.chan[i].pending_rel_inc = 0;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+}
+
+/*
  * Deliver a release that was decoded before the channel's counter slot was
  * registered (see pending_rel_pay). Called whenever a slot becomes known and
  * from the per-pass publish path, so a queued payload lands as soon as the
- * injected pin is visible to the service.
+ * injected pin is visible to the service. An increment-queued payload adds to
+ * the slot (UVM INC semantics), an absolute one writes its value.
  */
 static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
 				      struct svc_channel *ch)
 {
 	u32 payload = ch->pending_rel_pay;
+	bool inc = ch->pending_rel_inc;
 	u32 cur = 0;
+	u32 nv;
 
 	if (!payload || !ch->counter_pa)
 		return;
 	if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur,
 			sizeof(cur)))
 		return;
-	if (payload <= cur || payload - cur > 0x100000) {
+	if (inc) {
+		if (payload > 0x100000) {
+			ch->pending_rel_pay = 0;
+			ch->pending_rel_inc = 0;
+			svc_counter_pending_clear(ch->h_object);
+			return;
+		}
+	} else if (payload <= cur || payload - cur > 0x100000) {
 		ch->pending_rel_pay = 0;
+		ch->pending_rel_inc = 0;
+		svc_counter_pending_clear(ch->h_object);
 		if (cur > ch->counter_last)
 			ch->counter_last = cur;
 		return;
 	}
-	if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &payload,
-			 sizeof(payload)))
+	nv = inc ? cur + payload : payload;
+	if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &nv,
+			 sizeof(nv)))
 		return;
 	ch->pending_rel_pay = 0;
-	if (payload > ch->counter_last)
-		ch->counter_last = payload;
+	ch->pending_rel_inc = 0;
+	svc_counter_pending_clear(ch->h_object);
+	if (nv > ch->counter_last)
+		ch->counter_last = nv;
 	svc.releases_applied++;
 	svc.counter_rel_applied++;
 	svc.gr_rel_applied++;
@@ -4494,33 +4563,61 @@ static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
 			return;
 		derived = true;
 	}
-	ch->counter_pa = pa & ~0x3ULL;
+	pa &= ~0x3ULL;
 	/*
-	 * Persist the registration into the authoritative channel array
-	 * immediately, under the lock, instead of relying on the pump's
-	 * end-of-pass write-back: the pump works on a snapshot and several
-	 * early `continue`s (USERD not yet resolvable, ring not yet
-	 * readable) skip that write-back. When that happened every pass
-	 * re-registered the same slot - the "counter auto" line flooded the
-	 * log ring (1.17M lines, noclaim27) and, worse, the publish path
-	 * never saw a registered counter (`pub_calls=0`), so the client's
-	 * tracker semaphores were never advanced and cudaMalloc hung. The
-	 * authoritative store makes the registration durable and the log
-	 * one-shot.
+	 * Never install a slot another live channel already owns: the
+	 * rank-derived formula is evaluated while channels are still
+	 * registering and can name a slot a lower rank already registered
+	 * (observed: rank3 -> rank0's slot ...ec0). The collision makes every
+	 * pass flip the slot (svc_counter_learn writes the exact slot, the
+	 * pass write-back restores the guess) and the client's tracker never
+	 * completes; leave counter_pa zero for a later pass instead.
 	 */
 	{
 		unsigned long flags;
 		int j;
+		bool collision = false;
 
 		spin_lock_irqsave(&svc.chan_lock, flags);
 		for (j = 0; j < SVC_MAX_CHANNELS; j++) {
 			if (svc.chan[j].valid &&
-			    svc.chan[j].h_object == ch->h_object) {
-				svc.chan[j].counter_pa = ch->counter_pa;
+			    svc.chan[j].h_object != ch->h_object &&
+			    svc.chan[j].counter_pa == pa) {
+				collision = true;
 				break;
 			}
 		}
+		if (!collision) {
+			/*
+			 * Persist the registration into the authoritative
+			 * channel array immediately, under the lock, instead
+			 * of relying on the pump's end-of-pass write-back:
+			 * the pump works on a snapshot and several early
+			 * `continue`s (USERD not yet resolvable, ring not
+			 * yet readable) skip that write-back. When that
+			 * happened every pass re-registered the same slot
+			 * (1.17M log lines, noclaim27) and, worse, the
+			 * publish path never saw a registered counter
+			 * (`pub_calls=0`), so the client's tracker
+			 * semaphores were never advanced and cudaMalloc
+			 * hung. The authoritative store makes the
+			 * registration durable and the log one-shot.
+			 */
+			ch->counter_pa = pa;
+			for (j = 0; j < SVC_MAX_CHANNELS; j++) {
+				if (svc.chan[j].valid &&
+				    svc.chan[j].h_object == ch->h_object) {
+					svc.chan[j].counter_pa = pa;
+					break;
+				}
+			}
+		}
 		spin_unlock_irqrestore(&svc.chan_lock, flags);
+		if (collision) {
+			pr_info_ratelimited("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx rejected (slot owned by another channel; not installed)\n",
+					    ch->h_object, ch->gr_index, pa);
+			return;
+		}
 	}
 	if (derived)
 		pr_info("sema-chan: counter derived h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
@@ -4570,6 +4667,11 @@ static void svc_counter_learn(struct nv_vgpu_device *vgpu, u32 h, u64 sem_va)
 		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
 			if (svc.chan[i].counter_pa != (pa & ~0x3ULL)) {
 				svc.chan[i].counter_pa = pa & ~0x3ULL;
+				svc.chan[i].relearn_n++;
+				if (svc.chan[i].relearn_n == 64)
+					pr_warn_ratelimited("sema-chan: counter re-learn storm h=0x%x n=%u (slot clobbered; UVM will stall)\n",
+							    h,
+							    svc.chan[i].relearn_n);
 				if (derived)
 					pr_info("sema-chan: counter derived h=0x%x pa=0x%llx (from release va=0x%llx)\n",
 						h, pa & ~0x3ULL, sem_va);
@@ -4600,7 +4702,8 @@ static bool svc_rel_apply_906f(struct nv_vgpu_device *vgpu,
 
 	if (ch->h_object && svc_counter_va(va)) {
 		svc_counter_learn(vgpu, ch->h_object, va);
-		svc_counter_release_apply(vgpu, ch->h_object, va, payload);
+		svc_counter_release_apply(vgpu, ch->h_object, va, payload,
+					  op == HOST_SEM_OP_REDUCTION);
 		return false;
 	}
 	if (!svc_sem_read(vgpu, va, &cur) && payload > cur) {
@@ -4623,7 +4726,8 @@ static bool svc_rel_apply_reportsem(struct nv_vgpu_device *vgpu,
 
 	if (ch->h_object && svc_counter_va(va)) {
 		svc_counter_learn(vgpu, ch->h_object, va);
-		svc_counter_release_apply(vgpu, ch->h_object, va, payload);
+		svc_counter_release_apply(vgpu, ch->h_object, va, payload,
+					  false);
 		return false;
 	}
 	if (svc_sem_read(vgpu, va, &cur)) {
@@ -4668,7 +4772,8 @@ static bool svc_rel_apply_launch(struct nv_vgpu_device *vgpu,
 	}
 	if (ch->engine_type == 0)
 		svc_counter_learn(vgpu, ch->h_object, sem_va);
-	svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload);
+	svc_counter_release_apply(vgpu, ch->h_object, sem_va, payload,
+				  inc && inc_apply);
 	return false;
 }
 
@@ -11226,9 +11331,18 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				 * snapshot: without this the next pass re-derives
 				 * it (with a different live GR count during
 				 * teardown) and the publishes target the wrong
-				 * slot.
+				 * slot. Never let the pass-start snapshot revert
+				 * a slot learned mid-pass (svc_counter_learn) or
+				 * registered from the injected pin: the snapshot
+				 * may still hold the rank-derived autoreg guess
+				 * (or 0), which made the registration
+				 * non-durable (24,834 re-learn lines in one
+				 * stalled run vs <=35 per channel in passing
+				 * runs, 2026-10-06). Only fill a missing slot; a
+				 * live authoritative value wins.
 				 */
-				svc.chan[k].counter_pa = ch->counter_pa;
+				if (!svc.chan[k].counter_pa)
+					svc.chan[k].counter_pa = ch->counter_pa;
 				break;
 			}
 		}