sema-svc: route 906f/report-semaphore releases to the exact GR counter slot (all 20 channel releases now land), queue payloads that arrive before the slot pin

1 files changed, 136 insertions, 10 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 2874cc1..9a765c8 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -151,6 +151,11 @@ struct svc_channel {
 				 * (registered via the sema_counter debug param);
 				 * 0 = not registered */
 	u32	counter_last;	/* last value published to counter_pa */
+	u32	pending_rel_pay;	/* release payload decoded while counter_pa
+				 * was still unknown (the injected pin for the
+				 * counter page can land a moment after the
+				 * first push decode); flushed as soon as the
+				 * slot is registered. 0 = none */
 };
 
 /*
@@ -2088,6 +2093,20 @@ static bool svc_method_stream_plausible(const u8 *buf, u32 len)
 static u64 svc_rel_va_seen;
 static u32 svc_dbg_streams;
 
+static void svc_counter_learn(u32 h, u64 sem_va);
+
+/* GR per-channel completion counters: one 16-byte-strided u32 slot per GR
+ * channel in a client semaphore page. Releases to this range must be written
+ * through the injected-pin-backed exact slot, never through the general
+ * content-scanned VA map (which holds stale translations for it). */
+#define SVC_COUNTER_VA_LO	0x204000000ULL
+#define SVC_COUNTER_VA_HI	0x204100000ULL
+
+static inline bool svc_counter_va(u64 va)
+{
+	return va >= SVC_COUNTER_VA_LO && va < SVC_COUNTER_VA_HI;
+}
+
 /*
  * Apply a decoded release directly to a channel's registered completion
  * counter slot.
@@ -2103,7 +2122,7 @@ static u32 svc_dbg_streams;
  * already-completed push cannot rewind the counter.
  */
 static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
-				      u32 payload)
+				      u64 sem_va, u32 payload)
 {
 	unsigned long flags;
 	u64 pa = 0;
@@ -2114,6 +2133,15 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 	if (!h || !payload)
 		return;
 
+	/*
+	 * The injected pin for the counter page can land a moment after the
+	 * first push of a channel is decoded, so the slot may still be
+	 * unknown here even though the release VA names it exactly. Resolve
+	 * it from the pin table first.
+	 */
+	if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL)
+		svc_counter_learn(h, sem_va);
+
 	spin_lock_irqsave(&svc.chan_lock, flags);
 	for (i = 0; i < SVC_MAX_CHANNELS; i++) {
 		if (svc.chan[i].valid && svc.chan[i].h_object == h) {
@@ -2123,10 +2151,36 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 	}
 	spin_unlock_irqrestore(&svc.chan_lock, flags);
 	if (!pa) {
-		if (svc.counter_rel_dbg < 32) {
+		/*
+		 * Still no slot: remember the payload so the slot
+		 * registration (learn/auto/publish) flushes it. Dropping it
+		 * leaves the client's tracker polling a value that never
+		 * arrives and its context-init wait expires. Only counter-page
+		 * releases are queued here; pool releases (CE channels) have
+		 * no counter slot and are delivered by the pool/fake paths.
+		 */
+		if (sem_va >= 0x204000000ULL && sem_va < 0x204100000ULL) {
+			spin_lock_irqsave(&svc.chan_lock, flags);
+			for (i = 0; i < SVC_MAX_CHANNELS; i++) {
+				if (svc.chan[i].valid &&
+				    svc.chan[i].h_object == h) {
+					if (payload >
+					    svc.chan[i].pending_rel_pay)
+						svc.chan[i].pending_rel_pay =
+							payload;
+					break;
+				}
+			}
+			spin_unlock_irqrestore(&svc.chan_lock, flags);
+			if (svc.counter_rel_dbg < 96) {
+				svc.counter_rel_dbg++;
+				pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> slot pending\n",
+					h, sem_va, payload);
+			}
+		} else if (svc.counter_rel_dbg < 96) {
 			svc.counter_rel_dbg++;
-			pr_info("sema-chan: rel-apply h=0x%x pay=%u -> no slot registered\n",
-				h, payload);
+			pr_info("sema-chan: rel-apply h=0x%x va=0x%llx pay=%u -> no slot registered\n",
+				h, sem_va, payload);
 		}
 		return;
 	}
@@ -2163,6 +2217,46 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
 }
 
 /*
+ * Deliver a release that was decoded before the channel's counter slot was
+ * registered (see pending_rel_pay). Called whenever a slot becomes known and
+ * from the per-pass publish path, so a queued payload lands as soon as the
+ * injected pin is visible to the service.
+ */
+static void svc_counter_flush_pending(struct nv_vgpu_device *vgpu,
+				      struct svc_channel *ch)
+{
+	u32 payload = ch->pending_rel_pay;
+	u32 cur = 0;
+
+	if (!payload || !ch->counter_pa)
+		return;
+	if (svc_read_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &cur,
+			sizeof(cur)))
+		return;
+	if (payload <= cur || payload - cur > 0x100000) {
+		ch->pending_rel_pay = 0;
+		if (cur > ch->counter_last)
+			ch->counter_last = cur;
+		return;
+	}
+	if (svc_write_ap(vgpu, ch->counter_pa, SVC_APERTURE_SYS, &payload,
+			 sizeof(payload)))
+		return;
+	ch->pending_rel_pay = 0;
+	if (payload > ch->counter_last)
+		ch->counter_last = payload;
+	svc.releases_applied++;
+	svc.counter_rel_applied++;
+	svc.gr_rel_applied++;
+	svc.post_hint_until = jiffies + 10 * HZ;
+	if (svc.counter_rel_dbg < 96) {
+		svc.counter_rel_dbg++;
+		pr_info("sema-chan: rel-flush h=0x%x pa=0x%llx pay=%u cur=%u -> written\n",
+			ch->h_object, ch->counter_pa, payload, cur);
+	}
+}
+
+/*
  * Auto-register a GR channel's completion-counter slot from the observed
  * layout, without waiting for a readable push: the client zips GR channels
  * in ascending handle order with descending counter slots, so with n_gr
@@ -2173,13 +2267,18 @@ static void svc_counter_release_apply(struct nv_vgpu_device *vgpu, u32 h,
  * lower ranks' 676-byte context-init pushes stayed unresolved while the
  * other 13 channels' counters advanced, and the client timed out waiting).
  */
-static void svc_counter_autoreg(struct svc_channel *ch, int n_gr)
+static void svc_counter_autoreg(struct nv_vgpu_device *vgpu,
+				struct svc_channel *ch, int n_gr)
 {
 	u64 va, pa = 0;
 	int ap = SVC_APERTURE_SYS;
 
-	if (ch->engine_type || ch->counter_pa || n_gr <= 0)
+	if (ch->engine_type || n_gr <= 0)
+		return;
+	if (ch->counter_pa) {
+		svc_counter_flush_pending(vgpu, ch);
 		return;
+	}
 	if (ch->gr_index >= n_gr)
 		return;
 	va = 0x20400fec0ULL + (u64)(n_gr - 1 - ch->gr_index) * 0x10ULL;
@@ -2195,6 +2294,7 @@ static void svc_counter_autoreg(struct svc_channel *ch, int n_gr)
 	ch->counter_pa = pa & ~0x3ULL;
 	pr_info("sema-chan: counter auto h=0x%x rank=%u pa=0x%llx (slot va=0x%llx)\n",
 		ch->h_object, ch->gr_index, ch->counter_pa, va);
+	svc_counter_flush_pending(vgpu, ch);
 }
 
 /*
@@ -2389,8 +2489,21 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					ops_runs++;
 				}
 				n_ops = 0;
-				if (!svc_sem_read(vgpu, va, &cur) &&
-				    payload > cur) {
+				/*
+				 * A release to a GR completion counter must go
+				 * through the injected-pin-backed exact slot:
+				 * the general map's translation for that VA is
+				 * frequently stale or absent (the counter page
+				 * is a client UVM allocation outside every
+				 * scanned window), and a dropped release leaves
+				 * the client's tracker polling forever.
+				 */
+				if (ch_h && svc_counter_va(va)) {
+					svc_counter_learn(ch_h, va);
+					svc_counter_release_apply(vgpu, ch_h,
+								  va, payload);
+				} else if (!svc_sem_read(vgpu, va, &cur) &&
+					   payload > cur) {
 					svc.pushes_pending++;
 					cur = (op == HOST_SEM_OP_REDUCTION) ?
 						cur + payload : payload;
@@ -2472,6 +2585,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					ops_runs++;
 				}
 					n_ops = 0;
+					if (ch_h && svc_counter_va(va)) {
+						/*
+						 * Exact-slot delivery for GR
+						 * counters (see the 906f SEM
+						 * block above).
+						 */
+						svc_counter_learn(ch_h, va);
+						svc_counter_release_apply(vgpu,
+									  ch_h, va,
+									  st.rs_payload);
+						break;
+					}
 					if (svc_sem_read(vgpu, va, &cur))
 						break;
 					if (st.rs_payload <= cur)
@@ -2705,7 +2830,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				 */
 				if (ch_eng == 0)
 					svc_counter_learn(ch_h, sem_va);
-				svc_counter_release_apply(vgpu, ch_h,
+				svc_counter_release_apply(vgpu, ch_h, sem_va,
 							  st.sem_payload);
 				/* release ends the push */
 				n_ops = 0;
@@ -3989,6 +4114,7 @@ static void svc_counter_publish(struct nv_vgpu_device *vgpu,
 	if (!ch->counter_pa)
 		return;
 	svc.counter_pub_calls++;
+	svc_counter_flush_pending(vgpu, ch);
 
 	val = put;
 	if (val == ch->counter_last)
@@ -4457,7 +4583,7 @@ if (gsp_sema_fake_rel &&
 		}
 
 		/* Advance the client-visible completion counter. */
-		svc_counter_autoreg(ch, n_gr);
+		svc_counter_autoreg(vgpu, ch, n_gr);
 		svc_counter_publish(vgpu, ch, put);
 
 		spin_lock_irqsave(&svc.chan_lock, flags);