vgsp: kick-family USERD acceptance + ring revalidate + notify event post - notify lane executes (n=512), client c46f/c597 allocs appear (SESSION 153 r972/r973)

1 files changed, 154 insertions, 4 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index e414967..45bfdf5 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -666,6 +666,14 @@ struct svc_channel {
 	 */
 	bool	gr_ram_ring;
 	/*
+	 * SESSION 153-CONTINUED-19 (kick-family USERD acceptance): ring
+	 * liveness re-validation pace for the kick channel (h 0xbfef00xx).
+	 * While the kick USERD stays unresolved this channel's ring latch is
+	 * re-checked each pass (at most once a second) and re-armed when it
+	 * no longer decodes entries - see svc_kick_ring_revalidate.
+	 */
+	unsigned long gr_kick_ring_reval_at;
+	/*
 	 * r496b: the FB-scan USERD (see svc_userd_fb_scan). Latched when the
 	 * registered/ring-derived candidates all fail; the scan validates the
 	 * hit against the ring entries the put claims, so a foreign channel's
@@ -7894,6 +7902,17 @@ static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu,
 			(unsigned long long)ts, o0, n0, changed ? 1 : 0,
 			(unsigned long long)svc_kick_count, rc);
 	}
+	/*
+	 * SESSION 153-CONTINUED-19: the completed NOTIFY is the submitting
+	 * client's completion edge - post the channel event so its OS event
+	 * fires (r934's lesson on the notify2d path: without the event the
+	 * GL client's kick submit wait never completes and the app retries
+	 * on a ~16-22 s timer; r972 measured exactly that cadence with the
+	 * event missing here). Only for a real channel (the dev probe
+	 * passes ch=NULL).
+	 */
+	if (ok && ch)
+		gsp_rm_post_channel_event(vgpu, ch->h_object);
 	return ok;
 }
 
@@ -19381,6 +19400,99 @@ static bool svc_gr_ram_ring_scan(struct nv_vgpu_device *vgpu,
 }
 
 /*
+ * SESSION 153-CONTINUED-19 (kick-family USERD acceptance).
+ *
+ * The glxgears/X GLX client's kick channel (h 0xbfef00xx, GR engine, ring
+ * found by the guest-RAM content scan) posts a per-frame completion kick
+ * the client's flush loop paces on. Its USERD is a sysmem control block
+ * the driver allocates far from the ring - r971 measured the ring at pa
+ * 0x12f9ed000 and the live USERD at pa 0x1052ae000 (~681 MB away; the
+ * same nomination was +0.6..+2.3 GB across r966-r971) - while the general
+ * RAM USERD scan's allocation-locality gate only admits candidates within
+ * +/-64 MB of the ring (SESSION 153-CONT-7, built for the r936-r939
+ * nine-decoy class). That gate rejected the channel's real USERD on every
+ * boot since the gate landed, the scan retired ("RAM scan found no
+ * candidate in 20 slice(s)"), the pump never iterated the kick entries
+ * and the kick-notify lane never ran. For this family the ring-entry
+ * match (every PUT-claimed ring entry must decode to a pushbuffer VA
+ * inside the ring's own allocation window) is the ownership proof; the
+ * locality distance is not - so the gate is waived for these channels and
+ * the distance is logged instead. The pump's speculative-latch
+ * confirmation (a +1/+2 PUT step, svc_pump_channels) remains the final
+ * judge, and the rejected-latch memory prevents a frozen block from ever
+ * pinning the scan.
+ */
+static bool svc_kick_family(const struct svc_channel *ch)
+{
+	return ch && !ch->engine_type &&
+	       ch->h_object >= 0xbfef0000u && ch->h_object <= 0xbfef00ffu;
+}
+
+/*
+ * Re-validate the kick channel's ring latch each pass (bounded to once a
+ * second per channel). The RAM content scan latches a ring page once; if
+ * the driver later zeroes or reuses that page (the r966-r971 class the
+ * design brief describes: the USERD scan then validates every candidate
+ * against a dead ring and adopts none), the latch is dropped here so the
+ * content scan re-runs and re-latches the channel's current ring page.
+ * The stale entry pair is logged on re-arm so the transition is visible
+ * in the record. A live ring (a first entry that still decodes to a
+ * window-bounded pushbuffer VA) is a cheap no-op.
+ */
+static void svc_kick_ring_revalidate(struct nv_vgpu_device *vgpu,
+				     struct svc_channel *ch)
+{
+	u8 e[SVC_GPFIFO_ENTRY_SIZE];
+	u32 e0 = 0, e1 = 0;
+	unsigned long flags;
+	int i;
+
+	if (!svc_kick_family(ch) || !ch->gr_ram_ring || ch->userd_ok)
+		return;
+	if (!time_after(jiffies, ch->gr_kick_ring_reval_at))
+		return;
+	ch->gr_kick_ring_reval_at = jiffies + msecs_to_jiffies(1000);
+	if (!svc_read_va(vgpu, ch->gpfifo_va, e, sizeof(e))) {
+		u64 eva;
+		u32 elen;
+
+		memcpy(&e0, e, 4);
+		memcpy(&e1, e + 4, 4);
+		eva = ((u64)(e1 & 0xff) << 32) | (u64)(e0 & 0xfffffffc);
+		elen = ((e1 >> 10) & 0x1fffff) << 2;
+		if (eva && elen >= 4 && elen <= 0x100000)
+			return;	/* ring page live; nothing to do */
+	}
+	/*
+	 * The latched page no longer decodes entries: drop the one-shot
+	 * latch state so the RAM ring content scan re-runs. The pin is left
+	 * in place - a stale pin only shadows VA reads, while the content
+	 * scan validates against raw RAM and re-pins on a hit.
+	 */
+	clear_bit(ch->gr_index, &svc_gr_ring_done);
+	clear_bit(ch->gr_index, &svc_pb_gr_done);
+	ch->gr_ram_cursor = 0;
+	ch->gr_ram_passes = 0;
+	ch->gr_ram_logged = false;
+	ch->gr_ram_ring = false;
+	pr_info("sema-chan: kick ring re-arm h=0x%x va=0x%llx (latched entry e0=0x%x e1=0x%x no longer decodes; content scan re-runs)\n",
+		ch->h_object, ch->gpfifo_va, e0, e1);
+	spin_lock_irqsave(&svc.chan_lock, flags);
+	for (i = 0; i < SVC_MAX_CHANNELS; i++)
+		if (svc.chan[i].valid && svc.chan[i].vgpu == vgpu &&
+		    svc.chan[i].h_object == ch->h_object) {
+			svc.chan[i].gr_ram_cursor = ch->gr_ram_cursor;
+			svc.chan[i].gr_ram_passes = ch->gr_ram_passes;
+			svc.chan[i].gr_ram_logged = ch->gr_ram_logged;
+			svc.chan[i].gr_ram_ring = ch->gr_ram_ring;
+			svc.chan[i].gr_kick_ring_reval_at =
+				ch->gr_kick_ring_reval_at;
+			break;
+		}
+	spin_unlock_irqrestore(&svc.chan_lock, flags);
+}
+
+/*
  * r912 (M-GL1.2): guest-RAM USERD shape scan for a GR channel whose ring
  * was content-validated in guest RAM (gr_ram_ring; svc_gr_ram_ring_scan).
  * Such a client's USERD is a system-memory object like its ring, so the
@@ -19420,6 +19532,7 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 	u64 live_pa = 0;
 	u32 live_put = 0, live_get = 0, live_prev = 0;
 	int ring_ap = SVC_APERTURE_SYS;
+	bool kick;
 	u32 wi;
 	int i;
 	unsigned long flags;
@@ -19438,6 +19551,14 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 		if (!buf)
 			return false;
 	}
+	kick = svc_kick_family(ch);
+	/*
+	 * SESSION 153-CONTINUED-19: re-check the kick channel's ring latch
+	 * before validating candidates against it (svc_kick_ring_revalidate
+	 * re-latches a ring page the driver has zeroed since the latch; see
+	 * its comment).
+	 */
+	svc_kick_ring_revalidate(vgpu, ch);
 	(void)svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap);
 	/*
 	 * SESSION 154 (M-GL1.2): liveness re-check of the watched candidates.
@@ -19464,9 +19585,13 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 		 * mapped by NO process, and never advanced on any pass
 		 * (two frozen snapshots that cost a 15 s reject/re-latch
 		 * cycle all boot). A control block this far from its own
-		 * ring is not this channel's USERD.
+		 * ring is not this channel's USERD. SESSION 153-CONT-19:
+		 * waived for the kick family - its real USERD is measured
+		 * 0.6-2.3 GB from the ring, and the ring-entry match (not
+		 * proximity) is the ownership proof there (see
+		 * svc_kick_family).
 		 */
-		if (ring_pa &&
+		if (ring_pa && !kick &&
 		    (wpa < ring_pa - (SVC_GR_RAM_WIN << 5) ||
 		     wpa > ring_pa + (SVC_GR_RAM_WIN << 5))) {
 			ch->gr_ram_userd_watch_n--;
@@ -19596,8 +19721,14 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 			 * above. Excludes the r936-r939 nine-decoy pair
 			 * (~460 MB from the ring) before it can occupy a
 			 * watch slot and burn a reject/re-latch cycle.
+			 * SESSION 153-CONT-19: waived for the kick family
+			 * (h 0xbfef00xx) - its real USERD is measured far
+			 * outside this window, so admission there is by the
+			 * ring-entry match alone (the entry loop below) and
+			 * the pump's +1/+2-step confirmation; the distance
+			 * is logged at the latch.
 			 */
-			if (ring_pa &&
+			if (ring_pa && !kick &&
 			    (gpa + o < ring_pa - (SVC_GR_RAM_WIN << 5) ||
 			     gpa + o > ring_pa + (SVC_GR_RAM_WIN << 5)))
 				continue;
@@ -19686,6 +19817,12 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 			live_put = hp;
 			live_get = hg;
 			ch->gr_ram_userd_watch_n = 0;
+			if (kick)
+				pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family accept pa=0x%llx put=%u get=%u (ring_pa=0x%llx delta=%lld MB; ring-entry validated; locality gate waived)\n",
+					ch->h_object, ch->gr_index, live_pa,
+					live_put, live_get, ring_pa,
+					(long long)(live_pa - ring_pa) /
+						(1024 * 1024));
 			pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put step across passes) -> pa=0x%llx put=%u get=%u\n",
 				ch->h_object, ch->gr_index, live_pa,
 				live_put, live_get);
@@ -30530,8 +30667,19 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		 * probably locked" while every other channel kept running.
 		 * Writing the ack on every pass that sees work pending is
 		 * idempotent.
+		 *
+		 * SESSION 153-CONTINUED-19: NOT for the kick family. Their
+		 * notify2d_va is the kick record's own NOTIFY target (the
+		 * shared completion word the GL client polls), and the kick
+		 * lane authors that word (a completed NvNotification with a
+		 * nonzero timestamp, svc_kick_notify_exec). The per-pass ack
+		 * zeroed it 14029 times on r972 - the last kick write
+		 * survived at most one pass (~60 ms) before being wiped, so
+		 * the client's poll kept seeing "not fired". The GPU never
+		 * clears this notification; the client does (re-arm).
 		 */
-		if (!ch->engine_type && ch->notify2d_va && put != get)
+		if (!ch->engine_type && ch->notify2d_va && put != get &&
+		    !svc_kick_family(ch))
 			svc_notify2d_write(vgpu, ch, false);
 		/*
 		 * SESSION 148 (r864): registration-based keep-alive that
@@ -33401,6 +33549,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				svc.chan[k].gr_ram_refresh_at =
 					ch->gr_ram_refresh_at;
 				svc.chan[k].gr_ram_ring = ch->gr_ram_ring;
+				svc.chan[k].gr_kick_ring_reval_at =
+					ch->gr_kick_ring_reval_at;
 				svc.chan[k].userd_scans = ch->userd_scans;
 				svc.chan[k].notify2d_va = ch->notify2d_va;
 				svc.chan[k].zero_held = ch->zero_held;