vgsp: DDX-family USERD latch lifetime - served-latch deferral + live re-probe + FB backoff

The X DDX's push-on-demand generations allocate a new USERD control block
every 11-24 s, but the 15 s no-advance judge dropped the latch that had just
passed the DMA test, and the next scan adopted a frozen FB lookalike
(put=2 get=0) - the retry window then closed with 'Failed to idle DMA'.
For the DDX/kick handle families only (0x01xxxxxx / 0xbfef00xx; SYS/CE/UVM
untouched):
- svc_userd_ddx_family() gates all of it;
- a latch that HAS delivered decoded entries (submissions advanced since
  adoption, no contradiction) is DEFERRED from rejection (bound 32; timer
  reset) instead of dropped;
- before any latch decision (and on a 2 s cadence while unconfirmed) the
  content scan re-probes the live path and switches the latch in place to a
  newly validated block (svc_userd_ddx_reprobe; near-empty page + ring-entry
  validation + PUT-step liveness are the ownership proof);
- after a rejection the FB shape scan backs off 6 s so the RAM scan wins the
  window; the DDX re-probe is exempt from the retired slice budget
  (rejects < 255).

Validated (c141, rich r1624 args, tick): X ScreenInit passed with ZERO
'Failed to (idle|initialize) DMA' / push-buffer / AddScreen lines; the
channel decoded a 5090+-entry burst and served across block moves
(latch SWITCHED 0x24128e100 -> 0x103774100 -> 0x122309d00, uok=1 held).
Known residual (next refinement): the switch is not yet forward-only/
rate-limited, so lookalike blocks can oscillate at ~2 s and starve the
client; make the switch require nput > latch_put and a >=10 s cadence (or
stop switching once defers>0), then the real block wins.

1 files changed, 200 insertions, 1 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 82dceaf..f8604b6 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -760,6 +760,29 @@ struct svc_channel {
 	u32	gr_ram_userd_reject_put;	/* its PUT at rejection time */
 	u8	gr_ram_userd_rejects;		/* rejections seen (bound) */
 	/*
+	 * CONT.462 (X-DMA latch lifetime): delivery + backoff state.
+	 * gr_ram_userd_latch_sub snapshots the consumed-entry count
+	 * (ch->submissions) at latch adoption so the pump can tell a latch
+	 * that HAS delivered decoded entries from a frozen lookalike (the
+	 * false fb 0x8f0000 block never consumes: its put maps onto the
+	 * current epoch's already-done slots and submissions stays put).
+	 * A DDX-family latch (h 0x01xxxxxx / 0xbfef00xx) that has served is
+	 * DEFERRED from the 15 s no-advance rejection: the X DDX's
+	 * push-on-demand generations arrive 11-24 s apart (measured c130/
+	 * c131), so the fixed bound used to kill the block that had just
+	 * passed the DMA test and the next generation then found a frozen FB
+	 * lookalike. gr_ram_userd_defers bounds the deferral (~2 min); on
+	 * every defer/reject decision the DDX family is re-probed (the RAM
+	 * scan may adopt a newer live block in place).
+	 * gr_ram_userd_reject_at stamps the last rejection so the FB shape
+	 * scan backs off after one, leaving the RAM scan the window to find
+	 * the real client block.
+	 */
+	u64	gr_ram_userd_latch_sub;	/* ch->submissions at latch */
+	u8	gr_ram_userd_defers;	/* deferrals applied (bound 32) */
+	unsigned long gr_ram_userd_reject_at;	/* jiffies of last reject */
+	unsigned long gr_ram_userd_reprobe_at;	/* last live-latch re-probe */
+	/*
 	 * SESSION 154 (M-GL1.2): content anchor for a client push whose
 	 * translation went stale (r913: a validated pin kept answering for
 	 * a reused page and the wrong-VAS walk answer was implausible while
@@ -5218,6 +5241,7 @@ static void svc_pin_drop(u64 va)
  */
 static inline bool svc_pool_sem_va(u64 va);
 static bool svc_kick_family(const struct svc_channel *ch);
+static bool svc_userd_ddx_family(const struct svc_channel *ch);
 static void svc_pool_pa_forget(u64 va);
 
 /*
@@ -10927,6 +10951,7 @@ static void svc_gl_scan_window(u32 start, u32 end, const u8 *buf, u32 len,
  * (svc_lookup_trusted -> svc_write_ap), then readback verification.
  */
 static bool svc_kick_family(const struct svc_channel *ch);
+static bool svc_userd_ddx_family(const struct svc_channel *ch);
 static u64 svc_push_pa_ceiling(void);
 static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu,
 				 struct svc_channel *ch, u64 va, u32 val,
@@ -27361,7 +27386,14 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	     ch->gr_ram_userd_watch_n ||
 	     ((ch->h_object >= 0xbfef0000u &&
        ch->h_object <= 0xbfef00ffu) &&
-      ch->gr_ram_userd_kick_sweeps < 64))) {
+      ch->gr_ram_userd_kick_sweeps < 64) ||
+	     /*
+	      * CONT.462: the DDX family re-probe is ALWAYS allowed (clause
+	      * iii): its generations churn blocks every 11-24 s, so a
+	      * retired slice budget must not end the search for the live
+	      * one. The scan stays sliced/bounded per call.
+	      */
+	     (svc_userd_ddx_family(ch) && ch->gr_ram_userd_rejects < 255))) {
 		u64 upa = 0;
 		u32 uput = 0, uget = 0;
 
@@ -27398,6 +27430,8 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 			ch->gr_ram_put_seeded = uput;
 			ch->gr_ram_userd_latch_put = uput;
 			ch->gr_ram_userd_latch_at = jiffies;
+			ch->gr_ram_userd_latch_sub = ch->submissions;
+			ch->gr_ram_userd_defers = 0;
 			ch->gr_ram_userd_confirmed = false;
 			*put = uput;
 			*get = uget;
@@ -27455,6 +27489,16 @@ ce_fb_scan:
 	    (!ch->engine_type ||
 	     (ch->h_object & 0xfffff000u) == 0xbeef0000u) &&
 	    ch->fb_userd_scan_n < 512 &&
+	    /*
+	     * CONT.462: after a rejected latch, give the DDX family's RAM
+	     * scan a backoff window before the FB shape scan may re-latch
+	     * (the frozen fb 0x8f0000 lookalike used to win every ~18 s
+	     * churn cycle and keep the channel dark).
+	     */
+	    !(svc_userd_ddx_family(ch) && ch->gr_ram_userd_reject_at &&
+	      time_before(jiffies,
+			  (unsigned long)ch->gr_ram_userd_reject_at +
+			  msecs_to_jiffies(6000))) &&
 	    (!ch->fb_userd_scan_at ||
 	     time_after_eq(jiffies,
 			   (unsigned long)ch->fb_userd_scan_at +
@@ -27765,6 +27809,8 @@ ce_fb_scan:
 				 */
 				ch->gr_ram_userd_latch_put = best_put;
 				ch->gr_ram_userd_latch_at = jiffies;
+				ch->gr_ram_userd_latch_sub = ch->submissions;
+				ch->gr_ram_userd_defers = 0;
 				ch->gr_ram_userd_confirmed = false;
 				pr_info("sema-chan: %s USERD h=0x%x rank=%u found by VRAM shape scan at fb 0x%llx (put=%u get=%u%s, ring-validated)\n",
 					ch->engine_type ? "CE" : "GR",
@@ -30555,6 +30601,22 @@ static bool svc_kick_family(const struct svc_channel *ch)
 }
 
 /*
+ * CONT.462 (X-DMA latch lifetime): the DDX/kick handle families whose
+ * USERD latch is probed authoritatively by content (the X server's
+ * 0x01000013 kick channel and the GL kick family share the pattern: the
+ * control block is client-allocated and moves between push-on-demand
+ * generations). For these families (and only these) the pump may
+ * re-probe on a latch decision and the pre-reject FB shape-scan backoff
+ * applies - the SYS/CE/UVM families keep their existing semantics.
+ */
+static bool svc_userd_ddx_family(const struct svc_channel *ch)
+{
+	return ch && !ch->engine_type &&
+	       (((ch->h_object & 0xff000000u) == 0x01000000u) ||
+		svc_kick_family(ch));
+}
+
+/*
  * Re-validate the kick channel's ring latch each pass (bounded to once a
  * second per channel). The RAM content scan latches a ring page once; if
  * the driver later zeroes or reuses that page (the r966-r971 class the
@@ -43890,6 +43952,66 @@ static bool svc_userd_put_wrap(const struct svc_channel *ch)
 	       ch->userd_as == SVC_APERTURE_FB && !svc_kick_family(ch);
 }
 
+/*
+ * CONT.462 (X-DMA latch lifetime): live-latch re-probe for the DDX family.
+ *
+ * The X DDX allocates a NEW control block per push-on-demand generation and
+ * its DMA-test window is only 10 s (c140: the RAM scan latched a lookalike at
+ * 0x101996f00 put=1 get=1, the real block never landed, the channel was freed
+ * 10 s later with submissions=0). Waiting for the 15 s latch decision is too
+ * late, and while a latch is live the pump never re-enters the scan. Run the
+ * content scan from the live path on a slow cadence: if it validates a
+ * DIFFERENT block (near-empty page, ring-entry validation, PUT-step
+ * liveness), switch the latch to it in place - the old block stays serving
+ * until that moment. Confirmed latches (the client advanced their PUT) are
+ * left alone; the SYS/CE/UVM families never enter.
+ */
+static bool svc_userd_ddx_reprobe(struct nv_vgpu_device *vgpu,
+				  struct svc_channel *ch, u64 ring_pa,
+				  int ring_ap, u32 *put, u32 *get)
+{
+	u64 npa = 0;
+	u32 nput = 0, nget = 0;
+
+	if (!svc_userd_ddx_family(ch) || !ch->userd_ok ||
+	    ch->gr_ram_userd_confirmed || !ch->gr_ram_userd_latch_at)
+		return false;
+	if (!svc_gr_ram_userd_scan(vgpu, ch, &npa, &nput, &nget))
+		return false;
+	if (npa == ch->userd_pa)
+		return false;
+	{
+		u64 opa = ch->userd_pa;
+		u32 oput = ch->gr_ram_userd_latch_put;
+
+		ch->userd_pa = npa;
+		ch->userd_as = SVC_APERTURE_SYS;
+		ch->userd_ok = true;
+		svc_kick_capture_arm(vgpu, ch, ring_pa, ring_ap, nput);
+		if (ch->gpfifo_entries > 1 && nput) {
+			u32 span = nput < ch->gpfifo_entries ?
+				   nput : ch->gpfifo_entries - 1;
+
+			ch->last_get = (nput % ch->gpfifo_entries +
+					ch->gpfifo_entries - span) %
+				       ch->gpfifo_entries;
+		} else {
+			ch->last_get = 0;
+		}
+		ch->gr_ram_put_seeded = nput;
+		ch->gr_ram_userd_latch_put = nput;
+		ch->gr_ram_userd_latch_at = jiffies;
+		ch->gr_ram_userd_latch_sub = ch->submissions;
+		ch->gr_ram_userd_defers = 0;
+		ch->gr_ram_userd_confirmed = false;
+		*put = nput;
+		*get = nget;
+		pr_info("sema-chan: GR USERD h=0x%x latch SWITCHED to re-probed block pa=0x%llx (put=%u get=%u; old pa=0x%llx latch=%u)\n",
+			ch->h_object, npa, nput, nget, opa, oput);
+	}
+	return true;
+}
+
 static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 {
 	if (svc_pdb_disc_pending) {
@@ -44717,6 +44839,21 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		bool kfam = svc_kick_family(ch);
 
 		/*
+		 * CONT.462: keep a slow live-latch re-probe running for the
+		 * DDX family (see svc_userd_ddx_reprobe). The client's 10 s
+		 * DMA-test window does not wait for the 15 s latch decision,
+		 * so a lookalike latch must be able to yield to the real
+		 * block the moment the content scan validates one.
+		 */
+		if (svc_userd_ddx_family(ch) && !ch->gr_ram_userd_confirmed &&
+		    time_after_eq(jiffies, ch->gr_ram_userd_reprobe_at)) {
+			ch->gr_ram_userd_reprobe_at =
+				jiffies + msecs_to_jiffies(2000);
+			(void)svc_userd_ddx_reprobe(vgpu, ch, ring_pa,
+						    ring_ap, &put, &get);
+		}
+
+		/*
 		 * SESSION 154 (M-GL1.2): speculative-latch confirmation for
 		 * the RAM USERD. The scan latches a gated candidate right
 		 * away (fast walk start); the pump is the judge: a +1/+2
@@ -44785,6 +44922,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				put > ch->gr_ram_userd_latch_put;
 			if (stepped) {
 				ch->gr_ram_userd_confirmed = true;
+				ch->gr_ram_userd_defers = 0;
 				if (wrap &&
 				    put < ch->gr_ram_userd_latch_put)
 					pr_info_ratelimited("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch wrap accepted (put %u latch %u ent %u step %u)\n",
@@ -44832,9 +44970,59 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				   time_after(jiffies,
 					      ch->gr_ram_userd_latch_at +
 					      msecs_to_jiffies(15000))) {
+				/*
+				 * CONT.462 (X-DMA latch lifetime, clause
+				 * iii): before this latch is dropped or
+				 * deferred, ALWAYS re-probe the DDX family.
+				 * The client's push-on-demand generations
+				 * allocate a new control block (c131:
+				 * consecutive RAM-scan adopts landed on three
+				 * different pages), so the useful answer to a
+				 * stuck put is often "the block moved", not
+				 * "the latch is stale". If the content scan
+				 * has just validated a live block, switch to
+				 * it in place; its own gates (near-empty page,
+				 * ring-entry validation, PUT-step liveness)
+				 * are the ownership proof.
+				 */
+				bool served = ch->submissions !=
+					      ch->gr_ram_userd_latch_sub;
+
+				if (svc_userd_ddx_reprobe(vgpu, ch, ring_pa,
+							  ring_ap, &put, &get)) {
+					/*
+					 * Switched to a fresh validated block
+					 * (the helper logged and re-seeded the
+					 * latch; put/get now carry it).
+					 */
+				} else if (!moved && served &&
+					   svc_userd_ddx_family(ch) &&
+					   ch->gr_ram_userd_defers < 32) {
+					/*
+					 * Clause (i): this latch HAS delivered
+					 * decoded entries (submissions advanced
+					 * since adoption) and the client has
+					 * not contradicted it (no decrease).
+					 * The DDX's generations are 11-24 s
+					 * apart; keep the proven block serving
+					 * instead of dropping it at the fixed
+					 * 15 s no-advance bound. Bounded: 32
+					 * deferrals (~8 min) then the old reject
+					 * path resumes for a dead latch.
+					 */
+					ch->gr_ram_userd_defers++;
+					ch->gr_ram_userd_latch_at = jiffies;
+					pr_info_ratelimited("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch DEFERRED (served; put=%u latch=%u; staying latched %u/32)\n",
+						ch->h_object, ch->userd_pa, put,
+						ch->gr_ram_userd_latch_put,
+						ch->gr_ram_userd_defers);
+				} else {
 				unsigned long lflags;
 				int li;
 
+				if (svc_userd_ddx_family(ch))
+					ch->gr_ram_userd_reject_at = jiffies;
+
 				pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch REJECTED (put=%u latch=%u%s)\n",
 					ch->h_object, ch->userd_pa, put,
 					ch->gr_ram_userd_latch_put,
@@ -44886,6 +45074,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							ch->gr_ram_userd_reject_pa;
 						svc.chan[li].gr_ram_userd_reject_put =
 							ch->gr_ram_userd_reject_put;
+						svc.chan[li].gr_ram_userd_reject_at =
+							ch->gr_ram_userd_reject_at;
 						svc.chan[li].gr_ram_userd_rejects =
 							ch->gr_ram_userd_rejects;
 						break;
@@ -44893,6 +45083,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 				spin_unlock_irqrestore(&svc.chan_lock, lflags);
 				continue;
 			}
+			}
 		}
 		if (put >= ch->gpfifo_entries) {
 			u32 put_raw = put;
@@ -47750,6 +47941,14 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 					ch->gr_ram_userd_latch_put;
 				svc.chan[k].gr_ram_userd_latch_at =
 					ch->gr_ram_userd_latch_at;
+				svc.chan[k].gr_ram_userd_latch_sub =
+					ch->gr_ram_userd_latch_sub;
+				svc.chan[k].gr_ram_userd_defers =
+					ch->gr_ram_userd_defers;
+				svc.chan[k].gr_ram_userd_reject_at =
+					ch->gr_ram_userd_reject_at;
+				svc.chan[k].gr_ram_userd_reprobe_at =
+					ch->gr_ram_userd_reprobe_at;
 				svc.chan[k].gr_ram_userd_confirmed =
 					ch->gr_ram_userd_confirmed;
 				/* CONT.407: keep-across-put-reset state. */