vgsp: kick USERD watch table 64 + adopt-on-PUT-step + churn monitor + FB->SYS notify retry - frozen alternation solved (SESSION 153 r974-r976)

1 files changed, 248 insertions, 14 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 45bfdf5..15f728c 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -206,7 +206,14 @@ struct sema_map_entry {
 #define SVC_GR_RAM_REFRESH_MS	10000		/* minimum spacing between them */
 #define SVC_GR_RAM_WIN		(2ULL << 20)	/* allocation window pinned */
 /* SESSION 154 (M-GL1.2): watched USERD candidates for the liveness latch. */
-#define SVC_GR_USERD_WATCH	8
+#define SVC_GR_USERD_WATCH	64
+/*
+ * SESSION 153-CONTINUED-20: a kick-family (h 0xbfef00xx) watched candidate
+ * whose PUT does not move for this long is a frozen pool lookalike (the
+ * r972/r973 descriptor-pool class: put=7 forever) - dropped so the ~50
+ * frozen entries can never crowd out the real advancing control block.
+ */
+#define SVC_KICK_WATCH_STALE_MS	20000
 
 /*
  * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM
@@ -612,7 +619,23 @@ struct svc_channel {
 	u64	gr_ram_userd_watch[SVC_GR_USERD_WATCH];		/* watched PAs */
 	u32	gr_ram_userd_watch_put[SVC_GR_USERD_WATCH];	/* last PUT seen */
 	u8	gr_ram_userd_watch_hits[SVC_GR_USERD_WATCH];	/* +1/+2 streak */
+	/*
+	 * SESSION 153-CONTINUED-20: jiffies when the entry was added or last
+	 * stepped; the stale-entry timeout drops a candidate whose PUT has
+	 * not moved for SVC_KICK_WATCH_STALE_MS (the frozen descriptor-pool
+	 * class the kick-family watch exists to defeat).
+	 */
+	unsigned long gr_ram_userd_watch_at[SVC_GR_USERD_WATCH];
 	u8	gr_ram_userd_watch_n;
+	u32	gr_ram_userd_watch_log;		/* bounded add-log counter */
+	/*
+	 * SESSION 153-CONTINUED-20: bounded budget re-arms for the
+	 * kick-family watch sweep. When every watch entry has gone stale
+	 * (frozen pool) and the slice budget is spent, one re-arm keeps the
+	 * sweep alive so the real block can still be found; capped like the
+	 * reject counter so a dead channel stops sweeping in bounded time.
+	 */
+	u8	gr_ram_userd_kick_sweeps;
 	u32	gr_ram_put_seeded;	/* last PUT the walk seed consumed */
 	/*
 	 * SESSION 154 (M-GL1.2): speculative-latch confirmation state. The
@@ -7879,6 +7902,32 @@ static bool svc_kick_notify_exec(struct nv_vgpu_device *vgpu,
 	if (!svc_lookup_trusted(vgpu, va, &pa, &ap)) {
 		int r1, r2;
 
+		/*
+		 * SESSION 153-CONTINUED-20: prefer the client-visible sysmem
+		 * mapping. The target is the completion word the client's
+		 * flush loop polls through its own memfd; r974/r975 measured
+		 * the serviced-channel VAS walk answering this VA with the FB
+		 * shadow (ap=2, pa=0x20e0b0) - a write no client can observe
+		 * (r973's SYS answer, pa=0x12ae340b0, was the readable one,
+		 * and the ch=NULL dev probe on r975 re-resolved the same VA
+		 * to SYS pa=0x104b640b0 = the kick ring page + 0x1000 + 0xb0,
+		 * exactly the r973 ring-adjacent layout). When the biased
+		 * chain lands in FB, re-resolve with the serviced-channel VAS
+		 * bias cleared and take the sysmem answer when it exists.
+		 */
+		if (svc_ap_of(ap) == SVC_APERTURE_FB) {
+			u64 pa2 = 0;
+			int ap2 = SVC_APERTURE_FB;
+			u32 save = svc_cur_vas;
+
+			svc_cur_vas = 0;
+			if (!svc_lookup_trusted(vgpu, va, &pa2, &ap2) &&
+			    svc_ap_of(ap2) == SVC_APERTURE_SYS) {
+				pa = pa2;
+				ap = ap2;
+			}
+			svc_cur_vas = save;
+		}
 		r1 = svc_read_ap(vgpu, pa, svc_ap_of(ap), old, sizeof(old));
 		r2 = svc_write_ap(vgpu, pa, svc_ap_of(ap), nw, sizeof(nw));
 		if (!r1 && !r2 &&
@@ -17531,10 +17580,18 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	 * (see svc_gr_ram_userd_scan; SESSION 154 liveness). The budget
 	 * gate stays open while candidates are under watch, so the
 	 * liveness confirmation cannot be starved by slice accounting.
+	 * SESSION 153-CONTINUED-20: for the kick family it also stays open
+	 * while the bounded sweep re-arm budget lasts - the watch set can
+	 * go empty for stretches (all frozen pool entries timed out), and
+	 * the in-scan re-arm needs a next call to re-run the sweep that
+	 * finds the advancing block.
 	 */
 	if (!ch->engine_type && ch->gr_ram_ring && !ch->userd_ok &&
 	    (ch->gr_ram_userd_passes < svc_gr_ram_pass_budget() ||
-	     ch->gr_ram_userd_watch_n)) {
+	     ch->gr_ram_userd_watch_n ||
+	     ((ch->h_object >= 0xbfef0000u &&
+	       ch->h_object <= 0xbfef00ffu) &&
+	      ch->gr_ram_userd_kick_sweeps < 64))) {
 		u64 upa = 0;
 		u32 uput = 0, uget = 0;
 
@@ -19616,10 +19673,47 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 		}
 		last = ch->gr_ram_userd_watch_put[wi];
 		if (hp == last) {
+			/*
+			 * SESSION 153-CONTINUED-20: stale-entry timeout. A
+			 * kick-family watched block whose PUT has not moved
+			 * for the bound is a frozen lookalike (the r972/r973
+			 * descriptor-pool class, put=7 forever) - drop it so
+			 * the watch set keeps room for the advancing control
+			 * block. Non-kick entries keep the old semantics
+			 * (they are only added by the speculative latch's
+			 * own re-arm today).
+			 */
+			if (kick && time_after(jiffies,
+					ch->gr_ram_userd_watch_at[wi] +
+					msecs_to_jiffies(SVC_KICK_WATCH_STALE_MS))) {
+				ch->gr_ram_userd_watch_n--;
+				ch->gr_ram_userd_watch[wi] =
+					ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n];
+				ch->gr_ram_userd_watch_put[wi] =
+					ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n];
+				ch->gr_ram_userd_watch_hits[wi] =
+					ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n];
+				ch->gr_ram_userd_watch_at[wi] =
+					ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n];
+				continue;
+			}
 			wi++;
 			continue;
 		}
-		if (hp != last + 1) {
+		if (kick && hp < last) {
+			/* A reset is churn: drop (same class as the jump). */
+			ch->gr_ram_userd_watch_n--;
+			ch->gr_ram_userd_watch[wi] =
+				ch->gr_ram_userd_watch[ch->gr_ram_userd_watch_n];
+			ch->gr_ram_userd_watch_put[wi] =
+				ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n];
+			ch->gr_ram_userd_watch_hits[wi] =
+				ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n];
+			ch->gr_ram_userd_watch_at[wi] =
+				ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n];
+			continue;
+		}
+		if (!kick && hp != last + 1) {
 			/*
 			 * SESSION 154: a jump or a reset is the churn
 			 * signature (the r916-r919 lookalike pools move by
@@ -19634,9 +19728,37 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 				ch->gr_ram_userd_watch_put[ch->gr_ram_userd_watch_n];
 			ch->gr_ram_userd_watch_hits[wi] =
 				ch->gr_ram_userd_watch_hits[ch->gr_ram_userd_watch_n];
+			ch->gr_ram_userd_watch_at[wi] =
+				ch->gr_ram_userd_watch_at[ch->gr_ram_userd_watch_n];
 			continue;
 		}
 		ch->gr_ram_userd_watch_put[wi] = hp;
+		ch->gr_ram_userd_watch_at[wi] = jiffies;
+		/*
+		 * SESSION 153-CONTINUED-20: THE kick-family adoption rule -
+		 * any positive PUT step across passes. The real control block
+		 * is authored by the client's frame loop (measured 140/s), so
+		 * between two sightings its PUT steps by several positions;
+		 * the frozen descriptor-pool entries never step and time out
+		 * above. Do NOT require the +1 step here: for this family a
+		 * free-running put is the LIVE signature, a stall is the
+		 * lookalike.
+		 */
+		if (kick) {
+			(void)svc_read(vgpu, wpa + SVC_USERD_GP_GET, &hg,
+				       sizeof(hg));
+			if (hg > hp)
+				continue;	/* impossible; keep watching */
+			live_pa = wpa;
+			live_put = hp;
+			live_get = hg;
+			live_prev = last;
+			ch->gr_ram_userd_watch_n = 0;
+			pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family ADOPT on PUT step (put %u -> %u across passes) -> pa=0x%llx put=%u get=%u\n",
+				ch->h_object, ch->gr_index, live_prev,
+				live_put, live_pa, live_put, live_get);
+			break;
+		}
 		/*
 		 * SESSION 154: ONE observed +1 step adopts. The live block
 		 * advances ~2-8 s per step (measured) while the client's
@@ -19693,9 +19815,15 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 			 * latches read 12/8 and 16/16 and every previously
 			 * measured lookalike counted 16 or more. Also accept
 			 * a free-running put (the old put <= entries bound
-			 * would reject the live block).
+			 * would reject the live block). SESSION 153-
+			 * CONTINUED-20: for the kick family the cap is
+			 * LIFTED to the watch loop's sanity bound (0x100000)
+			 * - its control block is authored by the client's
+			 * frame loop and its PUT free-runs with the frame
+			 * rate; the step-based watch adoption (not the cap)
+			 * is the discriminator there.
 			 */
-			if (!hp || hp > 8 || hg > hp)
+			if (!hp || hp > (kick ? 0x100000 : 8) || hg > hp)
 				continue;
 			{
 				u32 z = 0, zi;
@@ -19800,6 +19928,64 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 			    hp == ch->gr_ram_userd_reject_put)
 				continue;
 			/*
+			 * SESSION 153-CONTINUED-20 (watch-adopt-on-PUT-
+			 * step): a kick-family candidate is NOT adopted on
+			 * sight. This family's RAM is littered with frozen
+			 * descriptor-pool lookalikes whose +0x88/+0x8c
+			 * words happen to form the put/get shape (r972/973:
+			 * put=7 frozen, 65 pump rejections in one boot,
+			 * the latch alternating between two pool entries).
+			 * Add the candidate to the watch set (deduped, LRU
+			 * replace when full) and KEEP scanning the slice;
+			 * the watch loop at the top adopts the block only
+			 * when its PUT takes a step across passes, and a
+			 * stale-entry timeout (SVC_KICK_WATCH_STALE_MS)
+			 * evicts frozen entries so the pool can never crowd
+			 * out the real control block. The r971-r973
+			 * immediate latch is retired for this family.
+			 */
+			if (kick) {
+				u32 wn = ch->gr_ram_userd_watch_n;
+				bool known = false;
+				u32 wj;
+
+				for (wj = 0; wj < wn; wj++)
+					if (ch->gr_ram_userd_watch[wj] ==
+					    gpa + o) {
+						known = true;
+						break;
+					}
+				if (!known) {
+					if (wn >= SVC_GR_USERD_WATCH) {
+						u32 old = 0, si;
+
+						for (si = 1; si < wn; si++)
+							if (time_before(
+							    ch->gr_ram_userd_watch_at[si],
+							    ch->gr_ram_userd_watch_at[old]))
+								old = si;
+						wn = old;
+					}
+					ch->gr_ram_userd_watch[wn] = gpa + o;
+					ch->gr_ram_userd_watch_put[wn] = hp;
+					ch->gr_ram_userd_watch_hits[wn] = 0;
+					ch->gr_ram_userd_watch_at[wn] = jiffies;
+					if (wn == ch->gr_ram_userd_watch_n)
+						ch->gr_ram_userd_watch_n++;
+					if (ch->gr_ram_userd_watch_log < 48) {
+						ch->gr_ram_userd_watch_log++;
+						pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family watch[%u] pa=0x%llx put=%u get=%u (ring_pa=0x%llx delta=%lld MB; ring-entry validated; adopt on PUT step)\n",
+							ch->h_object,
+							ch->gr_index, wn,
+							gpa + o, hp, hg,
+							ring_pa,
+							(long long)(gpa + o - ring_pa) /
+								(1024 * 1024));
+					}
+				}
+				continue;	/* keep scanning; adopt is deferred */
+			}
+			/*
 			 * SESSION 154 (M-GL1.2): a candidate that passed the
 			 * shape gate, the entry validation, the page-density
 			 * gate and the put cap is latched SPECULATIVELY. The
@@ -19817,12 +20003,6 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 			live_put = hp;
 			live_get = hg;
 			ch->gr_ram_userd_watch_n = 0;
-			if (kick)
-				pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family accept pa=0x%llx put=%u get=%u (ring_pa=0x%llx delta=%lld MB; ring-entry validated; locality gate waived)\n",
-					ch->h_object, ch->gr_index, live_pa,
-					live_put, live_get, ring_pa,
-					(long long)(live_pa - ring_pa) /
-						(1024 * 1024));
 			pr_info("sema-chan: GR USERD h=0x%x rank=%u LIVE (put step across passes) -> pa=0x%llx put=%u get=%u\n",
 				ch->h_object, ch->gr_index, live_pa,
 				live_put, live_get);
@@ -19833,6 +20013,23 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 	}
 	ch->gr_ram_userd_cursor = (end >= top) ? 0 : end;
 	ch->gr_ram_userd_passes++;
+	/*
+	 * SESSION 153-CONTINUED-20: kick-family sweep re-arm. When every
+	 * watched candidate has gone stale (frozen pool) and the slice
+	 * budget is spent, one bounded re-arm keeps the sweep alive so the
+	 * real advancing block can still be found; capped like the reject
+	 * counter so a truly dead channel stops sweeping in bounded time.
+	 */
+	if (kick && !live_pa && !ch->gr_ram_userd_watch_n &&
+	    ch->gr_ram_userd_passes >= svc_gr_ram_pass_budget() &&
+	    ch->gr_ram_userd_kick_sweeps < 64) {
+		ch->gr_ram_userd_kick_sweeps++;
+		ch->gr_ram_userd_passes = 0;
+		ch->gr_ram_userd_logged = false;
+		pr_info("sema-chan: GR USERD h=0x%x rank=%u kick-family sweep re-arm %u/64 (watch empty, budget spent; content scan re-runs)\n",
+			ch->h_object, ch->gr_index,
+			ch->gr_ram_userd_kick_sweeps);
+	}
 	if (ch->gr_ram_userd_cursor == 0 && !ch->gr_ram_userd_logged &&
 	    !ch->gr_ram_userd_watch_n) {
 		ch->gr_ram_userd_logged = true;
@@ -19865,6 +20062,10 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 				ch->gr_ram_userd_rejects;
 			svc.chan[i].gr_ram_userd_watch_n =
 				ch->gr_ram_userd_watch_n;
+			svc.chan[i].gr_ram_userd_watch_log =
+				ch->gr_ram_userd_watch_log;
+			svc.chan[i].gr_ram_userd_kick_sweeps =
+				ch->gr_ram_userd_kick_sweeps;
 			for (wj = 0; wj < SVC_GR_USERD_WATCH; wj++) {
 				svc.chan[i].gr_ram_userd_watch[wj] =
 					ch->gr_ram_userd_watch[wj];
@@ -19872,6 +20073,8 @@ static bool svc_gr_ram_userd_scan(struct nv_vgpu_device *vgpu,
 					ch->gr_ram_userd_watch_put[wj];
 				svc.chan[i].gr_ram_userd_watch_hits[wj] =
 					ch->gr_ram_userd_watch_hits[wj];
+				svc.chan[i].gr_ram_userd_watch_at[wj] =
+					ch->gr_ram_userd_watch_at[wj];
 			}
 			break;
 		}
@@ -30772,6 +30975,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		/* Initialize expected semaphore payload from the pool (if mapped). */
 		svc_init_expected_payload(vgpu, ch);
 
+		/* SESSION 153-CONTINUED-20: the family test shared by the
+		 * kick-confirm relaxation and the churn monitor below. */
+		bool kfam = svc_kick_family(ch);
+
 		/*
 		 * SESSION 154 (M-GL1.2): speculative-latch confirmation for
 		 * the RAM USERD. The scan latches a gated candidate right
@@ -30782,13 +30989,34 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		 * candidate. The live block advances 1->2->3 in 2-10 s
 		 * (measured); the client's own release waits are ~11 s, so
 		 * the no-advance bound must sit above those.
+		 *
+		 * SESSION 153-CONTINUED-20: for the kick family ANY positive
+		 * PUT step confirms (its put is authored by the client's
+		 * frame loop), only a decrease or the no-advance bound
+		 * rejects - and a CONFIRMED kick latch is not exempt from the
+		 * churn rule: the r974 false confirm (pa=0x11c62e000, put
+		 * walked 1->2->3, then reset to 0/1 and toggled for minutes)
+		 * shows a slowly resetting pool block can pass any
+		 * single-step test. Track its PUT while confirmed and
+		 * unlatch on the first decrease - a client-authored
+		 * free-running put never decreases (the rewind path owns the
+		 * session-reset case).
 		 */
+		if (kfam && ch->gr_ram_userd_confirmed &&
+		    put > ch->gr_ram_userd_latch_put)
+			ch->gr_ram_userd_latch_put = put;
 		if (!ch->engine_type && ch->gr_ram_ring && ch->userd_ok &&
-		    !ch->gr_ram_userd_confirmed) {
+		    (!ch->gr_ram_userd_confirmed ||
+		     (kfam && put < ch->gr_ram_userd_latch_put))) {
+			bool stepped;
+
 			if (!ch->gr_ram_userd_latch_at)
 				ch->gr_ram_userd_latch_at = jiffies;
-			if (put == ch->gr_ram_userd_latch_put + 1 ||
-			    put == ch->gr_ram_userd_latch_put + 2) {
+			stepped = kfam ?
+				put > ch->gr_ram_userd_latch_put :
+				(put == ch->gr_ram_userd_latch_put + 1 ||
+				 put == ch->gr_ram_userd_latch_put + 2);
+			if (stepped) {
 				ch->gr_ram_userd_confirmed = true;
 				pr_info("sema-chan: GR USERD h=0x%x pa=0x%llx speculative latch CONFIRMED (put %u -> %u)\n",
 					ch->h_object, ch->userd_pa,
@@ -33513,6 +33741,10 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 
 					svc.chan[k].gr_ram_userd_watch_n =
 						ch->gr_ram_userd_watch_n;
+					svc.chan[k].gr_ram_userd_watch_log =
+						ch->gr_ram_userd_watch_log;
+					svc.chan[k].gr_ram_userd_kick_sweeps =
+						ch->gr_ram_userd_kick_sweeps;
 					for (wj = 0; wj < SVC_GR_USERD_WATCH;
 					     wj++) {
 						svc.chan[k].gr_ram_userd_watch[wj] =
@@ -33521,6 +33753,8 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							ch->gr_ram_userd_watch_put[wj];
 						svc.chan[k].gr_ram_userd_watch_hits[wj] =
 							ch->gr_ram_userd_watch_hits[wj];
+						svc.chan[k].gr_ram_userd_watch_at[wj] =
+							ch->gr_ram_userd_watch_at[wj];
 					}
 				}
 				svc.chan[k].gr_ram_put_seeded =