vgsp: X/GL wall broken - PROMOTE_CTX registry, small trusted-NOP acceptance, push-scoped release deferral

The X server now starts on the emulated vGPU and GLX runs on the real
NVIDIA driver: Xorg :0 passes its DMA sanity push and ScreenInit, and
DISPLAY=:0 glxinfo reports 'OpenGL 4.6.0 NVIDIA 570.144' with direct
rendering; GZDOOM 4.14.2 launches and renders.  Before this, every X
start died at 'NVIDIA(GPU-0): Failed to initialize DMA' (the
__nvPushTestPushBuffer 10 s GP_GET wait) and no nvidia-push kernel
buffer could be resolved.

What each piece fixes (all observed live on the r817->r827 cycles):

- sema: register the driver's PROMOTE_CTX context-buffer statements as
  a VA->PA registry, consulted as the LAST resort after every walk.
  A promoted (va, pa) pair is a VA-space booking, not a mapping: its
  span can swallow other buffers' VAs (the ctx MAIN booking covered the
  channel GPFIFO VA and range arithmetic answered the entry validation
  with MAIN.pa+offset, a zero page - the latch never fired, r823), and
  even its base page can host a separately mapped buffer (the PATCH
  booking's base VA hosts nvidia-push's 4 KB progress semaphore; the
  release landed in the booking's page while the driver's own polled
  page stayed zero, r826).  The walks describe the real mappings and
  must answer first.

- sema: an all-zero GR push of <= 16 bytes resolved by the trusted
  chain (src=4) is accepted as the driver's NOP sanity push
  (NVA16F_DMA_NOP is a single zero dword; real launches are hundreds of
  bytes).  The r496e acceptance required a content-scanned ring, which
  the X/GL channel is not.

- sema: the wave-1.5 release deferral is push-scoped.  A release whose
  push owns no captured forward candidate has nothing to wait for; it
  applies inline (the progress-tracker semaphore release used to be
  recorded and dropped at every pass reset while an unrelated lingering
  candidate armed the global defer).

- sema: the m164 no-capture hold no longer arms for the accepted zero
  NOP - it held the NOP entry for its 6 s window 'forcing capture' of a
  launch that cannot exist.

- sema: once the ring resolves, the USERD latch scan may retry past its
  8-scan cap (the client's VAS directory registration lost the race
  against the 10 s DMA window when a rebuild stretched the pass cadence,
  r820).

- sema: dbg-gated USERD-scan decision witnesses (rejection reason, sweep
  summary, per-call state), GR-only and budgeted.

Known next issue (not this change): sustained X/GL rendering traffic
wedges the guest RC watchdog ('NVRM: krcWatchdog_IMPL: GPU is probably
locked') after minutes of GZDOOM/GLX activity; the FIFO service itself
keeps consuming.

1 files changed, 316 insertions, 6 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 939d1bb..1fa09d0 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -2753,6 +2753,139 @@ static void svc_lat_store(u64 va, u64 addr, int ap)
 }
 
 /*
+ * r820: driver-declared kernel-buffer translations (PROMOTE_CTX).
+ *
+ * The guest RM announces the physical backing of a channel's context
+ * buffers through NV2080_CTRL_CMD_GPU_PROMOTE_CTX: every entry carries
+ * (gpuVirtAddr, gpuPhysAddr, size, physAttr, bufferId). These are
+ * KERNEL-owned buffers - the nvidia-push pushbuffer, the progress-tracker
+ * segment, the FECS/patch/access-map blocks - so the userspace pin
+ * scanner can never cover them (/proc/<pid>/pagemap has no kernel VAs)
+ * and the page-table walk can answer with a stale/wrong page. Observed
+ * r817 (Xorg GL init): the walk resolved the 64 KB pushbuffer VA
+ * 0x120010000 to pa=0x103fed000, a page reading all zeros, while the
+ * driver's own promote entry said pa=0x11e120000. The NOP sanity push of
+ * __nvPushTestPushBuffer then never decoded, GP_GET never advanced and
+ * the X server died with "Failed to initialize DMA".
+ *
+ * The registry is a small table keyed by the entry VA, newest wins: a
+ * client that re-promotes the same buffer - observed between two Xorg
+ * starts, the same channel object announced a NEW pa for the same VA -
+ * replaces the old record, so a stale backing is never preferred over
+ * the live statement. VA-only bookings (bInitialize=0 re-promotes of
+ * already-initialized buffers, zero PA/size) and bNonmapped entries
+ * (va=0) carry no backing and must not erase the last statement.
+ *
+ * Lookups run before the learned cache in svc_lookup_trusted: the wrong
+ * r817 walk answer had been cached in the learned table (svc_lat_store
+ * after every walk success), so a promote hit must both pre-empt and
+ * overwrite that entry; the hit re-stores the corrected page into the
+ * learned table so the direct svc_lat_lookup callers heal too.
+ */
+#define SVC_PROMO_MAX 256
+struct svc_promo_rec {
+	u64 va;
+	u64 pa;
+	u64 size;
+	unsigned long seq;
+	u32 phys_attr;
+	u16 buffer_id;
+	bool valid;
+};
+static struct svc_promo_rec svc_promo[SVC_PROMO_MAX];
+static unsigned long svc_promo_seq;
+static DEFINE_SPINLOCK(svc_promo_lock);
+static u32 svc_promo_hit_log;	/* bounded "answered from PROMOTE_CTX" witness */
+
+void gsp_sema_service_promote_add(u64 va, u64 pa, u64 size, u32 phys_attr,
+				  u32 buffer_id)
+{
+	struct svc_promo_rec *rec = NULL, *free_rec = NULL, *oldest = NULL;
+	unsigned long flags;
+	int i;
+
+	if (!va || !pa || !size)
+		return;
+
+	spin_lock_irqsave(&svc_promo_lock, flags);
+	for (i = 0; i < SVC_PROMO_MAX; i++) {
+		if (svc_promo[i].valid && svc_promo[i].va == va) {
+			rec = &svc_promo[i];
+			break;
+		}
+		if (!svc_promo[i].valid && !free_rec)
+			free_rec = &svc_promo[i];
+		if (svc_promo[i].valid &&
+		    (!oldest || svc_promo[i].seq < oldest->seq))
+			oldest = &svc_promo[i];
+	}
+	if (!rec)
+		rec = free_rec ? free_rec : oldest;
+	if (!rec) {
+		spin_unlock_irqrestore(&svc_promo_lock, flags);
+		return;
+	}
+	rec->va = va;
+	rec->pa = pa;
+	rec->size = size;
+	rec->phys_attr = phys_attr;
+	rec->buffer_id = (u16)buffer_id;
+	rec->seq = ++svc_promo_seq;
+	rec->valid = true;
+	spin_unlock_irqrestore(&svc_promo_lock, flags);
+}
+
+static int svc_promo_lookup(u64 va, u64 *addr, int *ap)
+{
+	unsigned long flags;
+	u64 best_seq = 0;
+	int rc = -ENOENT;
+	int i;
+
+	spin_lock_irqsave(&svc_promo_lock, flags);
+	for (i = 0; i < SVC_PROMO_MAX; i++) {
+		const struct svc_promo_rec *rec = &svc_promo[i];
+
+		/*
+		 * r823: answer ONLY the base page. The promoted (va, size)
+		 * pair is a VA-space booking, not a contiguous mapping: the
+		 * ctx MAIN buffer's span [0x120020000, 0x120153000) contains
+		 * OTHER buffers' VAs (the channel GPFIFO at 0x120090000), and
+		 * the range arithmetic answered such an interior VA with
+		 * MAIN.pa + offset - a zero page (observed r823: the GPFIFO
+		 * VA resolved to 0x11ce70000, the ring-entry validation read
+		 * va=0x0 len=0 and the latch never fired). Only the page at
+		 * the declared VA has a proven claim to the declared PA; the
+		 * pushbuffer's NOP sits exactly there (nvPushHeader writes at
+		 * the segment base).
+		 */
+		if (!rec->valid || va < rec->va ||
+		    va - rec->va >= SEMA_PAGE_SIZE)
+			continue;
+		if (best_seq && rec->seq <= best_seq)
+			continue;
+		best_seq = rec->seq;
+		*addr = rec->pa + (va - rec->va);
+		/*
+		 * physAttr 0x4 was observed only on vidmem entries (pa in
+		 * the emulated FB aperture, e.g. 0x1f4000000), 0x5/0x6 on
+		 * system memory. Unknown values fall to SYS, whose read
+		 * guard (pa < SEMA_PA_MAX) still rejects nonsense.
+		 */
+		*ap = (rec->phys_attr == 0x4) ? SVC_APERTURE_FB :
+						SVC_APERTURE_SYS;
+		rc = 0;
+	}
+	spin_unlock_irqrestore(&svc_promo_lock, flags);
+	if (!rc && svc_promo_hit_log < 64) {
+		svc_promo_hit_log++;
+		pr_info("sema-promo: va=0x%llx -> pa=0x%llx ap=%d (driver PROMOTE_CTX statement)\n",
+			va, *addr, *ap);
+	}
+	return rc;
+}
+
+/*
  * VA shadow: page-granular backing for device VAs whose VA->PA translation
  * cannot be derived from the guest's page tables.
  *
@@ -3088,9 +3221,24 @@ static u64 svc_pushalt_hits;	/* push reads served by a displaced pair */
 static u32 svc_pushalt_log;
 /* r707x: per-client push-read attempt witness (pump read site). */
 static u32 svc_rdw_log;
+/* r820: bounded per-session USERD-scan decision witness (svc_chan_userd).
+ * GR channels only and capped: a hot CE channel re-running this every
+ * ~16 ms burned the whole budget before the channel under diagnosis ever
+ * appeared. Each scan reports its decision, a scan that found nothing
+ * reportable logs a sweep summary. */
+static u32 svc_userdp_log;
 static u8 svc_read_last_src;
 /* r715: PA the winning source of the last push read resolved to. */
 static u64 svc_read_last_pa;
+/*
+ * r820: set when the last svc_read_va_push_ex chunk was answered by a
+ * driver-declared PROMOTE_CTX kernel buffer. The push decode uses it to
+ * accept the NOP sanity push: nvidia-push's __nvPushTestPushBuffer writes
+ * NVA16F_DMA_NOP (0x00000000) into a promoted kernel pushbuffer, so the
+ * whole small push is zeros - unparseable as a method stream, but exactly
+ * right from a translation the driver itself declared.
+ */
+static bool svc_read_last_promote;
 /* r715: per-client push-decode branch witness (pump read site). */
 static u32 svc_pushdec_log;
 /* r707d: bounded push-read failure witness (see svc_read_va_push_ex). */
@@ -3444,6 +3592,23 @@ static int svc_lookup_trusted(struct nv_vgpu_device *vgpu, u64 va, u64 *addr,
 		svc_lat_store(va, pa, SVC_APERTURE_SYS);
 		return 0;
 	}
+	/*
+	 * r826: driver-declared PROMOTE_CTX buffers are the LAST resort, not
+	 * the first. The promoted (va, pa) pair names a BOOKING, and even its
+	 * base page can belong to a different buffer than the one mapped at
+	 * that VA (r823: the ctx-MAIN booking's span swallowed the GPFIFO VA;
+	 * r826: the PATCH booking's base VA 0x120001000 hosts nvidia-push's
+	 * separately mapped 4 KB progress semaphore - writing the release to
+	 * the booking's pa left the driver's own polled page at zero and the
+	 * X server failed at its 10 s wait while the release value sat in the
+	 * wrong page). The page-table walks describe the real mappings and
+	 * must answer first; the promote record is only a fallback for VAs
+	 * the walks cannot resolve at all.
+	 */
+	if (!svc_promo_lookup(va, addr, ap)) {
+		svc_lat_store(va, *addr, *ap);
+		return 0;
+	}
 	return -ENOENT;
 }
 
@@ -6533,6 +6698,31 @@ static bool svc_fwd_defer_arm(void)
 	       gsp_phys_work_defer_release_enabled();
 }
 
+/*
+ * r824: the release deferral must be owned by a candidate of THIS push.
+ * svc_fwd_defer_arm() answers globally (any unoffered candidate of the
+ * boot's ring); an unrelated lingering candidate then parked the release
+ * of a push that has no forwarding at all. Observed: the X/GL DMA test's
+ * progress-tracker release (the kernel's progress semaphore) was recorded
+ * into svc_fwd_rel and dropped at the next pass's reset, every pass, so
+ * the semaphore never advanced and Xorg failed at its 10 s wait - while
+ * every other stage of the chain (latch, NOP acceptance, decode) had
+ * just started working. A release whose push owns no candidate cannot
+ * wait for a forwarded copy that will never exist; it applies inline.
+ */
+static bool svc_fwd_cand_owns_push(u64 push_va)
+{
+	u32 ri;
+
+	for (ri = 0; ri < SVC_GR_CAND_RING; ri++) {
+		if (svc_gr_cand_ring[ri].valid &&
+		    !svc_gr_cand_ring[ri].offered &&
+		    svc_gr_cand_ring[ri].push_va == push_va)
+			return true;
+	}
+	return false;
+}
+
 
 /*
  * P6b wave-1.5 deferred guest releases: every release group the current
@@ -11194,7 +11384,16 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					or.va = va;
 					or.payload = payload;
 					or.entry_idx = entry_idx;
+					/*
+					 * r824: the deferral needs a
+					 * candidate that OWNS this push
+					 * (see svc_fwd_cand_owns_push) -
+					 * a release from a push with no
+					 * forwarding has nothing to wait
+					 * for and applies inline.
+					 */
 					if (svc_fwd_defer_arm() &&
+					    svc_fwd_cand_owns_push(push_va) &&
 					    !svc.op_tr_pending) {
 						svc_opdef_drop(vgpu, ch,
 							       entry_idx);
@@ -13147,6 +13346,16 @@ static void svc_session_flush(void)
 	 * name the previous client's pages (ISSUE-0002 fan-out).
 	 */
 	memset(svc_ipin_alt, 0, sizeof(svc_ipin_alt));
+	/*
+	 * r820: PROMOTE_CTX records name one client's kernel buffers, and a
+	 * new client's channels re-announce their own (the same VAs can back
+	 * with different PAs across sessions): session state. A flush skips
+	 * entirely while any channel is live, so a running client's records
+	 * cannot be wiped under it.
+	 */
+	memset(svc_promo, 0, sizeof(svc_promo));
+	svc_promo_seq = 0;
+	svc_promo_hit_log = 0;
 	memset(svc_cnt_der, 0, sizeof(svc_cnt_der));
 	svc_cnt_der_n = 0;
 	/* r551: delivered-release marks are per-session (same handles reused). */
@@ -14278,6 +14487,9 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
 {
 	u8 *p = buf;
 
+	/* r820: per-read; the decode reads it right after the call. */
+	svc_read_last_promote = false;
+
 	while (len) {
 		u64 page_va = va & ~(u64)(SEMA_PAGE_SIZE - 1);
 		size_t chunk = min_t(size_t, len,
@@ -14406,6 +14618,19 @@ static int svc_read_va_push_ex(struct nv_vgpu_device *vgpu,
 				continue;
 			svc_read_last_src = (u8)src;
 			svc_read_last_pa = pa;
+			/*
+			 * r820: name whether the trusted answer came from a
+			 * driver-declared PROMOTE_CTX buffer; the decode uses
+			 * it for the zero-NOP acceptance (see the flag).
+			 */
+			if (src == 4) {
+				u64 promo_pa = 0;
+				int promo_ap = SVC_APERTURE_SYS;
+
+				svc_read_last_promote =
+					!svc_promo_lookup(page_va, &promo_pa,
+							  &promo_ap);
+			}
 			done = true;
 		}
 		if (!done) {
@@ -14496,6 +14721,22 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 						  : SVC_APERTURE_FB;
 
 	/*
+	 * r820 diag: the Xorg DMA-test window is 10 s and the latch hinges on
+	 * this function's bounded scan re-runs (userd_scans < 8); a pass
+	 * cadence stretched by a rebuild can spend every scan before the VAS
+	 * candidate registers (r820: the ring resolved at +1 s but no latch
+	 * line followed). One bounded row per call names the state the scan
+	 * decides on.
+	 */
+	if (gsp_sema_dbg && !ch->engine_type && svc_userdp_log < 512) {
+		svc_userdp_log++;
+		pr_info("sema-chan: userdp h=0x%x eng=%u scans=%u rrc=%d ring_pa=0x%llx ring_ap=%d userd_pa=0x%llx as=%d ok=%u\n",
+			ch->h_object, ch->engine_type, ch->userd_scans,
+			rrc, ring_pa, ring_ap, ch->userd_pa, ch->userd_as,
+			ch->userd_ok ? 1 : 0);
+	}
+
+	/*
 	 * ISSUE-0002 (r40): prefer the layout-derived CE USERD and accept it
 	 * while it is still idle. In a second session the registered userdMem
 	 * offset (pool-relative, see below) aliases a DIFFERENT rank's real
@@ -14691,12 +14932,27 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 	 * USERD cannot pass that test. One scan per channel (the result
 	 * persists through the pump's write-back), logged on latch.
 	 */
-	if (!ch->engine_type && ch->userd_scans < 8 && ring_pa) {
+	/*
+	 * r820: the latch scan is capped per channel so a channel that
+	 * never anchors cannot sweep forever. But the resolution the scan
+	 * needs can arrive LATE: the Xorg GL channel's ring resolves only
+	 * when the client's VAS page directory registers (COPY_SERVER_PDES
+	 * age-out), and r820 showed every one of the 8 pre-resolution scans
+	 * spent before that moment (the pass cadence stretched by a
+	 * rebuild), so the 10 s DMA-test window closed with the ring
+	 * resolved and no retry left. Once ring_pa is non-zero the scan is
+	 * cheap (the sweep validates against a resolved ring), so it may
+	 * keep retrying up to a wider cap; the client's own timeout still
+	 * bounds the channel's life.
+	 */
+	if (!ch->engine_type && ring_pa &&
+	    ch->userd_scans < (ring_pa ? 64u : 8u)) {
 		u64 off, lim;
 		u8 *base = vgpu->bars[1].vaddr;
 		u64 best = 0, best_dist = ~0ULL;
 		u32 best_put = 0, best_get = 0;
 		bool best_pending = false;
+		u32 gate_n = 0, vfail_n = 0;
 
 		ch->userd_scans++;
 		lim = min_t(u64, svc_fb_shadow_size(vgpu), 0x20000000ULL);
@@ -14707,23 +14963,33 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 				u32 k;
 				u64 dist;
 				bool pending;
+				int rck = -1;
+				u64 rkva = 0;
+				u32 rklen = 0;
 
 				memcpy(&hp, base + off + SVC_USERD_GP_PUT, 4);
 				memcpy(&hg, base + off + SVC_USERD_GP_GET, 4);
 				if (!hp || hp > ch->gpfifo_entries || hg > hp)
 					continue;
+				gate_n++;
 				/* Validate: every claimed entry decodes to a
 				 * pushbuffer VA in the ring's window. */
 				for (k = 0; k < hp && k < 64; k++) {
 					u8 e[8];
 					u64 eva;
 					u32 elen, a0, a1;
+					int rdrc;
 
-					if (svc_read_va(vgpu,
+					rdrc = svc_read_va(vgpu,
 							ch->gpfifo_va +
 							(u64)k * 8,
-							e, sizeof(e))) {
+							e, sizeof(e));
+					if (rdrc) {
 						ok = false;
+						rck = (int)k;
+						/* -1: read failure rc; else the
+						 * window check below. */
+						rkva = (u64)rdrc;
 						break;
 					}
 					memcpy(&a0, e, 4);
@@ -14744,11 +15010,29 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 						SVC_GR_RAM_WIN / 2 ||
 					    elen < 4 || elen > 0x100000) {
 						ok = false;
+						rck = (int)k;
+						rkva = eva;
+						rklen = elen;
 						break;
 					}
 				}
-				if (!ok)
+				if (!ok) {
+					/*
+					 * r820 diag: name why a scanned
+					 * block was rejected (read rc or
+					 * the entry VA/len window).
+					 */
+					vfail_n++;
+					if (gsp_sema_dbg && !ch->engine_type &&
+					    svc_userdp_log < 512) {
+						svc_userdp_log++;
+						pr_info("sema-chan: userdp scan off=0x%llx put=%u rejected k=%d (read_rc=%lld va=0x%llx len=%u)\n",
+							off, hp, rck,
+							(long long)rkva,
+							rkva, rklen);
+					}
 					continue;
+				}
 				/*
 				 * Several blocks can validate: a previous
 				 * session leaves its old USERD holding the same
@@ -14788,6 +15072,17 @@ static int svc_chan_userd(struct nv_vgpu_device *vgpu, struct svc_channel *ch,
 					best_pending ? "" : ", idle");
 				return 0;
 			}
+			/*
+			 * r820 diag: the sweep found nothing adoptable. gates
+			 * counts put/get-coherent blocks seen, vfails the ones
+			 * rejected by the ring-entry validation.
+			 */
+			if (gsp_sema_dbg && svc_userdp_log < 512) {
+				svc_userdp_log++;
+				pr_info("sema-chan: userdp sweep h=0x%x scans=%u gates=%u vfails=%u best=0x%llx\n",
+					ch->h_object, ch->userd_scans,
+					gate_n, vfail_n, best);
+			}
 		}
 	}
 	return -ENOENT;
@@ -27048,8 +27343,23 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 						 * dword - NVA06F_DMA_NOP), not a bad
 						 * translation: accept it and let the
 						 * decode find no ops.
+						 * r820/r826: the same push arrives on
+						 * KERNEL channels (nvidia-push's
+						 * __nvPushTestPushBuffer writes the
+						 * NOP into the kernel pushbuffer); its
+						 * promote entry is a booking, so the
+						 * trusted walk answers - a SMALL
+						 * all-zero GR push resolved by the
+						 * trusted chain (src=4) is the NOP.
+						 * Real launches are large (a QMD push
+						 * is hundreds of bytes), so the size
+						 * bound keeps an unread legitimate
+						 * push from being consumed as empty.
 						 */
-						zero_nop = ch->gr_ram_ring &&
+						zero_nop = (ch->gr_ram_ring ||
+							    (!ch->engine_type &&
+							     svc_read_last_src == 4 &&
+							     len <= 16)) &&
 							len >= 4 &&
 							!memchr_inv(pb, 0, len);
 
@@ -28083,7 +28393,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 							if (!ch->engine_type && va && len &&
 							    gsp_phys_gr_forward_enabled() &&
 							    (svc_gr_qmd_seen > 0 ||
-							     all_zero) &&
+							     (all_zero && !zero_nop)) &&
 							    (!entry_replay ||
 							     svc_gr_cand_push_owed(va)) &&
 							    !(svc_gr_cand.valid &&