vgsp: native >6GiB client page resolution (PA ceiling to scan_top), kick 2D-notifier release decode, zero_nop bound (SESSION 153 r933/r934)

1 files changed, 145 insertions, 52 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index 52820b7..3d14d4b 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -80,8 +80,17 @@ static inline u64 svc_fb_shadow_size(const struct nv_vgpu_device *vgpu)
  * hole), so the top of RAM is around 6 GB, not 4 GB. Anything below this is
  * RAM; the emulated vidmem lives in its own aperture and must not be confused
  * with it.
- */
-#define SEMA_PA_MAX		0x180000000ULL
+ *
+ * r933: the 8 GiB GL guest's top of RAM is 0x280000000, and the GL client's
+ * kick/push/ring pages were measured in the 6-8 GiB band (kick gpa
+ * 0x20ad9b000 > the old 0x180000000 6 GiB cap), so every walk leaf there was
+ * rejected by the aperture bound (walk-probe rc=-2 pa=0x229c1a000) and only
+ * per-session pin injection could read them. The compile-time constant sizes
+ * the fixed tables (hot-page bitmap); the live acceptance bound is
+ * svc_pa_max(), driven by the sema_gr_scan_top module parameter so a cycle on
+ * a smaller guest shrinks both together.
+ */
+#define SEMA_PA_MAX		0x280000000ULL
 #define SEMA_SCAN_PAGES_MAX	0x20000	/* 512 MB worth of pages per pass */
 
 /* Plausible GPU VA window for the client's system-memory objects (Turing). */
@@ -201,21 +210,42 @@ struct sema_map_entry {
 
 /*
  * r908: guest-RAM ring-scan top (guest physical). The guest's usable RAM
- * can extend above SEMA_PA_MAX: the 8 GiB guest's e820 top is
+ * can extend above the old 6 GB service cap: the 8 GiB guest's e820 top is
  * 0x280000000, and the GL client channel's GPFIFO ring page was measured
- * at gpa 0x231e5d000 (readable: svc_read -> vfio_dma_rw answered there) -
- * above the old 6 GB cap, so the content scan could never reach it. Only
- * the ring scan uses this wider top; every other table/bound in the
- * service stays sized on SEMA_PA_MAX. Reads past mapped RAM (the
- * 0x80000000-0x100000000 PCI hole, or a smaller guest) fail fast, so a
- * too-large top costs a few failed probes per sweep. The module parameter
- * is the per-cycle guard: shrink it to the boot's real RAM top.
+ * at gpa 0x231e5d000 (readable: svc_read -> vfio_dma_rw answered there).
+ * Reads past mapped RAM (the 0x80000000-0x100000000 PCI hole, or a smaller
+ * guest) fail fast, so a too-large top costs a few failed probes per sweep.
+ * The module parameter is the per-cycle guard: shrink it to the boot's real
+ * RAM top.
+ *
+ * r933: this parameter is now the single live PA acceptance bound for the
+ * whole service (svc_pa_max), not just the ring scan - the walk leaves in
+ * the 6-8 GiB band were exactly what the old separate 6 GB SEMA_PA_MAX
+ * rejected (the GL client's kick pages). SEMA_PA_MAX remains the
+ * compile-time maximum that sizes fixed tables; the param clamps it.
  */
 unsigned long long gsp_sema_gr_scan_top = 0x280000000ULL;
 module_param_named(sema_gr_scan_top, gsp_sema_gr_scan_top, ullong, 0644);
 #define SVC_GR_SCAN_TOP_MAX	0x400000000ULL	/* 16 GiB sanity clamp */
 
 /*
+ * Live guest-PA acceptance bound: min(param, compile-time max). Every
+ * runtime range check that used to compare against SEMA_PA_MAX uses this
+ * instead, so raising the ceiling for a bigger guest (r933: 8 GiB guest,
+ * GL client's pages above 6 GiB) is one module parameter, and shrinking it
+ * for a smaller guest shrinks the scans and the walk acceptance together.
+ * Never 0 (a zero param falls back to the compile-time max).
+ */
+static inline u64 svc_pa_max(void)
+{
+	u64 top = READ_ONCE(gsp_sema_gr_scan_top);
+
+	if (!top || top > SEMA_PA_MAX)
+		return SEMA_PA_MAX;
+	return top;
+}
+
+/*
  * Ring-scan slice budget: two full sweeps at the configured top, with the
  * original 24-slice floor and a u8-safe cap (gr_ram_passes).
  */
@@ -2342,7 +2372,7 @@ static bool svc_ap_ok(struct nv_vgpu_device *vgpu, u64 addr, int ap)
 {
 	if (ap == SVC_APERTURE_FB)
 		return vgpu->bars[1].vaddr && addr + 8 <= svc_fb_shadow_size(vgpu);
-	return addr + 8 <= SEMA_PA_MAX;
+	return addr + 8 <= svc_pa_max();
 }
 
 static int svc_read_ent(struct nv_vgpu_device *vgpu, u64 table, u64 off,
@@ -2738,7 +2768,7 @@ static void svc_pd_discover(struct nv_vgpu_device *vgpu)
 	 * matters is the first one of the page. */
 	for (pass = 0; pass < 2 && !svc_pd_found; pass++) {
 		int ap = pass == 0 ? SVC_APERTURE_SYS : SVC_APERTURE_FB;
-		u64 limit = pass == 0 ? SEMA_PA_MAX : svc_fb_shadow_size(vgpu);
+		u64 limit = pass == 0 ? svc_pa_max() : svc_fb_shadow_size(vgpu);
 		u8 *fb = (u8 *)vgpu->bars[1].vaddr;
 		u64 gpa;
 
@@ -2838,8 +2868,8 @@ static void svc_pd_ring_discover(struct nv_vgpu_device *vgpu)
 	buf = kmalloc(chunk, GFP_KERNEL);
 	if (!buf)
 		return;
-	for (gpa = 0; gpa < SEMA_PA_MAX && !svc_pd_r_found; gpa += chunk) {
-		u32 len = (u32)min_t(u64, chunk, SEMA_PA_MAX - gpa);
+	for (gpa = 0; gpa < svc_pa_max() && !svc_pd_r_found; gpa += chunk) {
+		u32 len = (u32)min_t(u64, chunk, svc_pa_max() - gpa);
 		u64 off;
 
 		if (svc_read(vgpu, gpa, buf, len))
@@ -3673,8 +3703,8 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va,
 		return -ENOMEM;
 
 	end = tr_cursor + SVC_TR_SLICE;
-	if (end > SEMA_PA_MAX)
-		end = SEMA_PA_MAX;
+	if (end > svc_pa_max())
+		end = svc_pa_max();
 
 	for (gpa = tr_cursor; gpa < end; gpa += SVC_TR_CHUNK) {
 		u64 chunk = min_t(u64, SVC_TR_CHUNK, end - gpa);
@@ -3703,7 +3733,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va,
 			if (cpu_va < direct_base)
 				continue;
 			pa = cpu_va - direct_base;
-			if (pa >= SEMA_PA_MAX || (pa & 0xfff))
+			if (pa >= svc_pa_max() || (pa & 0xfff))
 				continue;
 
 			*pa_out = pa | (gpu_va & (SEMA_PAGE_SIZE - 1));
@@ -3721,7 +3751,7 @@ static int svc_targeted_resolve(struct nv_vgpu_device *vgpu, u64 gpu_va,
 
 	tr_cursor = end;
 	kfree(buf);
-	if (tr_cursor >= SEMA_PA_MAX) {
+	if (tr_cursor >= svc_pa_max()) {
 		pr_info("sema-svc: targeted resolve va=0x%llx: full RAM scan found nothing (scan #%llu)\n",
 			page_va, tr_scans);
 		svc_tr_note_failure(page_va);
@@ -4963,7 +4993,7 @@ static void svc_scan_pass(struct nv_vgpu_device *vgpu, u8 *buf,
 	 * a second with identical pair/base detection (the inner page loop
 	 * is byte-for-byte the old per-page body).
 	 */
-	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
+	for (gpa = 0; gpa < svc_pa_max(); gpa += SVC_SCAN_CHUNK) {
 		u32 po;
 
 		if (READ_ONCE(svc_sess_flush))
@@ -5087,7 +5117,7 @@ static void svc_add_pair_merge(u64 base, const struct raw_pair *p, u32 sorted_n)
 	if (p->cpu_va < base)
 		return;
 	pa = p->cpu_va - base;
-	if (pa >= SEMA_PA_MAX || (pa & 0xfff))
+	if (pa >= svc_pa_max() || (pa & 0xfff))
 		return;
 	if (p->gpu_va < SEMA_VA_LO || p->gpu_va >= SEMA_VA_HI)
 		return;
@@ -11825,8 +11855,8 @@ static void svc_push_raw_dump(const u8 *buf, u32 len, u64 push_va,
 static u64 svc_notify2d_done;
 static u32 svc_notify2d_log;
 
-static void svc_notify2d_write(struct nv_vgpu_device *vgpu,
-			       struct svc_channel *ch)
+static bool svc_notify2d_write(struct nv_vgpu_device *vgpu,
+			       struct svc_channel *ch, bool completion)
 {
 	/* 16 bytes = the NvNotification (status word at +14). */
 	u8 zero[16] = { 0 };
@@ -11834,17 +11864,17 @@ static void svc_notify2d_write(struct nv_vgpu_device *vgpu,
 	int ap = SVC_APERTURE_SYS;
 
 	if (!ch->notify2d_va)
-		return;
+		return false;
 	if (svc_lookup(vgpu, ch->notify2d_va, &pa, &ap)) {
 		if (gsp_sema_dbg && svc_notify2d_log < 16) {
 			svc_notify2d_log++;
 			pr_info("sema-notify2d: h=0x%x va=0x%llx unresolved (RC watchdog notifier; guest re-posts every second)\n",
 				ch->h_object, ch->notify2d_va);
 		}
-		return;
+		return false;
 	}
 	if (svc_write_ap(vgpu, pa, svc_ap_of(ap), zero, sizeof(zero)))
-		return;
+		return false;
 	svc_notify2d_done++;
 	/*
 	 * r834: pin the page that accepted the ack; the same stale-map
@@ -11852,11 +11882,22 @@ static void svc_notify2d_write(struct nv_vgpu_device *vgpu,
 	 */
 	svc_pin_store(ch->notify2d_va & ~(u64)(SEMA_PAGE_SIZE - 1),
 		      pa & ~(u64)(SEMA_PAGE_SIZE - 1), svc_ap_of(ap));
+	/*
+	 * r934: a NOTIFY decoded from a push is that push's completion
+	 * edge on real hardware - post the channel event so the submitting
+	 * client's OS event fires (without it the GL client's kick that
+	 * arms 0x12000e0b0 and NOTIFYs never completes its submit wait
+	 * and the app retries every ~22 s). The periodic idempotent ack
+	 * (r832) passes completion=false so it stays silent.
+	 */
+	if (completion)
+		gsp_rm_post_channel_event(vgpu, ch->h_object);
 	if (gsp_sema_dbg && svc_notify2d_log < 16) {
 		svc_notify2d_log++;
 		pr_info("sema-notify2d: h=0x%x va=0x%llx -> pa=0x%llx zeroed (RC watchdog acked)\n",
 			ch->h_object, ch->notify2d_va, pa);
 	}
+	return true;
 }
 
 /*
@@ -12616,6 +12657,18 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 			case SVC_2D_SET_NOTIFY_A:
 				ch->notify2d_va = (u64)(v0 & 0x1ffffff) << 32;
 				/*
+				 * r934: the GL client's kick pushes the
+				 * pair as ONE count-2 run at 0x104 (the
+				 * methods 0x104 + 0x108 are contiguous):
+				 * `20026041 00000001 2000e0b0` =
+				 * SET_NOTIFY_A=1, SET_NOTIFY_B=0x2000e0b0
+				 * -> notification VA 0x12000e0b0. Fold the
+				 * second data word in; the watchdog's
+				 * separate count-1 runs are unchanged.
+				 */
+				if (count >= 2)
+					ch->notify2d_va |= (u64)v1;
+				/*
 				 * r834: this method stream reached us through
 				 * a REAL read (svc_read_last_pa); pin its page
 				 * so the watchdog's push translation cannot go
@@ -12635,8 +12688,20 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 				ch->notify2d_va |= (u64)v0;
 				break;
 			case SVC_2D_NOTIFY:
-				if (count == 1 && v0 == 0)	/* WRITE_ONLY */
-					svc_notify2d_write(vgpu, ch);
+				if (count == 1 && v0 == 0) {	/* WRITE_ONLY */
+					if (svc_notify2d_write(vgpu, ch,
+							       true)) {
+						/*
+						 * r934: the notifier write IS
+						 * this push's release edge
+						 * (the GL client's kick car-
+						 * ries no semaphore form);
+						 * mark it so the push line
+						 * shows have_rel=1.
+						 */
+						st.have_rel = 1;
+					}
+				}
 				break;
 			default:
 				break;
@@ -13254,7 +13319,7 @@ static void svc_decode_stream(struct nv_vgpu_device *vgpu, const u8 *buf,
 					st.have_dst_mode &&
 					st.dst_phys_mode == CE_APERTURE_SYS &&
 					st.offset_out + (u64)st.line_length <=
-						SEMA_PA_MAX;
+						svc_pa_max();
 				/* r365: vidmem-window fill (see gsp_sema_winfill). */
 				bool winfill = gsp_sema_winfill &&
 					st.offset_out >= SVC_VIDMEM_VA_LO &&
@@ -15294,8 +15359,8 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
 		if (!buf)
 			return;
 	}
-	if (end > SEMA_PA_MAX)
-		end = SEMA_PA_MAX;
+	if (end > svc_pa_max())
+		end = svc_pa_max();
 
 	svc_apply_hot(vgpu);
 
@@ -15313,7 +15378,7 @@ static void svc_apply_pushes(struct nv_vgpu_device *vgpu)
 		}
 	}
 
-	svc_scan_cursor = (end >= SEMA_PA_MAX) ? 0 : end;
+	svc_scan_cursor = (end >= svc_pa_max()) ? 0 : end;
 }
 
 /*
@@ -15340,9 +15405,9 @@ static void svc_apply_sweep_slice(struct nv_vgpu_device *vgpu)
 	u32 pages = 0;
 	bool complete;
 
-	if (end > SEMA_PA_MAX)
-		end = SEMA_PA_MAX;
-	complete = (end >= SEMA_PA_MAX);
+	if (end > svc_pa_max())
+		end = svc_pa_max();
+	complete = (end >= svc_pa_max());
 
 	/*
 	 * Back off after a full pass that applied nothing: a stalled channel
@@ -15487,7 +15552,7 @@ static void svc_pb_discover(struct nv_vgpu_device *vgpu)
 	svc_pb_passes++;
 	svc_pb_last = jiffies;
 	svc_pb_last_n = svc_pb_n;
-	for (gpa = 0; gpa < SEMA_PA_MAX; gpa += SVC_SCAN_CHUNK) {
+	for (gpa = 0; gpa < svc_pa_max(); gpa += SVC_SCAN_CHUNK) {
 		u32 o;
 
 		if (READ_ONCE(svc_sess_flush))
@@ -15744,7 +15809,7 @@ static int svc_pb_win_lookup(const struct svc_channel *ch, u64 va_page,
 		    va_page >= svc_pb[i].va + 0x100000ULL)
 			continue;
 		pa = svc_pb[i].pa_page + (va_page - svc_pb[i].va);
-		if (pa >= SEMA_PA_MAX)
+		if (pa >= svc_pa_max())
 			return -ENOENT;
 		*pa_page = pa;
 		return 0;
@@ -17604,7 +17669,7 @@ static void svc_gr_content_discover(struct nv_vgpu_device *vgpu)
 	if (!buf)
 		return;
 
-	for (gpa = SEMA_PA_MAX - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) {
+	for (gpa = svc_pa_max() - SVC_SCAN_CHUNK;; gpa -= SVC_SCAN_CHUNK) {
 		if (svc_read(vgpu, gpa, buf, SVC_SCAN_CHUNK))
 			goto next_chunk;
 		for (o = 0; o + 24 <= SVC_SCAN_CHUNK; o += 4) {
@@ -18831,7 +18896,7 @@ static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va,
 	 * window phase already covered.
 	 */
 	if (ring_pa >= SVC_GR_RAM_WIN / 2 &&
-	    ring_pa + SVC_GR_RAM_WIN / 2 <= SEMA_PA_MAX) {
+	    ring_pa + SVC_GR_RAM_WIN / 2 <= svc_pa_max()) {
 		u64 lo = (ring_pa - SVC_GR_RAM_WIN / 2) & ~0xfffULL;
 		u64 hi = (ring_pa + SVC_GR_RAM_WIN / 2) & ~0xfffULL;
 
@@ -18848,7 +18913,7 @@ static void svc_xvas_pdb_discover(struct nv_vgpu_device *vgpu, u64 ring_va,
 
 		for (mi = 0; mi < nmap; mi++) {
 			off = svc.map[mi].pa_page & ~0xfffULL;
-			if (!off || off >= SEMA_PA_MAX)
+			if (!off || off >= svc_pa_max())
 				continue;
 			if (ring_pa >= SVC_GR_RAM_WIN / 2 &&
 			    off >= ring_pa - SVC_GR_RAM_WIN / 2 &&
@@ -19619,7 +19684,7 @@ static void svc_pb_pair_gr(struct svc_channel *ch)
 	if (svc_pin_lookup_any(ch->gpfifo_va, &ring_pa, &ring_ap))
 		return;
 	ring_pa &= ~(u64)(SEMA_PAGE_SIZE - 1);
-	if (!ring_pa || ring_pa >= SEMA_PA_MAX)
+	if (!ring_pa || ring_pa >= svc_pa_max())
 		return;
 	ring_va = ch->gpfifo_va & ~(u64)(SEMA_PAGE_SIZE - 1);
 	want = SVC_PB_GR_SLOT_BASE | ch->gr_index;
@@ -19654,7 +19719,7 @@ static void svc_pb_pair_gr(struct svc_channel *ch)
 	}
 	/* Upper half: [ring, ring + 1 MB) -> [ring_pa, ring_pa + 1 MB). */
 	if (svc_pb_n < SVC_PB_MAX &&
-	    ring_pa + SVC_GR_RAM_WIN / 2 < SEMA_PA_MAX) {
+	    ring_pa + SVC_GR_RAM_WIN / 2 < svc_pa_max()) {
 		svc_pb[svc_pb_n].va = ring_va;
 		svc_pb[svc_pb_n].pa_page = ring_pa;
 		svc_pb[svc_pb_n].slot = want;
@@ -20044,7 +20109,7 @@ static int svc_rescue_run_try(struct nv_vgpu_device *vgpu,
 			break;
 		}
 		pa = (u64)((s64)anchor_pa + delta);
-		if (pa + SEMA_PAGE_SIZE > SEMA_PA_MAX) {
+		if (pa + SEMA_PAGE_SIZE > svc_pa_max()) {
 			rc = -EINVAL;
 			break;
 		}
@@ -20380,8 +20445,8 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 		lo = ALIGN(svc_pb[i].pa_page > (512ULL << 20) ?
 			   svc_pb[i].pa_page - (512ULL << 20) :
 			   (u64)SVC_SCAN_CHUNK, SVC_SCAN_CHUNK);
-		if (hi > SEMA_PA_MAX - SVC_SCAN_CHUNK)
-			hi = SEMA_PA_MAX - SVC_SCAN_CHUNK;
+		if (hi > svc_pa_max() - SVC_SCAN_CHUNK)
+			hi = svc_pa_max() - SVC_SCAN_CHUNK;
 		if (lo <= hi &&
 		    !svc_rescue_scan_range(vgpu, ch, entry_va, entry_len,
 					   slot, hi, lo, buf, cand, cand_va)) {
@@ -20406,7 +20471,7 @@ static int svc_rescue_push_page(struct nv_vgpu_device *vgpu,
 	 * few hundred MB.
 	 */
 	i = svc_rescue_scan_range(vgpu, ch, entry_va, entry_len, slot,
-				  SEMA_PA_MAX - SVC_SCAN_CHUNK, 0, buf, cand,
+				  svc_pa_max() - SVC_SCAN_CHUNK, 0, buf, cand,
 				  cand_va);
 	kfree(buf);
 	if (!i)
@@ -29122,7 +29187,7 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 		 * idempotent.
 		 */
 		if (!ch->engine_type && ch->notify2d_va && put != get)
-			svc_notify2d_write(vgpu, ch);
+			svc_notify2d_write(vgpu, ch, false);
 		/*
 		 * SESSION 148 (r864): registration-based keep-alive that
 		 * needs no decode - ack the driver-registered notifier when
@@ -29860,11 +29925,39 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 						 * bound keeps an unread legitimate
 						 * push from being consumed as empty.
 						 */
+						/*
+						 * r934 (SESSION 153-CONT-3):
+						 * bound the gr_ram_ring
+						 * zero-NOP acceptance by
+						 * SIZE. The NOP sanity push
+						 * is one zero dword
+						 * (NVA06F_DMA_NOP; observed
+						 * len=4 on h=0x1000013);
+						 * without the bound a
+						 * 64-byte ALL-ZERO read
+						 * from a stale validated pin
+						 * (the GL client's kick page
+						 * after the guest reused it,
+						 * r933: src=2
+						 * pa=0x1285a3000 zeros while
+						 * the real kick 0001fff0
+						 * 20010000 sat at
+						 * pa=0x12c94b000) was
+						 * consumed as empty and the
+						 * entry never fell through
+						 * to the VAS walk that
+						 * answers correctly. With
+						 * the bound the implausible
+						 * push takes the r707v
+						 * viawalk retry (and, if
+						 * that fails, the stale pin
+						 * is dropped by the
+						 * rescue path).
+						 */
 						zero_nop = (ch->gr_ram_ring ||
 							    (!ch->engine_type &&
-							     svc_read_last_src == 4 &&
-							     len <= 16)) &&
-							len >= 4 &&
+							     svc_read_last_src == 4)) &&
+							len >= 4 && len <= 16 &&
 							!memchr_inv(pb, 0, len);
 
 						if (!svc_method_stream_plausible(pb, len) &&
@@ -33006,7 +33099,7 @@ int gsp_sema_service_counter_add(u32 h_object, u64 pa)
 	if (!svc.task)
 		return -ENODEV;
 	/* The counter is a u32: require a 4-byte aligned guest PA. */
-	if (!h_object || !pa || (pa & 0x3) || pa >= SEMA_PA_MAX)
+	if (!h_object || !pa || (pa & 0x3) || pa >= svc_pa_max())
 		return -EINVAL;
 
 	spin_lock_irqsave(&svc.chan_lock, flags);
@@ -33051,9 +33144,9 @@ int gsp_sema_service_scrubbuf_add(u64 fin_pa, u64 hsem_pa)
 	if (!svc.task)
 		return -ENODEV;
 	/* Both words are u32: require 4-byte alignment and a RAM-shaped PA. */
-	if (!fin_pa || (fin_pa & 0x3) || fin_pa >= SEMA_PA_MAX)
+	if (!fin_pa || (fin_pa & 0x3) || fin_pa >= svc_pa_max())
 		return -EINVAL;
-	if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= SEMA_PA_MAX)
+	if (!hsem_pa || (hsem_pa & 0x3) || hsem_pa >= svc_pa_max())
 		return -EINVAL;
 
 	mutex_lock(&nv_vgpu_driver.lock);