vgsp-sema: hold GR launches whose device argument pointers do not translate; derive the CE vmap pairing per allocation

Two second-client zeros/stall countermeasures:

- svc_gr_data_fill now reports when a launch carries high (device-space)
  argument pointers and none of them resolves through any trusted source.
  svc_gr_request refuses to forward such a launch and leaves the capture
  unconsumed; the pump holds the ring entry for one bounded window per
  entry (gsp_sema_gr_grace_ms) and retries it, so a translation that
  appears a pass later (scanner pin, published directory, learned pair)
  is used instead of executing the kernel against unresolved pointers.
  On window expiry the entry is consumed exactly as before (bounded
  zeros, never a hang). Bounded per-session fill diagnostics name the
  source that serves each candidate (ipin/pin/lat/map/vas/disc/win) and
  dump the published vaspace walk results for misses.

- The CE push-buffer kcore chain resolver derives its vmap page offset
  per allocation instead of using a fixed shift: the rm_mem size is read
  from its real offset (570.144 layout) and paired with the vmallocinfo
  area page count, so a leading header page is only skipped when it
  actually exists. With an unreadable size the two candidate offsets are
  content-probed with the host's method-stream plausibility test and an
  ambiguous result keeps the status-quo pairing. This removes the
  off-by-one that fed wrong push pages to the pump (content-rescue
  crawl).

2 files changed, 317 insertions, 13 deletions

diff --git a/kernel/gsp_sema_service.c b/kernel/gsp_sema_service.c
index a0efef0..4bed8ab 100644
--- a/kernel/gsp_sema_service.c
+++ b/kernel/gsp_sema_service.c
@@ -339,6 +339,11 @@ struct svc_channel {
 				 * slot a bounded number of times, while a
 				 * clobbered slot loops forever (storm
 				 * detector) */
+	unsigned long gr_hold_until;	/* m150: GR data-arg retry window expiry */
+	u32	gr_hold_idx;	/* GPFIFO slot the window was armed for (identity:
+				 * an entry holds at most once, then is consumed
+				 * even if still unresolved - bounded zeros, never
+				 * a hang) */
 };
 
 /*
@@ -759,6 +764,7 @@ struct sema_svc {
 	u64			fwd_gr_ok;
 	u64			fwd_gr_fallback;
 	u64			fwd_gr_refused;
+	u64			fwd_gr_defer;	/* m150: held for data-arg retry */
 	u64			fwd_gr_timeout;
 	u64			fwd_gr_fault;
 	u64			fwd_gr_bridge;
@@ -6998,11 +7004,14 @@ static void svc_pending_add(u64 va, u32 payload, u32 op)
  * is reproducibly green. Held releases belong to waiters that no longer
  * exist; drop them instead of resolving them against the next session.
  */
+static void svc_gr_diag_reset(void);	/* m150: defined with the fill diag */
+
 static void svc_session_flush(void)
 {
 	unsigned long flags;
 	u32 i;
 
+	svc_gr_diag_reset();		/* m150: per-session diagnostic budgets */
 	svc.map_n = 0;
 	svc_pb_n = 0;
 	svc_pd_found = 0;
@@ -12203,6 +12212,78 @@ static bool svc_gr_page_inline_covers(u64 page)
 #define SVC_GR_DATA_BOUNCE_MAX	100u
 
 static u32 svc_gr_data_log;	/* m142: bounded per-page attribution log */
+static u32 svc_gr_src_log;	/* m150: bounded per-candidate source log */
+static bool svc_gr_data_stalled;	/* m150: high data args unresolved */
+static u64 svc_gr_hold_log;	/* m150: bounded GR-hold proof lines */
+
+/* m150: re-arm both fill diagnostics for each client session. */
+static void svc_gr_diag_reset(void)
+{
+	svc_gr_data_log = 0;
+	svc_gr_src_log = 0;
+}
+
+/*
+ * m150 (second-client zeros diagnosis): identify, read-only, which trusted
+ * source answers a candidate VA at fill time. Mirrors svc_resolve_va's source
+ * order without storing anything, so the fill's choice can be attributed in
+ * the dmesg record (the second client's data VAs resolve through no source -
+ * this names the one that serves the green client and the one missing).
+ * Codes: 0 none, 1 ipin, 2 pin, 3 lat, 4 map, 5 published vaspace,
+ * 6 discovered directory, 7 vidmem window convention.
+ */
+static const char *svc_gr_src_probe(struct nv_vgpu_device *vgpu, u64 va,
+				    u64 *pa_out, u32 *ap_out)
+{
+	u64 pa = 0;
+	int ap = SVC_APERTURE_SYS;
+	u64 wpa;
+	int wap;
+
+	if (!svc_ipin_lookup(va, &pa, &ap)) {
+		*pa_out = pa;
+		*ap_out = svc_ap_of(ap);
+		return "ipin";
+	}
+	if (!svc_pin_lookup(va, &pa, &ap)) {
+		*pa_out = pa;
+		*ap_out = svc_ap_of(ap);
+		return "pin";
+	}
+	if (!svc_lat_lookup(va, &pa, &ap)) {
+		*pa_out = pa;
+		*ap_out = svc_ap_of(ap);
+		return "lat";
+	}
+	if (!svc_map_lookup(va, &pa)) {
+		*pa_out = pa;
+		*ap_out = SVC_APERTURE_SYS;
+		return "map";
+	}
+	wpa = 0;
+	wap = SVC_APERTURE_SYS;
+	if (!svc_walk_vaspaces(vgpu, va & ~0xfffULL, &wpa, &wap)) {
+		*pa_out = wpa + (va & 0xfff);
+		*ap_out = svc_ap_of(wap);
+		return "vas";
+	}
+	wpa = 0;
+	wap = SVC_APERTURE_SYS;
+	if (!svc_walk_discovered(vgpu, va & ~0xfffULL, &wpa, &wap)) {
+		*pa_out = wpa + (va & 0xfff);
+		*ap_out = svc_ap_of(wap);
+		return "disc";
+	}
+	if (va >= SVC_VIDMEM_VA_BASE &&
+	    va - SVC_VIDMEM_VA_BASE + 4 <= svc_fb_shadow_size(vgpu)) {
+		*pa_out = va - SVC_VIDMEM_VA_BASE;
+		*ap_out = SVC_APERTURE_FB;
+		return "win";
+	}
+	*pa_out = 0;
+	*ap_out = 0;
+	return "none";
+}
 
 /*
  * m144 (second-client launch-state divergence): collect device-VA-shaped
@@ -12257,12 +12338,23 @@ static void svc_gr_scan_inline(u64 *wins, u32 *n_cand, u64 page_va)
 static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 {
 	u64 wins[16];
+	bool is_arg[16];	/* m150: candidate came from the argument scan */
 	u32 n_win = 0, n_cand;
 	u32 off = 0, i, w;
 	u8 kind;
+	/*
+	 * m150: high (user/device-space) argument candidates and how many of
+	 * them produced at least one resolved page. None resolving is the
+	 * second-client zeros signature: the engine would run the kernel
+	 * against unresolved pointers. Such a launch is held for a bounded
+	 * retry instead of forwarded (svc_gr_data_stalled).
+	 */
+	u32 n_arg_hi = 0, n_arg_hi_res = 0;
 
 	svc_fwd_req.gr_n_data = 0;
 	svc_fwd_req.gr_n_data_wins = 0;
+	svc_gr_data_stalled = false;
+	memset(is_arg, 0, sizeof(is_arg));
 	/*
 	 * m100: scan every const-bank page the launch carries (QMD words
 	 * 32..47 are the bank LO/HI pairs; the demo's LDG.E base comes from
@@ -12315,6 +12407,11 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 		 * pointers. */
 		svc_gr_scan_inline(wins, &n_cand, pp->va);
 	}
+	/* m150: everything collected so far came from the c[0x0] argument
+	 * page (or its inline bytes) - these are the pointers the kernel
+	 * actually dereferences. */
+	for (w = 0; w < n_cand; w++)
+		is_arg[w] = true;
 	for (i = 0; i < svc_fwd_req.gr_n_pages; i++) {
 		const struct svc_gr_page *pp = &svc_fwd_req.gr_pages[i];
 		u64 hpa = 0;
@@ -12357,6 +12454,23 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 		u64 base = wins[w];
 		u32 p;
 		u32 before = svc_fwd_req.gr_n_data;
+		u64 pba = 0;
+		u32 pba_ap = 0;
+		const char *pba_src = "?";
+
+		if (is_arg[w] && base >= 0x40000000000ULL)
+			n_arg_hi++;
+
+		/* m150: name the source that answers this candidate at fill
+		 * time (read-only probe before the real resolution). */
+		if (svc_gr_src_log < 24) {
+			svc_gr_src_log++;
+			pba_src = svc_gr_src_probe(vgpu, base, &pba, &pba_ap);
+			pr_info("vgsp-sema: svc-gr cand[%u]%s base=0x%llx probe=%s pa=0x%llx ap=%u pd_found=%u ipin_n=%u\n",
+				w, is_arg[w] ? " arg" : "", base,
+				pba_src, pba, pba_ap,
+				svc_pd_found, svc.ipin_n);
+		}
 
 		for (p = 0; p < SVC_GR_DATA_PAGES_PER_PTR; p++) {
 			u64 va = wins[w] + (u64)p * SEMA_PAGE_SIZE;
@@ -12417,12 +12531,56 @@ static void svc_gr_data_fill(struct nv_vgpu_device *vgpu)
 			}
 		}
 		(void)base;
+		if (is_arg[w] && base >= 0x40000000000ULL &&
+		    svc_fwd_req.gr_n_data > before)
+			n_arg_hi_res++;
 		pr_info("vgsp-sema: svc-gr candidate[%u] base=0x%llx -> %u new pages (total %u)\n",
 			w, base, svc_fwd_req.gr_n_data - before,
 			svc_fwd_req.gr_n_data);
+		/*
+		 * m150: an argument pointer that resolves zero pages is the
+		 * second-client zeros signature; record the walk result per
+		 * published vaspace (pdb/entries/ap + rc) so the post-mortem
+		 * shows whether the client's user VA space directory is
+		 * published at fill time and whether the walk reaches it.
+		 */
+		if (is_arg[w] && svc_fwd_req.gr_n_data == before &&
+		    svc_gr_src_log < 64) {
+			int vi;
+			char vb[240];
+			int o = 0;
+
+			svc_gr_src_log++;
+			for (vi = 0; vi < SVC_MAX_VASPACES; vi++) {
+				u64 wa = 0;
+				int wap = SVC_APERTURE_SYS;
+				int wrc;
+
+				if (!svc.vas[vi].valid)
+					continue;
+				wrc = svc_gmmu_walk(vgpu, svc.vas[vi].pdb,
+						    svc.vas[vi].entries, base,
+						    svc.vas[vi].ap, &wa, &wap,
+						    false);
+				o += snprintf(vb + o, sizeof(vb) - o,
+					      " vas[%d] pdb=0x%llx ent=%u ap=%d rc=%d pa=0x%llx",
+					      vi, svc.vas[vi].pdb,
+					      svc.vas[vi].entries,
+					      svc.vas[vi].ap, wrc, wa);
+			}
+			pr_info("vgsp-sema: svc-gr arg-miss base=0x%llx probe=%s n_cand=%u pd_found=%u ipin_n=%u disc=0x%llx/%d%s\n",
+				base, pba_src, n_cand, svc_pd_found,
+				svc.ipin_n, svc_pd_base, svc_pd_ap, vb);
+		}
 	}
-	pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n",
-		n_cand, svc_fwd_req.gr_n_data, (u64)SVC_GR_DATA_RELOC_BASE);
+	svc_gr_data_stalled = (n_arg_hi > 0 && n_arg_hi_res == 0);
+	if (svc_gr_data_stalled)
+		pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved; %u high argument ptr(s), 0 resolved -> launch held for a bounded retry\n",
+			n_cand, svc_fwd_req.gr_n_data, n_arg_hi);
+	else
+		pr_info("vgsp-sema: svc-gr data fill: %u candidate ptrs, %u pages resolved (reloc 0x%llx+)\n",
+			n_cand, svc_fwd_req.gr_n_data,
+			(u64)SVC_GR_DATA_RELOC_BASE);
 }
 
 static int svc_gr_req_fill(struct nv_vgpu_device *vgpu,
@@ -12591,6 +12749,7 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 	if (!gsp_phys_gr_forward_enabled() || !svc_gr_cand.valid ||
 	    svc_gr_cand.h_object != ch->h_object)
 		return false;
+	svc_gr_data_stalled = false;	/* m150: only the fill sets this */
 	svc.fwd_gr_req++;
 	if (!gsp_phys_backend_gr_ready() || !gsp_phys_guest_mem_enabled()) {
 		svc.fwd_gr_refused++;
@@ -12641,6 +12800,26 @@ static bool svc_gr_request(struct nv_vgpu_device *vgpu,
 		return false;
 	}
 	/*
+	 * m150: the fill found high argument pointers that no source can
+	 * translate (the second-client zeros class). Do NOT forward a launch
+	 * whose kernel would dereference unresolved pointers: leave the
+	 * candidate unconsumed and return false so the pump holds the entry
+	 * and retries it on later passes (svc_gr_data_stalled), bounded by
+	 * the channel's gr_hold window at the call site. Once the window
+	 * expires the entry is consumed exactly as before.
+	 */
+	if (svc_gr_data_stalled) {
+		clear_bit_unlock(SVC_FWD_INFLIGHT, &svc_fwd_state);
+		mutex_unlock(&svc_fwd_lock);
+		svc.fwd_gr_refused++;
+		svc.fwd_gr_defer++;
+		if ((svc.fwd_gr_defer & SVC_FWD_LOG_MASK) == 1)
+			pr_info("vgsp-work: GR launch held (data args unresolved) h=0x%x push=0x%llx defer=%llu\n",
+				ch->h_object, svc_gr_cand.push_va,
+				svc.fwd_gr_defer);
+		return false;	/* candidate kept: the pump retries the entry */
+	}
+	/*
 	 * The candidate is consumed: the request copy owns it from here, so a
 	 * later capture/replay cannot re-offer it and the bridge is only
 	 * reached through the request's own latch.
@@ -13879,6 +14058,45 @@ static void svc_pump_channels(struct nv_vgpu_device *vgpu)
 								 */
 								fwd_guest_queued =
 									svc_fwd_request(vgpu, ch);
+								if (fwd_guest_queued &&
+								    !ch->engine_type)
+									ch->gr_hold_idx = ~0u;
+								/*
+								 * m150: the GR fill held this
+								 * launch because its high
+								 * argument pointers do not
+								 * translate yet. Leave the
+								 * entry unconsumed (do not
+								 * write GP_GET) so the pump
+								 * re-captures and retries it;
+								 * one bounded window per entry,
+								 * then fall through and consume
+								 * exactly as before. The slot's
+								 * identity makes the hold
+								 * one-shot per entry: after
+								 * expiry the same entry is
+								 * never re-held.
+								 */
+								if (!fwd_guest_queued &&
+								    !ch->engine_type &&
+								    svc_gr_data_stalled &&
+								    ch->gr_hold_idx != (u32)idx) {
+									u32 hold_ms = gsp_sema_gr_grace_ms ?
+										gsp_sema_gr_grace_ms : 8000;
+
+									ch->gr_hold_idx = (u32)idx;
+									ch->gr_hold_until = jiffies +
+										msecs_to_jiffies(hold_ms);
+									svc.fwd_gr_defer++;
+									if (svc_gr_hold_log < 16) {
+										svc_gr_hold_log++;
+										pr_info("sema-svc: GR hold h=0x%x entry=%u (data args unresolved; retry window %u ms)\n",
+											ch->h_object,
+											(u32)idx, hold_ms);
+									}
+									kfree(pb);
+									break;
+								}
 								/*
 								 * P6b wave 1.5: when
 								 * this pass recorded
diff --git a/tools/guest-diag/guest_pin_stream.py b/tools/guest-diag/guest_pin_stream.py
index 35cbfb9..1da78fb 100644
--- a/tools/guest-diag/guest_pin_stream.py
+++ b/tools/guest-diag/guest_pin_stream.py
@@ -266,11 +266,17 @@ def _vmalloc_pages_for(kc, va):
     return 0
 
 
-def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0):
+def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0,
+                     page_off=0):
     """uvm_rm_mem kptr -> GPU-VA/GPA runs (direct map or page-table walk).
 
     va_lo/va_hi optionally restrict the emitted GPU VA range; the kcore
-    chain resolver targets the CE push-buffer window only.
+    chain resolver targets the CE push-buffer window only. page_off shifts
+    the vmap page paired with GPU data page k: the CE push-buffer vmap is
+    one page longer than its GPU data window on some allocations (a leading
+    header page), so GPU data page k lives at vmap page k + page_off.
+    page_off is derived per allocation (mem_page_off): a fixed offset is
+    wrong (r116/r117 regressed client #1 with a hard +1).
     """
     if not is_kern_ptr(mem):
         return 0
@@ -281,7 +287,7 @@ def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0):
             and (not va_hi or v < va_hi)]
     if not cpu or not gpus:
         return 0
-    size = kc.read64(mem + 0x60) or 0
+    size = _mem_size(kc, mem)
     dm = _direct_map_pa(kc, cpu)
     emitted = 0
     if dm is not None:
@@ -290,14 +296,15 @@ def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0):
             out.append("0x%x,0x%x,%d\n" % (gv, dm, npages))
             emitted += 1
         return emitted
+    area = _vmalloc_pages_for(kc, cpu)
     npages = (max(1, min(size >> 12, 4098)) if size
-              else _vmalloc_pages_for(kc, cpu))
-    if not npages:
-        npages = 1
+              else area or default_pages)
+    if area and page_off + npages > area:
+        npages = max(1, area - page_off)
     runs = []
     cur = None
     for k in range(npages):
-        pa = kc.va_to_pa(cpu + k * 0x1000)
+        pa = kc.va_to_pa(cpu + (k + page_off) * 0x1000)
         if pa is None:
             if cur:
                 runs.append(cur)
@@ -318,6 +325,81 @@ def resolve_mem_pins(kc, mem, out, default_pages=16, va_lo=0, va_hi=0):
     return emitted
 
 
+def _mem_size(kc, mem):
+    """uvm_rm_mem size (bytes) from the 570.144 layout, with a fallback.
+
+    The struct is type(4)+pad(4)+mask(8)+vas[33] (264 B)+proxy_vas+
+    gpu_owner+size, so size lives at +0x128; +0x60 (inside vas[]) reads 0
+    for unused processors and is kept only as a fallback.
+    """
+    for off in (0x128, 0x60):
+        v = kc.read64(mem + off) or 0
+        if 0 < v <= (256 << 20):
+            return v
+    return 0
+
+
+def _push_prefix_ok(buf):
+    """First-4KB method-stream plausibility, mirroring the host's check."""
+    off = 0
+    while off + 4 <= len(buf):
+        hdr = struct.unpack_from('<I', buf, off)[0]
+        if hdr:
+            break
+        off += 4
+    if off + 4 > len(buf):
+        return True		# all zeros: not refutable
+    op = (hdr >> 29) & 0x7
+    if op in (4, 7):
+        return True
+    if op not in (1, 3, 5):
+        return False
+    return ((hdr >> 16) & 0xfff) != 0
+
+
+def _page_content(kc, va):
+    pa = kc.va_to_pa(va & ~0xFFF)
+    if pa is None:
+        return None
+    return kc.read(kc.page_offset_base + pa, 64)
+
+
+def mem_page_off(kc, mem):
+    """Per-allocation vmap offset for a uvm_rm_mem's GPU data pages.
+
+    Derived, never fixed: the vmap area's page count (vmallocinfo) minus
+    the rm_mem's data page count (size at +0x128). A 4097-page area with
+    4096 data pages means one leading header page (offset 1); an area that
+    exactly matches the data means offset 0. When the size is unreadable,
+    the two candidate offsets are content-probed with the host's method-
+    stream plausibility test and the decisively-passing one wins; an
+    ambiguous result keeps 0 (the status-quo pairing).
+    """
+    vas = [kc.read64(mem + 0x10 + i * 8) for i in range(16)]
+    cpu = next((v for v in vas if is_kern_ptr(v)), None)
+    if not cpu or _direct_map_pa(kc, cpu) is not None:
+        return 0
+    area = _vmalloc_pages_for(kc, cpu)
+    if area <= 1:
+        return 0
+    size = _mem_size(kc, mem)
+    if size:
+        data = min(size >> 12, 4098)
+        if 0 <= area - data <= 2:
+            return area - data
+        return 0
+    g = []
+    for off in (0, 1):
+        if off >= area:
+            g.append(0)
+            continue
+        buf = _page_content(kc, cpu + off * 0x1000)
+        g.append(1 if (buf is not None and _push_prefix_ok(buf)) else 0)
+    if g[0] != g[1]:
+        return 0 if g[0] else 1
+    return 0
+
+
 def _emit_ring_pins(kc, gpu_va, entries_cpu, entries, out):
     """Pin the GPFIFO ring pages (and the pages after it, where the GPPUT
     buffer usually sits) from the ring's CPU mapping."""
@@ -444,7 +526,8 @@ def resolve_pb_pins(kc, pb, out):
     mem = kc.read64(pb + 8)
     if not mem:
         return 0
-    return resolve_mem_pins(kc, mem, out, default_pages=64)
+    return resolve_mem_pins(kc, mem, out, default_pages=64,
+                            page_off=mem_page_off(kc, mem))
 
 
 # The CE push-buffer window is the one UVM region the client never maps into
@@ -509,13 +592,16 @@ def resolve_pb_chain(kc, sem, out):
             if mem in RESOLVED_KC_PB:
                 continue
             RESOLVED_KC_PB.add(mem)
+            off = mem_page_off(kc, mem)
             n = resolve_mem_pins(kc, mem, out, default_pages=64,
-                                 va_lo=PB_CE_LO, va_hi=PB_CE_HI)
+                                 va_lo=PB_CE_LO, va_hi=PB_CE_HI,
+                                 page_off=off)
             try:
                 with open('/tmp/guest_pin_stream.log', 'a') as f:
                     f.write('%s kc-chain CE push-buffer mem=0x%x sem=0x%x '
-                            'pin-runs=%d\n'
-                            % (time.strftime('%T'), mem, sem, n))
+                            'pin-runs=%d off=%d size=%d\n'
+                            % (time.strftime('%T'), mem, sem, n, off,
+                               _mem_size(kc, mem)))
             except OSError:
                 pass
     return found